mirrors/bibliogram
1
0
Fork 0
mirror of https://git.sr.ht/~cadence/bibliogram synced 2025-12-14 02:35:06 +00:00
Code Issues Projects Releases Wiki Activity

Add VPN list and applied-privacy subnet list

Browse Source
This commit is contained in:
Cadence Ember
2020-07-31 02:44:28 +12:00
parent db364721d8
commit 271afec9db
5 changed files with 51 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/lib/quota/get_identifier.js
View File

@@ -1,25 +1,53 @@
const {request} = require("../utils/request")
const {log} = require("pinski/util/common")
const constants = require("../constants.js")
const {createChecker} = require("is-in-subnet")
let addresses = []
let addressSet = new Set()
let subnets = []
let checker = createChecker([])
request("https://check.torproject.org/torbulkexitlist").text().then(text => {
const lines = text.split("\n").filter(l => l)
addresses = addresses.concat(lines)
log(`Loaded Tor exit node list (${addresses.length} total)`, "spam")
})
/*
request("https://meta.bibliogram.art/ip_proxy_list.txt").text().then(text => {
const lines = text.split("\n").filter(l => l)
addresses = addresses.concat(lines)
log(`Loaded Bibliogram proxy list (${addresses.length} total)`, "spam")
function getList(url, description) {
return request(url).text().then(text => {
// this is decently fast, but if you have ideas for optimisation, please go for it
let d = Date.now()
const lines = text.split("\n").filter(l => l && l[0] !== "#")
subnets = subnets.concat(lines.filter(l => l.includes("/")))
addresses = addresses.concat(lines.filter(l => !l.includes("/")))
log(`Loaded ${description} (entries: ${lines.length}) (${Date.now()-d} ms)`, "spam")
})
*/
}
if (constants.quota.enabled) {
Promise.all([
getList("https://check.torproject.org/torbulkexitlist", "Tor exit node list"),
getList("https://meta.bibliogram.art/quota-list/vpn-ipv4.txt", "VPN IPv4 list"),
getList("https://meta.bibliogram.art/quota-list/applied-privacy.txt", "applied-privacy.net subnets")
]).then(() => {
let d = Date.now()
checker = createChecker(subnets)
addressSet = new Set(addresses.values())
log(`Created subnet checker (${Date.now()-d} ms)`, "spam")
})
}
function getIdentifier(address) {
if (addresses.includes(address)) return "proxy"
else return address
let d = Date.now()
const result = (() => {
try {
if (address == undefined) return "missing"
else if (checker(address)) return "proxy"
else if (addressSet.has(address)) return "proxy"
else return address
} catch (e) {
// not a valid IP address, or some error like that
console.error(e)
throw e
}
})()
console.log(`identified ${address} -> ${result} in ${Date.now()-d} ms`)
return result
}
module.exports.getIdentifier = getIdentifier

4
src/lib/quota/index.js
View File

@@ -20,7 +20,7 @@ function remaining(req) {
if (!constants.quota.enabled) return Infinity // sure.
const ip = getIPFromReq(req)
const identifier = String(getIdentifier(ip))
const identifier = getIdentifier(ip)
const remaining = limiter.remaining(identifier)
if (constants.quota.track) {
@@ -34,7 +34,7 @@ function add(req, count) {
if (!constants.quota.enabled) return Infinity // why not.
const ip = getIPFromReq(req)
const identifier = String(getIdentifier(ip))
const identifier = getIdentifier(ip)
return limiter.add(identifier, count)
}