From 3efc4928a57873945708bc66a5f222e301762d46 Mon Sep 17 00:00:00 2001 From: Cadence Fish Date: Sun, 15 Mar 2020 19:50:29 +1300 Subject: [PATCH] Allow got as request backend --- package-lock.json | 237 ++++++++++++++++++++ package.json | 4 +- src/lib/collectors.js | 68 +++--- src/lib/constants.js | 1 + src/lib/utils/request.js | 41 +++- src/lib/utils/requestbackends/got.js | 46 ++++ src/lib/utils/requestbackends/node-fetch.js | 30 +++ src/lib/utils/requestbackends/reference.js | 45 ++++ src/lib/utils/torswitcher.js | 7 +- src/site/api/proxy.js | 11 +- 10 files changed, 439 insertions(+), 51 deletions(-) create mode 100644 src/lib/utils/requestbackends/got.js create mode 100644 src/lib/utils/requestbackends/node-fetch.js create mode 100644 src/lib/utils/requestbackends/reference.js diff --git a/package-lock.json b/package-lock.json index e7b1803..b7b8368 100644 --- a/package-lock.json +++ b/package-lock.json @@ -204,6 +204,21 @@ "ip-address": "^5.8.9" } }, + "@sindresorhus/is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.0.tgz", + "integrity": "sha512-lXKXfypKo644k4Da4yXkPCrwcvn6SlUW2X2zFbuflKHNjf0w9htru01bo26uMhleMXsDmnZ12eJLdrAZa9MANg==", + "optional": true + }, + "@szmarczak/http-timer": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz", + "integrity": "sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==", + "optional": true, + "requires": { + "defer-to-connect": "^2.0.0" + } + }, "@types/babel-types": { "version": "7.0.7", "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.7.tgz", @@ -217,12 +232,48 @@ "@types/babel-types": "*" } }, + "@types/cacheable-request": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.1.tgz", + "integrity": "sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ==", + "optional": true, + "requires": { + "@types/http-cache-semantics": "*", + "@types/keyv": "*", + "@types/node": "*", + "@types/responselike": "*" + } + }, + "@types/http-cache-semantics": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz", + "integrity": "sha512-c3Xy026kOF7QOTn00hbIllV1dLR9hG9NkSrLQgCVs8NF6sBU+VGWjD3wLPhmh1TYAc7ugCFsvHYMN4VcBN1U1A==", + "optional": true + }, + "@types/keyv": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.1.tgz", + "integrity": "sha512-MPtoySlAZQ37VoLaPcTHCu1RWJ4llDkULYZIzOYxlhxBqYPB0RsRlmMU0R6tahtFe27mIdkHV+551ZWV4PLmVw==", + "optional": true, + "requires": { + "@types/node": "*" + } + }, "@types/node": { "version": "13.7.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", "optional": true }, + "@types/responselike": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", + "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==", + "optional": true, + "requires": { + "@types/node": "*" + } + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -584,6 +635,30 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", "dev": true }, + "cacheable-lookup": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.0.tgz", + "integrity": "sha512-s2piO6LvA7xnL1AR03wuEdSx3BZT3tIJpZ56/lcJwzO/6DTJZlTs7X3lrvPxk6d1PlDe6PrVe2TjlUIZNFglAQ==", + "optional": true, + "requires": { + "keyv": "^4.0.0" + } + }, + "cacheable-request": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.1.tgz", + "integrity": "sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==", + "optional": true, + "requires": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^4.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^2.0.0" + } + }, "caching-transform": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-3.0.2.tgz", @@ -717,6 +792,23 @@ "wordwrap": "0.0.2" } }, + "clone-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", + "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "optional": true, + "requires": { + "mimic-response": "^1.0.0" + }, + "dependencies": { + "mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "optional": true + } + } + }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -933,6 +1025,12 @@ } } }, + "defer-to-connect": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.0.tgz", + "integrity": "sha512-bYL2d05vOSf1JEZNx5vSAtPuBMkX8K9EUutg7zlKvTqKXHt7RhWJFbmd7qakVuf13i+IkGmp6FwSsONOf6VYIg==", + "optional": true + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -1000,6 +1098,12 @@ "domelementtype": "1" } }, + "duplexer3": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", + "optional": true + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -1298,6 +1402,15 @@ "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=" }, + "get-stream": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", + "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==", + "optional": true, + "requires": { + "pump": "^3.0.0" + } + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -1349,6 +1462,46 @@ "minimatch": "~3.0.2" } }, + "got": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-10.6.0.tgz", + "integrity": "sha512-3LIdJNTdCFbbJc+h/EH0V5lpNpbJ6Bfwykk21lcQvQsEcrzdi/ltCyQehFHLzJ/ka0UMH4Slg0hkYvAZN9qUDg==", + "optional": true, + "requires": { + "@sindresorhus/is": "^2.0.0", + "@szmarczak/http-timer": "^4.0.0", + "@types/cacheable-request": "^6.0.1", + "cacheable-lookup": "^2.0.0", + "cacheable-request": "^7.0.1", + "decompress-response": "^5.0.0", + "duplexer3": "^0.1.4", + "get-stream": "^5.0.0", + "lowercase-keys": "^2.0.0", + "mimic-response": "^2.1.0", + "p-cancelable": "^2.0.0", + "p-event": "^4.0.0", + "responselike": "^2.0.0", + "to-readable-stream": "^2.0.0", + "type-fest": "^0.10.0" + }, + "dependencies": { + "decompress-response": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-5.0.0.tgz", + "integrity": "sha512-TLZWWybuxWgoW7Lykv+gq9xvzOsUjQ9tF09Tj6NSTYGMTCHNXzrPnD6Hi+TgZq19PyTAGH4Ll/NIM/eTGglnMw==", + "optional": true, + "requires": { + "mimic-response": "^2.0.0" + } + }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "optional": true + } + } + }, "graceful-fs": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", @@ -1442,6 +1595,12 @@ } } }, + "http-cache-semantics": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", + "optional": true + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -1841,6 +2000,12 @@ "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", "dev": true }, + "json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "optional": true + }, "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -1882,6 +2047,15 @@ "promise": "^7.0.1" } }, + "keyv": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.0.tgz", + "integrity": "sha512-U7ioE8AimvRVLfw4LffyOIRhL2xVgmE8T22L6i0BucSnBUyv4w+I7VN/zVZwRKHOI6ZRUcdMdWHQ8KSUvGpEog==", + "optional": true, + "requires": { + "json-buffer": "3.0.1" + } + }, "kind-of": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", @@ -1999,6 +2173,12 @@ "signal-exit": "^3.0.0" } }, + "lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "optional": true + }, "lru-cache": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", @@ -2351,6 +2531,12 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "normalize-url": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", + "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", + "optional": true + }, "npmlog": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", @@ -2607,6 +2793,27 @@ "own-or": "^1.0.0" } }, + "p-cancelable": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.0.0.tgz", + "integrity": "sha512-wvPXDmbMmu2ksjkB4Z3nZWTSkJEb9lqVdMaCKpZUGJG9TMiNp9XcbG3fn9fPKjem04fJMJnXoyFPk2FmgiaiNg==", + "optional": true + }, + "p-event": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-event/-/p-event-4.1.0.tgz", + "integrity": "sha512-4vAd06GCsgflX4wHN1JqrMzBh/8QZ4j+rzp0cd2scXRwuBEv+QR3wrVA5aLhWDLw4y2WgDKvzWF3CCLmVM1UgA==", + "optional": true, + "requires": { + "p-timeout": "^2.0.1" + } + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "optional": true + }, "p-limit": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.2.tgz", @@ -2625,6 +2832,15 @@ "p-limit": "^2.0.0" } }, + "p-timeout": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-2.0.1.tgz", + "integrity": "sha512-88em58dDVB/KzPEx1X0N3LwFfYZPyDc4B6eF38M1rk9VTZMbxXXgjugz8mmwpS9Ox4BDZ+t6t3QP5+/gazweIA==", + "optional": true, + "requires": { + "p-finally": "^1.0.0" + } + }, "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -3114,6 +3330,15 @@ "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, + "responselike": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz", + "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==", + "optional": true, + "requires": { + "lowercase-keys": "^2.0.0" + } + }, "right-align": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz", @@ -4761,6 +4986,12 @@ "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz", "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc=" }, + "to-readable-stream": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-2.1.0.tgz", + "integrity": "sha512-o3Qa6DGg1CEXshSdvWNX2sN4QHqg03SPq7U6jPXRahlQdl5dK8oXjkU/2/sGrnOZKeGV1zLSO8qPwyKklPPE7w==", + "optional": true + }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -4836,6 +5067,12 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, + "type-fest": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.10.0.tgz", + "integrity": "sha512-EUV9jo4sffrwlg8s0zDhP0T2WD3pru5Xi0+HTE3zTUmBaZNhfkite9PdSJwdXLwPVW0jnAHT56pZHIOYckPEiw==", + "optional": true + }, "typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", diff --git a/package.json b/package.json index c548892..0b13df0 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "name": "bibliogram", + "private": true, "version": "1.0.0", "description": "", "main": "index.js", @@ -23,7 +24,8 @@ "socks-proxy-agent": "github:cloudrac3r/node-socks-proxy-agent#6a26d274b12098dfef6cc2faafd25b0c051f2467" }, "optionalDependencies": { - "@deadcanaries/granax": "^3.2.5" + "@deadcanaries/granax": "^3.2.5", + "got": "^10.6.0" }, "devDependencies": { "tap": "^14.10.6" diff --git a/src/lib/collectors.js b/src/lib/collectors.js index aded837..914c6a0 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -71,41 +71,41 @@ function fetchUserFromHTML(username) { if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN if (res.status === 429) throw constants.symbols.RATE_LIMITED return res - }).then(res => { + }).then(async g => { + const res = await g.response() if (res.status === 404) { throw constants.symbols.NOT_FOUND } else { - return res.text().then(text => { - // require down here or have to deal with require loop. require cache will take care of it anyway. - // User -> Timeline -> TimelineEntry -> collectors -/> User - const User = require("./structures/User") - const sharedData = extractSharedData(text) - const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user) - history.report("user", true) - if (constants.caching.db_user_id) { - const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username) - db.prepare( - "REPLACE INTO Users (username, user_id, created, updated, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url) VALUES " - +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)" - ).run({ - username: user.data.username, - user_id: user.data.id, - created: existing && existing.updated_version === constants.database_version ? existing.created : Date.now(), - updated: Date.now(), - updated_version: constants.database_version, - biography: user.data.biography || null, - post_count: user.posts || 0, - following_count: user.following || 0, - followed_by_count: user.followedBy || 0, - external_url: user.data.external_url || null, - full_name: user.data.full_name || null, - is_private: +user.data.is_private, - is_verified: +user.data.is_verified, - profile_pic_url: user.data.profile_pic_url - }) - } - return user - }) + const text = await g.text() + // require down here or have to deal with require loop. require cache will take care of it anyway. + // User -> Timeline -> TimelineEntry -> collectors -/> User + const User = require("./structures/User") + const sharedData = extractSharedData(text) + const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user) + history.report("user", true) + if (constants.caching.db_user_id) { + const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username) + db.prepare( + "REPLACE INTO Users (username, user_id, created, updated, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url) VALUES " + +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)" + ).run({ + username: user.data.username, + user_id: user.data.id, + created: existing && existing.updated_version === constants.database_version ? existing.created : Date.now(), + updated: Date.now(), + updated_version: constants.database_version, + biography: user.data.biography || null, + post_count: user.posts || 0, + following_count: user.following || 0, + followed_by_count: user.followedBy || 0, + external_url: user.data.external_url || null, + full_name: user.data.full_name || null, + is_private: +user.data.is_private, + is_verified: +user.data.is_verified, + profile_pic_url: user.data.profile_pic_url + }) + } + return user } }).catch(error => { if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) { @@ -202,8 +202,7 @@ function fetchTimelinePage(userID, after) { return requestCache.getOrFetchPromise(`page/${userID}/${after}`, () => { return switcher.request("timeline_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { if (res.status === 429) throw constants.symbols.RATE_LIMITED - return res - }).then(res => res.json()).then(root => { + }).then(g => g.json()).then(root => { /** @type {import("./types").PagedEdges} */ const timeline = root.data.user.edge_owner_to_timeline_media history.report("timeline", true) @@ -259,7 +258,6 @@ function fetchShortcodeData(shortcode) { return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => { return switcher.request("post_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { if (res.status === 429) throw constants.symbols.RATE_LIMITED - return res }).then(res => res.json()).then(root => { /** @type {import("./types").TimelineEntryN3} */ const data = root.data.shortcode_media diff --git a/src/lib/constants.js b/src/lib/constants.js index 1891ee5..07fa04d 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -20,6 +20,7 @@ let constants = { reel_graphql: true } }, + request_backend: "node-fetch", // one of: "node-fetch", "got" // After setting your privacy policy, I suggest you read src/site/html/.well-known/dnt-policy.txt. If you comply with it, // change this to `true` to serve it, which will make extensions like Privacy Badger automatically whitelist the domain. does_not_track: false, diff --git a/src/lib/utils/request.js b/src/lib/utils/request.js index 1df5a46..ca5fa16 100644 --- a/src/lib/utils/request.js +++ b/src/lib/utils/request.js @@ -1,16 +1,39 @@ -const fetch = require("node-fetch").default +const NodeFetch = require("./requestbackends/node-fetch") +const Got = require("./requestbackends/got") +const constants = require("../constants") + +const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" + +const backendStatusLineMap = new Map([ + ["node-fetch", "NF "], + ["got", "GOT"] +]) + +/** + * @returns {import("./requestbackends/reference")} + */ function request(url, options = {}, settings = {}) { if (settings.statusLine === undefined) settings.statusLine = "OUT" if (settings.log === undefined) settings.log = true - if (settings.log) console.log(` -> [${settings.statusLine}] ${url}`) // todo: make more like pinski? - // @ts-ignore - return fetch(url, Object.assign({ - headers: { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" - }, - redirect: "manual" - }, options)) + if (settings.log) console.log(` -> [${settings.statusLine}-${backendStatusLineMap.get(constants.request_backend)}] ${url}`) // todo: make more like pinski? + if (constants.request_backend === "node-fetch") { + return new NodeFetch(url, Object.assign({ + headers: { + "User-Agent": userAgent + }, + redirect: "manual" + }, options)) + } else if (constants.request_backend === "got") { + return new Got(url, Object.assign({ + headers: { + "User-Agent": userAgent + }, + followRedirect: false + }, options)) + } else { + throw new Error("Invalid value for setting `request_backend`.") + } } module.exports.request = request diff --git a/src/lib/utils/requestbackends/got.js b/src/lib/utils/requestbackends/got.js new file mode 100644 index 0000000..a6dc342 --- /dev/null +++ b/src/lib/utils/requestbackends/got.js @@ -0,0 +1,46 @@ +try { + var got = require("got").default +} catch (e) {} + +class Got { + constructor(url, options, stream) { + if (!got) throw new Error("`got` is not installed, either install it or set a different request backend.") + this.url = url + this.options = options + } + + stream() { + return Promise.resolve(got.stream(this.url, this.options)) + } + + send() { + if (!this.instance) { + this.instance = got(this.url, this.options) + } + return this + } + + /** + * @returns {Promise} + */ + response() { + return this.send().instance.then(res => ({ + status: res.statusCode + })) + } + + async check(test) { + await this.send().response().then(res => test(res)) + return this + } + + json() { + return this.send().instance.json() + } + + text() { + return this.send().instance.text() + } +} + +module.exports = Got diff --git a/src/lib/utils/requestbackends/node-fetch.js b/src/lib/utils/requestbackends/node-fetch.js new file mode 100644 index 0000000..bc9287d --- /dev/null +++ b/src/lib/utils/requestbackends/node-fetch.js @@ -0,0 +1,30 @@ +const fetch = require("node-fetch").default + +class NodeFetch { + constructor(url, options) { + this.instance = fetch(url, options) + } + + stream() { + return this.instance.then(res => res.body) + } + + response() { + return this.instance + } + + json() { + return this.instance.then(res => res.json()) + } + + text() { + return this.instance.then(res => res.text()) + } + + async check(test) { + await this.response().then(res => test(res)) + return this + } +} + +module.exports = NodeFetch diff --git a/src/lib/utils/requestbackends/reference.js b/src/lib/utils/requestbackends/reference.js new file mode 100644 index 0000000..aaefdea --- /dev/null +++ b/src/lib/utils/requestbackends/reference.js @@ -0,0 +1,45 @@ +/** + * @typedef GrabResponse + * @property {number} status + */ + +// @ts-nocheck + +class GrabReference { + /** + * @param {string} url + * @param {any} options + */ + constructor(url, options) { + throw new Error("This is the reference class, do not instantiate it.") + } + + // Please help me type this + /** + * @returns {Promise} + */ + stream() {} + + /** + * @returns {Promise} + */ + response() {} + + /** + * @returns {Promise} + */ + json() {} + + /** + * @returns {Promise} + */ + text() {} + + /** + * @param {(res: GrabResponse) => any} + * @returns {Promise} + */ + check(test) {} +} + +module.exports = GrabReference diff --git a/src/lib/utils/torswitcher.js b/src/lib/utils/torswitcher.js index e76195a..9ae5ac8 100644 --- a/src/lib/utils/torswitcher.js +++ b/src/lib/utils/torswitcher.js @@ -21,15 +21,14 @@ class TorSwitcher { * If the test function fails, its error will be rejected here. * Only include rate limit logic in the test function! * @param {string} url - * @param {(res: import("node-fetch").Response) => Promise} test - * @returns {Promise} - * @template T the return value of the test function + * @param {(res: import("./requestbackends/reference").GrabResponse) => any} test + * @returns {Promise} */ request(type, url, test) { if (this.torManager && constants.tor.for[type]) { return this.torManager.request(url, test) } else { - return request(url).then(res => test(res)) + return request(url).check(test) } } } diff --git a/src/site/api/proxy.js b/src/site/api/proxy.js index bb11b80..27a5fdd 100644 --- a/src/site/api/proxy.js +++ b/src/site/api/proxy.js @@ -36,18 +36,25 @@ module.exports = [ Some thumbnails aren't square and would otherwise be stretched on the page without this. If I cropped the images client side, it would have to be done with CSS background-image, which means no . */ - return request(verifyResult.url, {}, {log: false}).then(res => { + return request(verifyResult.url, {}, {log: false}).stream().then(body => { const converter = sharp().resize(width, width, {position: "entropy"}) + body.on("error", error => { + console.error("Response stream emitted an error:", error) + }) converter.on("error", error => { console.error("Sharp instance emitted an error:", error) }) + const piped = body.pipe(converter) + piped.on("error", error => { + console.error("Piped stream emitted na error:", error) + }) return { statusCode: 200, contentType: "image/jpeg", headers: { "Cache-Control": constants.caching.image_cache_control }, - stream: res.body.pipe(converter) + stream: piped } }) } else {