From d780192adae4e42adb6bac081fae3701847815e2 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 29 Apr 2024 15:06:30 +0600 Subject: [PATCH 01/15] instagram: add three more ways to get post info (#469) for total of fucking SIX??? --- src/modules/processing/services/instagram.js | 105 +++++++++++++------ 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index c8f190a4..405086de 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -2,11 +2,38 @@ import { createStream } from "../../stream/manage.js"; import { genericUserAgent } from "../../config.js"; import { getCookie, updateCookie } from "../cookie/manager.js"; -const commonInstagramHeaders = { - 'user-agent': genericUserAgent, - 'sec-gpc': '1', - 'sec-fetch-site': 'same-origin', - 'x-ig-app-id': '936619743392459' +const commonHeaders = { + "user-agent": genericUserAgent, + "sec-gpc": "1", + "sec-fetch-site": "same-origin", + "x-ig-app-id": "936619743392459" +} +const mobileHeaders = { + "x-ig-app-locale": "en_US", + "x-ig-device-locale": "en_US", + "x-ig-mapped-locale": "en_US", + "user-agent": "Instagram 275.0.0.27.98 Android (33/13; 280dpi; 720x1423; Xiaomi; Redmi 7; onclite; qcom; en_US; 458229237)", + "accept-language": "en-US", + "x-fb-http-engine": "Liger", + "x-fb-client-ip": "True", + "x-fb-server-cluster": "True", + "content-length": "0", +} +const embedHeaders = { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "Accept-Language": "en-GB,en;q=0.9", + "Cache-Control": "max-age=0", + "Dnt": "1", + "Priority": "u=0, i", + "Sec-Ch-Ua": 'Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99', + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Platform": "macOS", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", } const cachedDtsg = { @@ -20,7 +47,7 @@ async function findDtsgId(cookie) { const data = await fetch('https://www.instagram.com/', { headers: { - ...commonInstagramHeaders, + ...commonHeaders, cookie } }).then(r => r.text()); @@ -38,7 +65,7 @@ async function findDtsgId(cookie) { async function request(url, cookie, method = 'GET', requestData) { let headers = { - ...commonInstagramHeaders, + ...commonHeaders, 'x-ig-www-claim': cookie?._wwwClaim || '0', 'x-csrftoken': cookie?.values()?.csrftoken, cookie @@ -60,26 +87,36 @@ async function request(url, cookie, method = 'GET', requestData) { return data.json(); } +async function requestMobileApi(id, cookie) { + const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/'); + oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`); + + const oembed = await fetch(oembedURL, { + headers: { + ...mobileHeaders, + cookie + } + }).then(r => r.json()).catch(() => {}); + + const mediaId = oembed?.media_id; + if (!mediaId) return false; + + const mediaInfo = await fetch(`https://i.instagram.com/api/v1/media/${mediaId}/info/`, { + headers: { + ...mobileHeaders, + cookie + } + }).then(r => r.json()).catch(() => {}); + + return mediaInfo?.items?.[0]; +} async function requestHTML(id, cookie) { const data = await fetch(`https://www.instagram.com/p/${id}/embed/captioned/`, { headers: { - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", - "Accept-Language": "en-GB,en;q=0.9", - "Cache-Control": "max-age=0", - "Dnt": "1", - "Priority": "u=0, i", - "Sec-Ch-Ua": 'Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99', - "Sec-Ch-Ua-Mobile": "?0", - "Sec-Ch-Ua-Platform": "macOS", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - ...cookie + ...embedHeaders, + cookie } - }).then(r => r.text()); + }).then(r => r.text()).catch(() => {}); let embedData = JSON.parse(data?.match(/"init",\[\],\[(.*?)\]\],/)[1]); @@ -196,25 +233,29 @@ function extractNewPost(data, id) { } async function getPost(id) { - let data, result, dataType = 'old'; + let data, result; try { const cookie = getCookie('instagram'); - data = await requestHTML(id); + // mobile api (no cookie, cookie) + data = await requestMobileApi(id); + if (!data && cookie) data = await requestMobileApi(id, cookie); + + // html embed (no cookie, cookie) + if (!data) data = await requestHTML(id); if (!data && cookie) data = await requestHTML(id, cookie); - if (!data) { - dataType = 'new'; - data = await requestGQL(id, cookie); - } + // web app graphql api (no cookie, cookie) + if (!data) data = await requestGQL(id); + if (!data && cookie) data = await requestGQL(id, cookie); } catch {} if (!data) return { error: 'ErrorCouldntFetch' }; - if (dataType === 'new') { - result = extractNewPost(data, id) - } else { + if (data?.gql_data) { result = extractOldPost(data, id) + } else { + result = extractNewPost(data, id) } if (result) return result; From 5fbf35a8d3a47de31fb7f6df2e983122d56f40e2 Mon Sep 17 00:00:00 2001 From: jsopn Date: Mon, 29 Apr 2024 18:56:05 +0700 Subject: [PATCH 02/15] refactor: centralize envs and their defaults in `modules/config` (#464) * feat(config): centralized env variables and their default values * fix: fip `corsWildcard` variable check in `corsConfig` * fix(config): use already declared variables and default some strings to undefined * fix: check processingPriority against NaN --- src/cobalt.js | 8 +++--- src/core/api.js | 18 +++++++------- src/core/web.js | 8 +++--- src/modules/config.js | 30 ++++++++++++++++++++++- src/modules/pageRender/elements.js | 4 +-- src/modules/pageRender/page.js | 20 +++++++-------- src/modules/processing/cookie/manager.js | 3 ++- src/modules/processing/services/tiktok.js | 7 +++--- src/modules/stream/manage.js | 6 ++--- src/modules/stream/types.js | 6 ++--- 10 files changed, 68 insertions(+), 42 deletions(-) diff --git a/src/cobalt.js b/src/cobalt.js index 050aec46..473c9b5b 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -6,6 +6,7 @@ import express from "express"; import { Bright, Green, Red } from "./modules/sub/consoleText.js"; import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; import { loadLoc } from "./localization/manager.js"; +import { mode } from "./modules/config.js" import path from 'path'; import { fileURLToPath } from 'url'; @@ -22,13 +23,10 @@ app.disable('x-powered-by'); await loadLoc(); -const apiMode = process.env.API_URL && !process.env.WEB_URL; -const webMode = process.env.WEB_URL && process.env.API_URL; - -if (apiMode) { +if (mode === 'API') { const { runAPI } = await import('./core/api.js'); runAPI(express, app, gitCommit, gitBranch, __dirname) -} else if (webMode) { +} else if (mode === 'WEB') { const { runWeb } = await import('./core/web.js'); await runWeb(express, app, gitCommit, gitBranch, __dirname) } else { diff --git a/src/core/api.js b/src/core/api.js index c7e06284..8ff0b5c1 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -4,7 +4,7 @@ import { randomBytes } from "crypto"; const ipSalt = randomBytes(64).toString('hex'); -import { version } from "../modules/config.js"; +import { env, version } from "../modules/config.js"; import { getJSON } from "../modules/api.js"; import { apiJSON, checkJSONPost, getIP, languageCode } from "../modules/sub/utils.js"; import { Bright, Cyan } from "../modules/sub/consoleText.js"; @@ -14,8 +14,8 @@ import { generateHmac } from "../modules/sub/crypto.js"; import { verifyStream, getInternalStream } from "../modules/stream/manage.js"; export function runAPI(express, app, gitCommit, gitBranch, __dirname) { - const corsConfig = process.env.CORS_WILDCARD === '0' ? { - origin: process.env.CORS_URL, + const corsConfig = !env.corsWildcard ? { + origin: env.corsURL, optionsSuccessStatus: 200 } : {}; @@ -163,9 +163,9 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { version: version, commit: gitCommit, branch: gitBranch, - name: process.env.API_NAME || "unknown", - url: process.env.API_URL, - cors: process.env?.CORS_WILDCARD === "0" ? 0 : 1, + name: env.apiName, + url: env.apiURL, + cors: Number(env.corsWildcard), startTime: `${startTimestamp}` }); default: @@ -194,12 +194,12 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { res.redirect('/api/json') }); - app.listen(process.env.API_PORT || 9000, () => { + app.listen(env.apiPort, () => { console.log(`\n` + `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + `Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` + - `URL: ${Cyan(`${process.env.API_URL}`)}\n` + - `Port: ${process.env.API_PORT || 9000}\n` + `URL: ${Cyan(`${env.apiURL}`)}\n` + + `Port: ${env.apiPort}\n` ) }); } diff --git a/src/core/web.js b/src/core/web.js index 7c0cbf33..626574a3 100644 --- a/src/core/web.js +++ b/src/core/web.js @@ -1,4 +1,4 @@ -import { genericUserAgent, version } from "../modules/config.js"; +import { genericUserAgent, version, env } from "../modules/config.js"; import { apiJSON, languageCode } from "../modules/sub/utils.js"; import { Bright, Cyan } from "../modules/sub/consoleText.js"; @@ -76,12 +76,12 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) { return res.redirect('/') }); - app.listen(process.env.WEB_PORT || 9001, () => { + app.listen(env.webPort, () => { console.log(`\n` + `${Cyan("cobalt")} WEB ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + `Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` + - `URL: ${Cyan(`${process.env.WEB_URL}`)}\n` + - `Port: ${process.env.WEB_PORT || 9001}\n` + `URL: ${Cyan(`${env.webURL}`)}\n` + + `Port: ${env.webPort}\n` ) }) } diff --git a/src/modules/config.js b/src/modules/config.js index 5e079536..b774a8b6 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -12,6 +12,31 @@ Object.values(servicesConfigJson.config).forEach(service => { ) }) +const + apiURL = process.env.API_URL || '', + + // WEB mode related environment variables + webEnvs = { + webPort: process.env.WEB_PORT || 9001, + webURL: process.env.WEB_URL || '', + showSponsors: !!process.env.SHOW_SPONSORS, + isBeta: !!process.env.IS_BETA, + plausibleHostname: process.env.PLAUSIBLE_HOSTNAME, + apiURL + }, + + // API mode related environment variables + apiEnvs = { + apiPort: process.env.API_PORT || 9000, + apiName: process.env.API_NAME || 'unknown', + corsWildcard: process.env.CORS_WILDCARD !== '0', + corsURL: process.env.CORS_URL, + cookiePath: process.env.COOKIE_PATH, + processingPriority: process.env.PROCESSING_PRIORITY && parseInt(process.env.PROCESSING_PRIORITY), + tiktokDeviceInfo: process.env.TIKTOK_DEVICE_INFO && JSON.parse(process.env.TIKTOK_DEVICE_INFO), + apiURL + } + export const services = servicesConfigJson.config, audioIgnore = servicesConfigJson.audioIgnore, @@ -26,4 +51,7 @@ export const supportedAudio = config.supportedAudio, celebrations = config.celebrations, links = config.links, - sponsors = config.sponsors + sponsors = config.sponsors, + mode = (apiURL && !webEnvs.webURL) ? 'API' : + (webEnvs.webURL && apiURL) ? 'WEB' : undefined, + env = mode === 'API' ? apiEnvs : webEnvs \ No newline at end of file diff --git a/src/modules/pageRender/elements.js b/src/modules/pageRender/elements.js index e59385e1..ae14cd88 100644 --- a/src/modules/pageRender/elements.js +++ b/src/modules/pageRender/elements.js @@ -1,4 +1,4 @@ -import { authorInfo, celebrations, sponsors } from "../config.js"; +import { authorInfo, celebrations, sponsors, env } from "../config.js"; import emoji from "../emoji.js"; import { loadFile } from "../sub/loadFromFs.js"; @@ -266,5 +266,5 @@ export function sponsoredList() { } export function betaTag() { - return process.env.IS_BETA ? 'β' : '' + return env.isBeta ? 'β' : '' } diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index 58bb3a97..9deb0a20 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -1,5 +1,5 @@ import { checkbox, collapsibleList, explanation, footerButtons, multiPagePopup, popup, popupWithBottomButtons, sep, settingsCategory, switcher, socialLink, socialLinks, urgentNotice, keyboardShortcuts, webLoc, sponsoredList, betaTag, linkSVG } from "./elements.js"; -import { services as s, authorInfo, version, repo, donations, supportedAudio, links } from "../config.js"; +import { services as s, authorInfo, version, repo, donations, supportedAudio, links, env } from "../config.js"; import { getCommitInfo } from "../sub/currentCommit.js"; import loc from "../../localization/manager.js"; import emoji from "../emoji.js"; @@ -48,10 +48,10 @@ export default function(obj) { ${t("AppTitleCobalt")} - + - + @@ -75,11 +75,11 @@ export default function(obj) { - ${process.env.PLAUSIBLE_HOSTNAME ? + ${env.plausibleHostname ? `` : ''} @@ -169,7 +169,7 @@ export default function(obj) { name: "privacy", title: `${emoji("🔒")} ${t("CollapsePrivacy")}`, body: t("PrivacyPolicy") + `${ - process.env.PLAUSIBLE_HOSTNAME ? `

${t("AnalyticsDescription")}` : '' + env.plausibleHostname ? `

${t("AnalyticsDescription")}` : '' }` }, { name: "legal", @@ -177,7 +177,7 @@ export default function(obj) { body: t("FairUse") }]) }, - ...(process.env.SHOW_SPONSORS ? + ...(env.showSponsors ? [{ text: t("SponsoredBy"), classes: ["sponsored-by-text"], @@ -499,7 +499,7 @@ export default function(obj) { }]) }) + (() => { - if (process.env.PLAUSIBLE_HOSTNAME) { + if (env.plausibleHostname) { return settingsCategory({ name: "privacy", title: t('PrivateAnalytics'), @@ -629,7 +629,7 @@ export default function(obj) {