From 46274c8da090d89c22dd7fcd61d401210c465052 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 09:18:29 +0000 Subject: [PATCH 01/46] youtube: add support for using OAuth2 tokens --- src/modules/processing/services/youtube.js | 33 +++++++++++++++++++++- src/modules/sub/generate-youtube-tokens.js | 30 ++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 src/modules/sub/generate-youtube-tokens.js diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 39a94f3a..301b37b7 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -24,6 +24,26 @@ const codecMatch = { } } +const transformSessionData = (cookie) => { + if (!cookie) + return; + + const values = cookie.values(); + const REQUIRED_VALUES = [ + 'access_token', 'refresh_token', + 'client_id', 'client_secret', + 'expires' + ]; + + if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) { + return; + } + return { + ...values, + expires: new Date(values.expires), + }; +} + const cloneInnertube = async (customFetch) => { const innertube = await ytBase; if (innertube instanceof Error) { @@ -41,6 +61,17 @@ const cloneInnertube = async (customFetch) => { innertube.session.cache ); + const oauthData = transformSessionData(getCookie('youtube_oauth')); + + if (!session.logged_in && oauthData) { + await session.oauth.init(oauthData); + session.logged_in = true; + } + + if (session.logged_in) { + await session.oauth.refreshIfRequired(); + } + const yt = new Innertube(session); return yt; } @@ -62,7 +93,7 @@ export default async function(o) { } try { - info = await yt.getBasicInfo(o.id, 'IOS'); + info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS'); } catch(e) { if (e?.message === 'This video is unavailable') { return { error: 'ErrorCouldntFetch' }; diff --git a/src/modules/sub/generate-youtube-tokens.js b/src/modules/sub/generate-youtube-tokens.js new file mode 100644 index 00000000..26a6a1cd --- /dev/null +++ b/src/modules/sub/generate-youtube-tokens.js @@ -0,0 +1,30 @@ +import { Innertube } from 'youtubei.js'; + +const bail = (...msg) => (console.error(...msg), process.exit(1)); +const tube = await Innertube.create(); + +tube.session.once( + 'auth-pending', + ({ verification_url, user_code }) => console.log( + `Open ${verification_url} in a browser and enter ${user_code} when asked for the code.` + ) +); + +tube.session.once('auth-error', (err) => bail('An error occurred:', err)); +tube.session.once('auth', ({ status, credentials, ...rest }) => { + if (status !== 'SUCCESS') { + bail('something went wrong', rest); + } + + console.log(credentials); + + console.log( + JSON.stringify( + Object.entries(credentials) + .map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`) + .join('; ') + ) + ); +}); + +await tube.session.signIn(); \ No newline at end of file From 18d43729385336a0df8671133724641fd41a5769 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 09:26:58 +0000 Subject: [PATCH 02/46] youtube: drop cookie support it never really worked --- src/modules/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 301b37b7..3ad956d9 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -56,7 +56,7 @@ const cloneInnertube = async (customFetch) => { innertube.session.api_version, innertube.session.account_index, innertube.session.player, - getCookie('youtube')?.toString(), + undefined, customFetch ?? innertube.session.http.fetch, innertube.session.cache ); From 2387fc2fbb85590efa8040400044e4b41a60d560 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 09:30:12 +0000 Subject: [PATCH 03/46] youtube: update access token on change --- src/modules/processing/services/youtube.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 3ad956d9..49de029b 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -2,7 +2,7 @@ import { Innertube, Session } from 'youtubei.js'; import { env } from '../../config.js'; import { cleanString } from '../../sub/utils.js'; import { fetch } from 'undici' -import { getCookie } from '../cookie/manager.js' +import { getCookie, updateCookieValues } from '../cookie/manager.js' const ytBase = Innertube.create().catch(e => e); @@ -61,7 +61,8 @@ const cloneInnertube = async (customFetch) => { innertube.session.cache ); - const oauthData = transformSessionData(getCookie('youtube_oauth')); + const cookie = getCookie('youtube_oauth'); + const oauthData = transformSessionData(cookie); if (!session.logged_in && oauthData) { await session.oauth.init(oauthData); @@ -70,6 +71,15 @@ const cloneInnertube = async (customFetch) => { if (session.logged_in) { await session.oauth.refreshIfRequired(); + const oldExpiry = new Date(cookie.values().expires); + const newExpiry = session.oauth.credentials.expires; + + if (oldExpiry.getTime() !== newExpiry.getTime()) { + updateCookieValues(cookie, { + ...session.oauth.credentials, + expires: session.oauth.credentials.expires.toISOString() + }); + } } const yt = new Innertube(session); From d08e2ac04f3393d3bc7c0b582ed1bfb4053824b0 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 09:32:23 +0000 Subject: [PATCH 04/46] generate-youtube-tokens: use throw instead of process.exit fuck off deepsource --- src/modules/sub/generate-youtube-tokens.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/modules/sub/generate-youtube-tokens.js b/src/modules/sub/generate-youtube-tokens.js index 26a6a1cd..65d64494 100644 --- a/src/modules/sub/generate-youtube-tokens.js +++ b/src/modules/sub/generate-youtube-tokens.js @@ -1,6 +1,10 @@ import { Innertube } from 'youtubei.js'; -const bail = (...msg) => (console.error(...msg), process.exit(1)); +const bail = (...msg) => { + console.error(...msg); + throw new Error(msg); +}; + const tube = await Innertube.create(); tube.session.once( From 9e09bcab6ea1c38dde3793a6df75933e095e3901 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 11:52:36 +0000 Subject: [PATCH 05/46] refactor: create `util` directory, move tests to it --- package.json | 4 ++-- src/{test => util}/test.js | 2 +- src/{test => util}/testFilenamePresets.js | 0 src/{test => util}/tests.json | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/{test => util}/test.js (98%) rename src/{test => util}/testFilenamePresets.js (100%) rename src/{test => util}/tests.json (100%) diff --git a/package.json b/package.json index 72104aba..52640702 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,9 @@ "scripts": { "start": "node src/cobalt", "setup": "node src/modules/setup", - "test": "node src/test/test", + "test": "node src/util/test", "build": "node src/modules/buildStatic", - "testFilenames": "node src/test/testFilenamePresets" + "testFilenames": "node src/util/testFilenamePresets" }, "repository": { "type": "git", diff --git a/src/test/test.js b/src/util/test.js similarity index 98% rename from src/test/test.js rename to src/util/test.js index f26f3b95..294ff7e7 100644 --- a/src/test/test.js +++ b/src/util/test.js @@ -9,7 +9,7 @@ import { normalizeRequest } from "../modules/processing/request.js"; import { env } from "../modules/config.js"; env.apiURL = 'http://localhost:9000' -let tests = loadJSON('./src/test/tests.json'); +let tests = loadJSON('./src/util/tests.json'); let noTest = []; let failed = []; diff --git a/src/test/testFilenamePresets.js b/src/util/testFilenamePresets.js similarity index 100% rename from src/test/testFilenamePresets.js rename to src/util/testFilenamePresets.js diff --git a/src/test/tests.json b/src/util/tests.json similarity index 100% rename from src/test/tests.json rename to src/util/tests.json From ebe6668bc0c631a34a94a853efea2ecf24f1087b Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 11:52:53 +0000 Subject: [PATCH 06/46] refactor: move generate-youtube-tokens to `util` --- src/{modules/sub => util}/generate-youtube-tokens.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{modules/sub => util}/generate-youtube-tokens.js (100%) diff --git a/src/modules/sub/generate-youtube-tokens.js b/src/util/generate-youtube-tokens.js similarity index 100% rename from src/modules/sub/generate-youtube-tokens.js rename to src/util/generate-youtube-tokens.js From 6c1d8ef6c75cfcdb1cdd66278c43bfd8b15138e7 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 11:54:33 +0000 Subject: [PATCH 07/46] generate-youtube-tokens: add more explanatory text and clean up logging --- src/util/generate-youtube-tokens.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/util/generate-youtube-tokens.js b/src/util/generate-youtube-tokens.js index 65d64494..924afb13 100644 --- a/src/util/generate-youtube-tokens.js +++ b/src/util/generate-youtube-tokens.js @@ -1,4 +1,5 @@ import { Innertube } from 'youtubei.js'; +import { Red } from '../modules/sub/consoleText.js' const bail = (...msg) => { console.error(...msg); @@ -9,9 +10,13 @@ const tube = await Innertube.create(); tube.session.once( 'auth-pending', - ({ verification_url, user_code }) => console.log( - `Open ${verification_url} in a browser and enter ${user_code} when asked for the code.` - ) + ({ verification_url, user_code }) => { + console.log(`${Red('[!]')} The token generated by this script is sensitive and you should not share it with anyone!`); + console.log(` By using this token, you are risking your Google account getting terminated.`); + console.log(` You should ${Red('NOT')} use your personal account!`); + console.log(); + console.log(`Open ${verification_url} in a browser and enter ${user_code} when asked for the code.`); + } ); tube.session.once('auth-error', (err) => bail('An error occurred:', err)); @@ -20,9 +25,8 @@ tube.session.once('auth', ({ status, credentials, ...rest }) => { bail('something went wrong', rest); } - console.log(credentials); - console.log( + 'add this cookie to the youtube_oauth array in your cookies file:', JSON.stringify( Object.entries(credentials) .map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`) From a84d0ddc772218fd5f74ec62aee8d95783425428 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 12:05:18 +0000 Subject: [PATCH 08/46] package.json: remove testFilenames script, add youtube token generation --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 52640702..7d271fea 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "setup": "node src/modules/setup", "test": "node src/util/test", "build": "node src/modules/buildStatic", - "testFilenames": "node src/util/testFilenamePresets" + "token:youtube": "node src/util/generate-youtube-tokens" }, "repository": { "type": "git", From 90e066ac22c8f3b6ab30166a550e532f1e4e5159 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 8 Jun 2024 18:14:10 +0600 Subject: [PATCH 09/46] package: bump version to 7.14.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7d271fea..1d44f380 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.14.3", + "version": "7.14.4", "author": "imput", "exports": "./src/cobalt.js", "type": "module", From 77d167ce1e343bba44477bb08c1a7542e2be6e6f Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 8 Jun 2024 18:15:31 +0600 Subject: [PATCH 10/46] package-lock: update version --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2636146d..a61508b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cobalt", - "version": "7.14.3", + "version": "7.14.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cobalt", - "version": "7.14.3", + "version": "7.14.4", "license": "AGPL-3.0", "dependencies": { "content-disposition-header": "0.6.0", From f3056c6dc36ebf7e4d796314ea4af576116ab4f2 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 8 Jun 2024 18:31:00 +0600 Subject: [PATCH 11/46] servicesConfig: enable reddit back --- src/modules/processing/servicesConfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index a725473e..ae4cd9f0 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -15,7 +15,7 @@ "alias": "reddit videos & gifs", "patterns": ["r/:sub/comments/:id/:title", "user/:user/comments/:id/:title"], "subdomains": "*", - "enabled": false + "enabled": true }, "twitter": { "alias": "twitter videos & voice", From 04d66946fc657cdf5b6ea4871c2a69d8e6c417e5 Mon Sep 17 00:00:00 2001 From: jj Date: Sat, 8 Jun 2024 18:34:18 +0200 Subject: [PATCH 12/46] internal-hls: correctly handle URL concatenation of all types (#560) --- src/modules/stream/internal-hls.js | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/modules/stream/internal-hls.js b/src/modules/stream/internal-hls.js index c6a11455..9fa37e6e 100644 --- a/src/modules/stream/internal-hls.js +++ b/src/modules/stream/internal-hls.js @@ -1,16 +1,27 @@ import { createInternalStream } from './manage.js'; import HLS from 'hls-parser'; -import path from "node:path"; + +function getURL(url) { + try { + return new URL(url); + } catch { + return null; + } +} function transformObject(streamInfo, hlsObject) { if (hlsObject === undefined) { return (object) => transformObject(streamInfo, object); } - const fullUrl = hlsObject.uri.startsWith("/") - ? new URL(hlsObject.uri, streamInfo.url).toString() - : new URL(path.join(streamInfo.url, "/../", hlsObject.uri)).toString(); - hlsObject.uri = createInternalStream(fullUrl, streamInfo); + let fullUrl; + if (getURL(hlsObject.uri)) { + fullUrl = hlsObject.uri; + } else { + fullUrl = new URL(hlsObject.uri, streamInfo.url); + } + + hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo); return hlsObject; } From 1d5fa62271627ab84ac0398f39a3373488ce24ac Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 8 Jun 2024 22:59:30 +0600 Subject: [PATCH 13/46] youtube: add ratelimit error, update sign in error --- src/localization/languages/en.json | 3 ++- src/localization/languages/ru.json | 6 ++++-- src/modules/processing/services/youtube.js | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index fc4d7f80..eecd9ac1 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -158,6 +158,7 @@ "ErrorInvalidContentType": "invalid content type header", "UpdateOneMillion": "1 million users and blazing speed", "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!", - "ErrorYTLogin": "couldn't get this youtube video because it requires sign in.\n\nthis limitation is an a/b test done by google to seemingly stop scraping, coincidentally affecting all 3rd party tools and even their own clients.\n\nyou can track the issue on github." + "ErrorYTLogin": "couldn't get this youtube video because it requires an account to view.\n\nthis limitation is done by google to seemingly stop scraping, affecting all 3rd party tools and even their own clients.\n\ntry again, but if issue persists, {ContactLink}.", + "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}." } } diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index 6487e531..083d7bbc 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -1,7 +1,7 @@ { "name": "русский", "substrings": { - "ContactLink": "глянь статус серверов или напиши о проблеме на github (можно на русском)" + "ContactLink": "глянь статус серверов или напиши о проблеме на github" }, "strings": { "AppTitleCobalt": "кобальт", @@ -158,6 +158,8 @@ "SettingsYoutubeDub": "использовать язык браузера", "SettingsYoutubeDubDescription": "использует главный язык браузера для аудиодорожек на youtube. работает даже если кобальт не переведён в твой язык.", "UpdateOneMillion": "миллион и невероятная скорость", - "ErrorYTAgeRestrict": "это видео ограничено по возрасту, поэтому я не могу его скачать. попробуй другое!" + "ErrorYTAgeRestrict": "это видео ограничено по возрасту, поэтому я не могу его скачать. попробуй другое!", + "ErrorYTLogin": "не удалось получить это видео с youtube, т. к. для его просмотра требуется учетная запись.\n\nтакое ограничение сделано google, чтобы, по-видимому, помешать скрапингу, но в итоге ломает сторонние программы и даже собственные клиенты.\n\nпопробуй ещё раз, но если проблема останется, то {ContactLink}.", + "ErrorYTRateLimit": "youtube ограничил мне частоту запросов. попробуй ещё раз через несколько секунд, но если проблема останется, то {ContactLink}." } } diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 49de029b..1c86b4ed 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -124,6 +124,9 @@ export default async function(o) { return { error: 'ErrorYTAgeRestrict' } } } + if (playability.status === "UNPLAYABLE" && playability.reason.endsWith('request limit.')) { + return { error: 'ErrorYTRateLimit' } + } if (playability.status !== 'OK') return { error: 'ErrorYTUnavailable' }; if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' }; From d2e5b6542f71f3809ba94d56c26f382b5cb62762 Mon Sep 17 00:00:00 2001 From: jj Date: Sat, 15 Jun 2024 18:20:33 +0200 Subject: [PATCH 14/46] api: randomize cipherlist for making requests to services (#574) this makes cobalt less prone to TLS client fingerprinting, as it avoids having the default node.js TLS fingerprint that is shared by all node.js applications. --- src/core/api.js | 4 ++++ src/modules/sub/randomize-ciphers.js | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 src/modules/sub/randomize-ciphers.js diff --git a/src/core/api.js b/src/core/api.js index c90f78f7..a366ad09 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -10,6 +10,7 @@ import loc from "../localization/manager.js"; import { createResponse, normalizeRequest, getIP } from "../modules/processing/request.js"; import { verifyStream, getInternalStream } from "../modules/stream/manage.js"; +import { randomizeCiphers } from '../modules/sub/randomize-ciphers.js'; import { extract } from "../modules/processing/url.js"; import match from "../modules/processing/match.js"; import stream from "../modules/stream/stream.js"; @@ -215,6 +216,9 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { res.redirect('/api/serverInfo') }) + randomizeCiphers(); + setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes + app.listen(env.apiPort, env.listenAddress, () => { console.log(`\n` + `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + diff --git a/src/modules/sub/randomize-ciphers.js b/src/modules/sub/randomize-ciphers.js new file mode 100644 index 00000000..e11e1ee2 --- /dev/null +++ b/src/modules/sub/randomize-ciphers.js @@ -0,0 +1,28 @@ +import tls from 'node:tls'; +import { randomBytes } from 'node:crypto'; + +const ORIGINAL_CIPHERS = tls.DEFAULT_CIPHERS; + +// How many ciphers from the top of the list to shuffle. +// The remaining ciphers are left in the original order. +const TOP_N_SHUFFLE = 8; + +// Modified variation of https://stackoverflow.com/a/12646864 +const shuffleArray = (array) => { + for (let i = array.length - 1; i > 0; i--) { + const j = randomBytes(4).readUint32LE() % array.length; + [array[i], array[j]] = [array[j], array[i]]; + } + + return array; +} + +export const randomizeCiphers = () => { + do { + const cipherList = ORIGINAL_CIPHERS.split(':'); + const shuffled = shuffleArray(cipherList.slice(0, TOP_N_SHUFFLE)); + const retained = cipherList.slice(TOP_N_SHUFFLE); + + tls.DEFAULT_CIPHERS = [ ...shuffled, ...retained ].join(':'); + } while (tls.DEFAULT_CIPHERS === ORIGINAL_CIPHERS); +} From 21d5b4b8d457c0bfe3dfe17029c235e115957200 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 16 Jun 2024 10:52:54 +0000 Subject: [PATCH 15/46] instagram: use correct id when requesting from mobile API --- src/modules/processing/services/instagram.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index 2fc533d8..6ea05178 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -256,11 +256,11 @@ export default function(obj) { if (!media_id && cookie) media_id = await getMediaId(id, { cookie }); // mobile api (bearer) - if (media_id && token) data = await requestMobileApi(id, { token }); + if (media_id && token) data = await requestMobileApi(media_id, { token }); // mobile api (no cookie, cookie) - if (!data && media_id) data = await requestMobileApi(id); - if (!data && media_id && cookie) data = await requestMobileApi(id, { cookie }); + if (media_id && !data) data = await requestMobileApi(media_id); + if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie }); // html embed (no cookie, cookie) if (!data) data = await requestHTML(id); From ef97ff06af7a7d77de943712cfd7176dfe8e5f90 Mon Sep 17 00:00:00 2001 From: jj Date: Sat, 22 Jun 2024 12:57:30 +0200 Subject: [PATCH 16/46] stream: fix some memory leaks in internal stream handling (#581) --- src/modules/stream/internal.js | 23 ++++++++++++++--------- src/modules/stream/manage.js | 11 ++++++++++- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index 535bba2d..ef8ec6f7 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -1,6 +1,5 @@ import { request } from 'undici'; import { Readable } from 'node:stream'; -import { assert } from 'console'; import { getHeaders, pipe } from './shared.js'; import { handleHlsPlaylist, isHlsRequest } from './internal-hls.js'; @@ -36,21 +35,17 @@ async function* readChunks(streamInfo, size) { read += received; } -} - -function chunkedStream(streamInfo, size) { - assert(streamInfo.controller instanceof AbortController); - const stream = Readable.from(readChunks(streamInfo, size)); - return stream; } async function handleYoutubeStream(streamInfo, res) { + const { signal } = streamInfo.controller; + try { const req = await fetch(streamInfo.url, { headers: getHeaders('youtube'), method: 'HEAD', dispatcher: streamInfo.dispatcher, - signal: streamInfo.controller.signal + signal }); streamInfo.url = req.url; @@ -60,7 +55,16 @@ async function handleYoutubeStream(streamInfo, res) { return res.end(); } - const stream = chunkedStream(streamInfo, size); + const generator = readChunks(streamInfo, size); + + const abortGenerator = () => { + generator.return(); + signal.removeEventListener('abort', abortGenerator); + } + + signal.addEventListener('abort', abortGenerator); + + const stream = Readable.from(generator); for (const headerName of ['content-type', 'content-length']) { const headerValue = req.headers.get(headerName); @@ -69,6 +73,7 @@ async function handleYoutubeStream(streamInfo, res) { pipe(stream, res, () => res.end()); } catch { + signal.abort(); res.end(); } } diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 0dec1972..b02c6084 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -78,16 +78,25 @@ export function createInternalStream(url, obj = {}) { } const streamID = nanoid(); + const controller = new AbortController(); internalStreamCache[streamID] = { url, service: obj.service, headers: obj.headers, - controller: new AbortController(), + controller, dispatcher }; let streamLink = new URL('/api/istream', `http://127.0.0.1:${env.apiPort}`); streamLink.searchParams.set('id', streamID); + + const cleanup = () => { + destroyInternalStream(streamLink); + controller.signal.removeEventListener('abort', cleanup); + } + + controller.signal.addEventListener('abort', cleanup); + return streamLink.toString(); } From a5e00be37655866dbe1f741a48e3c48f0cef3dbd Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 22 Jun 2024 17:02:50 +0600 Subject: [PATCH 17/46] services: add support for m.vk.com links closes #576 --- src/modules/processing/servicesConfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index ae4cd9f0..d727b9a5 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -33,6 +33,7 @@ "vk": { "alias": "vk video & clips", "patterns": ["video:userId_:videoId", "clip:userId_:videoId", "clips:duplicate?z=clip:userId_:videoId"], + "subdomains": ["m"], "enabled": true }, "ok": { From 7815838751799071bdc5b1c38637344a8ef28b86 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 22 Jun 2024 17:03:43 +0600 Subject: [PATCH 18/46] package: bump version to 7.14.5 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index a61508b0..9c5434d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cobalt", - "version": "7.14.4", + "version": "7.14.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cobalt", - "version": "7.14.4", + "version": "7.14.5", "license": "AGPL-3.0", "dependencies": { "content-disposition-header": "0.6.0", diff --git a/package.json b/package.json index 1d44f380..b2224a50 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.14.4", + "version": "7.14.5", "author": "imput", "exports": "./src/cobalt.js", "type": "module", From a6733ef0cc64545b99b97995f781f8bc8691a13b Mon Sep 17 00:00:00 2001 From: jj Date: Sun, 23 Jun 2024 14:51:36 +0200 Subject: [PATCH 19/46] stream/internal: refactor, abort controller in more places (#583) --- src/modules/stream/internal.js | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index ef8ec6f7..5a28af93 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -39,6 +39,7 @@ async function* readChunks(streamInfo, size) { async function handleYoutubeStream(streamInfo, res) { const { signal } = streamInfo.controller; + const cleanup = () => (res.end(), streamInfo.controller.abort()); try { const req = await fetch(streamInfo.url, { @@ -52,7 +53,7 @@ async function handleYoutubeStream(streamInfo, res) { const size = BigInt(req.headers.get('content-length')); if (req.status !== 200 || !size) { - return res.end(); + return cleanup(); } const generator = readChunks(streamInfo, size); @@ -71,17 +72,15 @@ async function handleYoutubeStream(streamInfo, res) { if (headerValue) res.setHeader(headerName, headerValue); } - pipe(stream, res, () => res.end()); + pipe(stream, res, cleanup); } catch { - signal.abort(); - res.end(); + cleanup(); } } -export async function internalStream(streamInfo, res) { - if (streamInfo.service === 'youtube') { - return handleYoutubeStream(streamInfo, res); - } +async function handleGenericStream(streamInfo, res) { + const { signal } = streamInfo.controller; + const cleanup = () => (res.end(), streamInfo.controller.abort()); try { const req = await request(streamInfo.url, { @@ -90,7 +89,7 @@ export async function internalStream(streamInfo, res) { host: undefined }, dispatcher: streamInfo.dispatcher, - signal: streamInfo.controller.signal, + signal, maxRedirections: 16 }); @@ -105,9 +104,17 @@ export async function internalStream(streamInfo, res) { if (isHlsRequest(req)) { await handleHlsPlaylist(streamInfo, req, res); } else { - pipe(req.body, res, () => res.end()); + pipe(req.body, res, cleanup); } } catch { - streamInfo.controller.abort(); + cleanup(); } +} + +export function internalStream(streamInfo, res) { + if (streamInfo.service === 'youtube') { + return handleYoutubeStream(streamInfo, res); + } + + return handleGenericStream(streamInfo, res); } \ No newline at end of file From 33c3c398fc6361ddc4f684d8334e7a41482de7bf Mon Sep 17 00:00:00 2001 From: jj Date: Sun, 23 Jun 2024 17:37:02 +0200 Subject: [PATCH 20/46] stream/internal: don't abort immediately after close for generic streams (#584) * stream: move closeRequest to shared functions * stream: use closeRequest instead of abort() directly * stream/internal: don't abort immediately after close for generic streams --- src/modules/stream/internal.js | 12 +++++++----- src/modules/stream/manage.js | 3 ++- src/modules/stream/shared.js | 4 ++++ src/modules/stream/types.js | 6 +----- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index 5a28af93..8ab2ec76 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -1,6 +1,6 @@ import { request } from 'undici'; import { Readable } from 'node:stream'; -import { getHeaders, pipe } from './shared.js'; +import { closeRequest, getHeaders, pipe } from './shared.js'; import { handleHlsPlaylist, isHlsRequest } from './internal-hls.js'; const CHUNK_SIZE = BigInt(8e6); // 8 MB @@ -26,7 +26,7 @@ async function* readChunks(streamInfo, size) { const received = BigInt(chunk.headers['content-length']); if (received < expected / 2n) { - streamInfo.controller.abort(); + closeRequest(streamInfo.controller); } for await (const data of chunk.body) { @@ -39,7 +39,7 @@ async function* readChunks(streamInfo, size) { async function handleYoutubeStream(streamInfo, res) { const { signal } = streamInfo.controller; - const cleanup = () => (res.end(), streamInfo.controller.abort()); + const cleanup = () => (res.end(), closeRequest(streamInfo.controller)); try { const req = await fetch(streamInfo.url, { @@ -80,7 +80,7 @@ async function handleYoutubeStream(streamInfo, res) { async function handleGenericStream(streamInfo, res) { const { signal } = streamInfo.controller; - const cleanup = () => (res.end(), streamInfo.controller.abort()); + const cleanup = () => res.end(); try { const req = await request(streamInfo.url, { @@ -94,12 +94,13 @@ async function handleGenericStream(streamInfo, res) { }); res.status(req.statusCode); + req.body.on('error', () => {}); for (const [ name, value ] of Object.entries(req.headers)) res.setHeader(name, value) if (req.statusCode < 200 || req.statusCode > 299) - return res.end(); + return cleanup(); if (isHlsRequest(req)) { await handleHlsPlaylist(streamInfo, req, res); @@ -107,6 +108,7 @@ async function handleGenericStream(streamInfo, res) { pipe(req.body, res, cleanup); } } catch { + closeRequest(streamInfo.controller); cleanup(); } } diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index b02c6084..031d6711 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -5,6 +5,7 @@ import { nanoid } from "nanoid"; import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; import { env } from "../config.js"; import { strict as assert } from "assert"; +import { closeRequest } from "./shared.js"; // optional dependency const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); @@ -109,7 +110,7 @@ export function destroyInternalStream(url) { const id = url.searchParams.get('id'); if (internalStreamCache[id]) { - internalStreamCache[id].controller.abort(); + closeRequest(internalStreamCache[id].controller); delete internalStreamCache[id]; } } diff --git a/src/modules/stream/shared.js b/src/modules/stream/shared.js index 1e2f2e25..fd7d1569 100644 --- a/src/modules/stream/shared.js +++ b/src/modules/stream/shared.js @@ -16,6 +16,10 @@ const serviceHeaders = { } } +export function closeRequest(controller) { + try { controller.abort() } catch {} +} + export function closeResponse(res) { if (!res.headersSent) { res.sendStatus(500); diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 2372d6c1..000b7f7f 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -6,7 +6,7 @@ import { create as contentDisposition } from "content-disposition-header"; import { metadataManager } from "../sub/utils.js"; import { destroyInternalStream } from "./manage.js"; import { env, ffmpegArgs, hlsExceptions } from "../config.js"; -import { getHeaders, closeResponse, pipe } from "./shared.js"; +import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js"; function toRawHeaders(headers) { return Object.entries(headers) @@ -14,10 +14,6 @@ function toRawHeaders(headers) { .join(''); } -function closeRequest(controller) { - try { controller.abort() } catch {} -} - function killProcess(p) { // ask the process to terminate itself gracefully p?.kill('SIGTERM'); From 08c7aa1ce11b5092e545ae26851554e08543fdd3 Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 23 Jun 2024 22:13:36 +0600 Subject: [PATCH 21/46] stream: add support for remuxing multiple m3u8 files --- src/modules/processing/matchActionDecider.js | 14 ++++++++++---- src/modules/stream/types.js | 4 ++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index f7ed3da9..74f0f8c7 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -24,7 +24,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di else if (r.isGif && toGif) action = "gif"; else if (isAudioMuted) action = "muteVideo"; else if (isAudioOnly) action = "audio"; - else if (r.isM3U8) action = "singleM3U8"; + else if (r.isM3U8) action = "m3u8"; else action = "video"; if (action === "picker" || action === "audio") { @@ -48,13 +48,19 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di params = { type: "gif" } break; - case "singleM3U8": - params = { type: "remux" } + case "m3u8": + params = { + type: Array.isArray(r.urls) ? "render" : "remux" + } break; case "muteVideo": + let muteType = "mute"; + if (Array.isArray(r.urls) && !r.isM3U8) { + muteType = "bridge"; + } params = { - type: Array.isArray(r.urls) ? "bridge" : "mute", + type: muteType, u: Array.isArray(r.urls) ? r.urls[0] : r.urls, mute: true } diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 000b7f7f..af4aa2e5 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -92,6 +92,10 @@ export function streamLiveRender(streamInfo, res) { args = args.concat(ffmpegArgs[format]); + if (hlsExceptions.includes(streamInfo.service)) { + args.push('-bsf:a', 'aac_adtstoasc') + } + if (streamInfo.metadata) { args = args.concat(metadataManager(streamInfo.metadata)) } From eb05c4b938cebc86a0702527f388e7e6395f4405 Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 23 Jun 2024 22:17:11 +0600 Subject: [PATCH 22/46] stream/internal-hls: transform HLS map when defined in playlist header --- src/modules/stream/internal-hls.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/modules/stream/internal-hls.js b/src/modules/stream/internal-hls.js index 9fa37e6e..3ed7e17b 100644 --- a/src/modules/stream/internal-hls.js +++ b/src/modules/stream/internal-hls.js @@ -23,6 +23,10 @@ function transformObject(streamInfo, hlsObject) { hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo); + if (hlsObject.map) { + hlsObject.map = transformObject(streamInfo, hlsObject.map); + } + return hlsObject; } From 0432232ea4211c9abc405159e722b5cddd51bf72 Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 23 Jun 2024 23:02:57 +0600 Subject: [PATCH 23/46] vimeo: use HLS playlists instead of dash manifest --- src/modules/processing/services/vimeo.js | 60 ++++++++++++++++-------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js index 69a36eca..96364dac 100644 --- a/src/modules/processing/services/vimeo.js +++ b/src/modules/processing/services/vimeo.js @@ -1,6 +1,8 @@ import { env } from "../../config.js"; import { cleanString } from '../../sub/utils.js'; +import HLS from "hls-parser"; + const resolutionMatch = { "3840": "2160", "2732": "1440", @@ -33,7 +35,7 @@ export default async function(obj) { url.searchParams.set('h', obj.password); } - let api = await fetch(url) + const api = await fetch(url) .then(r => r.json()) .catch(() => {}); if (!api) return { error: 'ErrorCouldntFetch' }; @@ -43,14 +45,15 @@ export default async function(obj) { if (!obj.isAudioOnly && JSON.stringify(api).includes('"progressive":[{')) downloadType = "progressive"; - let fileMetadata = { + const fileMetadata = { title: cleanString(api.video.title.trim()), artist: cleanString(api.video.owner.name.trim()), } if (downloadType !== "dash") { if (qualityMatch[quality]) quality = qualityMatch[quality]; - let all = api.request.files.progressive.sort((a, b) => Number(b.width) - Number(a.width)); + + const all = api.request.files.progressive.sort((a, b) => Number(b.width) - Number(a.width)); let best = all[0]; let bestQuality = all[0].quality.split('p')[0]; @@ -59,7 +62,7 @@ export default async function(obj) { } if (Number(quality) < Number(bestQuality)) { - best = all.find(i => i.quality.split('p')[0] === quality); + best = all.find(v => v.quality.split('p')[0] === quality); } if (!best) return { error: 'ErrorEmptyDownload' }; @@ -74,26 +77,43 @@ export default async function(obj) { if (api.video.duration > env.durationLimit) return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - let masterJSONURL = api.request.files.dash.cdns.akfire_interconnect_quic.url; - let masterJSON = await fetch(masterJSONURL).then(r => r.json()).catch(() => {}); + const urlMasterHLS = api.request.files.hls.cdns.akfire_interconnect_quic.url; - if (!masterJSON) return { error: 'ErrorCouldntFetch' }; - if (!masterJSON.video) return { error: 'ErrorEmptyDownload' }; + const masterHLS = await fetch(urlMasterHLS) + .then(r => r.text()) + .catch(() => {}); - let masterJSON_Video = masterJSON.video - .sort((a, b) => Number(b.width) - Number(a.width)) - .filter(a => ["dash", "mp42"].includes(a.format)); + if (!masterHLS) return { error: 'ErrorCouldntFetch' }; - let bestVideo = masterJSON_Video[0]; - if (Number(quality) < Number(resolutionMatch[bestVideo.width])) { - bestVideo = masterJSON_Video.find(i => resolutionMatch[i.width] === quality) + const variants = HLS.parse(masterHLS).variants.sort( + (a, b) => Number(b.bandwidth) - Number(a.bandwidth) + ); + if (!variants) return { error: 'ErrorEmptyDownload' }; + + let bestQuality; + if (Number(quality) < Number(resolutionMatch[variants[0].resolution.width])) { + bestQuality = variants.find(v => + (Number(quality) === Number(resolutionMatch[v.resolution.width])) + ); + } + if (!bestQuality) bestQuality = variants[0]; + + const expandLink = (url) => { + return new URL(url, urlMasterHLS).toString(); + }; + + let urls = expandLink(bestQuality.uri); + + const audioPath = bestQuality?.audio[0]?.uri; + if (audioPath) { + urls = [ + urls, + expandLink(audioPath) + ] } - let masterM3U8 = `${masterJSONURL.split("/sep/")[0]}/sep/video/${bestVideo.id}/master.m3u8`; - const fallbackResolution = bestVideo.height > bestVideo.width ? bestVideo.width : bestVideo.height; - return { - urls: masterM3U8, + urls, isM3U8: true, fileMetadata: fileMetadata, filenameAttributes: { @@ -101,8 +121,8 @@ export default async function(obj) { id: obj.id, title: fileMetadata.title, author: fileMetadata.artist, - resolution: `${bestVideo.width}x${bestVideo.height}`, - qualityLabel: `${resolutionMatch[bestVideo.width] || fallbackResolution}p`, + resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`, + qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`, extension: "mp4" } } From cc4abbb3e26491daf211f47b935e7566a095ecb1 Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 23 Jun 2024 23:15:05 +0600 Subject: [PATCH 24/46] vimeo: remove progressive parsing it's no longer returned by the api --- src/modules/processing/services/vimeo.js | 39 ------------------------ 1 file changed, 39 deletions(-) diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js index 96364dac..e48317d6 100644 --- a/src/modules/processing/services/vimeo.js +++ b/src/modules/processing/services/vimeo.js @@ -16,16 +16,6 @@ const resolutionMatch = { "426": "240" } -const qualityMatch = { - "2160": "4K", - "1440": "2K", - "480": "540", - - "4K": "2160", - "2K": "1440", - "540": "480" -} - export default async function(obj) { let quality = obj.quality === "max" ? "9000" : obj.quality; if (!quality || obj.isAudioOnly) quality = "9000"; @@ -40,40 +30,11 @@ export default async function(obj) { .catch(() => {}); if (!api) return { error: 'ErrorCouldntFetch' }; - let downloadType = "dash"; - - if (!obj.isAudioOnly && JSON.stringify(api).includes('"progressive":[{')) - downloadType = "progressive"; - const fileMetadata = { title: cleanString(api.video.title.trim()), artist: cleanString(api.video.owner.name.trim()), } - if (downloadType !== "dash") { - if (qualityMatch[quality]) quality = qualityMatch[quality]; - - const all = api.request.files.progressive.sort((a, b) => Number(b.width) - Number(a.width)); - let best = all[0]; - - let bestQuality = all[0].quality.split('p')[0]; - if (qualityMatch[bestQuality]) { - bestQuality = qualityMatch[bestQuality] - } - - if (Number(quality) < Number(bestQuality)) { - best = all.find(v => v.quality.split('p')[0] === quality); - } - - if (!best) return { error: 'ErrorEmptyDownload' }; - - return { - urls: best.url, - audioFilename: `vimeo_${obj.id}_audio`, - filename: `vimeo_${obj.id}_${best.width}x${best.height}.mp4` - } - } - if (api.video.duration > env.durationLimit) return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; From 850877369c05b1f6fb999bc2cdffc4a76e68e543 Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 23 Jun 2024 23:22:58 +0600 Subject: [PATCH 25/46] vimeo: clean up & fix 144p quality --- src/modules/processing/services/vimeo.js | 31 +++++++++++++----------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js index e48317d6..2198bae7 100644 --- a/src/modules/processing/services/vimeo.js +++ b/src/modules/processing/services/vimeo.js @@ -4,21 +4,22 @@ import { cleanString } from '../../sub/utils.js'; import HLS from "hls-parser"; const resolutionMatch = { - "3840": "2160", - "2732": "1440", - "2560": "1440", - "2048": "1080", - "1920": "1080", - "1366": "720", - "1280": "720", - "960": "480", - "640": "360", - "426": "240" + "3840": 2160, + "2732": 1440, + "2560": 1440, + "2048": 1080, + "1920": 1080, + "1366": 720, + "1280": 720, + "960": 480, + "640": 360, + "426": 240 } export default async function(obj) { - let quality = obj.quality === "max" ? "9000" : obj.quality; - if (!quality || obj.isAudioOnly) quality = "9000"; + let quality = obj.quality === "max" ? 9000 : Number(obj.quality); + if (quality < 240) quality = 240; + if (!quality || obj.isAudioOnly) quality = 9000; const url = new URL(`https://player.vimeo.com/video/${obj.id}/config`); if (obj.password) { @@ -52,9 +53,9 @@ export default async function(obj) { if (!variants) return { error: 'ErrorEmptyDownload' }; let bestQuality; - if (Number(quality) < Number(resolutionMatch[variants[0].resolution.width])) { + if (quality < resolutionMatch[variants[0].resolution.width]) { bestQuality = variants.find(v => - (Number(quality) === Number(resolutionMatch[v.resolution.width])) + (quality === resolutionMatch[v.resolution.width]) ); } if (!bestQuality) bestQuality = variants[0]; @@ -71,6 +72,8 @@ export default async function(obj) { urls, expandLink(audioPath) ] + } else if (obj.isAudioOnly) { + return { error: 'ErrorEmptyDownload' }; } return { From de7df94271e23e3cf5f7e3a0bf7400e7a1b4e166 Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 23 Jun 2024 23:26:43 +0600 Subject: [PATCH 26/46] vimeo: use proper local variable name in expandLink --- src/modules/processing/services/vimeo.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js index 2198bae7..a80561f9 100644 --- a/src/modules/processing/services/vimeo.js +++ b/src/modules/processing/services/vimeo.js @@ -60,8 +60,8 @@ export default async function(obj) { } if (!bestQuality) bestQuality = variants[0]; - const expandLink = (url) => { - return new URL(url, urlMasterHLS).toString(); + const expandLink = (path) => { + return new URL(path, urlMasterHLS).toString(); }; let urls = expandLink(bestQuality.uri); From b51bcc2a7cd247b88bd73cf20624ec4023de7eb9 Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 23 Jun 2024 23:35:29 +0600 Subject: [PATCH 27/46] vimeo: added more checks to avoid exceptions --- src/modules/processing/services/vimeo.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js index a80561f9..d79d2e92 100644 --- a/src/modules/processing/services/vimeo.js +++ b/src/modules/processing/services/vimeo.js @@ -29,6 +29,7 @@ export default async function(obj) { const api = await fetch(url) .then(r => r.json()) .catch(() => {}); + if (!api) return { error: 'ErrorCouldntFetch' }; const fileMetadata = { @@ -36,10 +37,12 @@ export default async function(obj) { artist: cleanString(api.video.owner.name.trim()), } - if (api.video.duration > env.durationLimit) + if (api.video?.duration > env.durationLimit) return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - const urlMasterHLS = api.request.files.hls.cdns.akfire_interconnect_quic.url; + const urlMasterHLS = api.request?.files?.hls?.cdns?.akfire_interconnect_quic?.url; + + if (!urlMasterHLS) return { error: 'ErrorCouldntFetch' } const masterHLS = await fetch(urlMasterHLS) .then(r => r.text()) @@ -47,17 +50,19 @@ export default async function(obj) { if (!masterHLS) return { error: 'ErrorCouldntFetch' }; - const variants = HLS.parse(masterHLS).variants.sort( + const variants = HLS.parse(masterHLS)?.variants?.sort( (a, b) => Number(b.bandwidth) - Number(a.bandwidth) ); - if (!variants) return { error: 'ErrorEmptyDownload' }; + if (!variants || variants.length === 0) return { error: 'ErrorEmptyDownload' }; let bestQuality; - if (quality < resolutionMatch[variants[0].resolution.width]) { + + if (quality < resolutionMatch[variants[0]?.resolution?.width]) { bestQuality = variants.find(v => (quality === resolutionMatch[v.resolution.width]) ); } + if (!bestQuality) bestQuality = variants[0]; const expandLink = (path) => { From ed905fd60be554bc9da20db7cb1b8c18b88078ac Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Mon, 24 Jun 2024 17:42:29 +0000 Subject: [PATCH 28/46] stream/manage: inherit controller from parent istream if it exists --- src/modules/stream/manage.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 031d6711..06c73a6b 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -1,6 +1,7 @@ import NodeCache from "node-cache"; import { randomBytes } from "crypto"; import { nanoid } from "nanoid"; +import { setMaxListeners } from "node:events"; import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; import { env } from "../config.js"; @@ -79,7 +80,13 @@ export function createInternalStream(url, obj = {}) { } const streamID = nanoid(); - const controller = new AbortController(); + let controller = obj.controller; + + if (!controller) { + controller = new AbortController(); + setMaxListeners(Infinity, controller.signal); + } + internalStreamCache[streamID] = { url, service: obj.service, From af80db36341955e49ef42514fc05b99e42457d35 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 29 Jun 2024 18:59:19 +0600 Subject: [PATCH 29/46] github: add issue template for service request --- .github/ISSUE_TEMPLATE/service-request.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/service-request.md diff --git a/.github/ISSUE_TEMPLATE/service-request.md b/.github/ISSUE_TEMPLATE/service-request.md new file mode 100644 index 00000000..61824423 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/service-request.md @@ -0,0 +1,18 @@ +--- +name: service request +about: request service support in cobalt +title: 'add support for [service name]' +labels: service request +assignees: '' + +--- + +**service name & description** +provide the service name and brief description of what it is. + +**link samples for the service you'd like cobalt to support** +list of links that cobalt should recognize. +could be regular video link, shared video link, mobile video link, shortened link, etc. + +**additional context** +any additional context or screenshots should go here. if there aren't any, just remove this part. From 5f60b8274e867c8ff434ac5cbe209ab3b0b652eb Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 29 Jun 2024 19:04:45 +0600 Subject: [PATCH 30/46] github: add an issue template for hosting help --- .github/ISSUE_TEMPLATE/hosting-help.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/hosting-help.md diff --git a/.github/ISSUE_TEMPLATE/hosting-help.md b/.github/ISSUE_TEMPLATE/hosting-help.md new file mode 100644 index 00000000..e340062d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/hosting-help.md @@ -0,0 +1,12 @@ +--- +name: instance hosting help +about: ask any question regarding cobalt instance hosting +title: '[short description of the problem]' +labels: instance hosting help +assignees: '' + +--- + +**problem description** +describe what issue you're having, clearly and concisely. +support your description with screenshots/links/etc when needed. From 40b24d30b31f116589ef20c6fdcb8eec74df6893 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 29 Jun 2024 19:11:13 +0600 Subject: [PATCH 31/46] github: update bug report template improved formatting & clarity --- .github/ISSUE_TEMPLATE/bug-report.md | 32 ++++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index e5429401..4370171c 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,32 +1,36 @@ --- name: bug report -about: report an issue with downloads or something else -title: '' +about: report a global issue with the cobalt codebase +title: '[short description of the bug]' labels: bug assignees: '' --- -**bug description** -a clear and concise description of what the bug is. +### bug description +clear and concise description of what the issue is. -**reproduction steps** -steps to reproduce the behavior: +### reproduction steps +steps to reproduce the described behavior. +here's an example of what it could look like: 1. go to '...' 2. click on '....' -3. download this video: **[link here]** +3. download [media type] from [service] 4. see error -**screenshots** -if applicable, add screenshots or screen recordings to help explain your problem. +### screenshots +if applicable, add screenshots or screen recordings to support your explanation. +if not, remove this section. -**links** -if applicable, add links that cause the issue. more = better. +### links +if applicable, add links that cause the issue. more = better. +if not, remove this section. -**platform** +### platform information - OS [e.g. iOS, windows] - browser [e.g. chrome, safari, firefox] - version [e.g. 115] -**additional context** -add any other context about the problem here. +### additional context +add any other context about the problem here if applicable. +if not, remove this section. From 59eabe2ada33f2d6e313344efe4c04e081735cc5 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 29 Jun 2024 19:11:52 +0600 Subject: [PATCH 32/46] github: update formatting in hosting help template --- .github/ISSUE_TEMPLATE/hosting-help.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/hosting-help.md b/.github/ISSUE_TEMPLATE/hosting-help.md index e340062d..d10a677b 100644 --- a/.github/ISSUE_TEMPLATE/hosting-help.md +++ b/.github/ISSUE_TEMPLATE/hosting-help.md @@ -7,6 +7,6 @@ assignees: '' --- -**problem description** +### problem description describe what issue you're having, clearly and concisely. support your description with screenshots/links/etc when needed. From c27d1bbbeb077c4dea30cd31de116cc74f553ef9 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 29 Jun 2024 19:12:19 +0600 Subject: [PATCH 33/46] github: update formatting in service request template --- .github/ISSUE_TEMPLATE/service-request.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/service-request.md b/.github/ISSUE_TEMPLATE/service-request.md index 61824423..ca948656 100644 --- a/.github/ISSUE_TEMPLATE/service-request.md +++ b/.github/ISSUE_TEMPLATE/service-request.md @@ -7,12 +7,12 @@ assignees: '' --- -**service name & description** +### service name & description provide the service name and brief description of what it is. -**link samples for the service you'd like cobalt to support** +### link samples for the service you'd like cobalt to support list of links that cobalt should recognize. could be regular video link, shared video link, mobile video link, shortened link, etc. -**additional context** +### additional context any additional context or screenshots should go here. if there aren't any, just remove this part. From 94f512f76808489bfcc01e9d9acb85f3b15f6358 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 29 Jun 2024 19:17:48 +0600 Subject: [PATCH 34/46] github: update feature request template better clarity & formatting --- .github/ISSUE_TEMPLATE/feature-request.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 18307f4f..806b2bdf 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -1,17 +1,15 @@ --- name: feature request about: suggest a feature for cobalt -title: '' +title: '[short feature request description]' labels: feature request assignees: '' --- -**describe the feature you'd like to see** -a clear and concise description of what you want to happen. +### describe the feature you'd like to see +clear and concise description of the feature you want to see in cobalt. -**describe alternatives you've considered** -a clear and concise description of any alternative solutions or features you've considered. - -**additional context** -add any other context or screenshots about the feature request here. +### additional context +if applicable, add any other context or screenshots related to the feature request here. +if not, remove this section. From ec786f2babf97974dd357ba2d9c7cc558a138576 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 29 Jun 2024 19:20:48 +0600 Subject: [PATCH 35/46] github: add an issue template for bugs only on main instance --- .github/ISSUE_TEMPLATE/bug-main-instance.md | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-main-instance.md diff --git a/.github/ISSUE_TEMPLATE/bug-main-instance.md b/.github/ISSUE_TEMPLATE/bug-main-instance.md new file mode 100644 index 00000000..088811b3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-main-instance.md @@ -0,0 +1,36 @@ +--- +name: main instance bug report +about: report an issue with cobalt.tools or api.cobalt.tools +title: '[short description of the bug]' +labels: main instance issue +assignees: '' + +--- + +### bug description +clear and concise description of what the issue is. + +### reproduction steps +steps to reproduce the described behavior. +here's an example of what it could look like: +1. go to '...' +2. click on '....' +3. download [media type] from [service] +4. see error + +### screenshots +if applicable, add screenshots or screen recordings to support your explanation. +if not, remove this section. + +### links +if applicable, add links that cause the issue. more = better. +if not, remove this section. + +### platform information +- OS [e.g. iOS, windows] +- browser [e.g. chrome, safari, firefox] +- version [e.g. 115] + +### additional context +add any other context about the problem here if applicable. +if not, remove this section. From 8276e51dbcc7afb8b3da7538a0185b6bb29db40a Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 4 Jul 2024 11:28:17 +0000 Subject: [PATCH 36/46] ci: add fast-forward merge action --- .github/workflows/fast-forward.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/fast-forward.yml diff --git a/.github/workflows/fast-forward.yml b/.github/workflows/fast-forward.yml new file mode 100644 index 00000000..adda8f35 --- /dev/null +++ b/.github/workflows/fast-forward.yml @@ -0,0 +1,22 @@ +name: fast-forward +on: + issue_comment: + types: [created, edited] +jobs: + fast-forward: + # Only run if the comment contains the /fast-forward command. + if: ${{ contains(github.event.comment.body, '/fast-forward') + && github.event.issue.pull_request }} + runs-on: ubuntu-latest + + permissions: + contents: write + pull-requests: write + issues: write + + steps: + - name: Fast forwarding + uses: sequoia-pgp/fast-forward@v1 + with: + merge: true + comment: 'on-error' \ No newline at end of file From 0fefc4ac27a980befd047426c6069c7ba3a81a94 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 6 Jul 2024 08:23:20 +0000 Subject: [PATCH 37/46] services/ok: fix video data extraction closes #589 --- src/modules/processing/services/ok.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/modules/processing/services/ok.js b/src/modules/processing/services/ok.js index 295d5b81..97bbcf82 100644 --- a/src/modules/processing/services/ok.js +++ b/src/modules/processing/services/ok.js @@ -20,14 +20,15 @@ export default async function(o) { }).then(r => r.text()).catch(() => {}); if (!html) return { error: 'ErrorCouldntFetch' }; - if (!html.includes(`
/) + ?.[1] + ?.replaceAll(""", '"'); + + if (!videoData) { return { error: 'ErrorEmptyDownload' }; } - let videoData = html.split(`
Date: Sat, 6 Jul 2024 08:33:02 +0000 Subject: [PATCH 38/46] services/soundcloud: properly check script hostname --- src/modules/processing/services/soundcloud.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/modules/processing/services/soundcloud.js b/src/modules/processing/services/soundcloud.js index 389eaf6c..c36ec61d 100644 --- a/src/modules/processing/services/soundcloud.js +++ b/src/modules/processing/services/soundcloud.js @@ -12,17 +12,19 @@ async function findClientID() { let scVersion = String(sc.match(/