From 9b4f49fcf6c7791dc1d8f54080eca9cf06273eed Mon Sep 17 00:00:00 2001 From: J4mez <58501310+J4mez@users.noreply.github.com> Date: Fri, 31 May 2024 14:53:25 +0200 Subject: [PATCH 01/24] web: stop password managers from autofilling data into url area (#533) fixed dashlane autofill phone in url input --- src/modules/pageRender/page.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index 19e08b50..2c166177 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -594,7 +594,7 @@ export default function(obj) {
- +
From 7ebd9bc0ffb8ab486fbbab650576f1324b10f78a Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 31 May 2024 20:10:16 +0600 Subject: [PATCH 02/24] servicesConfig: temporarily disable reddit support reddit's media server times out unexpectedly --- src/modules/processing/servicesConfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index ae4cd9f0..a725473e 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -15,7 +15,7 @@ "alias": "reddit videos & gifs", "patterns": ["r/:sub/comments/:id/:title", "user/:user/comments/:id/:title"], "subdomains": "*", - "enabled": true + "enabled": false }, "twitter": { "alias": "twitter videos & voice", From fe7d4974e434d68c6527e48988846f25e5b29a8c Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 6 Jun 2024 14:39:28 +0000 Subject: [PATCH 03/24] stream: move pipe to shared functions --- src/modules/stream/shared.js | 10 ++++++++++ src/modules/stream/types.js | 12 +----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/modules/stream/shared.js b/src/modules/stream/shared.js index 4dbd59d3..1e2f2e25 100644 --- a/src/modules/stream/shared.js +++ b/src/modules/stream/shared.js @@ -28,4 +28,14 @@ export function getHeaders(service) { // Converting all header values to strings return Object.entries({ ...defaultHeaders, ...serviceHeaders[service] }) .reduce((p, [key, val]) => ({ ...p, [key]: String(val) }), {}) +} + +export function pipe(from, to, done) { + from.on('error', done) + .on('close', done); + + to.on('error', done) + .on('close', done); + + from.pipe(to); } \ No newline at end of file diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 2036b360..2372d6c1 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -6,7 +6,7 @@ import { create as contentDisposition } from "content-disposition-header"; import { metadataManager } from "../sub/utils.js"; import { destroyInternalStream } from "./manage.js"; import { env, ffmpegArgs, hlsExceptions } from "../config.js"; -import { getHeaders, closeResponse } from "./shared.js"; +import { getHeaders, closeResponse, pipe } from "./shared.js"; function toRawHeaders(headers) { return Object.entries(headers) @@ -28,16 +28,6 @@ function killProcess(p) { }, 5000); } -function pipe(from, to, done) { - from.on('error', done) - .on('close', done); - - to.on('error', done) - .on('close', done); - - from.pipe(to); -} - function getCommand(args) { if (!isNaN(env.processingPriority)) { return ['nice', ['-n', env.processingPriority.toString(), ffmpeg, ...args]] From 85bed9aa7479dc8175fbcab3d590c7cc685b75d4 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 6 Jun 2024 14:45:36 +0000 Subject: [PATCH 04/24] stream/internal: use pipe() to handle internal streams --- src/modules/stream/internal.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index fae23f57..833adf7d 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -1,7 +1,7 @@ import { request } from 'undici'; import { Readable } from 'node:stream'; import { assert } from 'console'; -import { getHeaders } from './shared.js'; +import { getHeaders, pipe } from './shared.js'; const CHUNK_SIZE = BigInt(8e6); // 8 MB const min = (a, b) => a < b ? a : b; @@ -66,8 +66,7 @@ async function handleYoutubeStream(streamInfo, res) { if (headerValue) res.setHeader(headerName, headerValue); } - stream.pipe(res); - stream.on('error', () => res.end()); + pipe(stream, res, () => res.end()); } catch { res.end(); } @@ -97,8 +96,7 @@ export async function internalStream(streamInfo, res) { if (req.statusCode < 200 || req.statusCode > 299) return res.end(); - req.body.pipe(res); - req.body.on('error', () => res.end()); + pipe(req.body, res, () => res.end()); } catch { streamInfo.controller.abort(); } From 4c8cd9dd30181da072b1555a47fde705103610e7 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 7 Jun 2024 14:52:55 +0600 Subject: [PATCH 05/24] youtube: change innertube client to ios --- src/modules/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index fae4e04d..f350612a 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -61,7 +61,7 @@ export default async function(o) { } try { - info = await yt.getBasicInfo(o.id, 'WEB'); + info = await yt.getBasicInfo(o.id, 'IOS'); } catch(e) { if (e?.message === 'This video is unavailable') { return { error: 'ErrorCouldntFetch' }; From 68f311c31818b569d730e4678dccc0e211ad2afa Mon Sep 17 00:00:00 2001 From: Mikhail Serebryakov Date: Fri, 7 Jun 2024 16:08:20 +0500 Subject: [PATCH 06/24] stream: add hls support for internal streams (#525) --- src/modules/stream/internal-hls.js | 56 ++++++++++++++++++++++++++++++ src/modules/stream/internal.js | 7 +++- src/modules/stream/manage.js | 8 +---- 3 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 src/modules/stream/internal-hls.js diff --git a/src/modules/stream/internal-hls.js b/src/modules/stream/internal-hls.js new file mode 100644 index 00000000..c6a11455 --- /dev/null +++ b/src/modules/stream/internal-hls.js @@ -0,0 +1,56 @@ +import { createInternalStream } from './manage.js'; +import HLS from 'hls-parser'; +import path from "node:path"; + +function transformObject(streamInfo, hlsObject) { + if (hlsObject === undefined) { + return (object) => transformObject(streamInfo, object); + } + + const fullUrl = hlsObject.uri.startsWith("/") + ? new URL(hlsObject.uri, streamInfo.url).toString() + : new URL(path.join(streamInfo.url, "/../", hlsObject.uri)).toString(); + hlsObject.uri = createInternalStream(fullUrl, streamInfo); + + return hlsObject; +} + +function transformMasterPlaylist(streamInfo, hlsPlaylist) { + const makeInternalStream = transformObject(streamInfo); + + const makeInternalVariants = (variant) => { + variant = transformObject(streamInfo, variant); + variant.video = variant.video.map(makeInternalStream); + variant.audio = variant.audio.map(makeInternalStream); + return variant; + }; + hlsPlaylist.variants = hlsPlaylist.variants.map(makeInternalVariants); + + return hlsPlaylist; +} + +function transformMediaPlaylist(streamInfo, hlsPlaylist) { + const makeInternalSegments = transformObject(streamInfo); + hlsPlaylist.segments = hlsPlaylist.segments.map(makeInternalSegments); + hlsPlaylist.prefetchSegments = hlsPlaylist.prefetchSegments.map(makeInternalSegments); + return hlsPlaylist; +} + +const HLS_MIME_TYPES = ["application/vnd.apple.mpegurl", "audio/mpegurl", "application/x-mpegURL"]; + +export function isHlsRequest (req) { + return HLS_MIME_TYPES.includes(req.headers['content-type']); +} + +export async function handleHlsPlaylist(streamInfo, req, res) { + let hlsPlaylist = await req.body.text(); + hlsPlaylist = HLS.parse(hlsPlaylist); + + hlsPlaylist = hlsPlaylist.isMasterPlaylist + ? transformMasterPlaylist(streamInfo, hlsPlaylist) + : transformMediaPlaylist(streamInfo, hlsPlaylist); + + hlsPlaylist = HLS.stringify(hlsPlaylist); + + res.send(hlsPlaylist); +} diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index 833adf7d..535bba2d 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -2,6 +2,7 @@ import { request } from 'undici'; import { Readable } from 'node:stream'; import { assert } from 'console'; import { getHeaders, pipe } from './shared.js'; +import { handleHlsPlaylist, isHlsRequest } from './internal-hls.js'; const CHUNK_SIZE = BigInt(8e6); // 8 MB const min = (a, b) => a < b ? a : b; @@ -96,7 +97,11 @@ export async function internalStream(streamInfo, res) { if (req.statusCode < 200 || req.statusCode > 299) return res.end(); - pipe(req.body, res, () => res.end()); + if (isHlsRequest(req)) { + await handleHlsPlaylist(streamInfo, req, res); + } else { + pipe(req.body, res, () => res.end()); + } } catch { streamInfo.controller.abort(); } diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 05008077..0dec1972 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -3,7 +3,7 @@ import { randomBytes } from "crypto"; import { nanoid } from "nanoid"; import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; -import { env, hlsExceptions } from "../config.js"; +import { env } from "../config.js"; import { strict as assert } from "assert"; // optional dependency @@ -106,12 +106,6 @@ export function destroyInternalStream(url) { } function wrapStream(streamInfo) { - /* m3u8 links are currently not supported - * for internal streams, skip them */ - if (hlsExceptions.includes(streamInfo.service)) { - return streamInfo; - } - const url = streamInfo.urls; if (typeof url === 'string') { From 268b6a40a30c3cfadff9102990ad572b399d3ba1 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 7 Jun 2024 17:10:33 +0600 Subject: [PATCH 07/24] localization: update user count in donation text --- src/localization/languages/en.json | 2 +- src/localization/languages/ru.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index 1c1b857f..e2f26c5e 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -88,7 +88,7 @@ "ChangelogPressToHide": "collapse", "Donate": "donate", "DonateSub": "help it stay online", - "DonateExplanation": "cobalt doesn't shove ads in your face and doesn't sell your personal data, meaning that it's completely free to use for everyone. but development and maintenance of a media-heavy service used by over 750k people is quite costly. both in terms of time and money.\n\nif cobalt helped you in the past and you want to keep it growing and evolving, you can return the favor by making a donation!\n\nyour donation will help all cobalt users: educators, students, content creators, artists, musicians, and many, many more!\n\nin past, donations have let cobalt:\n*; increase stability and uptime to nearly 100%.\n*; speed up ALL downloads, especially heavier ones.\n*; open the api for free public use.\n*; withstand several huge user influxes with 0 downtime.\n*; add resource-intensive features (such as gif conversion).\n*; continue improving our infrastructure.\n*; keep developers happy.\n\nevery cent matters and is extremely appreciated, you can truly make a difference!\n\nif you can't donate, share cobalt with a friend! we don't get ads anywhere, so cobalt is spread by word of mouth.\nsharing is the easiest way to help achieve the goal of better internet for everyone.", + "DonateExplanation": "cobalt doesn't shove ads in your face and doesn't sell your personal data, meaning that it's completely free to use for everyone. but development and maintenance of a media-heavy service used by over 1 million people is quite costly. both in terms of time and money.\n\nif cobalt helped you in the past and you want to keep it growing and evolving, you can return the favor by making a donation!\n\nyour donation will help all cobalt users: educators, students, content creators, artists, musicians, and many, many more!\n\nin past, donations have let cobalt:\n*; increase stability and uptime to nearly 100%.\n*; speed up ALL downloads, especially heavier ones.\n*; open the api for free public use.\n*; withstand several huge user influxes with 0 downtime.\n*; add resource-intensive features (such as gif conversion).\n*; continue improving our infrastructure.\n*; keep developers happy.\n\nevery cent matters and is extremely appreciated, you can truly make a difference!\n\nif you can't donate, share cobalt with a friend! we don't get ads anywhere, so cobalt is spread by word of mouth.\nsharing is the easiest way to help achieve the goal of better internet for everyone.", "DonateVia": "donate via", "SettingsVideoMute": "mute audio", "SettingsVideoMuteExplanation": "removes audio from video downloads when possible.", diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index eeb663e1..e50725ec 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -89,7 +89,7 @@ "ChangelogPressToHide": "скрыть", "Donate": "донаты", "DonateSub": "ты можешь помочь!", - "DonateExplanation": "кобальт не пихает рекламу тебе в лицо и не продаёт твои личные данные, а значит работает совершенно бесплатно для всех. но разработка и поддержка медиа сервиса, которым пользуются более 750 тысяч людей, обходится довольно затратно.\n\nесли кобальт тебе помог и ты хочешь, чтобы он продолжал расти и развиваться, то это можно сделать через донаты!\n\nтвой донат поможет всем, кто пользуется кобальтом: преподавателям, студентам, музыкантам, художникам, контент-мейкерам и многим-многим другим!\n\nв прошлом донаты помогли кобальту:\n*; повысить стабильность и аптайм почти до 100%.\n*; ускорить ВСЕ загрузки, особенно наиболее тяжёлые.\n*; открыть api для бесплатного использования.\n*; выдержать несколько огромных наплывов пользователей без перебоев.\n*; добавить ресурсоемкие фичи (например конвертацию в gif).\n*; продолжать улучшать нашу инфраструктуру.\n*; радовать разработчиков.\n\nкаждый донат невероятно ценится и помогает кобальту развиваться!\n\nесли ты не можешь отправить донат, то поделись кобальтом с другом! мы нигде не размещаем рекламу, поэтому кобальт распространяется из уст в уста.\nподелиться - самый простой способ помочь достичь цели лучшего интернета для всех.", + "DonateExplanation": "кобальт не пихает рекламу тебе в лицо и не продаёт твои личные данные, а значит работает совершенно бесплатно для всех. но разработка и поддержка медиа сервиса, которым пользуются более миллиона людей, обходится довольно затратно.\n\nесли кобальт тебе помог и ты хочешь, чтобы он продолжал расти и развиваться, то это можно сделать через донаты!\n\nтвой донат поможет всем, кто пользуется кобальтом: преподавателям, студентам, музыкантам, художникам, контент-мейкерам и многим-многим другим!\n\nв прошлом донаты помогли кобальту:\n*; повысить стабильность и аптайм почти до 100%.\n*; ускорить ВСЕ загрузки, особенно наиболее тяжёлые.\n*; открыть api для бесплатного использования.\n*; выдержать несколько огромных наплывов пользователей без перебоев.\n*; добавить ресурсоемкие фичи (например конвертацию в gif).\n*; продолжать улучшать нашу инфраструктуру.\n*; радовать разработчиков.\n\nкаждый донат невероятно ценится и помогает кобальту развиваться!\n\nесли ты не можешь отправить донат, то поделись кобальтом с другом! мы нигде не размещаем рекламу, поэтому кобальт распространяется из уст в уста.\nподелиться - самый простой способ помочь достичь цели лучшего интернета для всех.", "DonateVia": "открыть", "SettingsVideoMute": "убрать аудио", "SettingsVideoMuteExplanation": "убирает звук при загрузке видео, но только когда это возможно.", From f6632e2d610a55b7c7ced0ed79cc2a5b71eff001 Mon Sep 17 00:00:00 2001 From: jj Date: Fri, 7 Jun 2024 15:02:07 +0200 Subject: [PATCH 08/24] youtube: add cookie support (#553) --- src/modules/processing/services/youtube.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index f350612a..aa34dcfc 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -2,6 +2,7 @@ import { Innertube, Session } from 'youtubei.js'; import { env } from '../../config.js'; import { cleanString } from '../../sub/utils.js'; import { fetch } from 'undici' +import { getCookie } from '../cookie/manager.js' const ytBase = Innertube.create().catch(e => e); @@ -35,7 +36,7 @@ const cloneInnertube = async (customFetch) => { innertube.session.api_version, innertube.session.account_index, innertube.session.player, - undefined, + getCookie('youtube'), customFetch ?? innertube.session.http.fetch, innertube.session.cache ); From f7b36713e00a96824a8b18743a3125ab79c48b86 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 7 Jun 2024 19:37:01 +0600 Subject: [PATCH 09/24] youtube: convert cookie to string --- src/modules/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index aa34dcfc..558fe691 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -36,7 +36,7 @@ const cloneInnertube = async (customFetch) => { innertube.session.api_version, innertube.session.account_index, innertube.session.player, - getCookie('youtube'), + getCookie('youtube')?.toString(), customFetch ?? innertube.session.http.fetch, innertube.session.cache ); From 7fb2e6d8d982c3708e669c596b8ebfe48be3ff08 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 7 Jun 2024 21:46:45 +0600 Subject: [PATCH 10/24] youtube: proper age & sign in limit errors --- src/localization/languages/en.json | 6 ++++-- src/localization/languages/ru.json | 3 ++- src/modules/processing/services/youtube.js | 13 ++++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index e2f26c5e..fc4d7f80 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -100,7 +100,7 @@ "FollowSupport": "keep in touch with cobalt for news, support, and more:", "SourceCode": "explore source code, report issues, star or fork the repo:", "PrivacyPolicy": "cobalt's privacy policy is simple: no data about you is ever collected or stored. zero, zilch, nada, nothing.\nwhat you download is solely your business, not mine or anyone else's.\n\nif your download requires rendering, then data about requested content is encrypted and temporarily stored in server's RAM. it's necessary for this feature to function.\n\nencrypted data is stored for 90 seconds and then permanently removed.\n\nstored data is only possible to decrypt with unique encryption keys from your download link. furthermore, the official cobalt codebase doesn't provide a way to read temporarily stored data outside of processing functions.\n\nyou can check cobalt's source code yourself and see that everything is as stated.", - "ErrorYTUnavailable": "this youtube video is unavailable. it could be age or region restricted. try another one!", + "ErrorYTUnavailable": "this youtube video is unavailable. it could be visibility or region restricted. try another one!", "ErrorYTTryOtherCodec": "i couldn't find anything to download with your settings. try another codec or quality in settings!", "SettingsCodecSubtitle": "youtube codec", "SettingsCodecDescription": "h264: best support across apps/platforms, average detail level. max quality is 1080p.\nav1: best quality, small file size, most detail. supports 8k & HDR.\nvp9: same quality as av1, but file is x2 bigger. supports 4k & HDR.\n\npick h264 if you want best compatibility.\npick av1 if you want best quality and efficiency.", @@ -156,6 +156,8 @@ "SettingsYoutubeDub": "use browser language", "SettingsYoutubeDubDescription": "uses your browser's default language for youtube dubbed audio tracks. works even if cobalt ui isn't translated to your language.", "ErrorInvalidContentType": "invalid content type header", - "UpdateOneMillion": "1 million users and blazing speed" + "UpdateOneMillion": "1 million users and blazing speed", + "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!", + "ErrorYTLogin": "couldn't get this youtube video because it requires sign in.\n\nthis limitation is an a/b test done by google to seemingly stop scraping, coincidentally affecting all 3rd party tools and even their own clients.\n\nyou can track the issue on github." } } diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index e50725ec..6487e531 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -157,6 +157,7 @@ "SettingsTikTokH265Description": "скачивает видео с tiktok в 1080p и h265/hevc, когда это возможно.", "SettingsYoutubeDub": "использовать язык браузера", "SettingsYoutubeDubDescription": "использует главный язык браузера для аудиодорожек на youtube. работает даже если кобальт не переведён в твой язык.", - "UpdateOneMillion": "миллион и невероятная скорость" + "UpdateOneMillion": "миллион и невероятная скорость", + "ErrorYTAgeRestrict": "это видео ограничено по возрасту, поэтому я не могу его скачать. попробуй другое!" } } diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 558fe691..39a94f3a 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -73,7 +73,18 @@ export default async function(o) { if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; - if (info.playability_status.status !== 'OK') return { error: 'ErrorYTUnavailable' }; + const playability = info.playability_status; + + if (playability.status === 'LOGIN_REQUIRED') { + if (playability.reason.endsWith('bot')) { + return { error: 'ErrorYTLogin' } + } + if (playability.reason.endsWith('age')) { + return { error: 'ErrorYTAgeRestrict' } + } + } + + if (playability.status !== 'OK') return { error: 'ErrorYTUnavailable' }; if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' }; // return a critical error if returned video is "Video Not Available" From 46274c8da090d89c22dd7fcd61d401210c465052 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 09:18:29 +0000 Subject: [PATCH 11/24] youtube: add support for using OAuth2 tokens --- src/modules/processing/services/youtube.js | 33 +++++++++++++++++++++- src/modules/sub/generate-youtube-tokens.js | 30 ++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 src/modules/sub/generate-youtube-tokens.js diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 39a94f3a..301b37b7 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -24,6 +24,26 @@ const codecMatch = { } } +const transformSessionData = (cookie) => { + if (!cookie) + return; + + const values = cookie.values(); + const REQUIRED_VALUES = [ + 'access_token', 'refresh_token', + 'client_id', 'client_secret', + 'expires' + ]; + + if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) { + return; + } + return { + ...values, + expires: new Date(values.expires), + }; +} + const cloneInnertube = async (customFetch) => { const innertube = await ytBase; if (innertube instanceof Error) { @@ -41,6 +61,17 @@ const cloneInnertube = async (customFetch) => { innertube.session.cache ); + const oauthData = transformSessionData(getCookie('youtube_oauth')); + + if (!session.logged_in && oauthData) { + await session.oauth.init(oauthData); + session.logged_in = true; + } + + if (session.logged_in) { + await session.oauth.refreshIfRequired(); + } + const yt = new Innertube(session); return yt; } @@ -62,7 +93,7 @@ export default async function(o) { } try { - info = await yt.getBasicInfo(o.id, 'IOS'); + info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS'); } catch(e) { if (e?.message === 'This video is unavailable') { return { error: 'ErrorCouldntFetch' }; diff --git a/src/modules/sub/generate-youtube-tokens.js b/src/modules/sub/generate-youtube-tokens.js new file mode 100644 index 00000000..26a6a1cd --- /dev/null +++ b/src/modules/sub/generate-youtube-tokens.js @@ -0,0 +1,30 @@ +import { Innertube } from 'youtubei.js'; + +const bail = (...msg) => (console.error(...msg), process.exit(1)); +const tube = await Innertube.create(); + +tube.session.once( + 'auth-pending', + ({ verification_url, user_code }) => console.log( + `Open ${verification_url} in a browser and enter ${user_code} when asked for the code.` + ) +); + +tube.session.once('auth-error', (err) => bail('An error occurred:', err)); +tube.session.once('auth', ({ status, credentials, ...rest }) => { + if (status !== 'SUCCESS') { + bail('something went wrong', rest); + } + + console.log(credentials); + + console.log( + JSON.stringify( + Object.entries(credentials) + .map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`) + .join('; ') + ) + ); +}); + +await tube.session.signIn(); \ No newline at end of file From 18d43729385336a0df8671133724641fd41a5769 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 09:26:58 +0000 Subject: [PATCH 12/24] youtube: drop cookie support it never really worked --- src/modules/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 301b37b7..3ad956d9 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -56,7 +56,7 @@ const cloneInnertube = async (customFetch) => { innertube.session.api_version, innertube.session.account_index, innertube.session.player, - getCookie('youtube')?.toString(), + undefined, customFetch ?? innertube.session.http.fetch, innertube.session.cache ); From 2387fc2fbb85590efa8040400044e4b41a60d560 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 09:30:12 +0000 Subject: [PATCH 13/24] youtube: update access token on change --- src/modules/processing/services/youtube.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 3ad956d9..49de029b 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -2,7 +2,7 @@ import { Innertube, Session } from 'youtubei.js'; import { env } from '../../config.js'; import { cleanString } from '../../sub/utils.js'; import { fetch } from 'undici' -import { getCookie } from '../cookie/manager.js' +import { getCookie, updateCookieValues } from '../cookie/manager.js' const ytBase = Innertube.create().catch(e => e); @@ -61,7 +61,8 @@ const cloneInnertube = async (customFetch) => { innertube.session.cache ); - const oauthData = transformSessionData(getCookie('youtube_oauth')); + const cookie = getCookie('youtube_oauth'); + const oauthData = transformSessionData(cookie); if (!session.logged_in && oauthData) { await session.oauth.init(oauthData); @@ -70,6 +71,15 @@ const cloneInnertube = async (customFetch) => { if (session.logged_in) { await session.oauth.refreshIfRequired(); + const oldExpiry = new Date(cookie.values().expires); + const newExpiry = session.oauth.credentials.expires; + + if (oldExpiry.getTime() !== newExpiry.getTime()) { + updateCookieValues(cookie, { + ...session.oauth.credentials, + expires: session.oauth.credentials.expires.toISOString() + }); + } } const yt = new Innertube(session); From d08e2ac04f3393d3bc7c0b582ed1bfb4053824b0 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 09:32:23 +0000 Subject: [PATCH 14/24] generate-youtube-tokens: use throw instead of process.exit fuck off deepsource --- src/modules/sub/generate-youtube-tokens.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/modules/sub/generate-youtube-tokens.js b/src/modules/sub/generate-youtube-tokens.js index 26a6a1cd..65d64494 100644 --- a/src/modules/sub/generate-youtube-tokens.js +++ b/src/modules/sub/generate-youtube-tokens.js @@ -1,6 +1,10 @@ import { Innertube } from 'youtubei.js'; -const bail = (...msg) => (console.error(...msg), process.exit(1)); +const bail = (...msg) => { + console.error(...msg); + throw new Error(msg); +}; + const tube = await Innertube.create(); tube.session.once( From 9e09bcab6ea1c38dde3793a6df75933e095e3901 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 11:52:36 +0000 Subject: [PATCH 15/24] refactor: create `util` directory, move tests to it --- package.json | 4 ++-- src/{test => util}/test.js | 2 +- src/{test => util}/testFilenamePresets.js | 0 src/{test => util}/tests.json | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/{test => util}/test.js (98%) rename src/{test => util}/testFilenamePresets.js (100%) rename src/{test => util}/tests.json (100%) diff --git a/package.json b/package.json index 72104aba..52640702 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,9 @@ "scripts": { "start": "node src/cobalt", "setup": "node src/modules/setup", - "test": "node src/test/test", + "test": "node src/util/test", "build": "node src/modules/buildStatic", - "testFilenames": "node src/test/testFilenamePresets" + "testFilenames": "node src/util/testFilenamePresets" }, "repository": { "type": "git", diff --git a/src/test/test.js b/src/util/test.js similarity index 98% rename from src/test/test.js rename to src/util/test.js index f26f3b95..294ff7e7 100644 --- a/src/test/test.js +++ b/src/util/test.js @@ -9,7 +9,7 @@ import { normalizeRequest } from "../modules/processing/request.js"; import { env } from "../modules/config.js"; env.apiURL = 'http://localhost:9000' -let tests = loadJSON('./src/test/tests.json'); +let tests = loadJSON('./src/util/tests.json'); let noTest = []; let failed = []; diff --git a/src/test/testFilenamePresets.js b/src/util/testFilenamePresets.js similarity index 100% rename from src/test/testFilenamePresets.js rename to src/util/testFilenamePresets.js diff --git a/src/test/tests.json b/src/util/tests.json similarity index 100% rename from src/test/tests.json rename to src/util/tests.json From ebe6668bc0c631a34a94a853efea2ecf24f1087b Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 11:52:53 +0000 Subject: [PATCH 16/24] refactor: move generate-youtube-tokens to `util` --- src/{modules/sub => util}/generate-youtube-tokens.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{modules/sub => util}/generate-youtube-tokens.js (100%) diff --git a/src/modules/sub/generate-youtube-tokens.js b/src/util/generate-youtube-tokens.js similarity index 100% rename from src/modules/sub/generate-youtube-tokens.js rename to src/util/generate-youtube-tokens.js From 6c1d8ef6c75cfcdb1cdd66278c43bfd8b15138e7 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 11:54:33 +0000 Subject: [PATCH 17/24] generate-youtube-tokens: add more explanatory text and clean up logging --- src/util/generate-youtube-tokens.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/util/generate-youtube-tokens.js b/src/util/generate-youtube-tokens.js index 65d64494..924afb13 100644 --- a/src/util/generate-youtube-tokens.js +++ b/src/util/generate-youtube-tokens.js @@ -1,4 +1,5 @@ import { Innertube } from 'youtubei.js'; +import { Red } from '../modules/sub/consoleText.js' const bail = (...msg) => { console.error(...msg); @@ -9,9 +10,13 @@ const tube = await Innertube.create(); tube.session.once( 'auth-pending', - ({ verification_url, user_code }) => console.log( - `Open ${verification_url} in a browser and enter ${user_code} when asked for the code.` - ) + ({ verification_url, user_code }) => { + console.log(`${Red('[!]')} The token generated by this script is sensitive and you should not share it with anyone!`); + console.log(` By using this token, you are risking your Google account getting terminated.`); + console.log(` You should ${Red('NOT')} use your personal account!`); + console.log(); + console.log(`Open ${verification_url} in a browser and enter ${user_code} when asked for the code.`); + } ); tube.session.once('auth-error', (err) => bail('An error occurred:', err)); @@ -20,9 +25,8 @@ tube.session.once('auth', ({ status, credentials, ...rest }) => { bail('something went wrong', rest); } - console.log(credentials); - console.log( + 'add this cookie to the youtube_oauth array in your cookies file:', JSON.stringify( Object.entries(credentials) .map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`) From a84d0ddc772218fd5f74ec62aee8d95783425428 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 8 Jun 2024 12:05:18 +0000 Subject: [PATCH 18/24] package.json: remove testFilenames script, add youtube token generation --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 52640702..7d271fea 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "setup": "node src/modules/setup", "test": "node src/util/test", "build": "node src/modules/buildStatic", - "testFilenames": "node src/util/testFilenamePresets" + "token:youtube": "node src/util/generate-youtube-tokens" }, "repository": { "type": "git", From 90e066ac22c8f3b6ab30166a550e532f1e4e5159 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 8 Jun 2024 18:14:10 +0600 Subject: [PATCH 19/24] package: bump version to 7.14.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7d271fea..1d44f380 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.14.3", + "version": "7.14.4", "author": "imput", "exports": "./src/cobalt.js", "type": "module", From 77d167ce1e343bba44477bb08c1a7542e2be6e6f Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 8 Jun 2024 18:15:31 +0600 Subject: [PATCH 20/24] package-lock: update version --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2636146d..a61508b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cobalt", - "version": "7.14.3", + "version": "7.14.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cobalt", - "version": "7.14.3", + "version": "7.14.4", "license": "AGPL-3.0", "dependencies": { "content-disposition-header": "0.6.0", From f3056c6dc36ebf7e4d796314ea4af576116ab4f2 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 8 Jun 2024 18:31:00 +0600 Subject: [PATCH 21/24] servicesConfig: enable reddit back --- src/modules/processing/servicesConfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index a725473e..ae4cd9f0 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -15,7 +15,7 @@ "alias": "reddit videos & gifs", "patterns": ["r/:sub/comments/:id/:title", "user/:user/comments/:id/:title"], "subdomains": "*", - "enabled": false + "enabled": true }, "twitter": { "alias": "twitter videos & voice", From 04d66946fc657cdf5b6ea4871c2a69d8e6c417e5 Mon Sep 17 00:00:00 2001 From: jj Date: Sat, 8 Jun 2024 18:34:18 +0200 Subject: [PATCH 22/24] internal-hls: correctly handle URL concatenation of all types (#560) --- src/modules/stream/internal-hls.js | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/modules/stream/internal-hls.js b/src/modules/stream/internal-hls.js index c6a11455..9fa37e6e 100644 --- a/src/modules/stream/internal-hls.js +++ b/src/modules/stream/internal-hls.js @@ -1,16 +1,27 @@ import { createInternalStream } from './manage.js'; import HLS from 'hls-parser'; -import path from "node:path"; + +function getURL(url) { + try { + return new URL(url); + } catch { + return null; + } +} function transformObject(streamInfo, hlsObject) { if (hlsObject === undefined) { return (object) => transformObject(streamInfo, object); } - const fullUrl = hlsObject.uri.startsWith("/") - ? new URL(hlsObject.uri, streamInfo.url).toString() - : new URL(path.join(streamInfo.url, "/../", hlsObject.uri)).toString(); - hlsObject.uri = createInternalStream(fullUrl, streamInfo); + let fullUrl; + if (getURL(hlsObject.uri)) { + fullUrl = hlsObject.uri; + } else { + fullUrl = new URL(hlsObject.uri, streamInfo.url); + } + + hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo); return hlsObject; } From 1d5fa62271627ab84ac0398f39a3373488ce24ac Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 8 Jun 2024 22:59:30 +0600 Subject: [PATCH 23/24] youtube: add ratelimit error, update sign in error --- src/localization/languages/en.json | 3 ++- src/localization/languages/ru.json | 6 ++++-- src/modules/processing/services/youtube.js | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index fc4d7f80..eecd9ac1 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -158,6 +158,7 @@ "ErrorInvalidContentType": "invalid content type header", "UpdateOneMillion": "1 million users and blazing speed", "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!", - "ErrorYTLogin": "couldn't get this youtube video because it requires sign in.\n\nthis limitation is an a/b test done by google to seemingly stop scraping, coincidentally affecting all 3rd party tools and even their own clients.\n\nyou can track the issue on github." + "ErrorYTLogin": "couldn't get this youtube video because it requires an account to view.\n\nthis limitation is done by google to seemingly stop scraping, affecting all 3rd party tools and even their own clients.\n\ntry again, but if issue persists, {ContactLink}.", + "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}." } } diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index 6487e531..083d7bbc 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -1,7 +1,7 @@ { "name": "русский", "substrings": { - "ContactLink": "глянь статус серверов или напиши о проблеме на github (можно на русском)" + "ContactLink": "глянь статус серверов или напиши о проблеме на github" }, "strings": { "AppTitleCobalt": "кобальт", @@ -158,6 +158,8 @@ "SettingsYoutubeDub": "использовать язык браузера", "SettingsYoutubeDubDescription": "использует главный язык браузера для аудиодорожек на youtube. работает даже если кобальт не переведён в твой язык.", "UpdateOneMillion": "миллион и невероятная скорость", - "ErrorYTAgeRestrict": "это видео ограничено по возрасту, поэтому я не могу его скачать. попробуй другое!" + "ErrorYTAgeRestrict": "это видео ограничено по возрасту, поэтому я не могу его скачать. попробуй другое!", + "ErrorYTLogin": "не удалось получить это видео с youtube, т. к. для его просмотра требуется учетная запись.\n\nтакое ограничение сделано google, чтобы, по-видимому, помешать скрапингу, но в итоге ломает сторонние программы и даже собственные клиенты.\n\nпопробуй ещё раз, но если проблема останется, то {ContactLink}.", + "ErrorYTRateLimit": "youtube ограничил мне частоту запросов. попробуй ещё раз через несколько секунд, но если проблема останется, то {ContactLink}." } } diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 49de029b..1c86b4ed 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -124,6 +124,9 @@ export default async function(o) { return { error: 'ErrorYTAgeRestrict' } } } + if (playability.status === "UNPLAYABLE" && playability.reason.endsWith('request limit.')) { + return { error: 'ErrorYTRateLimit' } + } if (playability.status !== 'OK') return { error: 'ErrorYTUnavailable' }; if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' }; From d2e5b6542f71f3809ba94d56c26f382b5cb62762 Mon Sep 17 00:00:00 2001 From: jj Date: Sat, 15 Jun 2024 18:20:33 +0200 Subject: [PATCH 24/24] api: randomize cipherlist for making requests to services (#574) this makes cobalt less prone to TLS client fingerprinting, as it avoids having the default node.js TLS fingerprint that is shared by all node.js applications. --- src/core/api.js | 4 ++++ src/modules/sub/randomize-ciphers.js | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 src/modules/sub/randomize-ciphers.js diff --git a/src/core/api.js b/src/core/api.js index c90f78f7..a366ad09 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -10,6 +10,7 @@ import loc from "../localization/manager.js"; import { createResponse, normalizeRequest, getIP } from "../modules/processing/request.js"; import { verifyStream, getInternalStream } from "../modules/stream/manage.js"; +import { randomizeCiphers } from '../modules/sub/randomize-ciphers.js'; import { extract } from "../modules/processing/url.js"; import match from "../modules/processing/match.js"; import stream from "../modules/stream/stream.js"; @@ -215,6 +216,9 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { res.redirect('/api/serverInfo') }) + randomizeCiphers(); + setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes + app.listen(env.apiPort, env.listenAddress, () => { console.log(`\n` + `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + diff --git a/src/modules/sub/randomize-ciphers.js b/src/modules/sub/randomize-ciphers.js new file mode 100644 index 00000000..e11e1ee2 --- /dev/null +++ b/src/modules/sub/randomize-ciphers.js @@ -0,0 +1,28 @@ +import tls from 'node:tls'; +import { randomBytes } from 'node:crypto'; + +const ORIGINAL_CIPHERS = tls.DEFAULT_CIPHERS; + +// How many ciphers from the top of the list to shuffle. +// The remaining ciphers are left in the original order. +const TOP_N_SHUFFLE = 8; + +// Modified variation of https://stackoverflow.com/a/12646864 +const shuffleArray = (array) => { + for (let i = array.length - 1; i > 0; i--) { + const j = randomBytes(4).readUint32LE() % array.length; + [array[i], array[j]] = [array[j], array[i]]; + } + + return array; +} + +export const randomizeCiphers = () => { + do { + const cipherList = ORIGINAL_CIPHERS.split(':'); + const shuffled = shuffleArray(cipherList.slice(0, TOP_N_SHUFFLE)); + const retained = cipherList.slice(TOP_N_SHUFFLE); + + tls.DEFAULT_CIPHERS = [ ...shuffled, ...retained ].join(':'); + } while (tls.DEFAULT_CIPHERS === ORIGINAL_CIPHERS); +}