From fe7d4974e434d68c6527e48988846f25e5b29a8c Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 6 Jun 2024 14:39:28 +0000 Subject: [PATCH 1/8] stream: move pipe to shared functions --- src/modules/stream/shared.js | 10 ++++++++++ src/modules/stream/types.js | 12 +----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/modules/stream/shared.js b/src/modules/stream/shared.js index 4dbd59d3..1e2f2e25 100644 --- a/src/modules/stream/shared.js +++ b/src/modules/stream/shared.js @@ -28,4 +28,14 @@ export function getHeaders(service) { // Converting all header values to strings return Object.entries({ ...defaultHeaders, ...serviceHeaders[service] }) .reduce((p, [key, val]) => ({ ...p, [key]: String(val) }), {}) +} + +export function pipe(from, to, done) { + from.on('error', done) + .on('close', done); + + to.on('error', done) + .on('close', done); + + from.pipe(to); } \ No newline at end of file diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 2036b360..2372d6c1 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -6,7 +6,7 @@ import { create as contentDisposition } from "content-disposition-header"; import { metadataManager } from "../sub/utils.js"; import { destroyInternalStream } from "./manage.js"; import { env, ffmpegArgs, hlsExceptions } from "../config.js"; -import { getHeaders, closeResponse } from "./shared.js"; +import { getHeaders, closeResponse, pipe } from "./shared.js"; function toRawHeaders(headers) { return Object.entries(headers) @@ -28,16 +28,6 @@ function killProcess(p) { }, 5000); } -function pipe(from, to, done) { - from.on('error', done) - .on('close', done); - - to.on('error', done) - .on('close', done); - - from.pipe(to); -} - function getCommand(args) { if (!isNaN(env.processingPriority)) { return ['nice', ['-n', env.processingPriority.toString(), ffmpeg, ...args]] From 85bed9aa7479dc8175fbcab3d590c7cc685b75d4 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 6 Jun 2024 14:45:36 +0000 Subject: [PATCH 2/8] stream/internal: use pipe() to handle internal streams --- src/modules/stream/internal.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index fae23f57..833adf7d 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -1,7 +1,7 @@ import { request } from 'undici'; import { Readable } from 'node:stream'; import { assert } from 'console'; -import { getHeaders } from './shared.js'; +import { getHeaders, pipe } from './shared.js'; const CHUNK_SIZE = BigInt(8e6); // 8 MB const min = (a, b) => a < b ? a : b; @@ -66,8 +66,7 @@ async function handleYoutubeStream(streamInfo, res) { if (headerValue) res.setHeader(headerName, headerValue); } - stream.pipe(res); - stream.on('error', () => res.end()); + pipe(stream, res, () => res.end()); } catch { res.end(); } @@ -97,8 +96,7 @@ export async function internalStream(streamInfo, res) { if (req.statusCode < 200 || req.statusCode > 299) return res.end(); - req.body.pipe(res); - req.body.on('error', () => res.end()); + pipe(req.body, res, () => res.end()); } catch { streamInfo.controller.abort(); } From 4c8cd9dd30181da072b1555a47fde705103610e7 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 7 Jun 2024 14:52:55 +0600 Subject: [PATCH 3/8] youtube: change innertube client to ios --- src/modules/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index fae4e04d..f350612a 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -61,7 +61,7 @@ export default async function(o) { } try { - info = await yt.getBasicInfo(o.id, 'WEB'); + info = await yt.getBasicInfo(o.id, 'IOS'); } catch(e) { if (e?.message === 'This video is unavailable') { return { error: 'ErrorCouldntFetch' }; From 68f311c31818b569d730e4678dccc0e211ad2afa Mon Sep 17 00:00:00 2001 From: Mikhail Serebryakov Date: Fri, 7 Jun 2024 16:08:20 +0500 Subject: [PATCH 4/8] stream: add hls support for internal streams (#525) --- src/modules/stream/internal-hls.js | 56 ++++++++++++++++++++++++++++++ src/modules/stream/internal.js | 7 +++- src/modules/stream/manage.js | 8 +---- 3 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 src/modules/stream/internal-hls.js diff --git a/src/modules/stream/internal-hls.js b/src/modules/stream/internal-hls.js new file mode 100644 index 00000000..c6a11455 --- /dev/null +++ b/src/modules/stream/internal-hls.js @@ -0,0 +1,56 @@ +import { createInternalStream } from './manage.js'; +import HLS from 'hls-parser'; +import path from "node:path"; + +function transformObject(streamInfo, hlsObject) { + if (hlsObject === undefined) { + return (object) => transformObject(streamInfo, object); + } + + const fullUrl = hlsObject.uri.startsWith("/") + ? new URL(hlsObject.uri, streamInfo.url).toString() + : new URL(path.join(streamInfo.url, "/../", hlsObject.uri)).toString(); + hlsObject.uri = createInternalStream(fullUrl, streamInfo); + + return hlsObject; +} + +function transformMasterPlaylist(streamInfo, hlsPlaylist) { + const makeInternalStream = transformObject(streamInfo); + + const makeInternalVariants = (variant) => { + variant = transformObject(streamInfo, variant); + variant.video = variant.video.map(makeInternalStream); + variant.audio = variant.audio.map(makeInternalStream); + return variant; + }; + hlsPlaylist.variants = hlsPlaylist.variants.map(makeInternalVariants); + + return hlsPlaylist; +} + +function transformMediaPlaylist(streamInfo, hlsPlaylist) { + const makeInternalSegments = transformObject(streamInfo); + hlsPlaylist.segments = hlsPlaylist.segments.map(makeInternalSegments); + hlsPlaylist.prefetchSegments = hlsPlaylist.prefetchSegments.map(makeInternalSegments); + return hlsPlaylist; +} + +const HLS_MIME_TYPES = ["application/vnd.apple.mpegurl", "audio/mpegurl", "application/x-mpegURL"]; + +export function isHlsRequest (req) { + return HLS_MIME_TYPES.includes(req.headers['content-type']); +} + +export async function handleHlsPlaylist(streamInfo, req, res) { + let hlsPlaylist = await req.body.text(); + hlsPlaylist = HLS.parse(hlsPlaylist); + + hlsPlaylist = hlsPlaylist.isMasterPlaylist + ? transformMasterPlaylist(streamInfo, hlsPlaylist) + : transformMediaPlaylist(streamInfo, hlsPlaylist); + + hlsPlaylist = HLS.stringify(hlsPlaylist); + + res.send(hlsPlaylist); +} diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index 833adf7d..535bba2d 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -2,6 +2,7 @@ import { request } from 'undici'; import { Readable } from 'node:stream'; import { assert } from 'console'; import { getHeaders, pipe } from './shared.js'; +import { handleHlsPlaylist, isHlsRequest } from './internal-hls.js'; const CHUNK_SIZE = BigInt(8e6); // 8 MB const min = (a, b) => a < b ? a : b; @@ -96,7 +97,11 @@ export async function internalStream(streamInfo, res) { if (req.statusCode < 200 || req.statusCode > 299) return res.end(); - pipe(req.body, res, () => res.end()); + if (isHlsRequest(req)) { + await handleHlsPlaylist(streamInfo, req, res); + } else { + pipe(req.body, res, () => res.end()); + } } catch { streamInfo.controller.abort(); } diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 05008077..0dec1972 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -3,7 +3,7 @@ import { randomBytes } from "crypto"; import { nanoid } from "nanoid"; import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; -import { env, hlsExceptions } from "../config.js"; +import { env } from "../config.js"; import { strict as assert } from "assert"; // optional dependency @@ -106,12 +106,6 @@ export function destroyInternalStream(url) { } function wrapStream(streamInfo) { - /* m3u8 links are currently not supported - * for internal streams, skip them */ - if (hlsExceptions.includes(streamInfo.service)) { - return streamInfo; - } - const url = streamInfo.urls; if (typeof url === 'string') { From 268b6a40a30c3cfadff9102990ad572b399d3ba1 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 7 Jun 2024 17:10:33 +0600 Subject: [PATCH 5/8] localization: update user count in donation text --- src/localization/languages/en.json | 2 +- src/localization/languages/ru.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index 1c1b857f..e2f26c5e 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -88,7 +88,7 @@ "ChangelogPressToHide": "collapse", "Donate": "donate", "DonateSub": "help it stay online", - "DonateExplanation": "cobalt doesn't shove ads in your face and doesn't sell your personal data, meaning that it's completely free to use for everyone. but development and maintenance of a media-heavy service used by over 750k people is quite costly. both in terms of time and money.\n\nif cobalt helped you in the past and you want to keep it growing and evolving, you can return the favor by making a donation!\n\nyour donation will help all cobalt users: educators, students, content creators, artists, musicians, and many, many more!\n\nin past, donations have let cobalt:\n*; increase stability and uptime to nearly 100%.\n*; speed up ALL downloads, especially heavier ones.\n*; open the api for free public use.\n*; withstand several huge user influxes with 0 downtime.\n*; add resource-intensive features (such as gif conversion).\n*; continue improving our infrastructure.\n*; keep developers happy.\n\nevery cent matters and is extremely appreciated, you can truly make a difference!\n\nif you can't donate, share cobalt with a friend! we don't get ads anywhere, so cobalt is spread by word of mouth.\nsharing is the easiest way to help achieve the goal of better internet for everyone.", + "DonateExplanation": "cobalt doesn't shove ads in your face and doesn't sell your personal data, meaning that it's completely free to use for everyone. but development and maintenance of a media-heavy service used by over 1 million people is quite costly. both in terms of time and money.\n\nif cobalt helped you in the past and you want to keep it growing and evolving, you can return the favor by making a donation!\n\nyour donation will help all cobalt users: educators, students, content creators, artists, musicians, and many, many more!\n\nin past, donations have let cobalt:\n*; increase stability and uptime to nearly 100%.\n*; speed up ALL downloads, especially heavier ones.\n*; open the api for free public use.\n*; withstand several huge user influxes with 0 downtime.\n*; add resource-intensive features (such as gif conversion).\n*; continue improving our infrastructure.\n*; keep developers happy.\n\nevery cent matters and is extremely appreciated, you can truly make a difference!\n\nif you can't donate, share cobalt with a friend! we don't get ads anywhere, so cobalt is spread by word of mouth.\nsharing is the easiest way to help achieve the goal of better internet for everyone.", "DonateVia": "donate via", "SettingsVideoMute": "mute audio", "SettingsVideoMuteExplanation": "removes audio from video downloads when possible.", diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index eeb663e1..e50725ec 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -89,7 +89,7 @@ "ChangelogPressToHide": "скрыть", "Donate": "донаты", "DonateSub": "ты можешь помочь!", - "DonateExplanation": "кобальт не пихает рекламу тебе в лицо и не продаёт твои личные данные, а значит работает совершенно бесплатно для всех. но разработка и поддержка медиа сервиса, которым пользуются более 750 тысяч людей, обходится довольно затратно.\n\nесли кобальт тебе помог и ты хочешь, чтобы он продолжал расти и развиваться, то это можно сделать через донаты!\n\nтвой донат поможет всем, кто пользуется кобальтом: преподавателям, студентам, музыкантам, художникам, контент-мейкерам и многим-многим другим!\n\nв прошлом донаты помогли кобальту:\n*; повысить стабильность и аптайм почти до 100%.\n*; ускорить ВСЕ загрузки, особенно наиболее тяжёлые.\n*; открыть api для бесплатного использования.\n*; выдержать несколько огромных наплывов пользователей без перебоев.\n*; добавить ресурсоемкие фичи (например конвертацию в gif).\n*; продолжать улучшать нашу инфраструктуру.\n*; радовать разработчиков.\n\nкаждый донат невероятно ценится и помогает кобальту развиваться!\n\nесли ты не можешь отправить донат, то поделись кобальтом с другом! мы нигде не размещаем рекламу, поэтому кобальт распространяется из уст в уста.\nподелиться - самый простой способ помочь достичь цели лучшего интернета для всех.", + "DonateExplanation": "кобальт не пихает рекламу тебе в лицо и не продаёт твои личные данные, а значит работает совершенно бесплатно для всех. но разработка и поддержка медиа сервиса, которым пользуются более миллиона людей, обходится довольно затратно.\n\nесли кобальт тебе помог и ты хочешь, чтобы он продолжал расти и развиваться, то это можно сделать через донаты!\n\nтвой донат поможет всем, кто пользуется кобальтом: преподавателям, студентам, музыкантам, художникам, контент-мейкерам и многим-многим другим!\n\nв прошлом донаты помогли кобальту:\n*; повысить стабильность и аптайм почти до 100%.\n*; ускорить ВСЕ загрузки, особенно наиболее тяжёлые.\n*; открыть api для бесплатного использования.\n*; выдержать несколько огромных наплывов пользователей без перебоев.\n*; добавить ресурсоемкие фичи (например конвертацию в gif).\n*; продолжать улучшать нашу инфраструктуру.\n*; радовать разработчиков.\n\nкаждый донат невероятно ценится и помогает кобальту развиваться!\n\nесли ты не можешь отправить донат, то поделись кобальтом с другом! мы нигде не размещаем рекламу, поэтому кобальт распространяется из уст в уста.\nподелиться - самый простой способ помочь достичь цели лучшего интернета для всех.", "DonateVia": "открыть", "SettingsVideoMute": "убрать аудио", "SettingsVideoMuteExplanation": "убирает звук при загрузке видео, но только когда это возможно.", From f6632e2d610a55b7c7ced0ed79cc2a5b71eff001 Mon Sep 17 00:00:00 2001 From: jj Date: Fri, 7 Jun 2024 15:02:07 +0200 Subject: [PATCH 6/8] youtube: add cookie support (#553) --- src/modules/processing/services/youtube.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index f350612a..aa34dcfc 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -2,6 +2,7 @@ import { Innertube, Session } from 'youtubei.js'; import { env } from '../../config.js'; import { cleanString } from '../../sub/utils.js'; import { fetch } from 'undici' +import { getCookie } from '../cookie/manager.js' const ytBase = Innertube.create().catch(e => e); @@ -35,7 +36,7 @@ const cloneInnertube = async (customFetch) => { innertube.session.api_version, innertube.session.account_index, innertube.session.player, - undefined, + getCookie('youtube'), customFetch ?? innertube.session.http.fetch, innertube.session.cache ); From f7b36713e00a96824a8b18743a3125ab79c48b86 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 7 Jun 2024 19:37:01 +0600 Subject: [PATCH 7/8] youtube: convert cookie to string --- src/modules/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index aa34dcfc..558fe691 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -36,7 +36,7 @@ const cloneInnertube = async (customFetch) => { innertube.session.api_version, innertube.session.account_index, innertube.session.player, - getCookie('youtube'), + getCookie('youtube')?.toString(), customFetch ?? innertube.session.http.fetch, innertube.session.cache ); From 7fb2e6d8d982c3708e669c596b8ebfe48be3ff08 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 7 Jun 2024 21:46:45 +0600 Subject: [PATCH 8/8] youtube: proper age & sign in limit errors --- src/localization/languages/en.json | 6 ++++-- src/localization/languages/ru.json | 3 ++- src/modules/processing/services/youtube.js | 13 ++++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index e2f26c5e..fc4d7f80 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -100,7 +100,7 @@ "FollowSupport": "keep in touch with cobalt for news, support, and more:", "SourceCode": "explore source code, report issues, star or fork the repo:", "PrivacyPolicy": "cobalt's privacy policy is simple: no data about you is ever collected or stored. zero, zilch, nada, nothing.\nwhat you download is solely your business, not mine or anyone else's.\n\nif your download requires rendering, then data about requested content is encrypted and temporarily stored in server's RAM. it's necessary for this feature to function.\n\nencrypted data is stored for 90 seconds and then permanently removed.\n\nstored data is only possible to decrypt with unique encryption keys from your download link. furthermore, the official cobalt codebase doesn't provide a way to read temporarily stored data outside of processing functions.\n\nyou can check cobalt's source code yourself and see that everything is as stated.", - "ErrorYTUnavailable": "this youtube video is unavailable. it could be age or region restricted. try another one!", + "ErrorYTUnavailable": "this youtube video is unavailable. it could be visibility or region restricted. try another one!", "ErrorYTTryOtherCodec": "i couldn't find anything to download with your settings. try another codec or quality in settings!", "SettingsCodecSubtitle": "youtube codec", "SettingsCodecDescription": "h264: best support across apps/platforms, average detail level. max quality is 1080p.\nav1: best quality, small file size, most detail. supports 8k & HDR.\nvp9: same quality as av1, but file is x2 bigger. supports 4k & HDR.\n\npick h264 if you want best compatibility.\npick av1 if you want best quality and efficiency.", @@ -156,6 +156,8 @@ "SettingsYoutubeDub": "use browser language", "SettingsYoutubeDubDescription": "uses your browser's default language for youtube dubbed audio tracks. works even if cobalt ui isn't translated to your language.", "ErrorInvalidContentType": "invalid content type header", - "UpdateOneMillion": "1 million users and blazing speed" + "UpdateOneMillion": "1 million users and blazing speed", + "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!", + "ErrorYTLogin": "couldn't get this youtube video because it requires sign in.\n\nthis limitation is an a/b test done by google to seemingly stop scraping, coincidentally affecting all 3rd party tools and even their own clients.\n\nyou can track the issue on github." } } diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index e50725ec..6487e531 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -157,6 +157,7 @@ "SettingsTikTokH265Description": "скачивает видео с tiktok в 1080p и h265/hevc, когда это возможно.", "SettingsYoutubeDub": "использовать язык браузера", "SettingsYoutubeDubDescription": "использует главный язык браузера для аудиодорожек на youtube. работает даже если кобальт не переведён в твой язык.", - "UpdateOneMillion": "миллион и невероятная скорость" + "UpdateOneMillion": "миллион и невероятная скорость", + "ErrorYTAgeRestrict": "это видео ограничено по возрасту, поэтому я не могу его скачать. попробуй другое!" } } diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 558fe691..39a94f3a 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -73,7 +73,18 @@ export default async function(o) { if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; - if (info.playability_status.status !== 'OK') return { error: 'ErrorYTUnavailable' }; + const playability = info.playability_status; + + if (playability.status === 'LOGIN_REQUIRED') { + if (playability.reason.endsWith('bot')) { + return { error: 'ErrorYTLogin' } + } + if (playability.reason.endsWith('age')) { + return { error: 'ErrorYTAgeRestrict' } + } + } + + if (playability.status !== 'OK') return { error: 'ErrorYTUnavailable' }; if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' }; // return a critical error if returned video is "Video Not Available"