From 4bd3f54f643f1545dc788da29b410d0d14db6d57 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 23 May 2024 09:21:38 +0600 Subject: [PATCH 01/22] twitter: update api domain, endpoint & params --- src/modules/processing/services/twitter.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js index e96b6dbc..210de976 100644 --- a/src/modules/processing/services/twitter.js +++ b/src/modules/processing/services/twitter.js @@ -2,10 +2,12 @@ import { genericUserAgent } from "../../config.js"; import { createStream } from "../../stream/manage.js"; import { getCookie, updateCookie } from "../cookie/manager.js"; -const graphqlURL = 'https://twitter.com/i/api/graphql/5GOHgZe-8U2j5sVHQzEm9A/TweetResultByRestId'; -const tokenURL = 'https://api.twitter.com/1.1/guest/activate.json'; +const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId'; +const tokenURL = 'https://api.x.com/1.1/guest/activate.json'; -const tweetFeatures = JSON.stringify({ "creator_subscriptions_tweet_preview_api_enabled": true, "c9s_tweet_anatomy_moderator_badge_enabled": true, "tweetypie_unmention_optimization_enabled": true, "responsive_web_edit_tweet_api_enabled": true, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, "view_counts_everywhere_api_enabled": true, "longform_notetweets_consumption_enabled": true, "responsive_web_twitter_article_tweet_consumption_enabled": false, "tweet_awards_web_tipping_enabled": false, "responsive_web_home_pinned_timelines_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": true, "standardized_nudges_misinfo": true, "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, "longform_notetweets_rich_text_read_enabled": true, "longform_notetweets_inline_media_enabled": true, "responsive_web_graphql_exclude_directive_enabled": true, "verified_phone_label_enabled": false, "responsive_web_media_download_video_enabled": false, "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, "responsive_web_graphql_timeline_navigation_enabled": true, "responsive_web_enhance_cards_enabled": false }); +const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}); + +const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false}); const commonHeaders = { "user-agent": genericUserAgent, @@ -79,6 +81,7 @@ const requestTweet = async(tweetId, token, cookie) => { }) ); graphqlTweetURL.searchParams.set('features', tweetFeatures); + graphqlTweetURL.searchParams.set('fieldToggles', tweetFieldToggles); let result = await fetch(graphqlTweetURL, { headers }); updateCookie(cookie, result.headers); From 930faea09a2f8134aa7c78dab5ac054f7267816b Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 23 May 2024 09:22:33 +0600 Subject: [PATCH 02/22] twitter: add freebind support --- src/modules/processing/match.js | 3 ++- src/modules/processing/services/twitter.js | 24 ++++++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 5dfd2765..7f9714b7 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -62,7 +62,8 @@ export default async function(host, patternMatch, lang, obj) { r = await twitter({ id: patternMatch.id, index: patternMatch.index - 1, - toGif: !!obj.twitterGif + toGif: !!obj.twitterGif, + dispatcher }); break; case "vk": diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js index 210de976..36a8669b 100644 --- a/src/modules/processing/services/twitter.js +++ b/src/modules/processing/services/twitter.js @@ -37,14 +37,15 @@ function bestQuality(arr) { } let _cachedToken; -const getGuestToken = async (forceReload = false) => { +const getGuestToken = async (dispatcher, forceReload = false) => { if (_cachedToken && !forceReload) { return _cachedToken; } const tokenResponse = await fetch(tokenURL, { method: 'POST', - headers: commonHeaders + headers: commonHeaders, + dispatcher }).then(r => r.status === 200 && r.json()).catch(() => {}) if (tokenResponse?.guest_token) { @@ -52,7 +53,7 @@ const getGuestToken = async (forceReload = false) => { } } -const requestTweet = async(tweetId, token, cookie) => { +const requestTweet = async(dispatcher, tweetId, token, cookie) => { const graphqlTweetURL = new URL(graphqlURL); let headers = { @@ -83,7 +84,7 @@ const requestTweet = async(tweetId, token, cookie) => { graphqlTweetURL.searchParams.set('features', tweetFeatures); graphqlTweetURL.searchParams.set('fieldToggles', tweetFieldToggles); - let result = await fetch(graphqlTweetURL, { headers }); + let result = await fetch(graphqlTweetURL, { headers, dispatcher }); updateCookie(cookie, result.headers); // we might have been missing the `ct0` cookie, retry @@ -92,25 +93,26 @@ const requestTweet = async(tweetId, token, cookie) => { headers: { ...headers, 'x-csrf-token': cookie.values().ct0 - } + }, + dispatcher }); } return result } -export default async function({ id, index, toGif }) { +export default async function({ id, index, toGif, dispatcher }) { const cookie = await getCookie('twitter'); - let guestToken = await getGuestToken(); + let guestToken = await getGuestToken(dispatcher); if (!guestToken) return { error: 'ErrorCouldntFetch' }; - let tweet = await requestTweet(id, guestToken); + let tweet = await requestTweet(dispatcher, id, guestToken); // get new token & retry if old one expired if ([403, 429].includes(tweet.status)) { - guestToken = await getGuestToken(true); - tweet = await requestTweet(id, guestToken) + guestToken = await getGuestToken(dispatcher, true); + tweet = await requestTweet(dispatcher, id, guestToken) } tweet = await tweet.json(); @@ -124,7 +126,7 @@ export default async function({ id, index, toGif }) { return { error: 'ErrorTweetProtected' } case "NsfwLoggedOut": if (cookie) { - tweet = await requestTweet(id, guestToken, cookie); + tweet = await requestTweet(dispatcher, id, guestToken, cookie); tweet = await tweet.json(); tweetTypename = tweet?.data?.tweetResult?.result?.__typename; } else return { error: 'ErrorTweetNSFW' } From 72246cbc1fd7d7fdf7b9ab4c1c918d04d89871e2 Mon Sep 17 00:00:00 2001 From: jj Date: Thu, 23 May 2024 18:29:39 +0200 Subject: [PATCH 03/22] tiktok: use new cookie for each request (#514) * api: allow passing headers from service handler * tiktok: use new cookie for each request --- src/core/api.js | 7 ++- src/modules/processing/matchActionDecider.js | 2 + src/modules/processing/services/tiktok.js | 46 +++++++++++++------- src/modules/stream/manage.js | 2 + src/modules/stream/shared.js | 4 -- 5 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/core/api.js b/src/core/api.js index 04e190cd..f9b84c7a 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -192,8 +192,11 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { const streamInfo = getInternalStream(req.query.id); if (!streamInfo) return res.sendStatus(404); - streamInfo.headers = req.headers; - + streamInfo.headers = { + ...req.headers, + ...streamInfo.headers + }; + return stream(res, { type: 'internal', ...streamInfo }); } catch { return res.destroy(); diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 6b44f8ab..42a35f1a 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -9,6 +9,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di responseType = "stream", defaultParams = { u: r.urls, + headers: r.headers, service: host, filename: r.filenameAttributes ? createFilename(r.filenameAttributes, filenamePattern, isAudioOnly, isAudioMuted) : r.filename, @@ -80,6 +81,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di service: "tiktok", type: audioStreamType, u: r.urls, + headers: r.headers, filename: r.audioFilename, isAudioOnly: true, audioFormat, diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js index 8edb2a5b..0984ec04 100644 --- a/src/modules/processing/services/tiktok.js +++ b/src/modules/processing/services/tiktok.js @@ -4,9 +4,9 @@ import { extract } from "../url.js"; import Cookie from "../cookie/cookie.js"; const shortDomain = "https://vt.tiktok.com/"; -export const cookie = new Cookie({}); export default async function(obj) { + const cookie = new Cookie({}); let postId = obj.postId; if (!postId) { @@ -75,32 +75,46 @@ export default async function(obj) { if (audio.includes("mime_type=audio_mpeg")) bestAudio = 'mp3'; } - if (video) return { - urls: video, - filename: videoFilename + if (video) { + return { + urls: video, + filename: videoFilename, + headers: { cookie } + } } - if (images && obj.isAudioOnly) return { - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - bestAudio + + if (images && obj.isAudioOnly) { + return { + urls: audio, + audioFilename: audioFilename, + isAudioOnly: true, + bestAudio, + headers: { cookie } + } } + if (images) { let imageLinks = images .map(i => i.imageURL.urlList.find(p => p.includes(".jpeg?"))) - .map(url => ({ url })) + .map(url => ({ url })); + return { picker: imageLinks, urls: audio, audioFilename: audioFilename, isAudioOnly: true, - bestAudio + bestAudio, + headers: { cookie } } } - if (audio) return { - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - bestAudio + + if (audio) { + return { + urls: audio, + audioFilename: audioFilename, + isAudioOnly: true, + bestAudio, + headers: { cookie } + } } } diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 6c9e9d3c..bccf5c80 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -38,6 +38,7 @@ export function createStream(obj) { filename: obj.filename, audioFormat: obj.audioFormat, isAudioOnly: !!obj.isAudioOnly, + headers: obj.headers, copy: !!obj.copy, mute: !!obj.mute, metadata: obj.fileMetadata || false, @@ -82,6 +83,7 @@ export function createInternalStream(url, obj = {}) { internalStreamCache[streamID] = { url, service: obj.service, + headers: obj.headers, controller: new AbortController(), dispatcher }; diff --git a/src/modules/stream/shared.js b/src/modules/stream/shared.js index 42df2758..8b4c9109 100644 --- a/src/modules/stream/shared.js +++ b/src/modules/stream/shared.js @@ -1,5 +1,4 @@ import { genericUserAgent } from "../config.js"; -import { cookie as tiktokCookie } from "../processing/services/tiktok.js"; const defaultHeaders = { 'user-agent': genericUserAgent @@ -14,9 +13,6 @@ const serviceHeaders = { origin: 'https://www.youtube.com', referer: 'https://www.youtube.com', DNT: '?1' - }, - tiktok: { - cookie: tiktokCookie } } From 89d9d555d1c4c111e40c6771a214e927562fd363 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Wed, 22 May 2024 15:08:56 +0000 Subject: [PATCH 04/22] stream/internal: use end() instead of destroy() to close response --- src/modules/stream/internal.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index 9578da9b..fae23f57 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -55,8 +55,9 @@ async function handleYoutubeStream(streamInfo, res) { streamInfo.url = req.url; const size = BigInt(req.headers.get('content-length')); - if (req.status !== 200 || !size) - return res.destroy(); + if (req.status !== 200 || !size) { + return res.end(); + } const stream = chunkedStream(streamInfo, size); @@ -66,9 +67,9 @@ async function handleYoutubeStream(streamInfo, res) { } stream.pipe(res); - stream.on('error', () => res.destroy()); + stream.on('error', () => res.end()); } catch { - res.destroy(); + res.end(); } } @@ -94,10 +95,10 @@ export async function internalStream(streamInfo, res) { res.setHeader(name, value) if (req.statusCode < 200 || req.statusCode > 299) - return res.destroy(); + return res.end(); req.body.pipe(res); - req.body.on('error', () => res.destroy()); + req.body.on('error', () => res.end()); } catch { streamInfo.controller.abort(); } From aba1d8061572e2fc62814aedf1acd93aacfad39a Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Wed, 22 May 2024 15:09:14 +0000 Subject: [PATCH 05/22] stream/shared: use res.end() instead of destroy for closing response --- src/modules/stream/shared.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/modules/stream/shared.js b/src/modules/stream/shared.js index 8b4c9109..4dbd59d3 100644 --- a/src/modules/stream/shared.js +++ b/src/modules/stream/shared.js @@ -17,8 +17,11 @@ const serviceHeaders = { } export function closeResponse(res) { - if (!res.headersSent) res.sendStatus(500); - return res.destroy(); + if (!res.headersSent) { + res.sendStatus(500); + } + + return res.end(); } export function getHeaders(service) { From d72be2c95a86b675a06f75948a7f0c136cc3653d Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Wed, 22 May 2024 16:45:32 +0000 Subject: [PATCH 06/22] api: clean up and remove unnecessary `res.destroy()`s --- src/core/api.js | 74 +++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/src/core/api.js b/src/core/api.js index f9b84c7a..d822136f 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -163,52 +163,48 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { const checkBaseLength = id.length === 21 && exp.length === 13; const checkSafeLength = sig.length === 43 && sec.length === 43 && iv.length === 22; - if (checkQueries && checkBaseLength && checkSafeLength) { - // rate limit probe, will not return json after 8.0 - if (req.query.p) { - return res.status(200).json({ - status: "continue" - }) - } - try { - const streamInfo = verifyStream(id, sig, exp, sec, iv); - if (!streamInfo?.service) { - return res.sendStatus(streamInfo.status); - } - return stream(res, streamInfo); - } catch { - return res.destroy(); - } + if (!checkQueries || !checkBaseLength || !checkSafeLength) { + return res.sendStatus(400); } - return res.sendStatus(400); + + // rate limit probe, will not return json after 8.0 + if (req.query.p) { + return res.status(200).json({ + status: "continue" + }) + } + + const streamInfo = verifyStream(id, sig, exp, sec, iv); + if (!streamInfo?.service) { + return res.sendStatus(streamInfo.status); + } + return stream(res, streamInfo); }) app.get('/api/istream', (req, res) => { - try { - if (!req.ip.endsWith('127.0.0.1')) - return res.sendStatus(403); - if (String(req.query.id).length !== 21) - return res.sendStatus(400); - - const streamInfo = getInternalStream(req.query.id); - if (!streamInfo) return res.sendStatus(404); - streamInfo.headers = { - ...req.headers, - ...streamInfo.headers - }; - - return stream(res, { type: 'internal', ...streamInfo }); - } catch { - return res.destroy(); + if (!req.ip.endsWith('127.0.0.1')) { + return res.sendStatus(403); } + + if (String(req.query.id).length !== 21) { + return res.sendStatus(400); + } + + const streamInfo = getInternalStream(req.query.id); + if (!streamInfo) { + return res.sendStatus(404); + } + + streamInfo.headers = { + ...req.headers, + ...streamInfo.headers + }; + + return stream(res, { type: 'internal', ...streamInfo }); }) - app.get('/api/serverInfo', (req, res) => { - try { - return res.status(200).json(serverInfo); - } catch { - return res.destroy(); - } + app.get('/api/serverInfo', (_, res) => { + return res.status(200).json(serverInfo); }) app.get('/favicon.ico', (req, res) => { From 5c11e57e00c8c4165240b7246fe789ce380e0a3a Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 23 May 2024 22:39:22 +0600 Subject: [PATCH 07/22] package: bump version to 7.14.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f5e17282..9fdefba4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.14.1", + "version": "7.14.2", "author": "imput", "exports": "./src/cobalt.js", "type": "module", From abd9f2eb8784ee5dea2f3e7c1adf30c75e586fcf Mon Sep 17 00:00:00 2001 From: jj Date: Sat, 25 May 2024 15:12:18 +0200 Subject: [PATCH 08/22] web/cobalt.js: fix copying text on click Signed-off-by: jj --- src/front/cobalt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/front/cobalt.js b/src/front/cobalt.js index 1efe64be..0c89dec2 100644 --- a/src/front/cobalt.js +++ b/src/front/cobalt.js @@ -128,7 +128,7 @@ const copy = (id, data) => { if (data) { navigator.clipboard.writeText(data) } else { - navigator.clipboard.writeText(e.innerText) + navigator.clipboard.writeText(target.textContent) } } From e93ad8a9c5dc7b6cd0a5943e49fbf3d5d641fa2d Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 28 May 2024 09:00:09 +0600 Subject: [PATCH 09/22] reddit: use correct id in filename & add sub name --- src/modules/processing/services/reddit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/reddit.js b/src/modules/processing/services/reddit.js index 9341adc1..41f9efb4 100644 --- a/src/modules/processing/services/reddit.js +++ b/src/modules/processing/services/reddit.js @@ -107,7 +107,7 @@ export default async function(obj) { }).catch(() => {}) } - let id = video.split('/')[3]; + let id = `${String(obj.sub).toLowerCase()}_${obj.id}`; if (!audio) return { typeId: "redirect", From 669ab65be95969603695927a3120c19c5d19e767 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 28 May 2024 09:45:43 +0600 Subject: [PATCH 10/22] servicesConfig: add player subdomain for vimeo closes #520 --- src/modules/processing/servicesConfig.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index a862da71..9e8e8b66 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -61,8 +61,9 @@ }, "vimeo": { "patterns": [":id", "video/:id", ":id/:password", "/channels/:user/:id"], - "enabled": true, - "bestAudio": "mp3" + "subdomains": ["player"], + "bestAudio": "mp3", + "enabled": true }, "soundcloud": { "patterns": [":author/:song/s-:accessKey", ":author/:song", ":shortLink"], From 7c39b104359aaeca9b9f392ebe69112f8f179512 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Tue, 28 May 2024 06:19:42 +0000 Subject: [PATCH 11/22] api/istream: flip priority of header sources closes #526 --- src/core/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/api.js b/src/core/api.js index d822136f..c90f78f7 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -196,8 +196,8 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { } streamInfo.headers = { - ...req.headers, - ...streamInfo.headers + ...streamInfo.headers, + ...req.headers }; return stream(res, { type: 'internal', ...streamInfo }); From 7f333ec681fed43b0f988246d89b88649c552d3b Mon Sep 17 00:00:00 2001 From: jj Date: Tue, 28 May 2024 09:00:58 +0200 Subject: [PATCH 12/22] build: add test to check if lockfile needs an update --- .github/workflows/test.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a02c125..193ddd01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,19 @@ on: branches: [ current ] jobs: + check-lockfile: + name: check lockfile correctness + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Check that lockfile does not need an update + run: | + cp package-lock.json before.json + npm ci + npm i --package-lock-only + diff before.json package-lock.json + test-web: name: web sanity check runs-on: ubuntu-latest @@ -22,4 +35,4 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - name: Run test script - run: .github/test.sh api \ No newline at end of file + run: .github/test.sh api From 806ad142669e17411f5ee8cc5debf6618cbbdd82 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Tue, 28 May 2024 07:01:49 +0000 Subject: [PATCH 13/22] chore: bump version in lockfile --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9052067e..88004217 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cobalt", - "version": "7.14.1", + "version": "7.14.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cobalt", - "version": "7.14.1", + "version": "7.14.2", "license": "AGPL-3.0", "dependencies": { "content-disposition-header": "0.6.0", From 64b5990d8155b742a0c26f0504dbb5eff117cdb5 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 28 May 2024 14:32:03 +0600 Subject: [PATCH 14/22] stream: move hls exceptions to servicesConfig (#527) --- src/modules/config.js | 1 + src/modules/processing/servicesConfig.json | 1 + src/modules/stream/manage.js | 6 ++---- src/modules/stream/types.js | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/modules/config.js b/src/modules/config.js index 67114b56..530c5f0b 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -53,6 +53,7 @@ const export const services = servicesConfigJson.config, + hlsExceptions = servicesConfigJson.hlsExceptions, audioIgnore = servicesConfigJson.audioIgnore, version = packageJson.version, genericUserAgent = config.genericUserAgent, diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 9e8e8b66..2daa7b63 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -1,5 +1,6 @@ { "audioIgnore": ["vk", "ok"], + "hlsExceptions": ["dailymotion", "vimeo", "rutube"], "config": { "bilibili": { "alias": "bilibili.com & bilibili.tv", diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index bccf5c80..05008077 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -3,14 +3,12 @@ import { randomBytes } from "crypto"; import { nanoid } from "nanoid"; import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; -import { env } from "../config.js"; +import { env, hlsExceptions } from "../config.js"; import { strict as assert } from "assert"; // optional dependency const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); -const M3U_SERVICES = ['dailymotion', 'vimeo', 'rutube']; - const streamCache = new NodeCache({ stdTTL: env.streamLifespan, checkperiod: 10, @@ -110,7 +108,7 @@ export function destroyInternalStream(url) { function wrapStream(streamInfo) { /* m3u8 links are currently not supported * for internal streams, skip them */ - if (M3U_SERVICES.includes(streamInfo.service)) { + if (hlsExceptions.includes(streamInfo.service)) { return streamInfo; } diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index a1db0c60..2036b360 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -5,7 +5,7 @@ import { create as contentDisposition } from "content-disposition-header"; import { metadataManager } from "../sub/utils.js"; import { destroyInternalStream } from "./manage.js"; -import { env, ffmpegArgs } from "../config.js"; +import { env, ffmpegArgs, hlsExceptions } from "../config.js"; import { getHeaders, closeResponse } from "./shared.js"; function toRawHeaders(headers) { @@ -215,7 +215,7 @@ export function streamVideoOnly(streamInfo, res) { args.push('-an') } - if (["vimeo", "rutube", "dailymotion"].includes(streamInfo.service)) { + if (hlsExceptions.includes(streamInfo.service)) { args.push('-bsf:a', 'aac_adtstoasc') } From 490bbf82ec88e5d94b210349d0e025fb890656e3 Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 29 May 2024 12:57:26 +0600 Subject: [PATCH 15/22] processing/url: clean up cleanURL query exceptions --- src/modules/processing/url.js | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/modules/processing/url.js b/src/modules/processing/url.js index a006402c..1a8bb927 100644 --- a/src/modules/processing/url.js +++ b/src/modules/processing/url.js @@ -78,19 +78,30 @@ function aliasURL(url) { function cleanURL(url) { assert(url instanceof URL); const host = psl.parse(url.hostname).sld; + let stripQuery = true; - if (host === 'pinterest') { - url.hostname = 'pinterest.com' - } else if (host === 'vk' && url.pathname.includes('/clip')) { - if (url.searchParams.get('z')) - url.search = '?z=' + encodeURIComponent(url.searchParams.get('z')); - stripQuery = false; - } else if (host === 'youtube' && url.searchParams.get('v')) { - url.search = '?v=' + encodeURIComponent(url.searchParams.get('v')); + const limitQuery = (param) => { + url.search = `?${param}=` + encodeURIComponent(url.searchParams.get(param)); stripQuery = false; } + switch (host) { + case "pinterest": + url.hostname = 'pinterest.com'; + break; + case "vk": + if (url.pathname.includes('/clip') && url.searchParams.get('z')) { + limitQuery('z') + } + break; + case "youtube": + if (url.searchParams.get('v')) { + limitQuery('v') + } + break; + } + if (stripQuery) { url.search = '' } From 2a2183aa8404694adbbdb17228a67e434fd8030d Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 29 May 2024 13:02:05 +0600 Subject: [PATCH 16/22] rutube: add support for private video links --- src/modules/processing/match.js | 1 + src/modules/processing/services/rutube.js | 15 ++++++++------- src/modules/processing/servicesConfig.json | 2 +- src/modules/processing/servicesPatternTesters.js | 1 + src/modules/processing/url.js | 5 +++++ src/test/tests.json | 8 ++++++++ 6 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 7f9714b7..c3091709 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -179,6 +179,7 @@ export default async function(host, patternMatch, lang, obj) { r = await rutube({ id: patternMatch.id, yappyId: patternMatch.yappyId, + key: patternMatch.key, quality: obj.vQuality, isAudioOnly: isAudioOnly }); diff --git a/src/modules/processing/services/rutube.js b/src/modules/processing/services/rutube.js index dd984c57..a8d0abbe 100644 --- a/src/modules/processing/services/rutube.js +++ b/src/modules/processing/services/rutube.js @@ -12,10 +12,10 @@ async function requestJSON(url) { export default async function(obj) { if (obj.yappyId) { - let yappy = await requestJSON( + const yappy = await requestJSON( `https://rutube.ru/pangolin/api/web/yappy/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=15` ) - let yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link; + const yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link; if (!yappyURL) return { error: 'ErrorEmptyDownload' }; return { @@ -25,11 +25,12 @@ export default async function(obj) { } } - let quality = obj.quality === "max" ? "9000" : obj.quality; + const quality = obj.quality === "max" ? "9000" : obj.quality; - let play = await requestJSON( - `https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2` - ) + const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`); + if (obj.key) requestURL.searchParams.set('p', obj.key); + + const play = await requestJSON(requestURL); if (!play) return { error: 'ErrorCouldntFetch' }; if (play.detail || !play.video_balancer) return { error: 'ErrorEmptyDownload' }; @@ -51,7 +52,7 @@ export default async function(obj) { bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height)); } - let fileMetadata = { + const fileMetadata = { title: cleanString(play.title.trim()), artist: cleanString(play.author.name.trim()), } diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 2daa7b63..c0a11817 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -105,7 +105,7 @@ "rutube": { "alias": "rutube videos", "tld": "ru", - "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId"], + "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId", "video/private/:id?p=:key", "video/private/:id"], "enabled": true }, "dailymotion": { diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 25b8943a..a4770c17 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -20,6 +20,7 @@ export const testers = { || (patternMatch.user?.length <= 22 && patternMatch.id?.length <= 10), "rutube": (patternMatch) => + (patternMatch.id?.length === 32 && patternMatch.key?.length <= 32) || patternMatch.id?.length === 32 || patternMatch.yappyId?.length === 32, "soundcloud": (patternMatch) => diff --git a/src/modules/processing/url.js b/src/modules/processing/url.js index 1a8bb927..111f1f6f 100644 --- a/src/modules/processing/url.js +++ b/src/modules/processing/url.js @@ -100,6 +100,11 @@ function cleanURL(url) { limitQuery('v') } break; + case "rutube": + if (url.searchParams.get('p')) { + limitQuery('p') + } + break; } if (stripQuery) { diff --git a/src/test/tests.json b/src/test/tests.json index 565fa5a4..36044cde 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -1089,6 +1089,14 @@ "code": 200, "status": "stream" } + }, { + "name": "private video", + "url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } }], "ok": [{ "name": "regular video", From e4d42fa86a7a2092af69e6dacd308a150421cea6 Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 29 May 2024 13:12:52 +0600 Subject: [PATCH 17/22] processing: add loom support (#530) --- README.md | 5 ++- src/modules/processing/match.js | 6 +++ src/modules/processing/matchActionDecider.js | 1 + src/modules/processing/services/loom.js | 39 +++++++++++++++++++ src/modules/processing/servicesConfig.json | 7 +++- .../processing/servicesPatternTesters.js | 7 +++- src/test/tests.json | 29 ++++++++++++++ 7 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 src/modules/processing/services/loom.js diff --git a/README.md b/README.md index c67631be..f4c9a701 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ this list is not final and keeps expanding over time. if support for a service y | bilibili.com & bilibili.tv | ✅ | ✅ | ✅ | ➖ | ➖ | | dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ | | instagram posts & reels | ✅ | ✅ | ✅ | ➖ | ➖ | -| ok video | ✅ | ❌ | ❌ | ✅ | ✅ | +| loom | ✅ | ❌ | ✅ | ✅ | ➖ | +| ok video | ✅ | ❌ | ✅ | ✅ | ✅ | | pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | | reddit | ✅ | ✅ | ✅ | ❌ | ❌ | | rutube | ✅ | ✅ | ✅ | ✅ | ✅ | @@ -31,7 +32,7 @@ this list is not final and keeps expanding over time. if support for a service y | twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | | vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | | vine archive | ✅ | ✅ | ✅ | ➖ | ➖ | -| vk videos & clips | ✅ | ❌ | ❌ | ✅ | ✅ | +| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | | youtube videos, shorts & music | ✅ | ✅ | ✅ | ✅ | ✅ | | emoji | meaning | diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index c3091709..3e38c4db 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -24,6 +24,7 @@ import streamable from "./services/streamable.js"; import twitch from "./services/twitch.js"; import rutube from "./services/rutube.js"; import dailymotion from "./services/dailymotion.js"; +import loom from "./services/loom.js"; let freebind; @@ -187,6 +188,11 @@ export default async function(host, patternMatch, lang, obj) { case "dailymotion": r = await dailymotion(patternMatch); break; + case "loom": + r = await loom({ + id: patternMatch.id + }); + break; default: return createResponse("error", { t: loc(lang, 'ErrorUnsupported') diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 42a35f1a..f7ed3da9 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -129,6 +129,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di case "tumblr": case "pinterest": case "streamable": + case "loom": responseType = "redirect"; break; } diff --git a/src/modules/processing/services/loom.js b/src/modules/processing/services/loom.js new file mode 100644 index 00000000..ecb6c534 --- /dev/null +++ b/src/modules/processing/services/loom.js @@ -0,0 +1,39 @@ +import { genericUserAgent } from "../../config.js"; + +export default async function({ id }) { + const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/transcoded-url`, { + method: "POST", + headers: { + "user-agent": genericUserAgent, + origin: "https://www.loom.com", + referer: `https://www.loom.com/share/${id}`, + cookie: `loom_referral_video=${id};`, + + "apollographql-client-name": "web", + "apollographql-client-version": "14c0b42", + "x-loom-request-source": "loom_web_14c0b42", + }, + body: JSON.stringify({ + force_original: false, + password: null, + anonID: null, + deviceID: null + }) + }) + .then(r => r.status === 200 ? r.json() : false) + .catch(() => {}); + + if (!gql) return { error: 'ErrorEmptyDownload' }; + + const videoUrl = gql?.url; + + if (videoUrl?.includes('.mp4?')) { + return { + urls: videoUrl, + filename: `loom_${id}.mp4`, + audioFilename: `loom_${id}_audio` + } + } + + return { error: 'ErrorEmptyDownload' } +} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index c0a11817..ae4cd9f0 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -1,5 +1,5 @@ { - "audioIgnore": ["vk", "ok"], + "audioIgnore": ["vk", "ok", "loom"], "hlsExceptions": ["dailymotion", "vimeo", "rutube"], "config": { "bilibili": { @@ -112,6 +112,11 @@ "alias": "dailymotion videos", "patterns": ["video/:id"], "enabled": true + }, + "loom": { + "alias": "loom videos", + "patterns": ["share/:id"], + "enabled": true } } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index a4770c17..ddeea31f 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -8,7 +8,10 @@ export const testers = { "instagram": (patternMatch) => patternMatch.postId?.length <= 12 || (patternMatch.username?.length <= 30 && patternMatch.storyId?.length <= 24), - + + "loom": (patternMatch) => + patternMatch.id?.length <= 32, + "ok": (patternMatch) => patternMatch.id?.length <= 16, @@ -29,7 +32,7 @@ export const testers = { "streamable": (patternMatch) => patternMatch.id?.length === 6, - + "tiktok": (patternMatch) => patternMatch.postId?.length <= 21 || patternMatch.id?.length <= 13, diff --git a/src/test/tests.json b/src/test/tests.json index 36044cde..501ac2c0 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -1131,5 +1131,34 @@ "code": 200, "status": "stream" } + }], + "loom": [{ + "name": "1080p video", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, { + "name": "1080p video (muted)", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "1080p video (audio only)", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } }] } From ed607a08bbbde00de01155286ec33eafebd1f1c8 Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 29 May 2024 13:15:10 +0600 Subject: [PATCH 18/22] readme: add additional info about rutube --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f4c9a701..d2bb064c 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ this list is not final and keeps expanding over time. if support for a service y | instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. | | pinterest | supports photos, gifs, videos and stories. | | reddit | supports gifs and videos. | +| rutube | supports yappy & private links. | | soundcloud | supports private links. | | tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. | | twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. | From 35ba3dc1a3560b6f35ecb1daf2b0fa6df262057b Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 29 May 2024 13:21:06 +0600 Subject: [PATCH 19/22] package: bump version to 7.14.3 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 88004217..2636146d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cobalt", - "version": "7.14.2", + "version": "7.14.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cobalt", - "version": "7.14.2", + "version": "7.14.3", "license": "AGPL-3.0", "dependencies": { "content-disposition-header": "0.6.0", diff --git a/package.json b/package.json index 9fdefba4..72104aba 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.14.2", + "version": "7.14.3", "author": "imput", "exports": "./src/cobalt.js", "type": "module", From 44ecfeeea7963c61deb3946f874bcb2856c7d465 Mon Sep 17 00:00:00 2001 From: jj Date: Wed, 29 May 2024 10:26:17 +0200 Subject: [PATCH 20/22] youtube: don't block api startup waiting for innertube to activate (#532) cobalt api has been getting blocked for several seconds during startup, and also crashing when unable to connect to youtube (e.g. when it's blocked); this should fix both those things --- src/modules/processing/services/youtube.js | 25 +++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 64f6d18f..c97d8c24 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -3,7 +3,7 @@ import { env } from '../../config.js'; import { cleanString } from '../../sub/utils.js'; import { fetch } from 'undici' -const ytBase = await Innertube.create(); +const ytBase = Innertube.create().catch(e => e); const codecMatch = { h264: { @@ -23,16 +23,21 @@ const codecMatch = { } } -const cloneInnertube = (customFetch) => { +const cloneInnertube = async (customFetch) => { + const innertube = await ytBase; + if (innertube instanceof Error) { + throw innertube; + } + const session = new Session( - ytBase.session.context, - ytBase.session.key, - ytBase.session.api_version, - ytBase.session.account_index, - ytBase.session.player, + innertube.session.context, + innertube.session.key, + innertube.session.api_version, + innertube.session.account_index, + innertube.session.player, undefined, - customFetch ?? ytBase.session.http.fetch, - ytBase.session.cache + customFetch ?? innertube.session.http.fetch, + innertube.session.cache ); const yt = new Innertube(session); @@ -40,7 +45,7 @@ const cloneInnertube = (customFetch) => { } export default async function(o) { - const yt = cloneInnertube( + const yt = await cloneInnertube( (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher }) ); From b7697268e50cc6de2b903dc326e6402ed34ce2e6 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Wed, 29 May 2024 08:28:17 +0000 Subject: [PATCH 21/22] youtube: return different error message if video doesn't exist --- src/modules/processing/services/youtube.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index c97d8c24..fae4e04d 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -62,8 +62,12 @@ export default async function(o) { try { info = await yt.getBasicInfo(o.id, 'WEB'); - } catch { - return { error: 'ErrorCantConnectToServiceAPI' }; + } catch(e) { + if (e?.message === 'This video is unavailable') { + return { error: 'ErrorCouldntFetch' }; + } else { + return { error: 'ErrorCantConnectToServiceAPI' }; + } } if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; From 03fda93f969d8512dad24f5bfe737279d1c886bd Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Wed, 29 May 2024 08:48:08 +0000 Subject: [PATCH 22/22] tiktok: fix error when user prefers h265 and downloads photo slideshow --- src/modules/processing/services/tiktok.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js index 0984ec04..33e7aef3 100644 --- a/src/modules/processing/services/tiktok.js +++ b/src/modules/processing/services/tiktok.js @@ -57,7 +57,7 @@ export default async function(obj) { let playAddr = detail.video.playAddr; if (obj.h265) { - const h265PlayAddr = detail.video.bitrateInfo.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0] + const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0] playAddr = h265PlayAddr || playAddr }