diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a02c125..193ddd01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,19 @@ on: branches: [ current ] jobs: + check-lockfile: + name: check lockfile correctness + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Check that lockfile does not need an update + run: | + cp package-lock.json before.json + npm ci + npm i --package-lock-only + diff before.json package-lock.json + test-web: name: web sanity check runs-on: ubuntu-latest @@ -22,4 +35,4 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - name: Run test script - run: .github/test.sh api \ No newline at end of file + run: .github/test.sh api diff --git a/README.md b/README.md index b95d55e5..2b9cc7de 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ this list is not final and keeps expanding over time. if support for a service y | bilibili.com & bilibili.tv | ✅ | ✅ | ✅ | ➖ | ➖ | | dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ | | instagram posts & reels | ✅ | ✅ | ✅ | ➖ | ➖ | -| ok video | ✅ | ❌ | ❌ | ✅ | ✅ | +| loom | ✅ | ❌ | ✅ | ✅ | ➖ | +| ok video | ✅ | ❌ | ✅ | ✅ | ✅ | | pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | | reddit | ✅ | ✅ | ✅ | ❌ | ❌ | | rutube | ✅ | ✅ | ✅ | ✅ | ✅ | @@ -32,7 +33,7 @@ this list is not final and keeps expanding over time. if support for a service y | twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | | vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | | vine archive | ✅ | ✅ | ✅ | ➖ | ➖ | -| vk videos & clips | ✅ | ❌ | ❌ | ✅ | ✅ | +| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | | youtube videos, shorts & music | ✅ | ✅ | ✅ | ✅ | ✅ | | emoji | meaning | @@ -48,6 +49,7 @@ this list is not final and keeps expanding over time. if support for a service y | pinterest | supports photos, gifs, videos and stories. | | reddit | supports gifs and videos. | | snapchat | supports spotlights and stories. lets you pick what to save from stories. | +| rutube | supports yappy & private links. | | soundcloud | supports private links. | | tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. | | twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. | diff --git a/package-lock.json b/package-lock.json index 9052067e..2636146d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cobalt", - "version": "7.14.1", + "version": "7.14.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cobalt", - "version": "7.14.1", + "version": "7.14.3", "license": "AGPL-3.0", "dependencies": { "content-disposition-header": "0.6.0", diff --git a/package.json b/package.json index f5e17282..72104aba 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.14.1", + "version": "7.14.3", "author": "imput", "exports": "./src/cobalt.js", "type": "module", diff --git a/src/core/api.js b/src/core/api.js index 04e190cd..c90f78f7 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -163,49 +163,48 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { const checkBaseLength = id.length === 21 && exp.length === 13; const checkSafeLength = sig.length === 43 && sec.length === 43 && iv.length === 22; - if (checkQueries && checkBaseLength && checkSafeLength) { - // rate limit probe, will not return json after 8.0 - if (req.query.p) { - return res.status(200).json({ - status: "continue" - }) - } - try { - const streamInfo = verifyStream(id, sig, exp, sec, iv); - if (!streamInfo?.service) { - return res.sendStatus(streamInfo.status); - } - return stream(res, streamInfo); - } catch { - return res.destroy(); - } + if (!checkQueries || !checkBaseLength || !checkSafeLength) { + return res.sendStatus(400); } - return res.sendStatus(400); + + // rate limit probe, will not return json after 8.0 + if (req.query.p) { + return res.status(200).json({ + status: "continue" + }) + } + + const streamInfo = verifyStream(id, sig, exp, sec, iv); + if (!streamInfo?.service) { + return res.sendStatus(streamInfo.status); + } + return stream(res, streamInfo); }) app.get('/api/istream', (req, res) => { - try { - if (!req.ip.endsWith('127.0.0.1')) - return res.sendStatus(403); - if (String(req.query.id).length !== 21) - return res.sendStatus(400); - - const streamInfo = getInternalStream(req.query.id); - if (!streamInfo) return res.sendStatus(404); - streamInfo.headers = req.headers; - - return stream(res, { type: 'internal', ...streamInfo }); - } catch { - return res.destroy(); + if (!req.ip.endsWith('127.0.0.1')) { + return res.sendStatus(403); } + + if (String(req.query.id).length !== 21) { + return res.sendStatus(400); + } + + const streamInfo = getInternalStream(req.query.id); + if (!streamInfo) { + return res.sendStatus(404); + } + + streamInfo.headers = { + ...streamInfo.headers, + ...req.headers + }; + + return stream(res, { type: 'internal', ...streamInfo }); }) - app.get('/api/serverInfo', (req, res) => { - try { - return res.status(200).json(serverInfo); - } catch { - return res.destroy(); - } + app.get('/api/serverInfo', (_, res) => { + return res.status(200).json(serverInfo); }) app.get('/favicon.ico', (req, res) => { diff --git a/src/front/cobalt.js b/src/front/cobalt.js index 1efe64be..0c89dec2 100644 --- a/src/front/cobalt.js +++ b/src/front/cobalt.js @@ -128,7 +128,7 @@ const copy = (id, data) => { if (data) { navigator.clipboard.writeText(data) } else { - navigator.clipboard.writeText(e.innerText) + navigator.clipboard.writeText(target.textContent) } } diff --git a/src/modules/config.js b/src/modules/config.js index 67114b56..530c5f0b 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -53,6 +53,7 @@ const export const services = servicesConfigJson.config, + hlsExceptions = servicesConfigJson.hlsExceptions, audioIgnore = servicesConfigJson.audioIgnore, version = packageJson.version, genericUserAgent = config.genericUserAgent, diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index f6c99ded..e18e992d 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -25,6 +25,7 @@ import twitch from "./services/twitch.js"; import rutube from "./services/rutube.js"; import dailymotion from "./services/dailymotion.js"; import snapchat from "./services/snapchat.js"; +import loom from "./services/loom.js"; let freebind; @@ -63,7 +64,8 @@ export default async function(host, patternMatch, lang, obj) { r = await twitter({ id: patternMatch.id, index: patternMatch.index - 1, - toGif: !!obj.twitterGif + toGif: !!obj.twitterGif, + dispatcher }); break; case "vk": @@ -179,6 +181,7 @@ export default async function(host, patternMatch, lang, obj) { r = await rutube({ id: patternMatch.id, yappyId: patternMatch.yappyId, + key: patternMatch.key, quality: obj.vQuality, isAudioOnly: isAudioOnly }); @@ -194,6 +197,10 @@ export default async function(host, patternMatch, lang, obj) { spotlightId: patternMatch.spotlightId, shortLink: patternMatch.shortLink || false }); + case "loom": + r = await loom({ + id: patternMatch.id + }); break; default: return createResponse("error", { diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index eb6df0b7..c7d3114a 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -9,6 +9,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di responseType = "stream", defaultParams = { u: r.urls, + headers: r.headers, service: host, filename: r.filenameAttributes ? createFilename(r.filenameAttributes, filenamePattern, isAudioOnly, isAudioMuted) : r.filename, @@ -81,6 +82,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di service: "tiktok", type: audioStreamType, u: r.urls, + headers: r.headers, filename: r.audioFilename, isAudioOnly: true, audioFormat, @@ -129,6 +131,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di case "pinterest": case "streamable": case "snapchat": + case "loom": responseType = "redirect"; break; } diff --git a/src/modules/processing/services/loom.js b/src/modules/processing/services/loom.js new file mode 100644 index 00000000..ecb6c534 --- /dev/null +++ b/src/modules/processing/services/loom.js @@ -0,0 +1,39 @@ +import { genericUserAgent } from "../../config.js"; + +export default async function({ id }) { + const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/transcoded-url`, { + method: "POST", + headers: { + "user-agent": genericUserAgent, + origin: "https://www.loom.com", + referer: `https://www.loom.com/share/${id}`, + cookie: `loom_referral_video=${id};`, + + "apollographql-client-name": "web", + "apollographql-client-version": "14c0b42", + "x-loom-request-source": "loom_web_14c0b42", + }, + body: JSON.stringify({ + force_original: false, + password: null, + anonID: null, + deviceID: null + }) + }) + .then(r => r.status === 200 ? r.json() : false) + .catch(() => {}); + + if (!gql) return { error: 'ErrorEmptyDownload' }; + + const videoUrl = gql?.url; + + if (videoUrl?.includes('.mp4?')) { + return { + urls: videoUrl, + filename: `loom_${id}.mp4`, + audioFilename: `loom_${id}_audio` + } + } + + return { error: 'ErrorEmptyDownload' } +} diff --git a/src/modules/processing/services/reddit.js b/src/modules/processing/services/reddit.js index 9341adc1..41f9efb4 100644 --- a/src/modules/processing/services/reddit.js +++ b/src/modules/processing/services/reddit.js @@ -107,7 +107,7 @@ export default async function(obj) { }).catch(() => {}) } - let id = video.split('/')[3]; + let id = `${String(obj.sub).toLowerCase()}_${obj.id}`; if (!audio) return { typeId: "redirect", diff --git a/src/modules/processing/services/rutube.js b/src/modules/processing/services/rutube.js index dd984c57..a8d0abbe 100644 --- a/src/modules/processing/services/rutube.js +++ b/src/modules/processing/services/rutube.js @@ -12,10 +12,10 @@ async function requestJSON(url) { export default async function(obj) { if (obj.yappyId) { - let yappy = await requestJSON( + const yappy = await requestJSON( `https://rutube.ru/pangolin/api/web/yappy/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=15` ) - let yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link; + const yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link; if (!yappyURL) return { error: 'ErrorEmptyDownload' }; return { @@ -25,11 +25,12 @@ export default async function(obj) { } } - let quality = obj.quality === "max" ? "9000" : obj.quality; + const quality = obj.quality === "max" ? "9000" : obj.quality; - let play = await requestJSON( - `https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2` - ) + const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`); + if (obj.key) requestURL.searchParams.set('p', obj.key); + + const play = await requestJSON(requestURL); if (!play) return { error: 'ErrorCouldntFetch' }; if (play.detail || !play.video_balancer) return { error: 'ErrorEmptyDownload' }; @@ -51,7 +52,7 @@ export default async function(obj) { bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height)); } - let fileMetadata = { + const fileMetadata = { title: cleanString(play.title.trim()), artist: cleanString(play.author.name.trim()), } diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js index 8edb2a5b..33e7aef3 100644 --- a/src/modules/processing/services/tiktok.js +++ b/src/modules/processing/services/tiktok.js @@ -4,9 +4,9 @@ import { extract } from "../url.js"; import Cookie from "../cookie/cookie.js"; const shortDomain = "https://vt.tiktok.com/"; -export const cookie = new Cookie({}); export default async function(obj) { + const cookie = new Cookie({}); let postId = obj.postId; if (!postId) { @@ -57,7 +57,7 @@ export default async function(obj) { let playAddr = detail.video.playAddr; if (obj.h265) { - const h265PlayAddr = detail.video.bitrateInfo.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0] + const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0] playAddr = h265PlayAddr || playAddr } @@ -75,32 +75,46 @@ export default async function(obj) { if (audio.includes("mime_type=audio_mpeg")) bestAudio = 'mp3'; } - if (video) return { - urls: video, - filename: videoFilename + if (video) { + return { + urls: video, + filename: videoFilename, + headers: { cookie } + } } - if (images && obj.isAudioOnly) return { - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - bestAudio + + if (images && obj.isAudioOnly) { + return { + urls: audio, + audioFilename: audioFilename, + isAudioOnly: true, + bestAudio, + headers: { cookie } + } } + if (images) { let imageLinks = images .map(i => i.imageURL.urlList.find(p => p.includes(".jpeg?"))) - .map(url => ({ url })) + .map(url => ({ url })); + return { picker: imageLinks, urls: audio, audioFilename: audioFilename, isAudioOnly: true, - bestAudio + bestAudio, + headers: { cookie } } } - if (audio) return { - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - bestAudio + + if (audio) { + return { + urls: audio, + audioFilename: audioFilename, + isAudioOnly: true, + bestAudio, + headers: { cookie } + } } } diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js index e96b6dbc..36a8669b 100644 --- a/src/modules/processing/services/twitter.js +++ b/src/modules/processing/services/twitter.js @@ -2,10 +2,12 @@ import { genericUserAgent } from "../../config.js"; import { createStream } from "../../stream/manage.js"; import { getCookie, updateCookie } from "../cookie/manager.js"; -const graphqlURL = 'https://twitter.com/i/api/graphql/5GOHgZe-8U2j5sVHQzEm9A/TweetResultByRestId'; -const tokenURL = 'https://api.twitter.com/1.1/guest/activate.json'; +const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId'; +const tokenURL = 'https://api.x.com/1.1/guest/activate.json'; -const tweetFeatures = JSON.stringify({ "creator_subscriptions_tweet_preview_api_enabled": true, "c9s_tweet_anatomy_moderator_badge_enabled": true, "tweetypie_unmention_optimization_enabled": true, "responsive_web_edit_tweet_api_enabled": true, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, "view_counts_everywhere_api_enabled": true, "longform_notetweets_consumption_enabled": true, "responsive_web_twitter_article_tweet_consumption_enabled": false, "tweet_awards_web_tipping_enabled": false, "responsive_web_home_pinned_timelines_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": true, "standardized_nudges_misinfo": true, "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, "longform_notetweets_rich_text_read_enabled": true, "longform_notetweets_inline_media_enabled": true, "responsive_web_graphql_exclude_directive_enabled": true, "verified_phone_label_enabled": false, "responsive_web_media_download_video_enabled": false, "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, "responsive_web_graphql_timeline_navigation_enabled": true, "responsive_web_enhance_cards_enabled": false }); +const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}); + +const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false}); const commonHeaders = { "user-agent": genericUserAgent, @@ -35,14 +37,15 @@ function bestQuality(arr) { } let _cachedToken; -const getGuestToken = async (forceReload = false) => { +const getGuestToken = async (dispatcher, forceReload = false) => { if (_cachedToken && !forceReload) { return _cachedToken; } const tokenResponse = await fetch(tokenURL, { method: 'POST', - headers: commonHeaders + headers: commonHeaders, + dispatcher }).then(r => r.status === 200 && r.json()).catch(() => {}) if (tokenResponse?.guest_token) { @@ -50,7 +53,7 @@ const getGuestToken = async (forceReload = false) => { } } -const requestTweet = async(tweetId, token, cookie) => { +const requestTweet = async(dispatcher, tweetId, token, cookie) => { const graphqlTweetURL = new URL(graphqlURL); let headers = { @@ -79,8 +82,9 @@ const requestTweet = async(tweetId, token, cookie) => { }) ); graphqlTweetURL.searchParams.set('features', tweetFeatures); + graphqlTweetURL.searchParams.set('fieldToggles', tweetFieldToggles); - let result = await fetch(graphqlTweetURL, { headers }); + let result = await fetch(graphqlTweetURL, { headers, dispatcher }); updateCookie(cookie, result.headers); // we might have been missing the `ct0` cookie, retry @@ -89,25 +93,26 @@ const requestTweet = async(tweetId, token, cookie) => { headers: { ...headers, 'x-csrf-token': cookie.values().ct0 - } + }, + dispatcher }); } return result } -export default async function({ id, index, toGif }) { +export default async function({ id, index, toGif, dispatcher }) { const cookie = await getCookie('twitter'); - let guestToken = await getGuestToken(); + let guestToken = await getGuestToken(dispatcher); if (!guestToken) return { error: 'ErrorCouldntFetch' }; - let tweet = await requestTweet(id, guestToken); + let tweet = await requestTweet(dispatcher, id, guestToken); // get new token & retry if old one expired if ([403, 429].includes(tweet.status)) { - guestToken = await getGuestToken(true); - tweet = await requestTweet(id, guestToken) + guestToken = await getGuestToken(dispatcher, true); + tweet = await requestTweet(dispatcher, id, guestToken) } tweet = await tweet.json(); @@ -121,7 +126,7 @@ export default async function({ id, index, toGif }) { return { error: 'ErrorTweetProtected' } case "NsfwLoggedOut": if (cookie) { - tweet = await requestTweet(id, guestToken, cookie); + tweet = await requestTweet(dispatcher, id, guestToken, cookie); tweet = await tweet.json(); tweetTypename = tweet?.data?.tweetResult?.result?.__typename; } else return { error: 'ErrorTweetNSFW' } diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 64f6d18f..fae4e04d 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -3,7 +3,7 @@ import { env } from '../../config.js'; import { cleanString } from '../../sub/utils.js'; import { fetch } from 'undici' -const ytBase = await Innertube.create(); +const ytBase = Innertube.create().catch(e => e); const codecMatch = { h264: { @@ -23,16 +23,21 @@ const codecMatch = { } } -const cloneInnertube = (customFetch) => { +const cloneInnertube = async (customFetch) => { + const innertube = await ytBase; + if (innertube instanceof Error) { + throw innertube; + } + const session = new Session( - ytBase.session.context, - ytBase.session.key, - ytBase.session.api_version, - ytBase.session.account_index, - ytBase.session.player, + innertube.session.context, + innertube.session.key, + innertube.session.api_version, + innertube.session.account_index, + innertube.session.player, undefined, - customFetch ?? ytBase.session.http.fetch, - ytBase.session.cache + customFetch ?? innertube.session.http.fetch, + innertube.session.cache ); const yt = new Innertube(session); @@ -40,7 +45,7 @@ const cloneInnertube = (customFetch) => { } export default async function(o) { - const yt = cloneInnertube( + const yt = await cloneInnertube( (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher }) ); @@ -57,8 +62,12 @@ export default async function(o) { try { info = await yt.getBasicInfo(o.id, 'WEB'); - } catch { - return { error: 'ErrorCantConnectToServiceAPI' }; + } catch(e) { + if (e?.message === 'This video is unavailable') { + return { error: 'ErrorCouldntFetch' }; + } else { + return { error: 'ErrorCantConnectToServiceAPI' }; + } } if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 51d06320..54aec2b0 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -1,5 +1,6 @@ { - "audioIgnore": ["vk", "ok"], + "audioIgnore": ["vk", "ok", "loom"], + "hlsExceptions": ["dailymotion", "vimeo", "rutube"], "config": { "bilibili": { "alias": "bilibili.com & bilibili.tv", @@ -61,8 +62,9 @@ }, "vimeo": { "patterns": [":id", "video/:id", ":id/:password", "/channels/:user/:id"], - "enabled": true, - "bestAudio": "mp3" + "subdomains": ["player"], + "bestAudio": "mp3", + "enabled": true }, "soundcloud": { "patterns": [":author/:song/s-:accessKey", ":author/:song", ":shortLink"], @@ -103,7 +105,7 @@ "rutube": { "alias": "rutube videos", "tld": "ru", - "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId"], + "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId", "video/private/:id?p=:key", "video/private/:id"], "enabled": true }, "dailymotion": { @@ -116,6 +118,11 @@ "subdomains": ["t", "story"], "patterns": [":shortLink", "spotlight/:spotlightId", "add/:username/:storyId", "u/:username/:storyId", "add/:username", "u/:username"], "enabled": true + }, + "loom": { + "alias": "loom videos", + "patterns": ["share/:id"], + "enabled": true } } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 6746b8c5..598114bd 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -8,7 +8,10 @@ export const testers = { "instagram": (patternMatch) => patternMatch.postId?.length <= 12 || (patternMatch.username?.length <= 30 && patternMatch.storyId?.length <= 24), - + + "loom": (patternMatch) => + patternMatch.id?.length <= 32, + "ok": (patternMatch) => patternMatch.id?.length <= 16, @@ -20,6 +23,7 @@ export const testers = { || (patternMatch.user?.length <= 22 && patternMatch.id?.length <= 10), "rutube": (patternMatch) => + (patternMatch.id?.length === 32 && patternMatch.key?.length <= 32) || patternMatch.id?.length === 32 || patternMatch.yappyId?.length === 32, "soundcloud": (patternMatch) => @@ -33,7 +37,7 @@ export const testers = { "streamable": (patternMatch) => patternMatch.id?.length === 6, - + "tiktok": (patternMatch) => patternMatch.postId?.length <= 21 || patternMatch.id?.length <= 13, diff --git a/src/modules/processing/url.js b/src/modules/processing/url.js index a006402c..111f1f6f 100644 --- a/src/modules/processing/url.js +++ b/src/modules/processing/url.js @@ -78,19 +78,35 @@ function aliasURL(url) { function cleanURL(url) { assert(url instanceof URL); const host = psl.parse(url.hostname).sld; + let stripQuery = true; - if (host === 'pinterest') { - url.hostname = 'pinterest.com' - } else if (host === 'vk' && url.pathname.includes('/clip')) { - if (url.searchParams.get('z')) - url.search = '?z=' + encodeURIComponent(url.searchParams.get('z')); - stripQuery = false; - } else if (host === 'youtube' && url.searchParams.get('v')) { - url.search = '?v=' + encodeURIComponent(url.searchParams.get('v')); + const limitQuery = (param) => { + url.search = `?${param}=` + encodeURIComponent(url.searchParams.get(param)); stripQuery = false; } + switch (host) { + case "pinterest": + url.hostname = 'pinterest.com'; + break; + case "vk": + if (url.pathname.includes('/clip') && url.searchParams.get('z')) { + limitQuery('z') + } + break; + case "youtube": + if (url.searchParams.get('v')) { + limitQuery('v') + } + break; + case "rutube": + if (url.searchParams.get('p')) { + limitQuery('p') + } + break; + } + if (stripQuery) { url.search = '' } diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index 9578da9b..fae23f57 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -55,8 +55,9 @@ async function handleYoutubeStream(streamInfo, res) { streamInfo.url = req.url; const size = BigInt(req.headers.get('content-length')); - if (req.status !== 200 || !size) - return res.destroy(); + if (req.status !== 200 || !size) { + return res.end(); + } const stream = chunkedStream(streamInfo, size); @@ -66,9 +67,9 @@ async function handleYoutubeStream(streamInfo, res) { } stream.pipe(res); - stream.on('error', () => res.destroy()); + stream.on('error', () => res.end()); } catch { - res.destroy(); + res.end(); } } @@ -94,10 +95,10 @@ export async function internalStream(streamInfo, res) { res.setHeader(name, value) if (req.statusCode < 200 || req.statusCode > 299) - return res.destroy(); + return res.end(); req.body.pipe(res); - req.body.on('error', () => res.destroy()); + req.body.on('error', () => res.end()); } catch { streamInfo.controller.abort(); } diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 6c9e9d3c..05008077 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -3,14 +3,12 @@ import { randomBytes } from "crypto"; import { nanoid } from "nanoid"; import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; -import { env } from "../config.js"; +import { env, hlsExceptions } from "../config.js"; import { strict as assert } from "assert"; // optional dependency const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); -const M3U_SERVICES = ['dailymotion', 'vimeo', 'rutube']; - const streamCache = new NodeCache({ stdTTL: env.streamLifespan, checkperiod: 10, @@ -38,6 +36,7 @@ export function createStream(obj) { filename: obj.filename, audioFormat: obj.audioFormat, isAudioOnly: !!obj.isAudioOnly, + headers: obj.headers, copy: !!obj.copy, mute: !!obj.mute, metadata: obj.fileMetadata || false, @@ -82,6 +81,7 @@ export function createInternalStream(url, obj = {}) { internalStreamCache[streamID] = { url, service: obj.service, + headers: obj.headers, controller: new AbortController(), dispatcher }; @@ -108,7 +108,7 @@ export function destroyInternalStream(url) { function wrapStream(streamInfo) { /* m3u8 links are currently not supported * for internal streams, skip them */ - if (M3U_SERVICES.includes(streamInfo.service)) { + if (hlsExceptions.includes(streamInfo.service)) { return streamInfo; } diff --git a/src/modules/stream/shared.js b/src/modules/stream/shared.js index 42df2758..4dbd59d3 100644 --- a/src/modules/stream/shared.js +++ b/src/modules/stream/shared.js @@ -1,5 +1,4 @@ import { genericUserAgent } from "../config.js"; -import { cookie as tiktokCookie } from "../processing/services/tiktok.js"; const defaultHeaders = { 'user-agent': genericUserAgent @@ -14,15 +13,15 @@ const serviceHeaders = { origin: 'https://www.youtube.com', referer: 'https://www.youtube.com', DNT: '?1' - }, - tiktok: { - cookie: tiktokCookie } } export function closeResponse(res) { - if (!res.headersSent) res.sendStatus(500); - return res.destroy(); + if (!res.headersSent) { + res.sendStatus(500); + } + + return res.end(); } export function getHeaders(service) { diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index a1db0c60..2036b360 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -5,7 +5,7 @@ import { create as contentDisposition } from "content-disposition-header"; import { metadataManager } from "../sub/utils.js"; import { destroyInternalStream } from "./manage.js"; -import { env, ffmpegArgs } from "../config.js"; +import { env, ffmpegArgs, hlsExceptions } from "../config.js"; import { getHeaders, closeResponse } from "./shared.js"; function toRawHeaders(headers) { @@ -215,7 +215,7 @@ export function streamVideoOnly(streamInfo, res) { args.push('-an') } - if (["vimeo", "rutube", "dailymotion"].includes(streamInfo.service)) { + if (hlsExceptions.includes(streamInfo.service)) { args.push('-bsf:a', 'aac_adtstoasc') } diff --git a/src/test/tests.json b/src/test/tests.json index 6c28ad2c..a7550df4 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -1089,6 +1089,14 @@ "code": 200, "status": "stream" } + }, { + "name": "private video", + "url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } }], "ok": [{ "name": "regular video", @@ -1148,5 +1156,34 @@ "code": 200, "status": "picker" } + }], + "loom": [{ + "name": "1080p video", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, { + "name": "1080p video (muted)", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "1080p video (audio only)", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } }] } \ No newline at end of file