diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a02c125..193ddd01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,19 @@ on: branches: [ current ] jobs: + check-lockfile: + name: check lockfile correctness + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Check that lockfile does not need an update + run: | + cp package-lock.json before.json + npm ci + npm i --package-lock-only + diff before.json package-lock.json + test-web: name: web sanity check runs-on: ubuntu-latest @@ -22,4 +35,4 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - name: Run test script - run: .github/test.sh api \ No newline at end of file + run: .github/test.sh api diff --git a/README.md b/README.md index c67631be..d2bb064c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ this list is not final and keeps expanding over time. if support for a service y | bilibili.com & bilibili.tv | ✅ | ✅ | ✅ | ➖ | ➖ | | dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ | | instagram posts & reels | ✅ | ✅ | ✅ | ➖ | ➖ | -| ok video | ✅ | ❌ | ❌ | ✅ | ✅ | +| loom | ✅ | ❌ | ✅ | ✅ | ➖ | +| ok video | ✅ | ❌ | ✅ | ✅ | ✅ | | pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | | reddit | ✅ | ✅ | ✅ | ❌ | ❌ | | rutube | ✅ | ✅ | ✅ | ✅ | ✅ | @@ -31,7 +32,7 @@ this list is not final and keeps expanding over time. if support for a service y | twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | | vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | | vine archive | ✅ | ✅ | ✅ | ➖ | ➖ | -| vk videos & clips | ✅ | ❌ | ❌ | ✅ | ✅ | +| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | | youtube videos, shorts & music | ✅ | ✅ | ✅ | ✅ | ✅ | | emoji | meaning | @@ -46,6 +47,7 @@ this list is not final and keeps expanding over time. if support for a service y | instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. | | pinterest | supports photos, gifs, videos and stories. | | reddit | supports gifs and videos. | +| rutube | supports yappy & private links. | | soundcloud | supports private links. | | tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. | | twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. | diff --git a/package-lock.json b/package-lock.json index 88004217..2636146d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cobalt", - "version": "7.14.2", + "version": "7.14.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cobalt", - "version": "7.14.2", + "version": "7.14.3", "license": "AGPL-3.0", "dependencies": { "content-disposition-header": "0.6.0", diff --git a/package.json b/package.json index 9fdefba4..72104aba 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.14.2", + "version": "7.14.3", "author": "imput", "exports": "./src/cobalt.js", "type": "module", diff --git a/src/core/api.js b/src/core/api.js index d822136f..c90f78f7 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -196,8 +196,8 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { } streamInfo.headers = { - ...req.headers, - ...streamInfo.headers + ...streamInfo.headers, + ...req.headers }; return stream(res, { type: 'internal', ...streamInfo }); diff --git a/src/modules/config.js b/src/modules/config.js index 67114b56..530c5f0b 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -53,6 +53,7 @@ const export const services = servicesConfigJson.config, + hlsExceptions = servicesConfigJson.hlsExceptions, audioIgnore = servicesConfigJson.audioIgnore, version = packageJson.version, genericUserAgent = config.genericUserAgent, diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 7f9714b7..3e38c4db 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -24,6 +24,7 @@ import streamable from "./services/streamable.js"; import twitch from "./services/twitch.js"; import rutube from "./services/rutube.js"; import dailymotion from "./services/dailymotion.js"; +import loom from "./services/loom.js"; let freebind; @@ -179,6 +180,7 @@ export default async function(host, patternMatch, lang, obj) { r = await rutube({ id: patternMatch.id, yappyId: patternMatch.yappyId, + key: patternMatch.key, quality: obj.vQuality, isAudioOnly: isAudioOnly }); @@ -186,6 +188,11 @@ export default async function(host, patternMatch, lang, obj) { case "dailymotion": r = await dailymotion(patternMatch); break; + case "loom": + r = await loom({ + id: patternMatch.id + }); + break; default: return createResponse("error", { t: loc(lang, 'ErrorUnsupported') diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 42a35f1a..f7ed3da9 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -129,6 +129,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di case "tumblr": case "pinterest": case "streamable": + case "loom": responseType = "redirect"; break; } diff --git a/src/modules/processing/services/loom.js b/src/modules/processing/services/loom.js new file mode 100644 index 00000000..ecb6c534 --- /dev/null +++ b/src/modules/processing/services/loom.js @@ -0,0 +1,39 @@ +import { genericUserAgent } from "../../config.js"; + +export default async function({ id }) { + const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/transcoded-url`, { + method: "POST", + headers: { + "user-agent": genericUserAgent, + origin: "https://www.loom.com", + referer: `https://www.loom.com/share/${id}`, + cookie: `loom_referral_video=${id};`, + + "apollographql-client-name": "web", + "apollographql-client-version": "14c0b42", + "x-loom-request-source": "loom_web_14c0b42", + }, + body: JSON.stringify({ + force_original: false, + password: null, + anonID: null, + deviceID: null + }) + }) + .then(r => r.status === 200 ? r.json() : false) + .catch(() => {}); + + if (!gql) return { error: 'ErrorEmptyDownload' }; + + const videoUrl = gql?.url; + + if (videoUrl?.includes('.mp4?')) { + return { + urls: videoUrl, + filename: `loom_${id}.mp4`, + audioFilename: `loom_${id}_audio` + } + } + + return { error: 'ErrorEmptyDownload' } +} diff --git a/src/modules/processing/services/reddit.js b/src/modules/processing/services/reddit.js index 9341adc1..41f9efb4 100644 --- a/src/modules/processing/services/reddit.js +++ b/src/modules/processing/services/reddit.js @@ -107,7 +107,7 @@ export default async function(obj) { }).catch(() => {}) } - let id = video.split('/')[3]; + let id = `${String(obj.sub).toLowerCase()}_${obj.id}`; if (!audio) return { typeId: "redirect", diff --git a/src/modules/processing/services/rutube.js b/src/modules/processing/services/rutube.js index dd984c57..a8d0abbe 100644 --- a/src/modules/processing/services/rutube.js +++ b/src/modules/processing/services/rutube.js @@ -12,10 +12,10 @@ async function requestJSON(url) { export default async function(obj) { if (obj.yappyId) { - let yappy = await requestJSON( + const yappy = await requestJSON( `https://rutube.ru/pangolin/api/web/yappy/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=15` ) - let yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link; + const yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link; if (!yappyURL) return { error: 'ErrorEmptyDownload' }; return { @@ -25,11 +25,12 @@ export default async function(obj) { } } - let quality = obj.quality === "max" ? "9000" : obj.quality; + const quality = obj.quality === "max" ? "9000" : obj.quality; - let play = await requestJSON( - `https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2` - ) + const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`); + if (obj.key) requestURL.searchParams.set('p', obj.key); + + const play = await requestJSON(requestURL); if (!play) return { error: 'ErrorCouldntFetch' }; if (play.detail || !play.video_balancer) return { error: 'ErrorEmptyDownload' }; @@ -51,7 +52,7 @@ export default async function(obj) { bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height)); } - let fileMetadata = { + const fileMetadata = { title: cleanString(play.title.trim()), artist: cleanString(play.author.name.trim()), } diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js index 0984ec04..33e7aef3 100644 --- a/src/modules/processing/services/tiktok.js +++ b/src/modules/processing/services/tiktok.js @@ -57,7 +57,7 @@ export default async function(obj) { let playAddr = detail.video.playAddr; if (obj.h265) { - const h265PlayAddr = detail.video.bitrateInfo.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0] + const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0] playAddr = h265PlayAddr || playAddr } diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 64f6d18f..fae4e04d 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -3,7 +3,7 @@ import { env } from '../../config.js'; import { cleanString } from '../../sub/utils.js'; import { fetch } from 'undici' -const ytBase = await Innertube.create(); +const ytBase = Innertube.create().catch(e => e); const codecMatch = { h264: { @@ -23,16 +23,21 @@ const codecMatch = { } } -const cloneInnertube = (customFetch) => { +const cloneInnertube = async (customFetch) => { + const innertube = await ytBase; + if (innertube instanceof Error) { + throw innertube; + } + const session = new Session( - ytBase.session.context, - ytBase.session.key, - ytBase.session.api_version, - ytBase.session.account_index, - ytBase.session.player, + innertube.session.context, + innertube.session.key, + innertube.session.api_version, + innertube.session.account_index, + innertube.session.player, undefined, - customFetch ?? ytBase.session.http.fetch, - ytBase.session.cache + customFetch ?? innertube.session.http.fetch, + innertube.session.cache ); const yt = new Innertube(session); @@ -40,7 +45,7 @@ const cloneInnertube = (customFetch) => { } export default async function(o) { - const yt = cloneInnertube( + const yt = await cloneInnertube( (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher }) ); @@ -57,8 +62,12 @@ export default async function(o) { try { info = await yt.getBasicInfo(o.id, 'WEB'); - } catch { - return { error: 'ErrorCantConnectToServiceAPI' }; + } catch(e) { + if (e?.message === 'This video is unavailable') { + return { error: 'ErrorCouldntFetch' }; + } else { + return { error: 'ErrorCantConnectToServiceAPI' }; + } } if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index a862da71..ae4cd9f0 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -1,5 +1,6 @@ { - "audioIgnore": ["vk", "ok"], + "audioIgnore": ["vk", "ok", "loom"], + "hlsExceptions": ["dailymotion", "vimeo", "rutube"], "config": { "bilibili": { "alias": "bilibili.com & bilibili.tv", @@ -61,8 +62,9 @@ }, "vimeo": { "patterns": [":id", "video/:id", ":id/:password", "/channels/:user/:id"], - "enabled": true, - "bestAudio": "mp3" + "subdomains": ["player"], + "bestAudio": "mp3", + "enabled": true }, "soundcloud": { "patterns": [":author/:song/s-:accessKey", ":author/:song", ":shortLink"], @@ -103,13 +105,18 @@ "rutube": { "alias": "rutube videos", "tld": "ru", - "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId"], + "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId", "video/private/:id?p=:key", "video/private/:id"], "enabled": true }, "dailymotion": { "alias": "dailymotion videos", "patterns": ["video/:id"], "enabled": true + }, + "loom": { + "alias": "loom videos", + "patterns": ["share/:id"], + "enabled": true } } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 25b8943a..ddeea31f 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -8,7 +8,10 @@ export const testers = { "instagram": (patternMatch) => patternMatch.postId?.length <= 12 || (patternMatch.username?.length <= 30 && patternMatch.storyId?.length <= 24), - + + "loom": (patternMatch) => + patternMatch.id?.length <= 32, + "ok": (patternMatch) => patternMatch.id?.length <= 16, @@ -20,6 +23,7 @@ export const testers = { || (patternMatch.user?.length <= 22 && patternMatch.id?.length <= 10), "rutube": (patternMatch) => + (patternMatch.id?.length === 32 && patternMatch.key?.length <= 32) || patternMatch.id?.length === 32 || patternMatch.yappyId?.length === 32, "soundcloud": (patternMatch) => @@ -28,7 +32,7 @@ export const testers = { "streamable": (patternMatch) => patternMatch.id?.length === 6, - + "tiktok": (patternMatch) => patternMatch.postId?.length <= 21 || patternMatch.id?.length <= 13, diff --git a/src/modules/processing/url.js b/src/modules/processing/url.js index a006402c..111f1f6f 100644 --- a/src/modules/processing/url.js +++ b/src/modules/processing/url.js @@ -78,19 +78,35 @@ function aliasURL(url) { function cleanURL(url) { assert(url instanceof URL); const host = psl.parse(url.hostname).sld; + let stripQuery = true; - if (host === 'pinterest') { - url.hostname = 'pinterest.com' - } else if (host === 'vk' && url.pathname.includes('/clip')) { - if (url.searchParams.get('z')) - url.search = '?z=' + encodeURIComponent(url.searchParams.get('z')); - stripQuery = false; - } else if (host === 'youtube' && url.searchParams.get('v')) { - url.search = '?v=' + encodeURIComponent(url.searchParams.get('v')); + const limitQuery = (param) => { + url.search = `?${param}=` + encodeURIComponent(url.searchParams.get(param)); stripQuery = false; } + switch (host) { + case "pinterest": + url.hostname = 'pinterest.com'; + break; + case "vk": + if (url.pathname.includes('/clip') && url.searchParams.get('z')) { + limitQuery('z') + } + break; + case "youtube": + if (url.searchParams.get('v')) { + limitQuery('v') + } + break; + case "rutube": + if (url.searchParams.get('p')) { + limitQuery('p') + } + break; + } + if (stripQuery) { url.search = '' } diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index c90e062e..900aeff3 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -3,11 +3,11 @@ import { randomBytes } from "crypto"; import { nanoid } from "nanoid"; import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; -import { env } from "../config.js"; +import { env, hlsExceptions } from "../config.js"; import { strict as assert } from "assert"; // optional dependency -const freebind = env.freebindCIDR && await import('freebind').catch(() => {}) +const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); const streamCache = new NodeCache({ stdTTL: env.streamLifespan, diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index a1db0c60..2036b360 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -5,7 +5,7 @@ import { create as contentDisposition } from "content-disposition-header"; import { metadataManager } from "../sub/utils.js"; import { destroyInternalStream } from "./manage.js"; -import { env, ffmpegArgs } from "../config.js"; +import { env, ffmpegArgs, hlsExceptions } from "../config.js"; import { getHeaders, closeResponse } from "./shared.js"; function toRawHeaders(headers) { @@ -215,7 +215,7 @@ export function streamVideoOnly(streamInfo, res) { args.push('-an') } - if (["vimeo", "rutube", "dailymotion"].includes(streamInfo.service)) { + if (hlsExceptions.includes(streamInfo.service)) { args.push('-bsf:a', 'aac_adtstoasc') } diff --git a/src/test/tests.json b/src/test/tests.json index 565fa5a4..501ac2c0 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -1089,6 +1089,14 @@ "code": 200, "status": "stream" } + }, { + "name": "private video", + "url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } }], "ok": [{ "name": "regular video", @@ -1123,5 +1131,34 @@ "code": 200, "status": "stream" } + }], + "loom": [{ + "name": "1080p video", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, { + "name": "1080p video (muted)", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "1080p video (audio only)", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } }] }