Merge branch 'current' into improvement/hls-istream

.github/workflows/test.yml

@@ -6,6 +6,19 @@ on:
     branches: [ current ]
 
 jobs:
+  check-lockfile:
+    name: check lockfile correctness
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Check that lockfile does not need an update
+        run: |
+          cp package-lock.json before.json
+          npm ci
+          npm i --package-lock-only
+          diff before.json package-lock.json
+
   test-web:
     name: web sanity check
     runs-on: ubuntu-latest

README.md

@@ -19,7 +19,8 @@ this list is not final and keeps expanding over time. if support for a service y
 | bilibili.com & bilibili.tv     | ✅            | ✅         | ✅         | ➖         | ➖              |
 | dailymotion                    | ✅            | ✅         | ✅         | ✅         | ✅              |
 | instagram posts & reels        | ✅            | ✅         | ✅         | ➖         | ➖              |
-| ok video                       | ✅            | ❌         | ❌         | ✅         | ✅              |
+| loom                           | ✅            | ❌         | ✅         | ✅         | ➖              |
+| ok video                       | ✅            | ❌         | ✅         | ✅         | ✅              |
 | pinterest                      | ✅            | ✅         | ✅         | ➖         | ➖              |
 | reddit                         | ✅            | ✅         | ✅         | ❌         | ❌              |
 | rutube                         | ✅            | ✅         | ✅         | ✅         | ✅              |
@@ -31,7 +32,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | twitter/x                      | ✅            | ✅         | ✅         | ➖         | ➖              |
 | vimeo                          | ✅            | ✅         | ✅         | ✅         | ✅              |
 | vine archive                   | ✅            | ✅         | ✅         | ➖         | ➖              |
-| vk videos & clips              | ✅            | ❌         | ❌         | ✅         | ✅              |
+| vk videos & clips              | ✅            | ❌         | ✅         | ✅         | ✅              |
 | youtube videos, shorts & music | ✅            | ✅         | ✅         | ✅         | ✅              |
 
 | emoji   | meaning                 |
@@ -46,6 +47,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | instagram  | supports reels, photos, and videos. lets you pick what to save from multi-media posts.                               |
 | pinterest  | supports photos, gifs, videos and stories.                                                                           |
 | reddit     | supports gifs and videos.                                                                                            |
+| rutube     | supports yappy & private links.                                                                                      |
 | soundcloud | supports private links.                                                                                              |
 | tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |
 | twitter/x  | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management.               |

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "cobalt",
-    "version": "7.14.2",
+    "version": "7.14.3",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "cobalt",
-            "version": "7.14.2",
+            "version": "7.14.3",
             "license": "AGPL-3.0",
             "dependencies": {
                 "content-disposition-header": "0.6.0",

package.json

@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "7.14.2",
+    "version": "7.14.3",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",

src/core/api.js

@@ -196,8 +196,8 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
         }
 
         streamInfo.headers = {
-            ...req.headers,
+            ...streamInfo.headers,
-            ...streamInfo.headers
+            ...req.headers
         };
 
         return stream(res, { type: 'internal', ...streamInfo });

src/modules/config.js

@@ -53,6 +53,7 @@ const
 
 export const
     services = servicesConfigJson.config,
+    hlsExceptions = servicesConfigJson.hlsExceptions,
     audioIgnore = servicesConfigJson.audioIgnore,
     version = packageJson.version,
     genericUserAgent = config.genericUserAgent,

src/modules/processing/match.js

@@ -24,6 +24,7 @@ import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
 import rutube from "./services/rutube.js";
 import dailymotion from "./services/dailymotion.js";
+import loom from "./services/loom.js";
 
 let freebind;
 
@@ -179,6 +180,7 @@ export default async function(host, patternMatch, lang, obj) {
                 r = await rutube({
                     id: patternMatch.id,
                     yappyId: patternMatch.yappyId,
+                    key: patternMatch.key,
                     quality: obj.vQuality,
                     isAudioOnly: isAudioOnly
                 });
@@ -186,6 +188,11 @@ export default async function(host, patternMatch, lang, obj) {
             case "dailymotion":
                 r = await dailymotion(patternMatch);
                 break;
+            case "loom":
+                r = await loom({
+                    id: patternMatch.id
+                });
+                break;
             default:
                 return createResponse("error", {
                     t: loc(lang, 'ErrorUnsupported')

src/modules/processing/matchActionDecider.js

@@ -129,6 +129,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                 case "tumblr":
                 case "pinterest":
                 case "streamable":
+                case "loom":
                     responseType = "redirect";
                     break;
             }

src/modules/processing/services/loom.js

@@ -0,0 +1,39 @@
+import { genericUserAgent } from "../../config.js";
+
+export default async function({ id }) {
+    const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/transcoded-url`, {
+        method: "POST",
+        headers: {
+            "user-agent": genericUserAgent,
+            origin: "https://www.loom.com",
+            referer: `https://www.loom.com/share/${id}`,
+            cookie: `loom_referral_video=${id};`,
+
+            "apollographql-client-name": "web",
+            "apollographql-client-version": "14c0b42",
+            "x-loom-request-source": "loom_web_14c0b42",
+        },
+        body: JSON.stringify({
+            force_original: false,
+            password: null,
+            anonID: null,
+            deviceID: null
+        })
+    })
+    .then(r => r.status === 200 ? r.json() : false)
+    .catch(() => {});
+
+    if (!gql) return { error: 'ErrorEmptyDownload' };
+
+    const videoUrl = gql?.url;
+
+    if (videoUrl?.includes('.mp4?')) {
+        return {
+            urls: videoUrl,
+            filename: `loom_${id}.mp4`,
+            audioFilename: `loom_${id}_audio`
+        }
+    }
+
+    return { error: 'ErrorEmptyDownload' }
+}

src/modules/processing/services/reddit.js

@@ -107,7 +107,7 @@ export default async function(obj) {
         }).catch(() => {})
     }
 
-    let id = video.split('/')[3];
+    let id = `${String(obj.sub).toLowerCase()}_${obj.id}`;
 
     if (!audio) return {
         typeId: "redirect",

src/modules/processing/services/rutube.js

@@ -12,10 +12,10 @@ async function requestJSON(url) {
 
 export default async function(obj) {
     if (obj.yappyId) {
-        let yappy = await requestJSON(
+        const yappy = await requestJSON(
             `https://rutube.ru/pangolin/api/web/yappy/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=15`
         )
-        let yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link;
+        const yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link;
         if (!yappyURL) return { error: 'ErrorEmptyDownload' };
 
         return {
@@ -25,11 +25,12 @@ export default async function(obj) {
         }
     }
 
-    let quality = obj.quality === "max" ? "9000" : obj.quality;
+    const quality = obj.quality === "max" ? "9000" : obj.quality;
 
-    let play = await requestJSON(
+    const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`);
-        `https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`
+    if (obj.key) requestURL.searchParams.set('p', obj.key);
-    )
+
+    const play = await requestJSON(requestURL);
     if (!play) return { error: 'ErrorCouldntFetch' };
 
     if (play.detail || !play.video_balancer) return { error: 'ErrorEmptyDownload' };
@@ -51,7 +52,7 @@ export default async function(obj) {
         bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height));
     }
 
-    let fileMetadata = {
+    const fileMetadata = {
         title: cleanString(play.title.trim()),
         artist: cleanString(play.author.name.trim()),
     }

src/modules/processing/services/tiktok.js

@@ -57,7 +57,7 @@ export default async function(obj) {
 
     let playAddr = detail.video.playAddr;
     if (obj.h265) {
-        const h265PlayAddr = detail.video.bitrateInfo.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
+        const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
         playAddr = h265PlayAddr || playAddr
     }
 

src/modules/processing/services/youtube.js

@@ -3,7 +3,7 @@ import { env } from '../../config.js';
 import { cleanString } from '../../sub/utils.js';
 import { fetch } from 'undici'
 
-const ytBase = await Innertube.create();
+const ytBase = Innertube.create().catch(e => e);
 
 const codecMatch = {
     h264: {
@@ -23,16 +23,21 @@ const codecMatch = {
     }
 }
 
-const cloneInnertube = (customFetch) => {
+const cloneInnertube = async (customFetch) => {
+    const innertube = await ytBase;
+    if (innertube instanceof Error) {
+        throw innertube;
+    }
+
     const session = new Session(
-        ytBase.session.context,
+        innertube.session.context,
-        ytBase.session.key,
+        innertube.session.key,
-        ytBase.session.api_version,
+        innertube.session.api_version,
-        ytBase.session.account_index,
+        innertube.session.account_index,
-        ytBase.session.player,
+        innertube.session.player,
         undefined,
-        customFetch ?? ytBase.session.http.fetch,
+        customFetch ?? innertube.session.http.fetch,
-        ytBase.session.cache
+        innertube.session.cache
     );
 
     const yt = new Innertube(session);
@@ -40,7 +45,7 @@ const cloneInnertube = (customFetch) => {
 }
 
 export default async function(o) {
-    const yt = cloneInnertube(
+    const yt = await cloneInnertube(
         (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher })
     );
 
@@ -57,9 +62,13 @@ export default async function(o) {
 
     try {
         info = await yt.getBasicInfo(o.id, 'WEB');
-    } catch {
+    } catch(e) {
+        if (e?.message === 'This video is unavailable') {
+            return { error: 'ErrorCouldntFetch' };
+        } else {
             return { error: 'ErrorCantConnectToServiceAPI' };
         }
+    }
 
     if (!info) return { error: 'ErrorCantConnectToServiceAPI' };
 

src/modules/processing/servicesConfig.json

@@ -1,5 +1,6 @@
 {
-    "audioIgnore": ["vk", "ok"],
+    "audioIgnore": ["vk", "ok", "loom"],
+    "hlsExceptions": ["dailymotion", "vimeo", "rutube"],
     "config": {
         "bilibili": {
             "alias": "bilibili.com & bilibili.tv",
@@ -61,8 +62,9 @@
         },
         "vimeo": {
             "patterns": [":id", "video/:id", ":id/:password", "/channels/:user/:id"],
-            "enabled": true,
+            "subdomains": ["player"],
-            "bestAudio": "mp3"
+            "bestAudio": "mp3",
+            "enabled": true
         },
         "soundcloud": {
             "patterns": [":author/:song/s-:accessKey", ":author/:song", ":shortLink"],
@@ -103,13 +105,18 @@
         "rutube": {
             "alias": "rutube videos",
             "tld": "ru",
-            "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId"],
+            "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId", "video/private/:id?p=:key", "video/private/:id"],
             "enabled": true
         },
         "dailymotion": {
             "alias": "dailymotion videos",
             "patterns": ["video/:id"],
             "enabled": true
+        },
+        "loom": {
+            "alias": "loom videos",
+            "patterns": ["share/:id"],
+            "enabled": true
         }
     }
 }

src/modules/processing/servicesPatternTesters.js

@@ -9,6 +9,9 @@ export const testers = {
         patternMatch.postId?.length <= 12
         || (patternMatch.username?.length <= 30 && patternMatch.storyId?.length <= 24),
 
+    "loom": (patternMatch) =>
+        patternMatch.id?.length <= 32,
+
     "ok": (patternMatch) =>
         patternMatch.id?.length <= 16,
 
@@ -20,6 +23,7 @@ export const testers = {
         || (patternMatch.user?.length <= 22 && patternMatch.id?.length <= 10),
 
     "rutube": (patternMatch) =>
+        (patternMatch.id?.length === 32 && patternMatch.key?.length <= 32) ||
         patternMatch.id?.length === 32 || patternMatch.yappyId?.length === 32,
 
     "soundcloud": (patternMatch) =>

src/modules/processing/url.js

@@ -78,19 +78,35 @@ function aliasURL(url) {
 function cleanURL(url) {
     assert(url instanceof URL);
     const host = psl.parse(url.hostname).sld;
+
     let stripQuery = true;
 
-    if (host === 'pinterest') {
+    const limitQuery = (param) => {
-        url.hostname = 'pinterest.com'
+        url.search = `?${param}=` + encodeURIComponent(url.searchParams.get(param));
-    } else if (host === 'vk' && url.pathname.includes('/clip')) {
-        if (url.searchParams.get('z'))
-            url.search = '?z=' + encodeURIComponent(url.searchParams.get('z'));
-        stripQuery = false;
-    } else if (host === 'youtube' && url.searchParams.get('v')) {
-        url.search = '?v=' + encodeURIComponent(url.searchParams.get('v'));
         stripQuery = false;
     }
 
+    switch (host) {
+        case "pinterest":
+            url.hostname = 'pinterest.com';
+            break;
+        case "vk":
+            if (url.pathname.includes('/clip') && url.searchParams.get('z')) {
+                limitQuery('z')
+            }
+            break;
+        case "youtube":
+            if (url.searchParams.get('v')) {
+                limitQuery('v')
+            }
+            break;
+        case "rutube":
+            if (url.searchParams.get('p')) {
+                limitQuery('p')
+            }
+            break;
+    }
+
     if (stripQuery) {
         url.search = ''
     }

src/modules/stream/manage.js

@@ -3,11 +3,11 @@ import { randomBytes } from "crypto";
 import { nanoid } from "nanoid";
 
 import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
-import { env } from "../config.js";
+import { env, hlsExceptions } from "../config.js";
 import { strict as assert } from "assert";
 
 // optional dependency
-const freebind = env.freebindCIDR && await import('freebind').catch(() => {})
+const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
 
 const streamCache = new NodeCache({
     stdTTL: env.streamLifespan,

src/modules/stream/types.js

@@ -5,7 +5,7 @@ import { create as contentDisposition } from "content-disposition-header";
 
 import { metadataManager } from "../sub/utils.js";
 import { destroyInternalStream } from "./manage.js";
-import { env, ffmpegArgs } from "../config.js";
+import { env, ffmpegArgs, hlsExceptions } from "../config.js";
 import { getHeaders, closeResponse } from "./shared.js";
 
 function toRawHeaders(headers) {
@@ -215,7 +215,7 @@ export function streamVideoOnly(streamInfo, res) {
             args.push('-an')
         }
 
-        if (["vimeo", "rutube", "dailymotion"].includes(streamInfo.service)) {
+        if (hlsExceptions.includes(streamInfo.service)) {
             args.push('-bsf:a', 'aac_adtstoasc')
         }
 

src/test/tests.json

@@ -1089,6 +1089,14 @@
             "code": 200,
             "status": "stream"
         }
+    }, {
+        "name": "private video",
+        "url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "stream"
+        }
     }],
     "ok": [{
         "name": "regular video",
@@ -1123,5 +1131,34 @@
             "code": 200,
             "status": "stream"
         }
+    }],
+    "loom": [{
+        "name": "1080p video",
+        "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "1080p video (muted)",
+        "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
+        "params": {
+            "isAudioMuted": true
+        },
+        "expected": {
+            "code": 200,
+            "status": "stream"
+        }
+    }, {
+        "name": "1080p video (audio only)",
+        "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
+        "params": {
+            "isAudioOnly": true
+        },
+        "expected": {
+            "code": 200,
+            "status": "stream"
+        }
     }]
 }