Merge branch 'current' into support/odysee

2025-07-18 11:18:28 +00:00 · 2024-07-25 11:18:51 +02:00 · 2024-07-25 11:18:51 +02:00 · 4be7a81ac2
commit 4be7a81ac2
parent bf3bb79b54 cb2bde5b51
26 changed files with 565 additions and 91 deletions
--- a/.github/test.sh
+++ b/.github/test.sh
@ -18,7 +18,7 @@ test_api() {
         -X POST \
         -H "Accept: application/json" \
         -H "Content-Type: application/json" \
-         -d '{"url":"https://www.youtube.com/watch?v=jNQXAC9IVRw"}')
+         -d '{"url":"https://vine.co/v/huwVJIEJW50", "isAudioOnly": true}')

    echo "$API_RESPONSE"
    STATUS=$(echo "$API_RESPONSE" | jq -r .status)
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -36,3 +36,27 @@ jobs:
        uses: actions/checkout@v4
      - name: Run test script
        run: .github/test.sh api
+
+  check-services:
+    name: test service functionality
+    runs-on: ubuntu-latest
+    outputs:
+      services: ${{ steps.checkServices.outputs.service_list }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - id: checkServices
+        run: npm ci && echo "service_list=$(node src/util/test-ci get-services)" >> "$GITHUB_OUTPUT"
+
+  test-services:
+    needs: check-services
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        service: ${{ fromJson(needs.check-services.outputs.services) }}
+    name: "test service: ${{ matrix.service }}"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - run: npm ci && node src/util/test-ci run-tests-for ${{ matrix.service }}
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -26,7 +26,7 @@ the scope is not strictly defined, you can write whatever you find most fitting

 if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.

-if your contribution has uninformative commit messages, you may be asked to interactively rebase your branch and amend each commit to include a meaningful message.
+if your contribution has uninformative commit titles, you may be asked to interactively rebase your branch and amend each commit to include a meaningful title.

 ### clean commit history
 if your branch is out of date and/or has some merge conflicts with the `current` branch, you should **rebase** it instead of merging. this prevents meaningless merge commits from being included in your branch, which would then end up in the cobalt git history.
--- a/README.md
+++ b/README.md
@ -19,12 +19,14 @@ this list is not final and keeps expanding over time. if support for a service y
 | bilibili.com & bilibili.tv     | ✅            | ✅         | ✅         | ➖         | ➖              |
 | dailymotion                    | ✅            | ✅         | ✅         | ✅         | ✅              |
 | instagram posts & reels        | ✅            | ✅         | ✅         | ➖         | ➖              |
+| facebook videos                | ✅            | ❌         | ❌         | ➖         | ➖              |
 | loom                           | ✅            | ❌         | ✅         | ✅         | ➖              |
 | ok video                       | ✅            | ❌         | ✅         | ✅         | ✅              |
 | odysee                         | ✅            | ❌         | ✅         | ➖         | ➖              |
 | pinterest                      | ✅            | ✅         | ✅         | ➖         | ➖              |
 | reddit                         | ✅            | ✅         | ✅         | ❌         | ❌              |
 | rutube                         | ✅            | ✅         | ✅         | ✅         | ✅              |
+| snapchat stories & spotlights  | ✅            | ✅         | ✅         | ➖         | ➖              |
 | soundcloud                     | ➖            | ✅         | ➖         | ✅         | ✅              |
 | streamable                     | ✅            | ✅         | ✅         | ➖         | ➖              |
 | tiktok                         | ✅            | ✅         | ✅         | ❌         | ❌              |
@ -46,8 +48,10 @@ this list is not final and keeps expanding over time. if support for a service y
 | service    | notes or features                                                                                                    |
 | :--------  | :-----                                                                                                               |
 | instagram  | supports reels, photos, and videos. lets you pick what to save from multi-media posts.                               |
+| facebook   | supports public accessible videos content only.                                                                      |
 | pinterest  | supports photos, gifs, videos and stories.                                                                           |
 | reddit     | supports gifs and videos.                                                                                            |
+| snapchat   | supports spotlights and stories. lets you pick what to save from stories.                                            |
 | rutube     | supports yappy & private links.                                                                                      |
 | soundcloud | supports private links.                                                                                              |
 | tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |
--- a/docs/run-an-instance.md
+++ b/docs/run-an-instance.md
@ -56,8 +56,9 @@ sudo service nscd start
 | `API_LISTEN_ADDRESS`  | `0.0.0.0` | `127.0.0.1`             | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** |
 | `API_URL`             | ➖        | `https://api.cobalt.tools/` | changes url from which api server is accessible. <br> ***REQUIRED TO RUN THE API***. |
 | `API_NAME`            | `unknown` | `ams-1`                 | api server name that is shown in `/api/serverInfo`. |
+| `API_EXTERNAL_PROXY`  | ➖        | `http://user:password@127.0.0.1:8080`| url of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only. |
 | `CORS_WILDCARD`       | `1`       | `0`                     | toggles cross-origin resource sharing. <br> `0`: disabled. `1`: enabled. |
-| `CORS_URL`            | not used  | `https://cobalt.tools/` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. |
+| `CORS_URL`            | not used  | `https://cobalt.tools`  | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. |
 | `COOKIE_PATH`         | not used  | `/cookies.json`         | path for cookie file relative to main folder. |
 | `PROCESSING_PRIORITY` | not used  | `10`                    | changes `nice` value* for ffmpeg subprocess. available only on unix systems. |
 | `FREEBIND_CIDR`       | ➖        | `2001:db8::/32`         | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. see below for more info. |
--- a/package-lock.json
+++ b/package-lock.json
@ -1,12 +1,12 @@
 {
    "name": "cobalt",
-    "version": "7.14.5",
+    "version": "7.14.6",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
        "": {
            "name": "cobalt",
-            "version": "7.14.5",
+            "version": "7.14.6",
            "license": "AGPL-3.0",
            "dependencies": {
                "content-disposition-header": "0.6.0",
@ -24,7 +24,7 @@
                "set-cookie-parser": "2.6.0",
                "undici": "^5.19.1",
                "url-pattern": "1.0.3",
-                "youtubei.js": "^9.3.0"
+                "youtubei.js": "^10.1.0"
            },
            "engines": {
                "node": ">=18"
@ -73,9 +73,9 @@
            }
        },
        "node_modules/acorn": {
-            "version": "8.11.3",
-            "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
-            "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==",
+            "version": "8.12.1",
+            "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
+            "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==",
            "bin": {
                "acorn": "bin/acorn"
            },
@ -683,9 +683,9 @@
            }
        },
        "node_modules/jintr": {
-            "version": "1.1.0",
-            "resolved": "https://registry.npmjs.org/jintr/-/jintr-1.1.0.tgz",
-            "integrity": "sha512-Tu9wk3BpN2v+kb8yT6YBtue+/nbjeLFv4vvVC4PJ7oCidHKbifWhvORrAbQfxVIQZG+67am/mDagpiGSVtvrZg==",
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/jintr/-/jintr-2.0.0.tgz",
+            "integrity": "sha512-RiVlevxttZ4eHEYB2dXKXDXluzHfRuw0DJQGsYuKCc5IvZj5/GbOakeqVX+Bar/G9kTty9xDJREcxukurkmYLA==",
            "funding": [
                "https://github.com/sponsors/LuanRT"
            ],
@ -1123,14 +1123,14 @@
            }
        },
        "node_modules/youtubei.js": {
-            "version": "9.4.0",
-            "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-9.4.0.tgz",
-            "integrity": "sha512-8plCOZD2WabqWSEgZU3RjzigIIeR7sF028EERJENYrC9xO/6awpLMZfeoE1gNrNEbKcA+bzbMvonqlvBdxGdKg==",
+            "version": "10.1.0",
+            "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-10.1.0.tgz",
+            "integrity": "sha512-MokZMAnpWH11VYvWuW6qjPiiPmgRl5rfDgPQOpif9qXcVHoVw1hi8ePuRSD0AZSZ+uvWGe8rvas2dzp+Jv5JKQ==",
            "funding": [
                "https://github.com/sponsors/LuanRT"
            ],
            "dependencies": {
-                "jintr": "^1.1.0",
+                "jintr": "^2.0.0",
                "tslib": "^2.5.0",
                "undici": "^5.19.1"
            }
--- a/package.json
+++ b/package.json
@ -1,7 +1,7 @@
 {
    "name": "cobalt",
    "description": "save what you love",
-    "version": "7.14.5",
+    "version": "7.14.6",
    "author": "imput",
    "exports": "./src/cobalt.js",
    "type": "module",
@ -40,7 +40,7 @@
        "set-cookie-parser": "2.6.0",
        "undici": "^5.19.1",
        "url-pattern": "1.0.3",
-        "youtubei.js": "^9.3.0"
+        "youtubei.js": "^10.1.0"
    },
    "optionalDependencies": {
        "freebind": "^0.2.2"
--- a/src/core/api.js
+++ b/src/core/api.js
@ -1,5 +1,6 @@
 import cors from "cors";
 import rateLimit from "express-rate-limit";
+import { setGlobalDispatcher, ProxyAgent } from "undici";

 import { env, version } from "../modules/config.js";

@ -87,32 +88,17 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
        next();
    })

-    app.use('/api/json', express.json({
-        verify: (req, res, buf) => {
-            if (String(req.header('Accept')) === "application/json") {
-                if (buf.length > 720) throw new Error();
-                JSON.parse(buf);
-            } else {
-                throw new Error();
-            }
-        }
-    }))
-
-    // handle express.json errors properly (https://github.com/expressjs/express/issues/4065)
-    app.use('/api/json', (err, req, res, next) => {
-        let errorText = "invalid json body";
-        const acceptHeader = String(req.header('Accept')) !== "application/json";
-
-        if (err || acceptHeader) {
-            if (acceptHeader) errorText = "invalid accept header";
+    app.use('/api/json', express.json({ limit: 1024 }));
+    app.use('/api/json', (err, _, res, next) => {
+        if (err) {
            return res.status(400).json({
                status: "error",
-                text: errorText
+                text: "invalid json body"
            });
-        } else {
-            next();
        }
-    })
+
+        next();
+    });

    app.post('/api/json', async (req, res) => {
        const request = req.body;
@ -123,6 +109,10 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
            res.status(status).json(body);
        }

+        if (!acceptRegex.test(req.header('Accept'))) {
+            return fail('ErrorInvalidAcceptHeader');
+        }
+
        if (!acceptRegex.test(req.header('Content-Type'))) {
            return fail('ErrorInvalidContentType');
        }
@ -219,6 +209,14 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
    randomizeCiphers();
    setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes

+    if (env.externalProxy) {
+        if (env.freebindCIDR) {
+            throw new Error('Freebind is not available when external proxy is enabled')
+        }
+
+        setGlobalDispatcher(new ProxyAgent(env.externalProxy))
+    }
+
    app.listen(env.apiPort, env.listenAddress, () => {
        console.log(`\n` +
            `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` +
--- a/src/localization/languages/en.json
+++ b/src/localization/languages/en.json
@ -159,6 +159,7 @@
        "UpdateOneMillion": "1 million users and blazing speed",
        "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!",
        "ErrorYTLogin": "couldn't get this youtube video because it requires an account to view.\n\nthis limitation is done by google to seemingly stop scraping, affecting all 3rd party tools and even their own clients.\n\ntry again, but if issue persists, {ContactLink}.",
-        "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}."
+        "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}.",
+        "ErrorInvalidAcceptHeader": "invalid accept header"
    }
 }
--- a/src/modules/config.js
+++ b/src/modules/config.js
@ -48,7 +48,9 @@ const

        processingPriority: process.platform !== 'win32'
                                && process.env.PROCESSING_PRIORITY
-                                && parseInt(process.env.PROCESSING_PRIORITY)
+                                && parseInt(process.env.PROCESSING_PRIORITY),
+
+        externalProxy: process.env.API_EXTERNAL_PROXY,
    }

 export const
--- a/src/modules/processing/match.js
+++ b/src/modules/processing/match.js
@ -24,7 +24,9 @@ import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
 import rutube from "./services/rutube.js";
 import dailymotion from "./services/dailymotion.js";
+import snapchat from "./services/snapchat.js";
 import loom from "./services/loom.js";
+import facebook from "./services/facebook.js";
 import odysee from "./services/odysee.js";

 let freebind;
@ -189,6 +191,12 @@ export default async function(host, patternMatch, lang, obj) {
            case "dailymotion":
                r = await dailymotion(patternMatch);
                break;
+            case "snapchat":
+                r = await snapchat({
+                    hostname: url.hostname,
+                    ...patternMatch
+                });
+                break;
            case "loom":
                r = await loom({
                    id: patternMatch.id
@ -199,6 +207,11 @@ export default async function(host, patternMatch, lang, obj) {
                    id: patternMatch.id
                });
                break;
+            case "facebook":
+                r = await facebook({
+                    ...patternMatch
+                });
+                break;
            default:
                return createResponse("error", {
                    t: loc(lang, 'ErrorUnsupported')
--- a/src/modules/processing/matchActionDecider.js
+++ b/src/modules/processing/matchActionDecider.js
@ -73,6 +73,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
            switch (host) {
                case "instagram":
                case "twitter":
+                case "snapchat":
                    params = { picker: r.picker };
                    break;
                case "tiktok":
@ -130,11 +131,13 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                    params = { type: "bridge" };
                    break;

+                case "facebook":
                case "vine":
                case "instagram":
                case "tumblr":
                case "pinterest":
                case "streamable":
+                case "snapchat":
                case "loom":
                case "odysee":
                    responseType = "redirect";
--- a/src/modules/processing/services/facebook.js
+++ b/src/modules/processing/services/facebook.js
@ -0,0 +1,56 @@
+import { genericUserAgent } from "../../config.js";
+
+const headers = {
+    'User-Agent': genericUserAgent,
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+}
+
+const resolveUrl = (url) => {
+    return fetch(url, { headers })
+        .then(r => {
+            if (r.headers.get('location')) {
+                return decodeURIComponent(r.headers.get('location'));
+            }
+            if (r.headers.get('link')) {
+                const linkMatch = r.headers.get('link').match(/<(.*?)\/>/);
+                return decodeURIComponent(linkMatch[1]);
+            }
+            return false;
+        })
+        .catch(() => false);
+}
+
+export default async function({ id, shareType, shortLink }) {
+    let url = `https://web.facebook.com/i/videos/${id}`;
+
+    if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
+    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`);
+
+    const html = await fetch(url, { headers })
+        .then(r => r.text())
+        .catch(() => false);
+
+    if (!html) return { error: 'ErrorCouldntFetch' };
+
+    const urls = [];
+    const hd = html.match('"browser_native_hd_url":(".*?")');
+    const sd = html.match('"browser_native_sd_url":(".*?")');
+
+    if (hd?.[1]) urls.push(JSON.parse(hd[1]));
+    if (sd?.[1]) urls.push(JSON.parse(sd[1]));
+
+    if (!urls.length) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    const baseFilename = `facebook_${id || shortLink}`;
+
+    return {
+        urls: urls[0],
+        filename: `${baseFilename}.mp4`,
+        audioFilename: `${baseFilename}_audio`,
+    };
+}
--- a/src/modules/processing/services/ok.js
+++ b/src/modules/processing/services/ok.js
@ -45,7 +45,7 @@ export default async function(o) {

    let fileMetadata = {
        title: cleanString(videoData.movie.title.trim()),
-        author: cleanString(videoData.author.name.trim()),
+        author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()),
    }

    if (bestVideo) return {
--- a/src/modules/processing/services/rutube.js
+++ b/src/modules/processing/services/rutube.js
@ -10,6 +10,8 @@ async function requestJSON(url) {
    } catch {}
 }

+const delta = (a, b) => Math.abs(a - b);
+
 export default async function(obj) {
    if (obj.yappyId) {
        const yappy = await requestJSON(
@ -25,7 +27,7 @@ export default async function(obj) {
        }
    }

-    const quality = obj.quality === "max" ? "9000" : obj.quality;
+    const quality = Number(obj.quality) || 9000;

    const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`);
    if (obj.key) requestURL.searchParams.set('p', obj.key);
@ -45,12 +47,16 @@ export default async function(obj) {

    if (!m3u8) return { error: 'ErrorCouldntFetch' };

-    m3u8 = HLS.parse(m3u8).variants.sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
+    m3u8 = HLS.parse(m3u8).variants;

-    let bestQuality = m3u8[0];
-    if (Number(quality) < bestQuality.resolution.height) {
-        bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height));
-    }
+    const matchingQuality = m3u8.reduce((prev, next) => {
+        const diff = {
+            prev: delta(quality, prev.resolution.height),
+            next: delta(quality, next.resolution.height)
+        };
+
+        return diff.prev < diff.next ? prev : next;
+    });

    const fileMetadata = {
        title: cleanString(play.title.trim()),
@ -58,15 +64,15 @@ export default async function(obj) {
    }

    return {
-        urls: bestQuality.uri,
+        urls: matchingQuality.uri,
        isM3U8: true,
        filenameAttributes: {
            service: "rutube",
            id: obj.id,
            title: fileMetadata.title,
            author: fileMetadata.artist,
-            resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
-            qualityLabel: `${bestQuality.resolution.height}p`,
+            resolution: `${matchingQuality.resolution.width}x${matchingQuality.resolution.height}`,
+            qualityLabel: `${matchingQuality.resolution.height}p`,
            extension: "mp4"
        },
        fileMetadata: fileMetadata
--- a/src/modules/processing/services/snapchat.js
+++ b/src/modules/processing/services/snapchat.js
@ -0,0 +1,96 @@
+import { genericUserAgent } from "../../config.js";
+import { getRedirectingURL } from "../../sub/utils.js";
+import { extract, normalizeURL } from "../url.js";
+
+const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="(https:\/\/cf-st\.sc-cdn\.net\/d\/[\w.?=]+&amp;uc=\d+)" as="video"\/>/;
+const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
+
+async function getSpotlight(id) {
+    const html = await fetch(`https://www.snapchat.com/spotlight/${id}`, {
+        headers: { 'User-Agent': genericUserAgent }
+    }).then((r) => r.text()).catch(() => null);
+    if (!html) {
+        return { error: 'ErrorCouldntFetch' };
+    }
+
+    const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1];
+    if (videoURL) {
+        return {
+            urls: videoURL,
+            filename: `snapchat_${id}.mp4`,
+            audioFilename: `snapchat_${id}_audio`
+        }
+    }
+}
+
+async function getStory(username, storyId) {
+    const html = await fetch(`https://www.snapchat.com/add/${username}${storyId ? `/${storyId}` : ''}`, {
+        headers: { 'User-Agent': genericUserAgent }
+    }).then((r) => r.text()).catch(() => null);
+    if (!html) {
+        return { error: 'ErrorCouldntFetch' };
+    }
+
+    const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
+    if (nextDataString) {
+        const data = JSON.parse(nextDataString);
+        const storyIdParam = data.query.profileParams[1];
+
+        if (storyIdParam && data.props.pageProps.story) {
+            const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
+            if (story) {
+                if (story.snapMediaType === 0) {
+                    return {
+                        urls: story.snapUrls.mediaUrl,
+                        isPhoto: true
+                    }
+                }
+
+                return {
+                    urls: story.snapUrls.mediaUrl,
+                    filename: `snapchat_${storyId}.mp4`,
+                    audioFilename: `snapchat_${storyId}_audio`
+                }
+            }
+        }
+
+        const defaultStory = data.props.pageProps.curatedHighlights[0];
+        if (defaultStory) {
+            return {
+                picker: defaultStory.snapList.map((snap) => ({
+                    type: snap.snapMediaType === 0 ? 'photo' : 'video',
+                    url: snap.snapUrls.mediaUrl,
+                    thumb: snap.snapUrls.mediaPreviewUrl.value
+                }))
+            }
+        }
+    }
+}
+
+export default async function (obj) {
+    let params = obj;
+    if (obj.hostname === 't.snapchat.com' && obj.shortLink) {
+        const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
+
+        if (!link?.startsWith('https://www.snapchat.com/')) {
+            return { error: 'ErrorCouldntFetch' };
+        }
+
+        const extractResult = extract(normalizeURL(link));
+        if (extractResult?.host !== 'snapchat') {
+            return { error: 'ErrorCouldntFetch' };
+        }
+
+        params = extractResult.patternMatch;
+    }
+
+    if (params.spotlightId) {
+        const result = await getSpotlight(params.spotlightId);
+        if (result) return result;
+    } else if (params.username) {
+        const result = await getStory(params.username, params.storyId);
+        if (result) return result;
+    }
+
+    return { error: 'ErrorCouldntFetch' };
+}
--- a/src/modules/processing/services/youtube.js
+++ b/src/modules/processing/services/youtube.js
@ -28,20 +28,21 @@ const transformSessionData = (cookie) => {
    if (!cookie)
        return;

-    const values = cookie.values();
-    const REQUIRED_VALUES = [
-        'access_token', 'refresh_token',
-        'client_id', 'client_secret',
-        'expires'
-    ];
+    const values = { ...cookie.values() };
+    const REQUIRED_VALUES = [ 'access_token', 'refresh_token' ];

    if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
        return;
    }
-    return {
-        ...values,
-        expires: new Date(values.expires),
-    };
+
+    if (values.expires) {
+        values.expiry_date = values.expires;
+        delete values.expires;
+    } else if (!values.expiry_date) {
+        return;
+    }
+
+    return values;
 }

 const cloneInnertube = async (customFetch) => {
@ -70,14 +71,19 @@ const cloneInnertube = async (customFetch) => {
    }

    if (session.logged_in) {
-        await session.oauth.refreshIfRequired();
-        const oldExpiry = new Date(cookie.values().expires);
-        const newExpiry = session.oauth.credentials.expires;
+        if (session.oauth.shouldRefreshToken()) {
+            await session.oauth.refreshAccessToken();
+        }
+
+        const cookieValues = cookie.values();
+        const oldExpiry = new Date(cookieValues.expiry_date);
+        const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);

        if (oldExpiry.getTime() !== newExpiry.getTime()) {
            updateCookieValues(cookie, {
-                ...session.oauth.credentials,
-                expires: session.oauth.credentials.expires.toISOString()
+                ...session.oauth.client_id,
+                ...session.oauth.oauth2_tokens,
+                expiry_date: newExpiry.toISOString()
            });
        }
    }
--- a/src/modules/processing/servicesConfig.json
+++ b/src/modules/processing/servicesConfig.json
@ -114,6 +114,12 @@
            "patterns": ["video/:id"],
            "enabled": true
        },
+        "snapchat": {
+            "alias": "snapchat stories & spotlights",
+            "subdomains": ["t", "story"],
+            "patterns": [":shortLink", "spotlight/:spotlightId", "add/:username/:storyId", "u/:username/:storyId", "add/:username", "u/:username"],
+            "enabled": true
+        },
        "loom": {
            "alias": "loom videos",
            "patterns": ["share/:id"],
@ -123,6 +129,19 @@
            "alias": "odysee videos",
            "patterns": [":creator/:id", ":id"],
            "enabled": true
+        },
+        "facebook": {
+            "alias": "facebook videos",
+            "altDomains": ["fb.watch"],
+            "subdomains": ["web"],
+            "patterns": [
+                "_shortLink/:shortLink",
+                ":username/videos/:caption/:id",
+                ":username/videos/:id",
+                "reel/:id",
+                "share/:shareType/:id"
+            ],
+            "enabled": true
        }
    }
 }
--- a/src/modules/processing/servicesPatternTesters.js
+++ b/src/modules/processing/servicesPatternTesters.js
@ -33,6 +33,11 @@ export const testers = {
        (patternMatch.author?.length <= 255 && patternMatch.song?.length <= 255)
        || patternMatch.shortLink?.length <= 32,

+    "snapchat": (patternMatch) =>
+        (patternMatch.username?.length <= 32 && (!patternMatch.storyId || patternMatch.storyId?.length <= 255)) 
+        || patternMatch.spotlightId?.length <= 255 
+        || patternMatch.shortLink?.length <= 16,
+
    "streamable": (patternMatch) =>
        patternMatch.id?.length === 6,

@ -61,4 +66,11 @@ export const testers = {

    "youtube": (patternMatch) =>
        patternMatch.id?.length <= 11,
+
+    "facebook": (patternMatch) =>
+        patternMatch.shortLink?.length <= 11
+        || patternMatch.username?.length <= 30
+        || patternMatch.caption?.length <= 255
+        || patternMatch.id?.length <= 20 && !patternMatch.shareType
+        || patternMatch.id?.length <= 20 && patternMatch.shareType?.length === 1,
 }
--- a/src/modules/processing/url.js
+++ b/src/modules/processing/url.js
@ -64,7 +64,17 @@ function aliasURL(url) {
            if (url.hostname === 'dai.ly' && parts.length === 2) {
                url = new URL(`https://dailymotion.com/video/${parts[1]}`)
            }
+
+        case "facebook":
+        case "fb":
+            if (url.searchParams.get('v')) {
+                url = new URL(`https://web.facebook.com/user/videos/${url.searchParams.get('v')}`)
+            }
+            if (url.hostname === 'fb.watch') {
+                url = new URL(`https://web.facebook.com/_shortLink/${parts[1]}`)
+            }
            break;
+
        case "ddinstagram":
            if (services.instagram.altDomains.includes(host.domain) && [null, 'd', 'g'].includes(host.subdomain)) {
                url.hostname = 'instagram.com';
--- a/src/modules/stream/types.js
+++ b/src/modules/stream/types.js
@ -25,7 +25,7 @@ function killProcess(p) {
 }

 function getCommand(args) {
-    if (!isNaN(env.processingPriority)) {
+    if (typeof env.processingPriority === 'number' && !isNaN(env.processingPriority)) {
        return ['nice', ['-n', env.processingPriority.toString(), ffmpeg, ...args]]
    }
    return [ffmpeg, args]
--- a/src/modules/sub/utils.js
+++ b/src/modules/sub/utils.js
@ -45,6 +45,13 @@ export function cleanHTML(html) {
    return clean
 }

+export function getRedirectingURL(url) {
+    return fetch(url, { redirect: 'manual' }).then((r) => {
+        if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
+            return r.headers.get('location');
+    }).catch(() => null);
+}
+
 export function merge(a, b) {
    for (const k of Object.keys(b)) {
        if (Array.isArray(b[k])) {
--- a/src/modules/test.js
+++ b/src/modules/test.js
@ -0,0 +1,42 @@
+import { normalizeRequest } from "../modules/processing/request.js";
+import match from "./processing/match.js";
+import { extract } from "./processing/url.js";
+
+export async function runTest(url, params, expect) {
+    const normalized = normalizeRequest({ url, ...params });
+    if (!normalized) {
+        throw "invalid request";
+    }
+
+    const parsed = extract(normalized.url);
+    if (parsed === null) {
+        throw `invalid url: ${normalized.url}`;
+    }
+
+    const result = await match(
+        parsed.host, parsed.patternMatch, "en", normalized
+    );
+
+    let error = [];
+    if (expect.status !== result.body.status) {
+        const detail = `${expect.status} (expected) != ${result.body.status} (actual)`;
+        error.push(`status mismatch: ${detail}`);
+    }
+
+    if (expect.code !== result.status) {
+        const detail = `${expect.code} (expected) != ${result.status} (actual)`;
+        error.push(`status code mismatch: ${detail}`);
+    }
+
+    if (error.length) {
+        if (result.body.text) {
+            error.push(`error message: ${result.body.text}`);
+        }
+
+        throw error.join('\n');
+    }
+
+    if (result.body.status === 'stream') {
+        // TODO: stream testing
+    }
+}
--- a/src/util/generate-youtube-tokens.js
+++ b/src/util/generate-youtube-tokens.js
@ -20,9 +20,9 @@ tube.session.once(
 );

 tube.session.once('auth-error', (err) => bail('An error occurred:', err));
-tube.session.once('auth', ({ status, credentials, ...rest }) => {
-    if (status !== 'SUCCESS') {
-        bail('something went wrong', rest);
+tube.session.once('auth', ({ credentials }) => {
+    if (!credentials.access_token) {
+        bail('something went wrong');
    }

    console.log(
--- a/src/util/test-ci.js
+++ b/src/util/test-ci.js
@ -0,0 +1,81 @@
+import { env } from "../modules/config.js";
+import { runTest } from "../modules/test.js";
+import { loadLoc } from "../localization/manager.js";
+import { loadJSON } from "../modules/sub/loadFromFs.js";
+import { Red, Bright } from "../modules/sub/consoleText.js";
+
+const tests = loadJSON('./src/util/tests.json');
+const services = loadJSON('./src/modules/processing/servicesConfig.json');
+
+// services that are known to frequently fail due to external
+// factors (e.g. rate limiting)
+const finnicky = new Set(['bilibili', 'instagram', 'youtube'])
+
+const action = process.argv[2];
+switch (action) {
+    case "get-services":
+        const fromConfig = Object.keys(services.config);
+
+        const missingTests = fromConfig.filter(
+            service => !tests[service] || tests[service].length === 0
+        );
+
+        if (missingTests.length) {
+            console.error('services have no tests:', missingTests);
+            console.log('[]');
+            process.exitCode = 1;
+            break;
+        }
+
+        console.log(JSON.stringify(fromConfig));
+        break;
+
+    case "run-tests-for":
+        const service = process.argv[3];
+        let failed = false;
+
+        if (!tests[service]) {
+            console.error('no such service:', service);
+        }
+
+        await loadLoc();
+        env.streamLifespan = 10000;
+        env.apiURL = 'http://x';
+
+        for (const test of tests[service]) {
+            const { name, url, params, expected } = test;
+            const canFail = test.canFail || finnicky.has(service);
+
+            try {
+                await runTest(url, params, expected);
+                console.log(`${service}/${name}: ok`);
+
+            } catch(e) {
+                failed = !canFail;
+
+                let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
+                if (canFail && process.env.GITHUB_ACTION) {
+                    console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
+                }
+
+                console.error(`${service}/${name}: ${failText}`);
+                const errorString = e.toString().split('\n');
+                let c = '┃';
+                errorString.forEach((line, index) => {
+                    line = line.replace('!=', Red('!='));
+
+                    if (index === errorString.length - 1) {
+                        c = '┗';
+                    }
+
+                    console.error(`   ${c}`, line);
+                });
+            }
+        }
+
+        process.exitCode = Number(failed);
+        break;
+    default:
+        console.error('invalid action:', action);
+        process.exitCode = 1;
+}
--- a/src/util/tests.json
+++ b/src/util/tests.json
@ -1,7 +1,7 @@
 {
    "twitter": [{
        "name": "regular video",
-        "url": "https://twitter.com/TwitterSpaces/status/1526955853743546372?s=20",
+        "url": "https://twitter.com/X/status/1697304622749086011",
        "params": {
            "aFormat": "mp3",
            "isAudioOnly": false,
@ -14,7 +14,7 @@
    },
    {
        "name": "video with mobile web mediaviewer",
-        "url": "https://twitter.com/XSpaces/status/1526955853743546372/mediaViewer?currentTweet=1526955853743546372&currentTweetUser=XSpaces&currentTweet=1526955853743546372&currentTweetUser=XSpaces",
+        "url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011&currentTweetUser=X&currentTweet=1697304622749086011&currentTweetUser=X",
        "params": {},
        "expected": {
            "code": 200,
@ -34,7 +34,7 @@
        }
    }, {
        "name": "mixed media (image + gif)",
-        "url": "https://twitter.com/Twitter/status/1580661436132757506?s=20",
+        "url": "https://twitter.com/sky_mj26/status/1807756010712428565",
        "params": {
            "aFormat": "mp3",
            "isAudioOnly": false,
@ -42,7 +42,7 @@
        },
        "expected": {
            "code": 200,
-            "status": "redirect"
+            "status": "picker"
        }
    }, {
        "name": "picker: mixed media (3 videos)",
@ -113,9 +113,10 @@
            "status": "redirect"
        }
    }, {
-        "name": "age-restricted video",
+        "name": "age restricted video",
        "url": "https://twitter.com/FckyeahCharli/status/1650987582749065220",
        "params": {},
+        "canFail": true,
        "expected": {
            "code": 200,
            "status": "redirect"
@ -650,23 +651,23 @@
        }
    }, {
        "name": "1080p dash parcel",
-        "url": "https://vimeo.com/774694040",
+        "url": "https://vimeo.com/967252742",
        "params": {
            "vQuality": "1440"
        },
        "expected": {
            "code": 200,
-            "status": "stream"
+            "status": "redirect"
        }
    }, {
        "name": "720p dash parcel",
-        "url": "https://vimeo.com/774694040",
+        "url": "https://vimeo.com/967252742",
        "params": {
            "vQuality": "360"
        },
        "expected": {
            "code": 200,
-            "status": "stream"
+            "status": "redirect"
        }
    }, {
        "name": "private video",
@ -676,6 +677,14 @@
            "code": 200,
            "status": "redirect"
        }
+    }, {
+        "name": "mature video",
+        "url": "https://vimeo.com/973212054",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
    }],
    "reddit": [{
        "name": "video with audio",
@ -947,7 +956,7 @@
    }],
    "streamable": [{
        "name": "regular video",
-        "url": "https://streamable.com/03r3c2",
+        "url": "https://streamable.com/p9cln4",
        "params": {},
        "expected": {
            "code": 200,
@ -963,7 +972,7 @@
        }
    }, {
        "name": "regular video (isAudioOnly)",
-        "url": "https://streamable.com/03r3c2",
+        "url": "https://streamable.com/p9cln4",
        "params": {
            "isAudioOnly": true
        },
@ -973,7 +982,7 @@
        }
    }, {
        "name": "regular video (isAudioMuted)",
-        "url": "https://streamable.com/03r3c2",
+        "url": "https://streamable.com/p9cln4",
        "params": {
            "isAudioMuted": true
        },
@ -1042,8 +1051,8 @@
        "url": "https://rutube.ru/video/b521653b4f71ece57b8ff54e57ca9b82/",
        "params": {},
        "expected": {
-            "code": 200,
-            "status": "stream"
+            "code": 400,
+            "status": "error"
        }
    }, {
        "name": "vertical video",
@ -1055,7 +1064,7 @@
        }
    }, {
        "name": "yappy",
-        "url": "https://rutube.ru/yappy/f1771e86294748eea8ecb2ac88e55740/",
+        "url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/",
        "params": {},
        "expected": {
            "code": 200,
@ -1132,6 +1141,31 @@
            "status": "stream"
        }
    }],
+    "snapchat": [{
+        "name": "spotlight",
+        "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shortlinked spotlight",
+        "url": "https://t.snapchat.com/4ZsiBLDi",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "story",
+        "url": "https://www.snapchat.com/add/bazerkmakane",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    }],
    "loom": [{
        "name": "1080p video",
        "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
@ -1156,9 +1190,68 @@
        "params": {
            "isAudioOnly": true
        },
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }],
+    "facebook": [{
+        "name": "direct video with username and id",
+        "url": "https://web.facebook.com/100048111287134/videos/1157798148685638/",
+        "params": {},
        "expected": {
            "code": 200,
-            "status": "stream"
+            "status": "redirect"
+        }
+    }, {
+        "name": "direct video with id as query param",
+        "url": "https://web.facebook.com/watch/?v=883839773514682&ref=sharing",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "direct video with caption",
+        "url": "https://web.facebook.com/wood57/videos/𝐒𝐞𝐛𝐚𝐬𝐤𝐨𝐦-𝐟𝐮𝐥𝐥/883839773514682",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shortlink video",
+        "url": "https://fb.watch/r1K6XHMfGT/",
+        "canFail": true,
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "reel video",
+        "url": "https://web.facebook.com/reel/730293269054758",
+        "canFail": true,
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shared video link",
+        "url": "https://www.facebook.com/share/v/NEf87jbPTvFE8LsL/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shared video link v2",
+        "url": "https://web.facebook.com/share/r/JFZfPVgLkiJQmWrr/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
        }
    }],
    "odysee": [{