Merge branch 'imputnet:main' into main

.github/workflows/test-services.yml

@@ -31,3 +31,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
       - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }}
+        env:
+          API_EXTERNAL_PROXY: ${{ secrets.API_EXTERNAL_PROXY }}
+          TEST_IGNORE_SERVICES: ${{ vars.TEST_IGNORE_SERVICES }}

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "10.7.2",
+    "version": "10.7.5",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",

api/src/misc/run-test.js

@@ -23,6 +23,10 @@ export async function runTest(url, params, expect) {
     if (expect.status !== result.body.status) {
         const detail = `${expect.status} (expected) != ${result.body.status} (actual)`;
         error.push(`status mismatch: ${detail}`);
+
+        if (result.body.status === 'error') {
+            error.push(`error code: ${result.body?.error?.code}`);
+        }
     }
 
     if (expect.errorCode && expect.errorCode !== result.body?.error?.code) {

api/src/processing/match.js

@@ -227,7 +227,8 @@ export default async function({ host, patternMatch, params }) {
 
             case "facebook":
                 r = await facebook({
-                    ...patternMatch
+                    ...patternMatch,
+                    dispatcher
                 });
                 break;
 

api/src/processing/services/facebook.js

@@ -8,8 +8,8 @@ const headers = {
     'Sec-Fetch-Site': 'none',
 }
 
-const resolveUrl = (url) => {
+const resolveUrl = (url, dispatcher) => {
-    return fetch(url, { headers })
+    return fetch(url, { headers, dispatcher })
         .then(r => {
             if (r.headers.get('location')) {
                 return decodeURIComponent(r.headers.get('location'));
@@ -23,13 +23,13 @@ const resolveUrl = (url) => {
         .catch(() => false);
 }
 
-export default async function({ id, shareType, shortLink }) {
+export default async function({ id, shareType, shortLink, dispatcher }) {
     let url = `https://web.facebook.com/i/videos/${id}`;
 
     if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
-    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`);
+    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`, dispatcher);
 
-    const html = await fetch(url, { headers })
+    const html = await fetch(url, { headers, dispatcher })
         .then(r => r.text())
         .catch(() => false);
 

api/src/processing/services/instagram.js

@@ -305,12 +305,12 @@ export default function instagram(obj) {
         if (sidecar) {
             const picker = sidecar.edges.filter(e => e.node?.display_url)
                 .map((e, i) => {
-                    const type = e.node?.is_video ? "video" : "photo";
+                    const type = e.node?.is_video && e.node?.video_url ? "video" : "photo";
 
                     let url;
-                    if (type === 'video') {
+                    if (type === "video") {
                         url = e.node?.video_url;
-                    } else if (type === 'photo') {
+                    } else if (type === "photo") {
                         url = e.node?.display_url;
                     }
 

api/src/processing/services/ok.js

@@ -44,7 +44,7 @@ export default async function(o) {
 
     let fileMetadata = {
         title: videoData.movie.title.trim(),
-        author: (videoData.author?.name || videoData.compilationTitle).trim(),
+        author: (videoData.author?.name || videoData.compilationTitle)?.trim(),
     }
 
     if (bestVideo) return {

api/src/processing/services/youtube.js

@@ -193,7 +193,7 @@ export default async function (o) {
             if (playability.reason.endsWith("bot")) {
                 return { error: "youtube.login" }
             }
-            if (playability.reason.endsWith("age")) {
+            if (playability.reason.endsWith("age") || playability.reason.endsWith("inappropriate for some users.")) {
                 return { error: "content.video.age" }
             }
             if (playability?.error_screen?.reason?.text === "Private video") {

api/src/processing/url.js

@@ -98,6 +98,14 @@ function aliasURL(url) {
             if (url.hostname === 'xhslink.com' && parts.length === 3) {
                 url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`);
             }
+            break;
+
+        case "loom":
+            const idPart = parts[parts.length - 1];
+            if (idPart.length > 32) {
+                url.pathname = `/share/${idPart.slice(-32)}`;
+            }
+            break;
     }
 
     return url;

api/src/util/test.js

@@ -4,6 +4,7 @@ import { env } from "../config.js";
 import { runTest } from "../misc/run-test.js";
 import { loadJSON } from "../misc/load-from-fs.js";
 import { Red, Bright } from "../misc/console-text.js";
+import { setGlobalDispatcher, ProxyAgent } from "undici";
 import { randomizeCiphers } from "../misc/randomize-ciphers.js";
 
 import { services } from "../processing/service-config.js";
@@ -13,7 +14,11 @@ const getTests = (service) => loadJSON(getTestPath(service));
 
 // services that are known to frequently fail due to external
 // factors (e.g. rate limiting)
-const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']);
+const finnicky = new Set(
+    process.env.TEST_IGNORE_SERVICES
+    ? process.env.TEST_IGNORE_SERVICES.split(',')
+    : ['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']
+);
 
 const runTestsFor = async (service) => {
     const tests = getTests(service);
@@ -64,6 +69,14 @@ const printHeader = (service, padLen) => {
     console.log(service + '='.repeat(50));
 }
 
+if (env.externalProxy) {
+    setGlobalDispatcher(new ProxyAgent(env.externalProxy));
+}
+
+env.streamLifespan = 10000;
+env.apiURL = 'http://x/';
+randomizeCiphers();
+
 const action = process.argv[2];
 switch (action) {
     case "get-services":
@@ -86,9 +99,6 @@ switch (action) {
         break;
 
     case "run-tests-for":
-        env.streamLifespan = 10000;
-        env.apiURL = 'http://x/';
-        randomizeCiphers();
 
         try {
             const { softFails } = await runTestsFor(process.argv[3]);
@@ -104,10 +114,6 @@ switch (action) {
         const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0);
         const failCounters = {};
 
-        env.streamLifespan = 10000;
-        env.apiURL = 'http://x/';
-        randomizeCiphers();
-
         for (const service in services) {
             printHeader(service, maxHeaderLen);
             const { fails, softFails } = await runTestsFor(service);

api/src/util/tests/facebook.json

@@ -29,7 +29,6 @@
     {
         "name": "shortlink video",
         "url": "https://fb.watch/r1K6XHMfGT/",
-        "canFail": true,
         "params": {},
         "expected": {
             "code": 200,
@@ -39,7 +38,6 @@
     {
         "name": "reel video",
         "url": "https://web.facebook.com/reel/730293269054758",
-        "canFail": true,
         "params": {},
         "expected": {
             "code": 200,
@@ -64,4 +62,4 @@
             "status": "redirect"
         }
     }
-]
+]

api/src/util/tests/twitter.json

@@ -102,9 +102,8 @@
     },
     {
         "name": "retweeted video",
-        "url": "https://twitter.com/uwukko/status/1696901469633421344",
+        "url": "https://twitter.com/schlizzawg/status/1869017025055793405",
         "params": {},
-        "canFail": true,
         "expected": {
             "code": 200,
             "status": "redirect"
@@ -145,7 +144,7 @@
         "params": {},
         "expected": {
             "code": 200,
-            "status": "redirect"
+            "status": "tunnel"
         }
     },
     {
@@ -203,11 +202,11 @@
     },
     {
         "name": "bookmarked photo",
-        "url": "https://twitter.com/i/bookmarks?post_id=1837430141179289876",
+        "url": "https://twitter.com/i/bookmarks?post_id=1887450602164396149",
         "params": {},
         "expected": {
             "code": 200,
-            "status": "redirect"
+            "status": "tunnel"
         }
     }
-]
+]

api/src/util/tests/xiaohongshu.json

@@ -48,7 +48,6 @@
     {
         "name": "short link, wrong id",
         "url": "https://xhslink.com/a/aaaaaa",
-        "canFail": true,
         "params": {},
         "expected": {
             "code": 400,

docs/run-an-instance.md

@@ -18,7 +18,7 @@ if you need help with installing docker, follow *only the first step* of these t
     ```
     i'm using `nano` in this example, it may not be available in your distro. you can use any other text editor.
 
-3. copy and paste the [sample config from here](examples/docker-compose.example.yml) for either web or api instance (or both, if you wish) and edit it to your needs.
+3. copy and paste the [sample config from here](examples/docker-compose.example.yml) and edit it to your needs.
     make sure to replace default URLs with your own or cobalt won't work correctly.
 
 4. finally, start the cobalt container (from cobalt directory):

web/i18n/en/error.json

@@ -30,7 +30,8 @@
     "api.capacity": "cobalt is at capacity and can't process your request at the moment. try again in a few seconds!",
 
     "api.generic": "something went wrong and i couldn't get anything for you, try again in a few seconds. if the issue sticks, please report it!",
-    "api.unknown_response": "couldn't read the response from the processing instance. this could be caused by a version mismatch between cobalt instances.",
+    "api.unknown_response": "couldn't read the response from the processing instance. this is probably caused by the web app being out of date. reload the app and try again!",
+    "api.invalid_body": "couldn't send the request to the processing instance. this is probably caused by the web app being out of date. reload the app and try again!",
 
     "api.service.unsupported": "this service is not supported yet. have you pasted the right link?",
     "api.service.disabled": "this service is generally supported by cobalt, but it's disabled on this processing instance. try a link from another service!",

web/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@imput/cobalt-web",
-    "version": "10.6",
+    "version": "10.7.5",
     "type": "module",
     "private": true,
     "scripts": {

web/src/components/dialog/PickerDialog.svelte

@@ -50,7 +50,9 @@
         <div class="picker-body">
             {#if items}
                 {#each items as item, i}
-                    <PickerItem {item} number={i + 1} />
+                    {#if item?.url}
+                        <PickerItem {item} number={i + 1} />
+                    {/if}
                 {/each}
             {/if}
         </div>

web/src/components/dialog/PickerItem.svelte

@@ -14,18 +14,28 @@
     export let number: number;
 
     let imageLoaded = false;
-    const isTunnel = new URL(item.url).pathname === "/tunnel";
+
+    let validUrl = false;
+    try {
+        new URL(item.url);
+        validUrl = true;
+    } catch {}
+
+    const isTunnel = validUrl && new URL(item.url).pathname === "/tunnel";
 
     $: itemType = item.type ?? "photo";
 </script>
 
 <button
     class="picker-item"
-    on:click={() =>
+    on:click={() => {
-        downloadFile({
+        if (validUrl) {
-            url: item.url,
+            downloadFile({
-            urlType: isTunnel ? "tunnel" : "redirect",
+                url: item.url,
-        })}
+                urlType: isTunnel ? "tunnel" : "redirect",
+            });
+        }
+    }}
 >
     <div class="picker-type">
         {#if itemType === "video"}

web/src/lib/settings/youtube-lang.ts

@@ -72,6 +72,9 @@ export const youtubeLanguages = [
     "ur",
     "uz",
     "vi",
+    "zh",
+    "zh-Hans",
+    "zh-Hant",
     "zh-CN",
     "zh-HK",
     "zh-TW",