diff --git a/.github/workflows/test-services.yml b/.github/workflows/test-services.yml index 77242cb8..ad6e25b7 100644 --- a/.github/workflows/test-services.yml +++ b/.github/workflows/test-services.yml @@ -31,3 +31,6 @@ jobs: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }} + env: + API_EXTERNAL_PROXY: ${{ secrets.API_EXTERNAL_PROXY }} + TEST_IGNORE_SERVICES: ${{ vars.TEST_IGNORE_SERVICES }} diff --git a/api/package.json b/api/package.json index bc30aade..9e58ef16 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.7.2", + "version": "10.7.5", "author": "imput", "exports": "./src/cobalt.js", "type": "module", diff --git a/api/src/misc/run-test.js b/api/src/misc/run-test.js index 2dc1a28a..6dd08183 100644 --- a/api/src/misc/run-test.js +++ b/api/src/misc/run-test.js @@ -23,6 +23,10 @@ export async function runTest(url, params, expect) { if (expect.status !== result.body.status) { const detail = `${expect.status} (expected) != ${result.body.status} (actual)`; error.push(`status mismatch: ${detail}`); + + if (result.body.status === 'error') { + error.push(`error code: ${result.body?.error?.code}`); + } } if (expect.errorCode && expect.errorCode !== result.body?.error?.code) { diff --git a/api/src/processing/match.js b/api/src/processing/match.js index b7be8456..e2d6aa07 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -227,7 +227,8 @@ export default async function({ host, patternMatch, params }) { case "facebook": r = await facebook({ - ...patternMatch + ...patternMatch, + dispatcher }); break; diff --git a/api/src/processing/services/facebook.js b/api/src/processing/services/facebook.js index 7bfd4751..9e9d060d 100644 --- a/api/src/processing/services/facebook.js +++ b/api/src/processing/services/facebook.js @@ -8,8 +8,8 @@ const headers = { 'Sec-Fetch-Site': 'none', } -const resolveUrl = (url) => { - return fetch(url, { headers }) +const resolveUrl = (url, dispatcher) => { + return fetch(url, { headers, dispatcher }) .then(r => { if (r.headers.get('location')) { return decodeURIComponent(r.headers.get('location')); @@ -23,13 +23,13 @@ const resolveUrl = (url) => { .catch(() => false); } -export default async function({ id, shareType, shortLink }) { +export default async function({ id, shareType, shortLink, dispatcher }) { let url = `https://web.facebook.com/i/videos/${id}`; if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`; - if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`); + if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`, dispatcher); - const html = await fetch(url, { headers }) + const html = await fetch(url, { headers, dispatcher }) .then(r => r.text()) .catch(() => false); diff --git a/api/src/processing/services/instagram.js b/api/src/processing/services/instagram.js index eab4776e..9cc7dbdf 100644 --- a/api/src/processing/services/instagram.js +++ b/api/src/processing/services/instagram.js @@ -305,12 +305,12 @@ export default function instagram(obj) { if (sidecar) { const picker = sidecar.edges.filter(e => e.node?.display_url) .map((e, i) => { - const type = e.node?.is_video ? "video" : "photo"; + const type = e.node?.is_video && e.node?.video_url ? "video" : "photo"; let url; - if (type === 'video') { + if (type === "video") { url = e.node?.video_url; - } else if (type === 'photo') { + } else if (type === "photo") { url = e.node?.display_url; } diff --git a/api/src/processing/services/ok.js b/api/src/processing/services/ok.js index 10fb785b..cfe18e49 100644 --- a/api/src/processing/services/ok.js +++ b/api/src/processing/services/ok.js @@ -44,7 +44,7 @@ export default async function(o) { let fileMetadata = { title: videoData.movie.title.trim(), - author: (videoData.author?.name || videoData.compilationTitle).trim(), + author: (videoData.author?.name || videoData.compilationTitle)?.trim(), } if (bestVideo) return { diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 6e009d13..5d655318 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -193,7 +193,7 @@ export default async function (o) { if (playability.reason.endsWith("bot")) { return { error: "youtube.login" } } - if (playability.reason.endsWith("age")) { + if (playability.reason.endsWith("age") || playability.reason.endsWith("inappropriate for some users.")) { return { error: "content.video.age" } } if (playability?.error_screen?.reason?.text === "Private video") { diff --git a/api/src/processing/url.js b/api/src/processing/url.js index f4a5c5b1..82299999 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -98,6 +98,14 @@ function aliasURL(url) { if (url.hostname === 'xhslink.com' && parts.length === 3) { url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`); } + break; + + case "loom": + const idPart = parts[parts.length - 1]; + if (idPart.length > 32) { + url.pathname = `/share/${idPart.slice(-32)}`; + } + break; } return url; diff --git a/api/src/util/test.js b/api/src/util/test.js index 9457c1ca..abb9b3cd 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -4,6 +4,7 @@ import { env } from "../config.js"; import { runTest } from "../misc/run-test.js"; import { loadJSON } from "../misc/load-from-fs.js"; import { Red, Bright } from "../misc/console-text.js"; +import { setGlobalDispatcher, ProxyAgent } from "undici"; import { randomizeCiphers } from "../misc/randomize-ciphers.js"; import { services } from "../processing/service-config.js"; @@ -13,7 +14,11 @@ const getTests = (service) => loadJSON(getTestPath(service)); // services that are known to frequently fail due to external // factors (e.g. rate limiting) -const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']); +const finnicky = new Set( + process.env.TEST_IGNORE_SERVICES + ? process.env.TEST_IGNORE_SERVICES.split(',') + : ['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit'] +); const runTestsFor = async (service) => { const tests = getTests(service); @@ -64,6 +69,14 @@ const printHeader = (service, padLen) => { console.log(service + '='.repeat(50)); } +if (env.externalProxy) { + setGlobalDispatcher(new ProxyAgent(env.externalProxy)); +} + +env.streamLifespan = 10000; +env.apiURL = 'http://x/'; +randomizeCiphers(); + const action = process.argv[2]; switch (action) { case "get-services": @@ -86,9 +99,6 @@ switch (action) { break; case "run-tests-for": - env.streamLifespan = 10000; - env.apiURL = 'http://x/'; - randomizeCiphers(); try { const { softFails } = await runTestsFor(process.argv[3]); @@ -104,10 +114,6 @@ switch (action) { const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0); const failCounters = {}; - env.streamLifespan = 10000; - env.apiURL = 'http://x/'; - randomizeCiphers(); - for (const service in services) { printHeader(service, maxHeaderLen); const { fails, softFails } = await runTestsFor(service); diff --git a/api/src/util/tests/facebook.json b/api/src/util/tests/facebook.json index 876ac7fe..d0c8cc7b 100644 --- a/api/src/util/tests/facebook.json +++ b/api/src/util/tests/facebook.json @@ -29,7 +29,6 @@ { "name": "shortlink video", "url": "https://fb.watch/r1K6XHMfGT/", - "canFail": true, "params": {}, "expected": { "code": 200, @@ -39,7 +38,6 @@ { "name": "reel video", "url": "https://web.facebook.com/reel/730293269054758", - "canFail": true, "params": {}, "expected": { "code": 200, @@ -64,4 +62,4 @@ "status": "redirect" } } -] \ No newline at end of file +] diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json index 0024d097..4fc5900f 100644 --- a/api/src/util/tests/twitter.json +++ b/api/src/util/tests/twitter.json @@ -102,9 +102,8 @@ }, { "name": "retweeted video", - "url": "https://twitter.com/uwukko/status/1696901469633421344", + "url": "https://twitter.com/schlizzawg/status/1869017025055793405", "params": {}, - "canFail": true, "expected": { "code": 200, "status": "redirect" @@ -145,7 +144,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { @@ -203,11 +202,11 @@ }, { "name": "bookmarked photo", - "url": "https://twitter.com/i/bookmarks?post_id=1837430141179289876", + "url": "https://twitter.com/i/bookmarks?post_id=1887450602164396149", "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } } -] \ No newline at end of file +] diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json index de632a77..0cca9393 100644 --- a/api/src/util/tests/xiaohongshu.json +++ b/api/src/util/tests/xiaohongshu.json @@ -48,7 +48,6 @@ { "name": "short link, wrong id", "url": "https://xhslink.com/a/aaaaaa", - "canFail": true, "params": {}, "expected": { "code": 400, diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 60a3c8aa..ea31cfc5 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -18,7 +18,7 @@ if you need help with installing docker, follow *only the first step* of these t ``` i'm using `nano` in this example, it may not be available in your distro. you can use any other text editor. -3. copy and paste the [sample config from here](examples/docker-compose.example.yml) for either web or api instance (or both, if you wish) and edit it to your needs. +3. copy and paste the [sample config from here](examples/docker-compose.example.yml) and edit it to your needs. make sure to replace default URLs with your own or cobalt won't work correctly. 4. finally, start the cobalt container (from cobalt directory): diff --git a/web/i18n/en/error.json b/web/i18n/en/error.json index 6b83ceb9..2788d9e4 100644 --- a/web/i18n/en/error.json +++ b/web/i18n/en/error.json @@ -30,7 +30,8 @@ "api.capacity": "cobalt is at capacity and can't process your request at the moment. try again in a few seconds!", "api.generic": "something went wrong and i couldn't get anything for you, try again in a few seconds. if the issue sticks, please report it!", - "api.unknown_response": "couldn't read the response from the processing instance. this could be caused by a version mismatch between cobalt instances.", + "api.unknown_response": "couldn't read the response from the processing instance. this is probably caused by the web app being out of date. reload the app and try again!", + "api.invalid_body": "couldn't send the request to the processing instance. this is probably caused by the web app being out of date. reload the app and try again!", "api.service.unsupported": "this service is not supported yet. have you pasted the right link?", "api.service.disabled": "this service is generally supported by cobalt, but it's disabled on this processing instance. try a link from another service!", diff --git a/web/package.json b/web/package.json index 0c621f02..24b59501 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "@imput/cobalt-web", - "version": "10.6", + "version": "10.7.5", "type": "module", "private": true, "scripts": { diff --git a/web/src/components/dialog/PickerDialog.svelte b/web/src/components/dialog/PickerDialog.svelte index 1fa5f5f5..b84ea4ad 100644 --- a/web/src/components/dialog/PickerDialog.svelte +++ b/web/src/components/dialog/PickerDialog.svelte @@ -50,7 +50,9 @@
{#if items} {#each items as item, i} - + {#if item?.url} + + {/if} {/each} {/if}
diff --git a/web/src/components/dialog/PickerItem.svelte b/web/src/components/dialog/PickerItem.svelte index 4a515a6e..04e301ec 100644 --- a/web/src/components/dialog/PickerItem.svelte +++ b/web/src/components/dialog/PickerItem.svelte @@ -14,18 +14,28 @@ export let number: number; let imageLoaded = false; - const isTunnel = new URL(item.url).pathname === "/tunnel"; + + let validUrl = false; + try { + new URL(item.url); + validUrl = true; + } catch {} + + const isTunnel = validUrl && new URL(item.url).pathname === "/tunnel"; $: itemType = item.type ?? "photo";