mirror of
https://github.com/imputnet/cobalt.git
synced 2025-06-28 09:28:29 +00:00
Merge branch 'imputnet:main' into main
This commit is contained in:
ZEREX
GitHub
commit
4ca38faca4
3
.github/workflows/test-services.yml
vendored
3
.github/workflows/test-services.yml
vendored
@ -31,3 +31,6 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: pnpm/action-setup@v4
|
||||
- run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }}
|
||||
env:
|
||||
API_EXTERNAL_PROXY: ${{ secrets.API_EXTERNAL_PROXY }}
|
||||
TEST_IGNORE_SERVICES: ${{ vars.TEST_IGNORE_SERVICES }}
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@imput/cobalt-api",
|
||||
"description": "save what you love",
|
||||
"version": "10.7.2",
|
||||
"version": "10.7.5",
|
||||
"author": "imput",
|
||||
"exports": "./src/cobalt.js",
|
||||
"type": "module",
|
||||
|
||||
@ -23,6 +23,10 @@ export async function runTest(url, params, expect) {
|
||||
if (expect.status !== result.body.status) {
|
||||
const detail = `${expect.status} (expected) != ${result.body.status} (actual)`;
|
||||
error.push(`status mismatch: ${detail}`);
|
||||
|
||||
if (result.body.status === 'error') {
|
||||
error.push(`error code: ${result.body?.error?.code}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (expect.errorCode && expect.errorCode !== result.body?.error?.code) {
|
||||
|
||||
@ -227,7 +227,8 @@ export default async function({ host, patternMatch, params }) {
|
||||
|
||||
case "facebook":
|
||||
r = await facebook({
|
||||
...patternMatch
|
||||
...patternMatch,
|
||||
dispatcher
|
||||
});
|
||||
break;
|
||||
|
||||
|
||||
@ -8,8 +8,8 @@ const headers = {
|
||||
'Sec-Fetch-Site': 'none',
|
||||
}
|
||||
|
||||
const resolveUrl = (url) => {
|
||||
return fetch(url, { headers })
|
||||
const resolveUrl = (url, dispatcher) => {
|
||||
return fetch(url, { headers, dispatcher })
|
||||
.then(r => {
|
||||
if (r.headers.get('location')) {
|
||||
return decodeURIComponent(r.headers.get('location'));
|
||||
@ -23,13 +23,13 @@ const resolveUrl = (url) => {
|
||||
.catch(() => false);
|
||||
}
|
||||
|
||||
export default async function({ id, shareType, shortLink }) {
|
||||
export default async function({ id, shareType, shortLink, dispatcher }) {
|
||||
let url = `https://web.facebook.com/i/videos/${id}`;
|
||||
|
||||
if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
|
||||
if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`);
|
||||
if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`, dispatcher);
|
||||
|
||||
const html = await fetch(url, { headers })
|
||||
const html = await fetch(url, { headers, dispatcher })
|
||||
.then(r => r.text())
|
||||
.catch(() => false);
|
||||
|
||||
|
||||
@ -305,12 +305,12 @@ export default function instagram(obj) {
|
||||
if (sidecar) {
|
||||
const picker = sidecar.edges.filter(e => e.node?.display_url)
|
||||
.map((e, i) => {
|
||||
const type = e.node?.is_video ? "video" : "photo";
|
||||
const type = e.node?.is_video && e.node?.video_url ? "video" : "photo";
|
||||
|
||||
let url;
|
||||
if (type === 'video') {
|
||||
if (type === "video") {
|
||||
url = e.node?.video_url;
|
||||
} else if (type === 'photo') {
|
||||
} else if (type === "photo") {
|
||||
url = e.node?.display_url;
|
||||
}
|
||||
|
||||
|
||||
@ -44,7 +44,7 @@ export default async function(o) {
|
||||
|
||||
let fileMetadata = {
|
||||
title: videoData.movie.title.trim(),
|
||||
author: (videoData.author?.name || videoData.compilationTitle).trim(),
|
||||
author: (videoData.author?.name || videoData.compilationTitle)?.trim(),
|
||||
}
|
||||
|
||||
if (bestVideo) return {
|
||||
|
||||
@ -193,7 +193,7 @@ export default async function (o) {
|
||||
if (playability.reason.endsWith("bot")) {
|
||||
return { error: "youtube.login" }
|
||||
}
|
||||
if (playability.reason.endsWith("age")) {
|
||||
if (playability.reason.endsWith("age") || playability.reason.endsWith("inappropriate for some users.")) {
|
||||
return { error: "content.video.age" }
|
||||
}
|
||||
if (playability?.error_screen?.reason?.text === "Private video") {
|
||||
|
||||
@ -98,6 +98,14 @@ function aliasURL(url) {
|
||||
if (url.hostname === 'xhslink.com' && parts.length === 3) {
|
||||
url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`);
|
||||
}
|
||||
break;
|
||||
|
||||
case "loom":
|
||||
const idPart = parts[parts.length - 1];
|
||||
if (idPart.length > 32) {
|
||||
url.pathname = `/share/${idPart.slice(-32)}`;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return url;
|
||||
|
||||
@ -4,6 +4,7 @@ import { env } from "../config.js";
|
||||
import { runTest } from "../misc/run-test.js";
|
||||
import { loadJSON } from "../misc/load-from-fs.js";
|
||||
import { Red, Bright } from "../misc/console-text.js";
|
||||
import { setGlobalDispatcher, ProxyAgent } from "undici";
|
||||
import { randomizeCiphers } from "../misc/randomize-ciphers.js";
|
||||
|
||||
import { services } from "../processing/service-config.js";
|
||||
@ -13,7 +14,11 @@ const getTests = (service) => loadJSON(getTestPath(service));
|
||||
|
||||
// services that are known to frequently fail due to external
|
||||
// factors (e.g. rate limiting)
|
||||
const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']);
|
||||
const finnicky = new Set(
|
||||
process.env.TEST_IGNORE_SERVICES
|
||||
? process.env.TEST_IGNORE_SERVICES.split(',')
|
||||
: ['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']
|
||||
);
|
||||
|
||||
const runTestsFor = async (service) => {
|
||||
const tests = getTests(service);
|
||||
@ -64,6 +69,14 @@ const printHeader = (service, padLen) => {
|
||||
console.log(service + '='.repeat(50));
|
||||
}
|
||||
|
||||
if (env.externalProxy) {
|
||||
setGlobalDispatcher(new ProxyAgent(env.externalProxy));
|
||||
}
|
||||
|
||||
env.streamLifespan = 10000;
|
||||
env.apiURL = 'http://x/';
|
||||
randomizeCiphers();
|
||||
|
||||
const action = process.argv[2];
|
||||
switch (action) {
|
||||
case "get-services":
|
||||
@ -86,9 +99,6 @@ switch (action) {
|
||||
break;
|
||||
|
||||
case "run-tests-for":
|
||||
env.streamLifespan = 10000;
|
||||
env.apiURL = 'http://x/';
|
||||
randomizeCiphers();
|
||||
|
||||
try {
|
||||
const { softFails } = await runTestsFor(process.argv[3]);
|
||||
@ -104,10 +114,6 @@ switch (action) {
|
||||
const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0);
|
||||
const failCounters = {};
|
||||
|
||||
env.streamLifespan = 10000;
|
||||
env.apiURL = 'http://x/';
|
||||
randomizeCiphers();
|
||||
|
||||
for (const service in services) {
|
||||
printHeader(service, maxHeaderLen);
|
||||
const { fails, softFails } = await runTestsFor(service);
|
||||
|
||||
@ -29,7 +29,6 @@
|
||||
{
|
||||
"name": "shortlink video",
|
||||
"url": "https://fb.watch/r1K6XHMfGT/",
|
||||
"canFail": true,
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
@ -39,7 +38,6 @@
|
||||
{
|
||||
"name": "reel video",
|
||||
"url": "https://web.facebook.com/reel/730293269054758",
|
||||
"canFail": true,
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
|
||||
@ -102,9 +102,8 @@
|
||||
},
|
||||
{
|
||||
"name": "retweeted video",
|
||||
"url": "https://twitter.com/uwukko/status/1696901469633421344",
|
||||
"url": "https://twitter.com/schlizzawg/status/1869017025055793405",
|
||||
"params": {},
|
||||
"canFail": true,
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
@ -145,7 +144,7 @@
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -203,11 +202,11 @@
|
||||
},
|
||||
{
|
||||
"name": "bookmarked photo",
|
||||
"url": "https://twitter.com/i/bookmarks?post_id=1837430141179289876",
|
||||
"url": "https://twitter.com/i/bookmarks?post_id=1887450602164396149",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
"status": "tunnel"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -48,7 +48,6 @@
|
||||
{
|
||||
"name": "short link, wrong id",
|
||||
"url": "https://xhslink.com/a/aaaaaa",
|
||||
"canFail": true,
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
|
||||
@ -18,7 +18,7 @@ if you need help with installing docker, follow *only the first step* of these t
|
||||
```
|
||||
i'm using `nano` in this example, it may not be available in your distro. you can use any other text editor.
|
||||
|
||||
3. copy and paste the [sample config from here](examples/docker-compose.example.yml) for either web or api instance (or both, if you wish) and edit it to your needs.
|
||||
3. copy and paste the [sample config from here](examples/docker-compose.example.yml) and edit it to your needs.
|
||||
make sure to replace default URLs with your own or cobalt won't work correctly.
|
||||
|
||||
4. finally, start the cobalt container (from cobalt directory):
|
||||
|
||||
@ -30,7 +30,8 @@
|
||||
"api.capacity": "cobalt is at capacity and can't process your request at the moment. try again in a few seconds!",
|
||||
|
||||
"api.generic": "something went wrong and i couldn't get anything for you, try again in a few seconds. if the issue sticks, please report it!",
|
||||
"api.unknown_response": "couldn't read the response from the processing instance. this could be caused by a version mismatch between cobalt instances.",
|
||||
"api.unknown_response": "couldn't read the response from the processing instance. this is probably caused by the web app being out of date. reload the app and try again!",
|
||||
"api.invalid_body": "couldn't send the request to the processing instance. this is probably caused by the web app being out of date. reload the app and try again!",
|
||||
|
||||
"api.service.unsupported": "this service is not supported yet. have you pasted the right link?",
|
||||
"api.service.disabled": "this service is generally supported by cobalt, but it's disabled on this processing instance. try a link from another service!",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@imput/cobalt-web",
|
||||
"version": "10.6",
|
||||
"version": "10.7.5",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
@ -50,7 +50,9 @@
|
||||
<div class="picker-body">
|
||||
{#if items}
|
||||
{#each items as item, i}
|
||||
<PickerItem {item} number={i + 1} />
|
||||
{#if item?.url}
|
||||
<PickerItem {item} number={i + 1} />
|
||||
{/if}
|
||||
{/each}
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
@ -14,18 +14,28 @@
|
||||
export let number: number;
|
||||
|
||||
let imageLoaded = false;
|
||||
const isTunnel = new URL(item.url).pathname === "/tunnel";
|
||||
|
||||
let validUrl = false;
|
||||
try {
|
||||
new URL(item.url);
|
||||
validUrl = true;
|
||||
} catch {}
|
||||
|
||||
const isTunnel = validUrl && new URL(item.url).pathname === "/tunnel";
|
||||
|
||||
$: itemType = item.type ?? "photo";
|
||||
</script>
|
||||
|
||||
<button
|
||||
class="picker-item"
|
||||
on:click={() =>
|
||||
downloadFile({
|
||||
url: item.url,
|
||||
urlType: isTunnel ? "tunnel" : "redirect",
|
||||
})}
|
||||
on:click={() => {
|
||||
if (validUrl) {
|
||||
downloadFile({
|
||||
url: item.url,
|
||||
urlType: isTunnel ? "tunnel" : "redirect",
|
||||
});
|
||||
}
|
||||
}}
|
||||
>
|
||||
<div class="picker-type">
|
||||
{#if itemType === "video"}
|
||||
|
||||
@ -72,6 +72,9 @@ export const youtubeLanguages = [
|
||||
"ur",
|
||||
"uz",
|
||||
"vi",
|
||||
"zh",
|
||||
"zh-Hans",
|
||||
"zh-Hant",
|
||||
"zh-CN",
|
||||
"zh-HK",
|
||||
"zh-TW",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user