mirrors/cobalt
1
0
Fork 0
mirror of https://github.com/imputnet/cobalt.git synced 2025-07-18 03:08:30 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'current' into improvement/hls-istream

Browse Source
This commit is contained in:
Mikhail Serebryakov 2024-05-31 16:51:49 +05:00 committed by GitHub
commit 0e853644ae
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 184 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
.github/workflows/test.yml vendored
View File

@ -6,6 +6,19 @@ on:
branches: [ current ]
jobs:
check-lockfile:
name: check lockfile correctness
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check that lockfile does not need an update
run: |
cp package-lock.json before.json
npm ci
npm i --package-lock-only
diff before.json package-lock.json
test-web:
name: web sanity check
runs-on: ubuntu-latest
@ -22,4 +35,4 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v4
- name: Run test script
run: .github/test.sh api
run: .github/test.sh api

6
README.md
View File

@ -19,7 +19,8 @@ this list is not final and keeps expanding over time. if support for a service y
| bilibili.com & bilibili.tv | ✅ | ✅ | ✅ | ➖ | ➖ |
| dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ |
| instagram posts & reels | ✅ | ✅ | ✅ | ➖ | ➖ |
| ok video | ✅ | ❌ | ❌ | ✅ | ✅ |
| loom | ✅ | ❌ | ✅ | ✅ | ➖ |
| ok video | ✅ | ❌ | ✅ | ✅ | ✅ |
| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ |
| reddit | ✅ | ✅ | ✅ | ❌ | ❌ |
| rutube | ✅ | ✅ | ✅ | ✅ | ✅ |
@ -31,7 +32,7 @@ this list is not final and keeps expanding over time. if support for a service y
| twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ |
| vimeo | ✅ | ✅ | ✅ | ✅ | ✅ |
| vine archive | ✅ | ✅ | ✅ | ➖ | ➖ |
| vk videos & clips | ✅ | ❌ | ❌ | ✅ | ✅ |
| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ |
| youtube videos, shorts & music | ✅ | ✅ | ✅ | ✅ | ✅ |
| emoji | meaning |
@ -46,6 +47,7 @@ this list is not final and keeps expanding over time. if support for a service y
| instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. |
| pinterest | supports photos, gifs, videos and stories. |
| reddit | supports gifs and videos. |
| rutube | supports yappy & private links. |
| soundcloud | supports private links. |
| tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. |
| twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. |

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "cobalt",
"version": "7.14.2",
"version": "7.14.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "cobalt",
"version": "7.14.2",
"version": "7.14.3",
"license": "AGPL-3.0",
"dependencies": {
"content-disposition-header": "0.6.0",

2
package.json
View File

@ -1,7 +1,7 @@
{
"name": "cobalt",
"description": "save what you love",
"version": "7.14.2",
"version": "7.14.3",
"author": "imput",
"exports": "./src/cobalt.js",
"type": "module",

4
src/core/api.js
View File

@ -196,8 +196,8 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
}
streamInfo.headers = {
...req.headers,
...streamInfo.headers
...streamInfo.headers,
...req.headers
};
return stream(res, { type: 'internal', ...streamInfo });

1
src/modules/config.js
View File

@ -53,6 +53,7 @@ const
export const
services = servicesConfigJson.config,
hlsExceptions = servicesConfigJson.hlsExceptions,
audioIgnore = servicesConfigJson.audioIgnore,
version = packageJson.version,
genericUserAgent = config.genericUserAgent,

7
src/modules/processing/match.js
View File

@ -24,6 +24,7 @@ import streamable from "./services/streamable.js";
import twitch from "./services/twitch.js";
import rutube from "./services/rutube.js";
import dailymotion from "./services/dailymotion.js";
import loom from "./services/loom.js";
let freebind;
@ -179,6 +180,7 @@ export default async function(host, patternMatch, lang, obj) {
r = await rutube({
id: patternMatch.id,
yappyId: patternMatch.yappyId,
key: patternMatch.key,
quality: obj.vQuality,
isAudioOnly: isAudioOnly
});
@ -186,6 +188,11 @@ export default async function(host, patternMatch, lang, obj) {
case "dailymotion":
r = await dailymotion(patternMatch);
break;
case "loom":
r = await loom({
id: patternMatch.id
});
break;
default:
return createResponse("error", {
t: loc(lang, 'ErrorUnsupported')

1
src/modules/processing/matchActionDecider.js
View File

@ -129,6 +129,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
case "tumblr":
case "pinterest":
case "streamable":
case "loom":
responseType = "redirect";
break;
}

39
src/modules/processing/services/loom.js Normal file
View File

@ -0,0 +1,39 @@
import { genericUserAgent } from "../../config.js";
export default async function({ id }) {
const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/transcoded-url`, {
method: "POST",
headers: {
"user-agent": genericUserAgent,
origin: "https://www.loom.com",
referer: `https://www.loom.com/share/${id}`,
cookie: `loom_referral_video=${id};`,
"apollographql-client-name": "web",
"apollographql-client-version": "14c0b42",
"x-loom-request-source": "loom_web_14c0b42",
},
body: JSON.stringify({
force_original: false,
password: null,
anonID: null,
deviceID: null
})
})
.then(r => r.status === 200 ? r.json() : false)
.catch(() => {});
if (!gql) return { error: 'ErrorEmptyDownload' };
const videoUrl = gql?.url;
if (videoUrl?.includes('.mp4?')) {
return {
urls: videoUrl,
filename: `loom_${id}.mp4`,
audioFilename: `loom_${id}_audio`
}
}
return { error: 'ErrorEmptyDownload' }
}

2
src/modules/processing/services/reddit.js
View File

@ -107,7 +107,7 @@ export default async function(obj) {
}).catch(() => {})
}
let id = video.split('/')[3];
let id = `${String(obj.sub).toLowerCase()}_${obj.id}`;
if (!audio) return {
typeId: "redirect",

15
src/modules/processing/services/rutube.js
View File

@ -12,10 +12,10 @@ async function requestJSON(url) {
export default async function(obj) {
if (obj.yappyId) {
let yappy = await requestJSON(
const yappy = await requestJSON(
`https://rutube.ru/pangolin/api/web/yappy/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=15`
)
let yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link;
const yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link;
if (!yappyURL) return { error: 'ErrorEmptyDownload' };
return {
@ -25,11 +25,12 @@ export default async function(obj) {
}
}
let quality = obj.quality === "max" ? "9000" : obj.quality;
const quality = obj.quality === "max" ? "9000" : obj.quality;
let play = await requestJSON(
`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`
)
const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`);
if (obj.key) requestURL.searchParams.set('p', obj.key);
const play = await requestJSON(requestURL);
if (!play) return { error: 'ErrorCouldntFetch' };
if (play.detail || !play.video_balancer) return { error: 'ErrorEmptyDownload' };
@ -51,7 +52,7 @@ export default async function(obj) {
bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height));
}
let fileMetadata = {
const fileMetadata = {
title: cleanString(play.title.trim()),
artist: cleanString(play.author.name.trim()),
}

2
src/modules/processing/services/tiktok.js
View File

@ -57,7 +57,7 @@ export default async function(obj) {
let playAddr = detail.video.playAddr;
if (obj.h265) {
const h265PlayAddr = detail.video.bitrateInfo.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
playAddr = h265PlayAddr || playAddr
}

33
src/modules/processing/services/youtube.js
View File

@ -3,7 +3,7 @@ import { env } from '../../config.js';
import { cleanString } from '../../sub/utils.js';
import { fetch } from 'undici'
const ytBase = await Innertube.create();
const ytBase = Innertube.create().catch(e => e);
const codecMatch = {
h264: {
@ -23,16 +23,21 @@ const codecMatch = {
}
}
const cloneInnertube = (customFetch) => {
const cloneInnertube = async (customFetch) => {
const innertube = await ytBase;
if (innertube instanceof Error) {
throw innertube;
}
const session = new Session(
ytBase.session.context,
ytBase.session.key,
ytBase.session.api_version,
ytBase.session.account_index,
ytBase.session.player,
innertube.session.context,
innertube.session.key,
innertube.session.api_version,
innertube.session.account_index,
innertube.session.player,
undefined,
customFetch ?? ytBase.session.http.fetch,
ytBase.session.cache
customFetch ?? innertube.session.http.fetch,
innertube.session.cache
);
const yt = new Innertube(session);
@ -40,7 +45,7 @@ const cloneInnertube = (customFetch) => {
}
export default async function(o) {
const yt = cloneInnertube(
const yt = await cloneInnertube(
(input, init) => fetch(input, { ...init, dispatcher: o.dispatcher })
);
@ -57,8 +62,12 @@ export default async function(o) {
try {
info = await yt.getBasicInfo(o.id, 'WEB');
} catch {
return { error: 'ErrorCantConnectToServiceAPI' };
} catch(e) {
if (e?.message === 'This video is unavailable') {
return { error: 'ErrorCouldntFetch' };
} else {
return { error: 'ErrorCantConnectToServiceAPI' };
}
}
if (!info) return { error: 'ErrorCantConnectToServiceAPI' };

15
src/modules/processing/servicesConfig.json
View File

@ -1,5 +1,6 @@
{
"audioIgnore": ["vk", "ok"],
"audioIgnore": ["vk", "ok", "loom"],
"hlsExceptions": ["dailymotion", "vimeo", "rutube"],
"config": {
"bilibili": {
"alias": "bilibili.com & bilibili.tv",
@ -61,8 +62,9 @@
},
"vimeo": {
"patterns": [":id", "video/:id", ":id/:password", "/channels/:user/:id"],
"enabled": true,
"bestAudio": "mp3"
"subdomains": ["player"],
"bestAudio": "mp3",
"enabled": true
},
"soundcloud": {
"patterns": [":author/:song/s-:accessKey", ":author/:song", ":shortLink"],
@ -103,13 +105,18 @@
"rutube": {
"alias": "rutube videos",
"tld": "ru",
"patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId"],
"patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId", "video/private/:id?p=:key", "video/private/:id"],
"enabled": true
},
"dailymotion": {
"alias": "dailymotion videos",
"patterns": ["video/:id"],
"enabled": true
},
"loom": {
"alias": "loom videos",
"patterns": ["share/:id"],
"enabled": true
}
}
}

8
src/modules/processing/servicesPatternTesters.js
View File

@ -8,7 +8,10 @@ export const testers = {
"instagram": (patternMatch) =>
patternMatch.postId?.length <= 12
|| (patternMatch.username?.length <= 30 && patternMatch.storyId?.length <= 24),
"loom": (patternMatch) =>
patternMatch.id?.length <= 32,
"ok": (patternMatch) =>
patternMatch.id?.length <= 16,
@ -20,6 +23,7 @@ export const testers = {
|| (patternMatch.user?.length <= 22 && patternMatch.id?.length <= 10),
"rutube": (patternMatch) =>
(patternMatch.id?.length === 32 && patternMatch.key?.length <= 32) ||
patternMatch.id?.length === 32 || patternMatch.yappyId?.length === 32,
"soundcloud": (patternMatch) =>
@ -28,7 +32,7 @@ export const testers = {
"streamable": (patternMatch) =>
patternMatch.id?.length === 6,
"tiktok": (patternMatch) =>
patternMatch.postId?.length <= 21 || patternMatch.id?.length <= 13,

32
src/modules/processing/url.js
View File

@ -78,19 +78,35 @@ function aliasURL(url) {
function cleanURL(url) {
assert(url instanceof URL);
const host = psl.parse(url.hostname).sld;
let stripQuery = true;
if (host === 'pinterest') {
url.hostname = 'pinterest.com'
} else if (host === 'vk' && url.pathname.includes('/clip')) {
if (url.searchParams.get('z'))
url.search = '?z=' + encodeURIComponent(url.searchParams.get('z'));
stripQuery = false;
} else if (host === 'youtube' && url.searchParams.get('v')) {
url.search = '?v=' + encodeURIComponent(url.searchParams.get('v'));
const limitQuery = (param) => {
url.search = `?${param}=` + encodeURIComponent(url.searchParams.get(param));
stripQuery = false;
}
switch (host) {
case "pinterest":
url.hostname = 'pinterest.com';
break;
case "vk":
if (url.pathname.includes('/clip') && url.searchParams.get('z')) {
limitQuery('z')
}
break;
case "youtube":
if (url.searchParams.get('v')) {
limitQuery('v')
}
break;
case "rutube":
if (url.searchParams.get('p')) {
limitQuery('p')
}
break;
}
if (stripQuery) {
url.search = ''
}

4
src/modules/stream/manage.js
View File

@ -3,11 +3,11 @@ import { randomBytes } from "crypto";
import { nanoid } from "nanoid";
import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
import { env } from "../config.js";
import { env, hlsExceptions } from "../config.js";
import { strict as assert } from "assert";
// optional dependency
const freebind = env.freebindCIDR && await import('freebind').catch(() => {})
const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
const streamCache = new NodeCache({
stdTTL: env.streamLifespan,

4
src/modules/stream/types.js
View File

@ -5,7 +5,7 @@ import { create as contentDisposition } from "content-disposition-header";
import { metadataManager } from "../sub/utils.js";
import { destroyInternalStream } from "./manage.js";
import { env, ffmpegArgs } from "../config.js";
import { env, ffmpegArgs, hlsExceptions } from "../config.js";
import { getHeaders, closeResponse } from "./shared.js";
function toRawHeaders(headers) {
@ -215,7 +215,7 @@ export function streamVideoOnly(streamInfo, res) {
args.push('-an')
}
if (["vimeo", "rutube", "dailymotion"].includes(streamInfo.service)) {
if (hlsExceptions.includes(streamInfo.service)) {
args.push('-bsf:a', 'aac_adtstoasc')
}

37
src/test/tests.json
View File

@ -1089,6 +1089,14 @@
"code": 200,
"status": "stream"
}
}, {
"name": "private video",
"url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}],
"ok": [{
"name": "regular video",
@ -1123,5 +1131,34 @@
"code": 200,
"status": "stream"
}
}],
"loom": [{
"name": "1080p video",
"url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
"params": {},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "1080p video (muted)",
"url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
"params": {
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "1080p video (audio only)",
"url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
"params": {
"isAudioOnly": true
},
"expected": {
"code": 200,
"status": "stream"
}
}]
}