diff --git a/api/README.md b/api/README.md index 5c281246..d740fb21 100644 --- a/api/README.md +++ b/api/README.md @@ -1,4 +1,66 @@ # cobalt api +this directory includes the source code for cobalt api. it's made with [express.js](https://www.npmjs.com/package/express) and love! + +## running your own instance +if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md). +we recommend to use docker compose unless you intend to run cobalt for developing/debugging purposes. + +## accessing the api +there is currently no publicly available pre-hosted api. +we recommend [deploying your own instance](/docs/run-an-instance.md) if you wish to use the cobalt api. + +you can read [the api documentation here](/docs/api.md). + +> [!WARNING] +> the v7 public api (/api/json) will be shut down on **november 11th, 2024**. +> you can access documentation for it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md). + +## supported services +this list is not final and keeps expanding over time. if support for a service you want is missing, create an issue (or a pull request 👀). + +| service | video + audio | only audio | only video | metadata | rich file names | +| :-------- | :-----------: | :--------: | :--------: | :------: | :-------------: | +| bilibili | ✅ | ✅ | ✅ | ➖ | ➖ | +| bluesky | ✅ | ✅ | ✅ | ➖ | ➖ | +| dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ | +| instagram | ✅ | ✅ | ✅ | ➖ | ➖ | +| facebook | ✅ | ❌ | ✅ | ➖ | ➖ | +| loom | ✅ | ❌ | ✅ | ✅ | ➖ | +| ok.ru | ✅ | ❌ | ✅ | ✅ | ✅ | +| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | +| reddit | ✅ | ✅ | ✅ | ❌ | ❌ | +| rutube | ✅ | ✅ | ✅ | ✅ | ✅ | +| snapchat | ✅ | ✅ | ✅ | ➖ | ➖ | +| soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ | +| streamable | ✅ | ✅ | ✅ | ➖ | ➖ | +| tiktok | ✅ | ✅ | ✅ | ❌ | ❌ | +| tumblr | ✅ | ✅ | ✅ | ➖ | ➖ | +| twitch clips | ✅ | ✅ | ✅ | ✅ | ✅ | +| twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | +| vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | +| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | +| youtube | ✅ | ✅ | ✅ | ✅ | ✅ | + +| emoji | meaning | +| :-----: | :---------------------- | +| ✅ | supported | +| ➖ | impossible/unreasonable | +| ❌ | not supported | + +### additional notes or features (per service) +| service | notes or features | +| :-------- | :----- | +| instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. | +| facebook | supports public accessible videos content only. | +| pinterest | supports photos, gifs, videos and stories. | +| reddit | supports gifs and videos. | +| snapchat | supports spotlights and stories. lets you pick what to save from stories. | +| rutube | supports yappy & private links. | +| soundcloud | supports private links. | +| tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. | +| twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. | +| vimeo | audio downloads are only available for dash. | +| youtube | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. | ## license cobalt api code is licensed under [AGPL-3.0](LICENSE). @@ -9,14 +71,38 @@ as long as you: - provide a link to the license and indicate if changes to the code were made, and - release the code under the **same license** -## running your own instance -if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md). -it's *highly* recommended to use a docker compose method unless you run for developing/debugging purposes. +## acknowledgements +### ffmpeg +cobalt heavily relies on ffmpeg for converting and merging media files. it's an absolutely amazing piece of software offered for anyone for free, yet doesn't receive as much credit as it should. -## accessing the api -currently, there is no publicly accessible main api. we plan on providing a public api for -cobalt 10 in some form in the future. we recommend deploying your own instance if you wish -to use the latest api. you can access [the documentation](/docs/api.md) for it here. +you can [support ffmpeg here](https://ffmpeg.org/donations.html)! -if you are looking for the documentation for the old (7.x) api, you can find -it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md) \ No newline at end of file +#### ffmpeg-static +we use [ffmpeg-static](https://github.com/eugeneware/ffmpeg-static) to get binaries for ffmpeg depending on the platform. + +you can support the developer via various methods listed on their github page! (linked above) + +### youtube.js +cobalt relies on [youtube.js](https://github.com/LuanRT/YouTube.js) for interacting with the innertube api, it wouldn't have been possible without it. + +you can support the developer via various methods listed on their github page! (linked above) + +### many others +cobalt also depends on: + +- [content-disposition-header](https://www.npmjs.com/package/content-disposition-header) to simplify the provision of `content-disposition` headers. +- [cors](https://www.npmjs.com/package/cors) to manage cross-origin resource sharing within expressjs. +- [dotenv](https://www.npmjs.com/package/dotenv) to load environment variables from the `.env` file. +- [esbuild](https://www.npmjs.com/package/esbuild) to minify the frontend files. +- [express](https://www.npmjs.com/package/express) as the backbone of cobalt servers. +- [express-rate-limit](https://www.npmjs.com/package/express-rate-limit) to rate limit api endpoints. +- [hls-parser](https://www.npmjs.com/package/hls-parser) to parse `m3u8` playlists for certain services. +- [ipaddr.js](https://www.npmjs.com/package/ipaddr.js) to parse ip addresses (for rate limiting). +- [nanoid](https://www.npmjs.com/package/nanoid) to generate unique (temporary) identifiers for each requested stream. +- [node-cache](https://www.npmjs.com/package/node-cache) to cache stream info in server ram for a limited amount of time. +- [psl](https://www.npmjs.com/package/psl) as the domain name parser. +- [set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser) to parse cookies that cobalt receives from certain services. +- [undici](https://www.npmjs.com/package/undici) for making http requests. +- [url-pattern](https://www.npmjs.com/package/url-pattern) to match provided links with supported patterns. + +...and many other packages that these packages rely on. diff --git a/api/package.json b/api/package.json index e37c839d..0498fe23 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.1.0", + "version": "10.4.3", "author": "imput", "exports": "./src/cobalt.js", "type": "module", @@ -10,9 +10,9 @@ }, "scripts": { "start": "node src/cobalt", - "setup": "node src/util/setup", "test": "node src/util/test", - "token:youtube": "node src/util/generate-youtube-tokens" + "token:youtube": "node src/util/generate-youtube-tokens", + "token:jwt": "node src/util/generate-jwt-secret" }, "repository": { "type": "git", @@ -24,26 +24,29 @@ }, "homepage": "https://github.com/imputnet/cobalt#readme", "dependencies": { + "@datastructures-js/priority-queue": "^6.3.1", + "@imput/psl": "^2.0.4", "@imput/version-info": "workspace:^", "content-disposition-header": "0.6.0", "cors": "^2.8.5", "dotenv": "^16.0.1", "esbuild": "^0.14.51", - "express": "^4.18.1", - "express-rate-limit": "^6.3.0", + "express": "^4.21.1", + "express-rate-limit": "^7.4.1", "ffmpeg-static": "^5.1.0", "hls-parser": "^0.10.7", - "ipaddr.js": "2.1.0", + "ipaddr.js": "2.2.0", "nanoid": "^4.0.2", "node-cache": "^5.1.2", - "psl": "1.9.0", "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^10.3.0", + "youtubei.js": "^11.0.1", "zod": "^3.23.8" }, "optionalDependencies": { - "freebind": "^0.2.2" + "freebind": "^0.2.2", + "rate-limit-redis": "^4.2.0", + "redis": "^4.7.0" } } diff --git a/api/src/cobalt.js b/api/src/cobalt.js index c548e792..5cac208d 100644 --- a/api/src/cobalt.js +++ b/api/src/cobalt.js @@ -1,27 +1,32 @@ import "dotenv/config"; import express from "express"; +import cluster from "node:cluster"; -import path from 'path'; -import { fileURLToPath } from 'url'; +import path from "path"; +import { fileURLToPath } from "url"; -import { env } from "./config.js" -import { Bright, Green, Red } from "./misc/console-text.js"; +import { env, isCluster } from "./config.js" +import { Red } from "./misc/console-text.js"; +import { initCluster } from "./misc/cluster.js"; const app = express(); const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename).slice(0, -4); -app.disable('x-powered-by'); +app.disable("x-powered-by"); if (env.apiURL) { - const { runAPI } = await import('./core/api.js'); - runAPI(express, app, __dirname) + const { runAPI } = await import("./core/api.js"); + + if (isCluster) { + await initCluster(); + } + + runAPI(express, app, __dirname, cluster.isPrimary); } else { console.log( - Red(`cobalt wasn't configured yet or configuration is invalid.\n`) - + Bright(`please run the setup script to fix this: `) - + Green(`npm run setup`) + Red("API_URL env variable is missing, cobalt api can't start.") ) } diff --git a/api/src/config.js b/api/src/config.js index 1f00231e..191e8441 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,5 +1,6 @@ import { getVersion } from "@imput/version-info"; import { services } from "./processing/service-config.js"; +import { supportsReusePort } from "./misc/cluster.js"; const version = await getVersion(); @@ -13,6 +14,7 @@ const enabledServices = new Set(Object.keys(services).filter(e => { const env = { apiURL: process.env.API_URL || '', apiPort: process.env.API_PORT || 9000, + tunnelPort: process.env.API_PORT || 9000, listenAddress: process.env.API_LISTEN_ADDRESS, freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR, @@ -43,12 +45,35 @@ const env = { && process.env.TURNSTILE_SECRET && process.env.JWT_SECRET, + apiKeyURL: process.env.API_KEY_URL && new URL(process.env.API_KEY_URL), + authRequired: process.env.API_AUTH_REQUIRED === '1', + redisURL: process.env.API_REDIS_URL, + instanceCount: (process.env.API_INSTANCE_COUNT && parseInt(process.env.API_INSTANCE_COUNT)) || 1, + keyReloadInterval: 900, + enabledServices, } const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`; +export const setTunnelPort = (port) => env.tunnelPort = port; +export const isCluster = env.instanceCount > 1; + +if (env.sessionEnabled && env.jwtSecret.length < 16) { + throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)"); +} + +if (env.instanceCount > 1 && !env.redisURL) { + throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2"); +} else if (env.instanceCount > 1 && !await supportsReusePort()) { + console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js'); + console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux'); + console.error('(or other OS that supports it). for more info, see `reusePort` option on'); + console.error('https://nodejs.org/api/net.html#serverlistenoptions-callback'); + throw new Error('SO_REUSEPORT is not supported'); +} + export { env, genericUserAgent, diff --git a/api/src/core/api.js b/api/src/core/api.js index e7eadd78..153f2ca6 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -1,4 +1,5 @@ import cors from "cors"; +import http from "node:http"; import rateLimit from "express-rate-limit"; import { setGlobalDispatcher, ProxyAgent } from "undici"; import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info"; @@ -7,16 +8,18 @@ import jwt from "../security/jwt.js"; import stream from "../stream/stream.js"; import match from "../processing/match.js"; -import { env } from "../config.js"; +import { env, isCluster, setTunnelPort } from "../config.js"; import { extract } from "../processing/url.js"; -import { languageCode } from "../misc/utils.js"; -import { Bright, Cyan } from "../misc/console-text.js"; -import { generateHmac, generateSalt } from "../misc/crypto.js"; +import { Green, Bright, Cyan } from "../misc/console-text.js"; +import { hashHmac } from "../security/secrets.js"; +import { createStore } from "../store/redis-ratelimit.js"; import { randomizeCiphers } from "../misc/randomize-ciphers.js"; import { verifyTurnstileToken } from "../security/turnstile.js"; import { friendlyServiceName } from "../processing/service-alias.js"; import { verifyStream, getInternalStream } from "../stream/manage.js"; import { createResponse, normalizeRequest, getIP } from "../processing/request.js"; +import * as APIKeys from "../security/api-keys.js"; +import * as Cookies from "../processing/cookie/manager.js"; const git = { branch: await getBranch(), @@ -28,7 +31,6 @@ const version = await getVersion(); const acceptRegex = /^application\/json(; charset=utf-8)?$/; -const ipSalt = generateSalt(); const corsConfig = env.corsWildcard ? {} : { origin: env.corsURL, optionsSuccessStatus: 200 @@ -39,7 +41,7 @@ const fail = (res, code, context) => { res.status(status).json(body); } -export const runAPI = (express, app, __dirname) => { +export const runAPI = async (express, app, __dirname, isPrimary = true) => { const startTime = new Date(); const startTimestamp = startTime.getTime(); @@ -57,35 +59,46 @@ export const runAPI = (express, app, __dirname) => { git, }) + const handleRateExceeded = (_, res) => { + const { status, body } = createResponse("error", { + code: "error.api.rate_exceeded", + context: { + limit: env.rateLimitWindow + } + }); + return res.status(status).json(body); + }; + + const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url'); + + const sessionLimiter = rateLimit({ + windowMs: 60000, + limit: 10, + standardHeaders: 'draft-6', + legacyHeaders: false, + keyGenerator, + store: await createStore('session'), + handler: handleRateExceeded + }); + const apiLimiter = rateLimit({ windowMs: env.rateLimitWindow * 1000, - max: env.rateLimitMax, - standardHeaders: true, + limit: (req) => req.rateLimitMax || env.rateLimitMax, + standardHeaders: 'draft-6', legacyHeaders: false, - keyGenerator: req => { - if (req.authorized) { - return generateHmac(req.header("Authorization"), ipSalt); - } - return generateHmac(getIP(req), ipSalt); - }, - handler: (req, res) => { - const { status, body } = createResponse("error", { - code: "error.api.rate_exceeded", - context: { - limit: env.rateLimitWindow - } - }); - return res.status(status).json(body); - } + keyGenerator: req => req.rateLimitKey || keyGenerator(req), + store: await createStore('api'), + handler: handleRateExceeded }) - const apiLimiterStream = rateLimit({ + const apiTunnelLimiter = rateLimit({ windowMs: env.rateLimitWindow * 1000, - max: env.rateLimitMax, - standardHeaders: true, + limit: (req) => req.rateLimitMax || env.rateLimitMax, + standardHeaders: 'draft-6', legacyHeaders: false, - keyGenerator: req => generateHmac(getIP(req), ipSalt), - handler: (req, res) => { + keyGenerator: req => req.rateLimitKey || keyGenerator(req), + store: await createStore('tunnel'), + handler: (_, res) => { return res.sendStatus(429) } }) @@ -103,9 +116,6 @@ export const runAPI = (express, app, __dirname) => { ...corsConfig, })); - app.post('/', apiLimiter); - app.use('/tunnel', apiLimiterStream); - app.post('/', (req, res, next) => { if (!acceptRegex.test(req.header('Accept'))) { return fail(res, "error.api.header.accept"); @@ -117,7 +127,34 @@ export const runAPI = (express, app, __dirname) => { }); app.post('/', (req, res, next) => { - if (!env.sessionEnabled) { + if (!env.apiKeyURL) { + return next(); + } + + const { success, error } = APIKeys.validateAuthorization(req); + if (!success) { + // We call next() here if either if: + // a) we have user sessions enabled, meaning the request + // will still need a Bearer token to not be rejected, or + // b) we do not require the user to be authenticated, and + // so they can just make the request with the regular + // rate limit configuration; + // otherwise, we reject the request. + if ( + (env.sessionEnabled || !env.authRequired) + && ['missing', 'not_api_key'].includes(error) + ) { + return next(); + } + + return fail(res, `error.api.auth.key.${error}`); + } + + return next(); + }); + + app.post('/', (req, res, next) => { + if (!env.sessionEnabled || req.rateLimitKey) { return next(); } @@ -127,26 +164,29 @@ export const runAPI = (express, app, __dirname) => { return fail(res, "error.api.auth.jwt.missing"); } - if (!authorization.startsWith("Bearer ") || authorization.length > 256) { + if (authorization.length >= 256) { return fail(res, "error.api.auth.jwt.invalid"); } - const verifyJwt = jwt.verify( - authorization.split("Bearer ", 2)[1] - ); - - if (!verifyJwt) { + const [ type, token, ...rest ] = authorization.split(" "); + if (!token || type.toLowerCase() !== 'bearer' || rest.length) { return fail(res, "error.api.auth.jwt.invalid"); } - req.authorized = true; + if (!jwt.verify(token)) { + return fail(res, "error.api.auth.jwt.invalid"); + } + + req.rateLimitKey = hashHmac(token, 'rate'); } catch { return fail(res, "error.api.generic"); } next(); }); + app.post('/', apiLimiter); app.use('/', express.json({ limit: 1024 })); + app.use('/', (err, _, res, next) => { if (err) { const { status, body } = createResponse("error", { @@ -158,7 +198,7 @@ export const runAPI = (express, app, __dirname) => { next(); }); - app.post("/session", async (req, res) => { + app.post("/session", sessionLimiter, async (req, res) => { if (!env.sessionEnabled) { return fail(res, "error.api.auth.not_configured") } @@ -187,16 +227,11 @@ export const runAPI = (express, app, __dirname) => { app.post('/', async (req, res) => { const request = req.body; - const lang = languageCode(req); if (!request.url) { return fail(res, "error.api.link.missing"); } - if (request.youtubeDubBrowserLang) { - request.youtubeDubLang = lang; - } - const { success, data: normalizedRequest } = await normalizeRequest(request); if (!success) { return fail(res, "error.api.invalid_body"); @@ -228,8 +263,7 @@ export const runAPI = (express, app, __dirname) => { } }) - app.get('/tunnel', (req, res) => { - console.log("come to tunnel===========================================>"); + app.get('/tunnel', apiTunnelLimiter, async (req, res) => { const id = String(req.query.id); const exp = String(req.query.exp); const sig = String(req.query.sig); @@ -248,7 +282,7 @@ export const runAPI = (express, app, __dirname) => { return res.status(200).end(); } - const streamInfo = verifyStream(id, sig, exp, sec, iv); + const streamInfo = await verifyStream(id, sig, exp, sec, iv); if (!streamInfo?.service) { return res.status(streamInfo.status).end(); } @@ -260,7 +294,7 @@ export const runAPI = (express, app, __dirname) => { return stream(res, streamInfo); }) - app.get('/itunnel', (req, res) => { + const itunnelHandler = (req, res) => { if (!req.ip.endsWith('127.0.0.1')) { return res.sendStatus(403); } @@ -280,7 +314,9 @@ export const runAPI = (express, app, __dirname) => { ]); return stream(res, { type: 'internal', ...streamInfo }); - }) + }; + + app.get('/itunnel', itunnelHandler); app.get('/', (_, res) => { res.type('json'); @@ -311,20 +347,48 @@ export const runAPI = (express, app, __dirname) => { setGlobalDispatcher(new ProxyAgent(env.externalProxy)) } - app.listen(env.apiPort, env.listenAddress, () => { - console.log(`\n` + - Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" + + http.createServer(app).listen({ + port: env.apiPort, + host: env.listenAddress, + reusePort: env.instanceCount > 1 || undefined + }, () => { + if (isPrimary) { + console.log(`\n` + + Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" + - "~~~~~~\n" + - Bright("version: ") + version + "\n" + - Bright("commit: ") + git.commit + "\n" + - Bright("branch: ") + git.branch + "\n" + - Bright("remote: ") + git.remote + "\n" + - Bright("start time: ") + startTime.toUTCString() + "\n" + - "~~~~~~\n" + + "~~~~~~\n" + + Bright("version: ") + version + "\n" + + Bright("commit: ") + git.commit + "\n" + + Bright("branch: ") + git.branch + "\n" + + Bright("remote: ") + git.remote + "\n" + + Bright("start time: ") + startTime.toUTCString() + "\n" + + "~~~~~~\n" + - Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" + - Bright("port: ") + env.apiPort + "\n" - ) - }) + Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" + + Bright("port: ") + env.apiPort + "\n" + ); + } + + if (env.apiKeyURL) { + APIKeys.setup(env.apiKeyURL); + } + + if (env.cookiePath) { + Cookies.setup(env.cookiePath); + } + }); + + if (isCluster) { + const istreamer = express(); + istreamer.get('/itunnel', itunnelHandler); + const server = istreamer.listen({ + port: 0, + host: '127.0.0.1', + exclusive: true + }, () => { + const { port } = server.address(); + console.log(`${Green('[✓]')} cobalt sub-instance running on 127.0.0.1:${port}`); + setTunnelPort(port); + }); + } } diff --git a/api/src/misc/cluster.js b/api/src/misc/cluster.js new file mode 100644 index 00000000..56664d15 --- /dev/null +++ b/api/src/misc/cluster.js @@ -0,0 +1,71 @@ +import cluster from "node:cluster"; +import net from "node:net"; +import { syncSecrets } from "../security/secrets.js"; +import { env, isCluster } from "../config.js"; + +export { isPrimary, isWorker } from "node:cluster"; + +export const supportsReusePort = async () => { + try { + await new Promise((resolve, reject) => { + const server = net.createServer().listen({ port: 0, reusePort: true }); + server.on('listening', () => server.close(resolve)); + server.on('error', (err) => (server.close(), reject(err))); + }); + + return true; + } catch { + return false; + } +} + +export const initCluster = async () => { + if (cluster.isPrimary) { + for (let i = 1; i < env.instanceCount; ++i) { + cluster.fork(); + } + } + + await syncSecrets(); +} + +export const broadcast = (message) => { + if (!isCluster || !cluster.isPrimary || !cluster.workers) { + return; + } + + for (const worker of Object.values(cluster.workers)) { + worker.send(message); + } +} + +export const send = (message) => { + if (!isCluster) { + return; + } + + if (cluster.isPrimary) { + return broadcast(message); + } else { + return process.send(message); + } +} + +export const waitFor = (key) => { + return new Promise(resolve => { + const listener = (message) => { + if (key in message) { + process.off('message', listener); + return resolve(message); + } + } + + process.on('message', listener); + }); +} + +export const mainOnMessage = (cb) => { + for (const worker of Object.values(cluster.workers)) { + worker.on('message', cb); + } +} diff --git a/api/src/misc/console-text.js b/api/src/misc/console-text.js index 014584ae..8df8fcc6 100644 --- a/api/src/misc/console-text.js +++ b/api/src/misc/console-text.js @@ -1,16 +1,36 @@ -function t(color, tt) { - return color + tt + "\x1b[0m" +const ANSI = { + RESET: "\x1b[0m", + BRIGHT: "\x1b[1m", + RED: "\x1b[31m", + GREEN: "\x1b[32m", + CYAN: "\x1b[36m", + YELLOW: "\x1b[93m" } -export function Bright(tt) { - return t("\x1b[1m", tt) +function wrap(color, text) { + if (!ANSI[color.toUpperCase()]) { + throw "invalid color"; + } + + return ANSI[color.toUpperCase()] + text + ANSI.RESET; } -export function Red(tt) { - return t("\x1b[31m", tt) + +export function Bright(text) { + return wrap('bright', text); } -export function Green(tt) { - return t("\x1b[32m", tt) + +export function Red(text) { + return wrap('red', text); } -export function Cyan(tt) { - return t("\x1b[36m", tt) + +export function Green(text) { + return wrap('green', text); +} + +export function Cyan(text) { + return wrap('cyan', text); +} + +export function Yellow(text) { + return wrap('yellow', text); } diff --git a/api/src/misc/crypto.js b/api/src/misc/crypto.js index 3a520156..e0f8858b 100644 --- a/api/src/misc/crypto.js +++ b/api/src/misc/crypto.js @@ -1,15 +1,7 @@ -import { createHmac, createCipheriv, createDecipheriv, randomBytes } from "crypto"; +import { createCipheriv, createDecipheriv } from "crypto"; const algorithm = "aes256"; -export function generateSalt() { - return randomBytes(64).toString('hex'); -} - -export function generateHmac(str, salt) { - return createHmac("sha256", salt).update(str).digest("base64url"); -} - export function encryptStream(plaintext, iv, secret) { const buff = Buffer.from(JSON.stringify(plaintext)); const key = Buffer.from(secret, "base64url"); diff --git a/api/src/misc/run-test.js b/api/src/misc/run-test.js index 10d19aef..21d97d04 100644 --- a/api/src/misc/run-test.js +++ b/api/src/misc/run-test.js @@ -41,4 +41,4 @@ export async function runTest(url, params, expect) { if (result.body.status === 'tunnel') { // TODO: stream testing } -} \ No newline at end of file +} diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index 34666d1c..fd497d18 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -1,50 +1,3 @@ -const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '=']; - -export function metadataManager(obj) { - const keys = Object.keys(obj); - const tags = [ - "album", - "copyright", - "title", - "artist", - "track", - "date" - ] - let commands = [] - - for (const i in keys) { - if (tags.includes(keys[i])) - commands.push('-metadata', `${keys[i]}=${obj[keys[i]]}`) - } - return commands; -} - -export function cleanString(string) { - for (const i in forbiddenCharsString) { - string = string.replaceAll("/", "_") - .replaceAll(forbiddenCharsString[i], '') - } - return string; -} -export function verifyLanguageCode(code) { - const langCode = String(code.slice(0, 2).toLowerCase()); - if (RegExp(/[a-z]{2}/).test(code)) { - return langCode - } - return "en" -} -export function languageCode(req) { - if (req.header('Accept-Language')) { - return verifyLanguageCode(req.header('Accept-Language')) - } - return "en" -} -export function cleanHTML(html) { - let clean = html.replace(/ {4}/g, ''); - clean = clean.replace(/\n/g, ''); - return clean -} - export function getRedirectingURL(url) { return fetch(url, { redirect: 'manual' }).then((r) => { if ([301, 302, 303].includes(r.status) && r.headers.has('location')) diff --git a/api/src/processing/cookie/cookie.js b/api/src/processing/cookie/cookie.js index 6dd95fc3..1d9636d5 100644 --- a/api/src/processing/cookie/cookie.js +++ b/api/src/processing/cookie/cookie.js @@ -4,16 +4,24 @@ export default class Cookie { constructor(input) { assert(typeof input === 'object'); this._values = {}; - this.set(input) + + for (const [ k, v ] of Object.entries(input)) + this.set(k, v); } - set(values) { - Object.entries(values).forEach( - ([ key, value ]) => this._values[key] = value - ) + + set(key, value) { + const old = this._values[key]; + if (old === value) + return false; + + this._values[key] = value; + return true; } + unset(keys) { for (const key of keys) delete this._values[key] } + static fromString(str) { const obj = {}; @@ -25,12 +33,15 @@ export default class Cookie { return new Cookie(obj) } + toString() { return Object.entries(this._values).map(([ name, value ]) => `${name}=${value}`).join('; ') } + toJSON() { return this.toString() } + values() { return Object.freeze({ ...this._values }) } diff --git a/api/src/processing/cookie/manager.js b/api/src/processing/cookie/manager.js index 25bf9c90..c2b37801 100644 --- a/api/src/processing/cookie/manager.js +++ b/api/src/processing/cookie/manager.js @@ -1,50 +1,144 @@ import Cookie from './cookie.js'; + import { readFile, writeFile } from 'fs/promises'; +import { Red, Green, Yellow } from '../../misc/console-text.js'; import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser'; -import { env } from '../../config.js'; +import * as cluster from '../../misc/cluster.js'; +import { isCluster } from '../../config.js'; -const WRITE_INTERVAL = 60000, - cookiePath = env.cookiePath, - COUNTER = Symbol('counter'); +const WRITE_INTERVAL = 60000; +const VALID_SERVICES = new Set([ + 'instagram', + 'instagram_bearer', + 'reddit', + 'twitter', + 'youtube_oauth' +]); +const invalidCookies = {}; let cookies = {}, dirty = false, intervalId; -const setup = async () => { - try { - if (!cookiePath) return; - - cookies = await readFile(cookiePath, 'utf8'); - cookies = JSON.parse(cookies); - intervalId = setInterval(writeChanges, WRITE_INTERVAL) - } catch { /* no cookies for you */ } -} - -setup(); - -function writeChanges() { +function writeChanges(cookiePath) { if (!dirty) return; dirty = false; - writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => { - clearInterval(intervalId) + const cookieData = JSON.stringify({ ...cookies, ...invalidCookies }, null, 4); + writeFile(cookiePath, cookieData).catch((e) => { + console.warn(`${Yellow('[!]')} failed writing updated cookies to storage`); + console.warn(e); + clearInterval(intervalId); + intervalId = null; }) } -export function getCookie(service) { - if (!cookies[service] || !cookies[service].length) return; +const setupMain = async (cookiePath) => { + try { + cookies = await readFile(cookiePath, 'utf8'); + cookies = JSON.parse(cookies); + for (const serviceName in cookies) { + if (!VALID_SERVICES.has(serviceName)) { + console.warn(`${Yellow('[!]')} ignoring unknown service in cookie file: ${serviceName}`); + } else if (!Array.isArray(cookies[serviceName])) { + console.warn(`${Yellow('[!]')} ${serviceName} in cookies file is not an array, ignoring it`); + } else if (cookies[serviceName].some(c => typeof c !== 'string')) { + console.warn(`${Yellow('[!]')} some cookie for ${serviceName} contains non-string value in cookies file`); + } else continue; - let n; - if (cookies[service][COUNTER] === undefined) { - n = cookies[service][COUNTER] = 0 - } else { - ++cookies[service][COUNTER] - n = (cookies[service][COUNTER] %= cookies[service].length) + invalidCookies[serviceName] = cookies[serviceName]; + delete cookies[serviceName]; + } + + if (!intervalId) { + intervalId = setInterval(() => writeChanges(cookiePath), WRITE_INTERVAL); + } + + cluster.broadcast({ cookies }); + + console.log(`${Green('[✓]')} cookies loaded successfully!`); + } catch (e) { + console.error(`${Yellow('[!]')} failed to load cookies.`); + console.error('error:', e); + } +} + +const setupWorker = async () => { + cookies = (await cluster.waitFor('cookies')).cookies; +} + +export const loadFromFile = async (path) => { + if (cluster.isPrimary) { + await setupMain(path); + } else if (cluster.isWorker) { + await setupWorker(); } - const cookie = cookies[service][n]; - if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie); + dirty = false; +} - return cookies[service][n] +export const setup = async (path) => { + await loadFromFile(path); + + if (isCluster) { + const messageHandler = (message) => { + if ('cookieUpdate' in message) { + const { cookieUpdate } = message; + + if (cluster.isPrimary) { + dirty = true; + cluster.broadcast({ cookieUpdate }); + } + + const { service, idx, cookie } = cookieUpdate; + cookies[service][idx] = cookie; + } + } + + if (cluster.isPrimary) { + cluster.mainOnMessage(messageHandler); + } else { + process.on('message', messageHandler); + } + } +} + +export function getCookie(service) { + if (!VALID_SERVICES.has(service)) { + console.error( + `${Red('[!]')} ${service} not in allowed services list for cookies.` + + ' if adding a new cookie type, include it there.' + ); + return; + } + + if (!cookies[service] || !cookies[service].length) return; + + const idx = Math.floor(Math.random() * cookies[service].length); + + const cookie = cookies[service][idx]; + if (typeof cookie === 'string') { + cookies[service][idx] = Cookie.fromString(cookie); + } + + cookies[service][idx].meta = { service, idx }; + return cookies[service][idx]; +} + +export function updateCookieValues(cookie, values) { + let changed = false; + + for (const [ key, value ] of Object.entries(values)) { + changed = cookie.set(key, value) || changed; + } + + if (changed && cookie.meta) { + dirty = true; + if (isCluster) { + const message = { cookieUpdate: { ...cookie.meta, cookie } }; + cluster.send(message); + } + } + + return changed; } export function updateCookie(cookie, headers) { @@ -57,10 +151,6 @@ export function updateCookie(cookie, headers) { cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name)); parsed.filter(c => !c.expires || c.expires > new Date()).forEach(c => values[c.name] = c.value); + updateCookieValues(cookie, values); } - -export function updateCookieValues(cookie, values) { - cookie.set(values); - if (Object.keys(values).length) dirty = true -} diff --git a/api/src/processing/create-filename.js b/api/src/processing/create-filename.js index 216b15a4..911b5603 100644 --- a/api/src/processing/create-filename.js +++ b/api/src/processing/create-filename.js @@ -1,3 +1,13 @@ +const illegalCharacters = ['}', '{', '%', '>', '<', '^', ';', ':', '`', '$', '"', "@", '=', '?', '|', '*']; + +const sanitizeString = (string) => { + for (const i in illegalCharacters) { + string = string.replaceAll("/", "_").replaceAll("\\", "_") + .replaceAll(illegalCharacters[i], '') + } + return string; +} + export default (f, style, isAudioOnly, isAudioMuted) => { let filename = ''; @@ -5,7 +15,11 @@ export default (f, style, isAudioOnly, isAudioMuted) => { let classicTags = [...infoBase]; let basicTags = []; - const title = `${f.title} - ${f.author}`; + let title = sanitizeString(f.title); + + if (f.author) { + title += ` - ${sanitizeString(f.author)}`; + } if (f.resolution) { classicTags.push(f.resolution); diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 4fdb24f6..8d2c1e38 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -9,7 +9,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab let action, responseType = "tunnel", defaultParams = { - u: r.urls, + url: r.urls, headers: r.headers, service: host, filename: r.filenameAttributes ? @@ -24,7 +24,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab else if (r.isGif && twitterGif) action = "gif"; else if (isAudioOnly) action = "audio"; else if (isAudioMuted) action = "muteVideo"; - else if (r.isM3U8) action = "m3u8"; + else if (r.isHLS) action = "hls"; else action = "video"; if (action === "picker" || action === "audio") { @@ -54,20 +54,21 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab params = { type: "gif" }; break; - case "m3u8": + case "hls": params = { - type: Array.isArray(r.urls) ? "merge" : "remux" + type: Array.isArray(r.urls) ? "merge" : "remux", + isHLS: true, } break; case "muteVideo": let muteType = "mute"; - if (Array.isArray(r.urls) && !r.isM3U8) { + if (Array.isArray(r.urls) && !r.isHLS) { muteType = "proxy"; } params = { type: muteType, - u: Array.isArray(r.urls) ? r.urls[0] : r.urls + url: Array.isArray(r.urls) ? r.urls[0] : r.urls } if (host === "reddit" && r.typeId === "redirect") { responseType = "redirect"; @@ -92,12 +93,12 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab } params = { picker: r.picker, - u: createStream({ + url: createStream({ service: "tiktok", type: audioStreamType, - u: r.urls, + url: r.urls, headers: r.headers, - filename: r.audioFilename, + filename: `${r.audioFilename}.${audioFormat}`, isAudioOnly: true, audioFormat, }) @@ -137,13 +138,13 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab } break; + case "ok": case "vk": case "tiktok": params = { type: "proxy" }; break; case "facebook": - case "vine": case "instagram": case "tumblr": case "pinterest": @@ -159,7 +160,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "audio": if (audioIgnore.includes(host) || (host === "reddit" && r.typeId === "redirect")) { return createResponse("error", { - code: "error.api.fetch.empty" + code: "error.api.service.audio_not_supported" }) } @@ -183,18 +184,20 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab } } - if (r.isM3U8 || host === "vimeo") { + if (r.isHLS || host === "vimeo") { copy = false; processType = "audio"; } params = { type: processType, - u: Array.isArray(r.urls) ? r.urls[1] : r.urls, + url: Array.isArray(r.urls) ? r.urls[1] : r.urls, audioBitrate, audioCopy: copy, audioFormat, + + isHLS: r.isHLS, } break; } diff --git a/api/src/processing/match.js b/api/src/processing/match.js index ffb92c23..57f04b36 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -19,7 +19,6 @@ import tumblr from "./services/tumblr.js"; import vimeo from "./services/vimeo.js"; import soundcloud from "./services/soundcloud.js"; import instagram from "./services/instagram.js"; -import vine from "./services/vine.js"; import pinterest from "./services/pinterest.js"; import streamable from "./services/streamable.js"; import twitch from "./services/twitch.js"; @@ -78,8 +77,9 @@ export default async function({ host, patternMatch, params }) { case "vk": r = await vk({ - userId: patternMatch.userId, + ownerId: patternMatch.ownerId, videoId: patternMatch.videoId, + accessKey: patternMatch.accessKey, quality: params.videoQuality }); break; @@ -97,13 +97,14 @@ export default async function({ host, patternMatch, params }) { case "youtube": let fetchInfo = { + dispatcher, id: patternMatch.id.slice(0, 11), quality: params.videoQuality, format: params.youtubeVideoCodec, isAudioOnly, isAudioMuted, dubLang: params.youtubeDubLang, - dispatcher + youtubeHLS: params.youtubeHLS, } if (url.hostname === "music.youtube.com" || isAudioOnly) { @@ -127,7 +128,7 @@ export default async function({ host, patternMatch, params }) { case "tiktok": r = await tiktok({ postId: patternMatch.postId, - id: patternMatch.id, + shortLink: patternMatch.shortLink, fullAudio: params.tiktokFullAudio, isAudioOnly, h265: params.tiktokH265, @@ -174,12 +175,6 @@ export default async function({ host, patternMatch, params }) { }) break; - case "vine": - r = await vine({ - id: patternMatch.id - }); - break; - case "pinterest": r = await pinterest({ id: patternMatch.id, @@ -239,7 +234,8 @@ export default async function({ host, patternMatch, params }) { case "bsky": r = await bluesky({ ...patternMatch, - alwaysProxy: params.alwaysProxy + alwaysProxy: params.alwaysProxy, + dispatcher }); break; diff --git a/api/src/processing/request.js b/api/src/processing/request.js index 4287267c..d512bfe5 100644 --- a/api/src/processing/request.js +++ b/api/src/processing/request.js @@ -37,7 +37,7 @@ export function createResponse(responseType, responseData) { case "redirect": response = { - url: responseData?.u, + url: responseData?.url, filename: responseData?.filename } break; @@ -52,7 +52,7 @@ export function createResponse(responseType, responseData) { case "picker": response = { picker: responseData?.picker, - audio: responseData?.u, + audio: responseData?.url, audioFilename: responseData?.filename } break; diff --git a/api/src/processing/schema.js b/api/src/processing/schema.js index 172d480c..48d8b058 100644 --- a/api/src/processing/schema.js +++ b/api/src/processing/schema.js @@ -1,7 +1,5 @@ import { z } from "zod"; - import { normalizeURL } from "./url.js"; -import { verifyLanguageCode } from "../misc/utils.js"; export const apiSchema = z.object({ url: z.string() @@ -33,15 +31,21 @@ export const apiSchema = z.object({ ).default("1080"), youtubeDubLang: z.string() - .length(2) - .transform(verifyLanguageCode) + .min(2) + .max(8) + .regex(/^[0-9a-zA-Z\-]+$/) .optional(), + // TODO: remove this variable as it's no longer used + // and is kept for schema compatibility reasons + youtubeDubBrowserLang: z.boolean().default(false), + alwaysProxy: z.boolean().default(false), disableMetadata: z.boolean().default(false), tiktokFullAudio: z.boolean().default(false), tiktokH265: z.boolean().default(false), twitterGif: z.boolean().default(true), - youtubeDubBrowserLang: z.boolean().default(false), + + youtubeHLS: z.boolean().default(false), }) .strict(); diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 8d8bf4ac..81afaf39 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -1,7 +1,7 @@ import UrlPattern from "url-pattern"; export const audioIgnore = ["vk", "ok", "loom"]; -export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky"]; +export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky", "youtube"]; export const services = { bilibili: { @@ -30,7 +30,7 @@ export const services = { "reel/:id", "share/:shareType/:id" ], - subdomains: ["web"], + subdomains: ["web", "m"], altDomains: ["fb.watch"], }, instagram: { @@ -46,7 +46,7 @@ export const services = { altDomains: ["ddinstagram.com"], }, loom: { - patterns: ["share/:id"], + patterns: ["share/:id", "embed/:id"], }, ok: { patterns: [ @@ -111,10 +111,10 @@ export const services = { tiktok: { patterns: [ ":user/video/:postId", - ":id", - "t/:id", + ":shortLink", + "t/:shortLink", ":user/photo/:postId", - "v/:id.html" + "v/:postId.html" ], subdomains: ["vt", "vm", "m"], }, @@ -143,10 +143,6 @@ export const services = { subdomains: ["mobile"], altDomains: ["x.com", "vxtwitter.com", "fixvx.com"], }, - vine: { - patterns: ["v/:id"], - tld: "co", - }, vimeo: { patterns: [ ":id", @@ -158,11 +154,17 @@ export const services = { }, vk: { patterns: [ - "video:userId_:videoId", - "clip:userId_:videoId", - "clips:duplicate?z=clip:userId_:videoId" + "video:ownerId_:videoId", + "clip:ownerId_:videoId", + "clips:duplicate?z=clip:ownerId_:videoId", + "videos:duplicate?z=video:ownerId_:videoId", + "video:ownerId_:videoId_:accessKey", + "clip:ownerId_:videoId_:accessKey", + "clips:duplicate?z=clip:ownerId_:videoId_:accessKey", + "videos:duplicate?z=video:ownerId_:videoId_:accessKey" ], subdomains: ["m"], + altDomains: ["vkvideo.ru", "vk.ru"], }, youtube: { patterns: [ diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index 2105a563..e8c46639 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -36,10 +36,10 @@ export const testers = { || pattern.shortLink?.length <= 16, "streamable": pattern => - pattern.id?.length === 6, + pattern.id?.length <= 6, "tiktok": pattern => - pattern.postId?.length <= 21 || pattern.id?.length <= 13, + pattern.postId?.length <= 21 || pattern.shortLink?.length <= 13, "tumblr": pattern => pattern.id?.length < 21 @@ -55,11 +55,9 @@ export const testers = { pattern.id?.length <= 11 && (!pattern.password || pattern.password.length < 16), - "vine": pattern => - pattern.id?.length <= 12, - "vk": pattern => - pattern.userId?.length <= 10 && pattern.videoId?.length <= 10, + (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) || + (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18), "youtube": pattern => pattern.id?.length <= 11, diff --git a/api/src/processing/services/bluesky.js b/api/src/processing/services/bluesky.js index 5f5cbcec..bc887437 100644 --- a/api/src/processing/services/bluesky.js +++ b/api/src/processing/services/bluesky.js @@ -2,12 +2,19 @@ import HLS from "hls-parser"; import { cobaltUserAgent } from "../../config.js"; import { createStream } from "../../stream/manage.js"; -const extractVideo = async ({ media, filename }) => { - const urlMasterHLS = media?.playlist; - if (!urlMasterHLS) return { error: "fetch.empty" }; - if (!urlMasterHLS.startsWith("https://video.bsky.app/")) return { error: "fetch.empty" }; +const extractVideo = async ({ media, filename, dispatcher }) => { + let urlMasterHLS = media?.playlist; - const masterHLS = await fetch(urlMasterHLS) + if (!urlMasterHLS || !urlMasterHLS.startsWith("https://video.bsky.app/")) { + return { error: "fetch.empty" }; + } + + urlMasterHLS = urlMasterHLS.replace( + "video.bsky.app/watch/", + "video.cdn.bsky.app/hls/" + ); + + const masterHLS = await fetch(urlMasterHLS, { dispatcher }) .then(r => { if (r.status !== 200) return; return r.text(); @@ -26,7 +33,7 @@ const extractVideo = async ({ media, filename }) => { urls: videoURL, filename: `${filename}.mp4`, audioFilename: `${filename}_audio`, - isM3U8: true, + isHLS: true, } } @@ -48,7 +55,7 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => { let proxiedImage = createStream({ service: "bluesky", type: "proxy", - u: url, + url, filename: `${filename}_${i + 1}.jpg`, }); @@ -64,7 +71,7 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => { return { picker }; } -export default async function ({ user, post, alwaysProxy }) { +export default async function ({ user, post, alwaysProxy, dispatcher }) { const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0"); apiEndpoint.searchParams.set( "uri", @@ -73,8 +80,9 @@ export default async function ({ user, post, alwaysProxy }) { const getPost = await fetch(apiEndpoint, { headers: { - "user-agent": cobaltUserAgent - } + "user-agent": cobaltUserAgent, + }, + dispatcher }).then(r => r.json()).catch(() => {}); if (!getPost) return { error: "fetch.empty" }; @@ -87,7 +95,7 @@ export default async function ({ user, post, alwaysProxy }) { case "InvalidRequest": return { error: "link.unsupported" }; default: - return { error: "fetch.empty" }; + return { error: "content.post.unavailable" }; } } diff --git a/api/src/processing/services/dailymotion.js b/api/src/processing/services/dailymotion.js index a403a16b..a30a8bc7 100644 --- a/api/src/processing/services/dailymotion.js +++ b/api/src/processing/services/dailymotion.js @@ -92,7 +92,7 @@ export default async function({ id }) { return { urls: bestQuality.uri, - isM3U8: true, + isHLS: true, filenameAttributes: { service: 'dailymotion', id: media.xid, diff --git a/api/src/processing/services/instagram.js b/api/src/processing/services/instagram.js index 17e78ec4..d9a646aa 100644 --- a/api/src/processing/services/instagram.js +++ b/api/src/processing/services/instagram.js @@ -177,7 +177,7 @@ export default function(obj) { if (alwaysProxy) proxyFile = createStream({ service: "instagram", type: "proxy", - u: url, + url, filename: `instagram_${id}_${i + 1}.${itemExt}` }); @@ -189,7 +189,7 @@ export default function(obj) { thumb: createStream({ service: "instagram", type: "proxy", - u: e.node?.display_url, + url: e.node?.display_url, filename: `instagram_${id}_${i + 1}.jpg` }) } @@ -230,7 +230,7 @@ export default function(obj) { if (alwaysProxy) proxyFile = createStream({ service: "instagram", type: "proxy", - u: url, + url, filename: `instagram_${id}_${i + 1}.${itemExt}` }); @@ -242,7 +242,7 @@ export default function(obj) { thumb: createStream({ service: "instagram", type: "proxy", - u: imageUrl, + url: imageUrl, filename: `instagram_${id}_${i + 1}.jpg` }) } @@ -266,6 +266,7 @@ export default function(obj) { } async function getPost(id, alwaysProxy) { + const hasData = (data) => data && data.gql_data !== null; let data, result; try { const cookie = getCookie('instagram'); @@ -282,16 +283,16 @@ export default function(obj) { if (media_id && token) data = await requestMobileApi(media_id, { token }); // mobile api (no cookie, cookie) - if (media_id && !data) data = await requestMobileApi(media_id); - if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie }); + if (media_id && !hasData(data)) data = await requestMobileApi(media_id); + if (media_id && cookie && !hasData(data)) data = await requestMobileApi(media_id, { cookie }); // html embed (no cookie, cookie) - if (!data) data = await requestHTML(id); - if (!data && cookie) data = await requestHTML(id, cookie); + if (!hasData(data)) data = await requestHTML(id); + if (!hasData(data) && cookie) data = await requestHTML(id, cookie); // web app graphql api (no cookie, cookie) - if (!data) data = await requestGQL(id); - if (!data && cookie) data = await requestGQL(id, cookie); + if (!hasData(data)) data = await requestGQL(id); + if (!hasData(data) && cookie) data = await requestGQL(id, cookie); } catch {} if (!data) return { error: "fetch.fail" }; diff --git a/api/src/processing/services/ok.js b/api/src/processing/services/ok.js index 2fb6082d..10fb785b 100644 --- a/api/src/processing/services/ok.js +++ b/api/src/processing/services/ok.js @@ -1,5 +1,4 @@ import { genericUserAgent, env } from "../../config.js"; -import { cleanString } from "../../misc/utils.js"; const resolutions = { "ultra": "2160", @@ -44,8 +43,8 @@ export default async function(o) { let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1]; let fileMetadata = { - title: cleanString(videoData.movie.title.trim()), - author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()), + title: videoData.movie.title.trim(), + author: (videoData.author?.name || videoData.compilationTitle).trim(), } if (bestVideo) return { diff --git a/api/src/processing/services/rutube.js b/api/src/processing/services/rutube.js index 4305241a..5b502452 100644 --- a/api/src/processing/services/rutube.js +++ b/api/src/processing/services/rutube.js @@ -1,7 +1,5 @@ import HLS from "hls-parser"; - import { env } from "../../config.js"; -import { cleanString } from "../../misc/utils.js"; async function requestJSON(url) { try { @@ -35,6 +33,10 @@ export default async function(obj) { const play = await requestJSON(requestURL); if (!play) return { error: "fetch.fail" }; + if (play.detail?.type === "blocking_rule") { + return { error: "content.video.region" }; + } + if (play.detail || !play.video_balancer) return { error: "fetch.empty" }; if (play.live_streams?.hls) return { error: "content.video.live" }; @@ -59,13 +61,13 @@ export default async function(obj) { }); const fileMetadata = { - title: cleanString(play.title.trim()), - artist: cleanString(play.author.name.trim()), + title: play.title.trim(), + artist: play.author.name.trim(), } return { urls: matchingQuality.uri, - isM3U8: true, + isHLS: true, filenameAttributes: { service: "rutube", id: obj.id, diff --git a/api/src/processing/services/snapchat.js b/api/src/processing/services/snapchat.js index acb6813a..4c62a5ff 100644 --- a/api/src/processing/services/snapchat.js +++ b/api/src/processing/services/snapchat.js @@ -73,7 +73,7 @@ async function getStory(username, storyId, alwaysProxy) { const proxy = createStream({ service: "snapchat", type: "proxy", - u: snapUrl, + url: snapUrl, filename: `snapchat_${username}_${snap.timestampInSec.value}.${snapExt}`, }); @@ -81,7 +81,7 @@ async function getStory(username, storyId, alwaysProxy) { if (snapType === "video") thumbProxy = createStream({ service: "snapchat", type: "proxy", - u: snap.snapUrls.mediaPreviewUrl.value, + url: snap.snapUrls.mediaPreviewUrl.value, }); if (alwaysProxy) snapUrl = proxy; diff --git a/api/src/processing/services/soundcloud.js b/api/src/processing/services/soundcloud.js index 394f7dfe..ad535479 100644 --- a/api/src/processing/services/soundcloud.js +++ b/api/src/processing/services/soundcloud.js @@ -1,5 +1,4 @@ import { env } from "../../config.js"; -import { cleanString } from "../../misc/utils.js"; const cachedID = { version: '', @@ -63,7 +62,17 @@ export default async function(obj) { if (!json) return { error: "fetch.fail" }; - if (!json.media.transcodings) return { error: "fetch.empty" }; + if (json?.policy === "BLOCK") { + return { error: "content.region" }; + } + + if (json?.policy === "SNIP") { + return { error: "content.paid" }; + } + + if (!json?.media?.transcodings || !json?.media?.transcodings.length === 0) { + return { error: "fetch.empty" }; + } let bestAudio = "opus", selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"), @@ -75,6 +84,10 @@ export default async function(obj) { bestAudio = "mp3" } + if (!selectedStream) { + return { error: "fetch.empty" }; + } + let fileUrlBase = selectedStream.url; let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`; @@ -91,8 +104,8 @@ export default async function(obj) { if (!file) return { error: "fetch.empty" }; let fileMetadata = { - title: cleanString(json.title.trim()), - artist: cleanString(json.user.username.trim()), + title: json.title.trim(), + artist: json.user.username.trim(), } return { diff --git a/api/src/processing/services/tiktok.js b/api/src/processing/services/tiktok.js index 3c70e033..6978e071 100644 --- a/api/src/processing/services/tiktok.js +++ b/api/src/processing/services/tiktok.js @@ -12,7 +12,7 @@ export default async function(obj) { let postId = obj.postId; if (!postId) { - let html = await fetch(`${shortDomain}${obj.id}`, { + let html = await fetch(`${shortDomain}${obj.shortLink}`, { redirect: "manual", headers: { "user-agent": genericUserAgent.split(' Chrome/1')[0] @@ -24,7 +24,7 @@ export default async function(obj) { if (html.startsWith('')[1] - .split('')[0] - const data = JSON.parse(json) - detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"] + .split('')[0]; + + const data = JSON.parse(json); + const videoDetail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]; + + if (!videoDetail) throw "no video detail found"; + + // status_deleted or etc + if (videoDetail.statusMsg) { + return { error: "content.post.unavailable"}; + } + + detail = videoDetail?.itemInfo?.itemStruct; } catch { return { error: "fetch.fail" }; } + if (detail.isContentClassified) { + return { error: "content.post.age" }; + } + + if (!detail.author) { + return { error: "fetch.empty" }; + } + let video, videoFilename, audioFilename, audio, images, - filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`, + filenameBase = `tiktok_${detail.author?.uniqueId}_${postId}`, bestAudio; // will get defaulted to m4a later on in match-action images = detail.imagePost?.images; - let playAddr = detail.video.playAddr; + let playAddr = detail.video?.playAddr; + if (obj.h265) { const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0] playAddr = h265PlayAddr || playAddr @@ -102,7 +121,7 @@ export default async function(obj) { if (obj.alwaysProxy) url = createStream({ service: "tiktok", type: "proxy", - u: url, + url, filename: `${filenameBase}_photo_${i + 1}.jpg` }) diff --git a/api/src/processing/services/tumblr.js b/api/src/processing/services/tumblr.js index b361b98c..2b8aa4ce 100644 --- a/api/src/processing/services/tumblr.js +++ b/api/src/processing/services/tumblr.js @@ -1,4 +1,4 @@ -import psl from "psl"; +import psl from "@imput/psl"; const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z'; const API_BASE = 'https://api-http2.tumblr.com'; diff --git a/api/src/processing/services/twitch.js b/api/src/processing/services/twitch.js index ac85fbcf..4b9d4551 100644 --- a/api/src/processing/services/twitch.js +++ b/api/src/processing/services/twitch.js @@ -1,5 +1,4 @@ import { env } from "../../config.js"; -import { cleanString } from '../../misc/utils.js'; const gqlURL = "https://gql.twitch.tv/gql"; const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; @@ -73,13 +72,13 @@ export default async function (obj) { token: req_token[0].data.clip.playbackAccessToken.value })}`, fileMetadata: { - title: cleanString(clipMetadata.title.trim()), + title: clipMetadata.title.trim(), artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, }, filenameAttributes: { service: "twitch", id: clipMetadata.id, - title: cleanString(clipMetadata.title.trim()), + title: clipMetadata.title.trim(), author: `${clipMetadata.broadcaster.login}, clipped by ${clipMetadata.curator.login}`, qualityLabel: `${format.quality}p`, extension: 'mp4' diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index 18866b49..b4a1d557 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -159,10 +159,10 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { const getFileExt = (url) => new URL(url).pathname.split(".", 2)[1]; - const proxyMedia = (u, filename) => createStream({ + const proxyMedia = (url, filename) => createStream({ service: "twitter", type: "proxy", - u, filename, + url, filename, }) switch (media?.length) { @@ -208,7 +208,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { let url = bestQuality(content.video_info.variants); const shouldRenderGif = content.type === "animated_gif" && toGif; - const videoFilename = `twitter_${id}_${i + 1}.mp4`; + const videoFilename = `twitter_${id}_${i + 1}.${shouldRenderGif ? "gif" : "mp4"}`; let type = "video"; if (shouldRenderGif) type = "gif"; @@ -217,7 +217,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { url = createStream({ service: "twitter", type: shouldRenderGif ? "gif" : "remux", - u: url, + url, filename: videoFilename, }) } else if (alwaysProxy) { diff --git a/api/src/processing/services/vimeo.js b/api/src/processing/services/vimeo.js index 23e84191..8d704771 100644 --- a/api/src/processing/services/vimeo.js +++ b/api/src/processing/services/vimeo.js @@ -1,7 +1,6 @@ import HLS from "hls-parser"; - import { env } from "../../config.js"; -import { cleanString, merge } from '../../misc/utils.js'; +import { merge } from '../../misc/utils.js'; const resolutionMatch = { "3840": 2160, @@ -122,7 +121,7 @@ const getHLS = async (configURL, obj) => { return { urls, - isM3U8: true, + isHLS: true, filenameAttributes: { resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`, qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`, @@ -152,8 +151,8 @@ export default async function(obj) { } const fileMetadata = { - title: cleanString(info.name), - artist: cleanString(info.user.name), + title: info.name, + artist: info.user.name, }; return merge( diff --git a/api/src/processing/services/vk.js b/api/src/processing/services/vk.js index e3c18e47..33224d69 100644 --- a/api/src/processing/services/vk.js +++ b/api/src/processing/services/vk.js @@ -1,63 +1,140 @@ -import { cleanString } from "../../misc/utils.js"; -import { genericUserAgent, env } from "../../config.js"; +import { env } from "../../config.js"; -const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"]; +const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"]; -export default async function(o) { - let html, url, quality = o.quality === "max" ? 2160 : o.quality; +const oauthUrl = "https://oauth.vk.com/oauth/get_anonym_token"; +const apiUrl = "https://api.vk.com/method"; - html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, { - headers: { - "user-agent": genericUserAgent - } - }) - .then(r => r.arrayBuffer()) - .catch(() => {}); +const clientId = "51552953"; +const clientSecret = "qgr0yWwXCrsxA1jnRtRX"; - if (!html) return { error: "fetch.fail" }; +// used in stream/shared.js for accessing media files +export const vkClientAgent = "com.vk.vkvideo.prod/822 (iPhone, iOS 16.7.7, iPhone10,4, Scale/2.0) SAK/1.119"; - // decode cyrillic from windows-1251 because vk still uses apis from prehistoric times - let decoder = new TextDecoder('windows-1251'); - html = decoder.decode(html); +const cachedToken = { + token: "", + expiry: 0, + device_id: "", +}; - if (!html.includes(`{"lang":`)) return { error: "fetch.empty" }; - - let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]); - - if (Number(js.mvData.is_active_live) !== 0) { - return { error: "content.video.live" }; +const getToken = async () => { + if (cachedToken.expiry - 10 > Math.floor(new Date().getTime() / 1000)) { + return cachedToken.token; } - if (js.mvData.duration > env.durationLimit) { + const randomDeviceId = crypto.randomUUID().toUpperCase(); + + const anonymOauth = new URL(oauthUrl); + anonymOauth.searchParams.set("client_id", clientId); + anonymOauth.searchParams.set("client_secret", clientSecret); + anonymOauth.searchParams.set("device_id", randomDeviceId); + + const oauthResponse = await fetch(anonymOauth.toString(), { + headers: { + "user-agent": vkClientAgent, + } + }).then(r => { + if (r.status === 200) { + return r.json(); + } + }); + + if (!oauthResponse) return; + + if (oauthResponse?.token && oauthResponse?.expired_at && typeof oauthResponse?.expired_at === "number") { + cachedToken.token = oauthResponse.token; + cachedToken.expiry = oauthResponse.expired_at; + cachedToken.device_id = randomDeviceId; + } + + if (!cachedToken.token) return; + + return cachedToken.token; +} + +const getVideo = async (ownerId, videoId, accessKey) => { + const video = await fetch(`${apiUrl}/video.get`, { + method: "POST", + headers: { + "content-type": "application/x-www-form-urlencoded; charset=utf-8", + "user-agent": vkClientAgent, + }, + body: new URLSearchParams({ + anonymous_token: cachedToken.token, + device_id: cachedToken.device_id, + lang: "en", + v: "5.244", + videos: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}` + }).toString() + }) + .then(r => { + if (r.status === 200) { + return r.json(); + } + }); + + return video; +} + +export default async function ({ ownerId, videoId, accessKey, quality }) { + const token = await getToken(); + if (!token) return { error: "fetch.fail" }; + + const videoGet = await getVideo(ownerId, videoId, accessKey); + + if (!videoGet || !videoGet.response || videoGet.response.items.length !== 1) { + return { error: "fetch.empty" }; + } + + const video = videoGet.response.items[0]; + + if (video.restriction) { + const title = video.restriction.title; + if (title.endsWith("country") || title.endsWith("region.")) { + return { error: "content.video.region" }; + } + if (title === "Processing video") { + return { error: "fetch.empty" }; + } + return { error: "content.video.unavailable" }; + } + + if (!video.files || !video.duration) { + return { error: "fetch.fail" }; + } + + if (video.duration > env.durationLimit) { return { error: "content.too_long" }; } - for (let i in resolutions) { - if (js.player.params[0][`url${resolutions[i]}`]) { - quality = resolutions[i]; + const userQuality = quality === "max" ? resolutions[0] : quality; + let pickedQuality; + + for (const resolution of resolutions) { + if (video.files[`mp4_${resolution}`] && +resolution <= +userQuality) { + pickedQuality = resolution; break } } - if (Number(quality) > Number(o.quality)) quality = o.quality; - url = js.player.params[0][`url${quality}`]; + const url = video.files[`mp4_${pickedQuality}`]; - let fileMetadata = { - title: cleanString(js.player.params[0].md_title.trim()), - author: cleanString(js.player.params[0].md_author.trim()), + if (!url) return { error: "fetch.fail" }; + + const fileMetadata = { + title: video.title.trim(), } - if (url) return { + return { urls: url, + fileMetadata, filenameAttributes: { service: "vk", - id: `${o.userId}_${o.videoId}`, + id: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`, title: fileMetadata.title, - author: fileMetadata.author, - resolution: `${quality}p`, - qualityLabel: `${quality}p`, + resolution: `${pickedQuality}p`, + qualityLabel: `${pickedQuality}p`, extension: "mp4" } } - return { error: "fetch.empty" } } diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index f08489ed..3af6ba94 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -1,16 +1,16 @@ -import { fetch } from "undici"; +import HLS from "hls-parser"; +import { fetch } from "undici"; import { Innertube, Session } from "youtubei.js"; import { env } from "../../config.js"; -import { cleanString } from "../../misc/utils.js"; import { getCookie, updateCookieValues } from "../cookie/manager.js"; const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms let innertube, lastRefreshedAt; -const codecMatch = { +const codecList = { h264: { videoCodec: "avc1", audioCodec: "mp4a", @@ -28,6 +28,21 @@ const codecMatch = { } } +const hlsCodecList = { + h264: { + videoCodec: "avc1", + audioCodec: "mp4a", + container: "mp4" + }, + vp9: { + videoCodec: "vp09", + audioCodec: "mp4a", + container: "webm" + } +} + +const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320]; + const transformSessionData = (cookie) => { if (!cookie) return; @@ -71,19 +86,9 @@ const cloneInnertube = async (customFetch) => { const cookie = getCookie('youtube_oauth'); const oauthData = transformSessionData(cookie); - - if (!session.logged_in && oauthData) { - const tokensMod = { - ...oauthData, // 复制 oauthData 中的所有属性 - client: { - client_id: oauthData.client_id, - client_secret: oauthData.client_secret - } - }; - delete tokensMod.client_id; - delete tokensMod.client_secret; - await session.oauth.init(tokensMod); + if (!session.logged_in && oauthData) { + await session.oauth.init(oauthData); session.logged_in = true; } @@ -118,7 +123,7 @@ export default async function(o) { dispatcher: o.dispatcher }) ); - } catch(e) { + } catch (e) { if (e.message?.endsWith("decipher algorithm")) { return { error: "youtube.decipher" } } else if (e.message?.includes("refresh access token")) { @@ -126,29 +131,33 @@ export default async function(o) { } else throw e; } - const quality = o.quality === "max" ? "9000" : o.quality; + let useHLS = o.youtubeHLS; - let info, isDubbed, - format = o.format || "h264"; - - function qual(i) { - if (!i.quality_label) { - return; - } - - return i.quality_label.split('p')[0].split('s')[0] + // HLS playlists don't contain the av1 video format, at least with the iOS client + if (useHLS && o.format === "av1") { + useHLS = false; } + let info; try { - info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS'); - } catch(e) { - if (e?.info?.reason === "This video is private") { - return { error: "content.video.private" }; - } else if (e?.message === "This video is unavailable") { - return { error: "content.video.unavailable" }; - } else { - return { error: "fetch.fail" }; + info = await yt.getBasicInfo(o.id, useHLS ? 'IOS' : 'ANDROID'); + } catch (e) { + if (e?.info) { + const errorInfo = JSON.parse(e?.info); + + if (errorInfo?.reason === "This video is private") { + return { error: "content.video.private" }; + } + if (["INVALID_ARGUMENT", "UNAUTHENTICATED"].includes(errorInfo?.error?.status)) { + return { error: "youtube.api_error" }; + } } + + if (e?.message === "This video is unavailable") { + return { error: "content.video.unavailable" }; + } + + return { error: "fetch.fail" }; } if (!info) return { error: "fetch.fail" }; @@ -156,37 +165,47 @@ export default async function(o) { const playability = info.playability_status; const basicInfo = info.basic_info; - if (playability.status === "LOGIN_REQUIRED") { - if (playability.reason.endsWith("bot")) { - return { error: "youtube.login" } - } - if (playability.reason.endsWith("age")) { - return { error: "content.video.age" } - } - if (playability?.error_screen?.reason?.text === "Private video") { - return { error: "content.video.private" } - } - } + switch(playability.status) { + case "LOGIN_REQUIRED": + if (playability.reason.endsWith("bot")) { + return { error: "youtube.login" } + } + if (playability.reason.endsWith("age")) { + return { error: "content.video.age" } + } + if (playability?.error_screen?.reason?.text === "Private video") { + return { error: "content.video.private" } + } + break; - if (playability.status === "UNPLAYABLE") { - if (playability?.reason?.endsWith("request limit.")) { - return { error: "fetch.rate" } - } - if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) { - return { error: "content.video.region" } - } - if (playability?.error_screen?.reason?.text === "Private video") { - return { error: "content.video.private" } - } + case "UNPLAYABLE": + if (playability?.reason?.endsWith("request limit.")) { + return { error: "fetch.rate" } + } + if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) { + return { error: "content.video.region" } + } + if (playability?.error_screen?.reason?.text === "Private video") { + return { error: "content.video.private" } + } + break; + + case "AGE_VERIFICATION_REQUIRED": + return { error: "content.video.age" }; } if (playability.status !== "OK") { return { error: "content.video.unavailable" }; } + if (basicInfo.is_live) { return { error: "content.video.live" }; } + if (basicInfo.duration > env.durationLimit) { + return { error: "content.too_long" }; + } + // return a critical error if returned video is "Video Not Available" // or a similar stub by youtube if (basicInfo.id !== o.id) { @@ -196,64 +215,200 @@ export default async function(o) { } } - const filterByCodec = (formats) => - formats - .filter(e => - e.mime_type.includes(codecMatch[format].videoCodec) - || e.mime_type.includes(codecMatch[format].audioCodec) - ) - .sort((a, b) => Number(b.bitrate) - Number(a.bitrate)); + const quality = o.quality === "max" ? 9000 : Number(o.quality); - let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats); - - if (adaptive_formats.length === 0 && format === "vp9") { - format = "h264" - adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats) + const normalizeQuality = res => { + const shortestSide = res.height > res.width ? res.width : res.height; + return videoQualities.find(qual => qual >= shortestSide); } - let bestQuality; + let video, audio, dubbedLanguage, + codec = o.format || "h264"; - const bestVideo = adaptive_formats.find(i => i.has_video && i.content_length); - const hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length); + if (useHLS) { + const hlsManifest = info.streaming_data.hls_manifest_url; - if (bestVideo) bestQuality = qual(bestVideo); + if (!hlsManifest) { + return { error: "youtube.no_hls_streams" }; + } - if ((!bestQuality && !o.isAudioOnly) || !hasAudio) - return { error: "youtube.codec" }; + const fetchedHlsManifest = await fetch(hlsManifest, { + dispatcher: o.dispatcher, + }).then(r => { + if (r.status === 200) { + return r.text(); + } else { + throw new Error("couldn't fetch the HLS playlist"); + } + }).catch(() => {}); - if (basicInfo.duration > env.durationLimit) - return { error: "content.too_long" }; + if (!fetchedHlsManifest) { + return { error: "youtube.no_hls_streams" }; + } - const checkBestAudio = (i) => (i.has_audio && !i.has_video); + const variants = HLS.parse(fetchedHlsManifest).variants.sort( + (a, b) => Number(b.bandwidth) - Number(a.bandwidth) + ); - let audio = adaptive_formats.find(i => - checkBestAudio(i) && i.is_original - ); + if (!variants || variants.length === 0) { + return { error: "youtube.no_hls_streams" }; + } - if (o.dubLang) { - let dubbedAudio = adaptive_formats.find(i => - checkBestAudio(i) - && i.language === o.dubLang - && i.audio_track - ) + const matchHlsCodec = codecs => ( + codecs.includes(hlsCodecList[codec].videoCodec) + ); - if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) { - audio = dubbedAudio; - isDubbed = true; + const best = variants.find(i => matchHlsCodec(i.codecs)); + + const preferred = variants.find(i => + matchHlsCodec(i.codecs) && normalizeQuality(i.resolution) === quality + ); + + let selected = preferred || best; + + if (!selected) { + codec = "h264"; + selected = variants.find(i => matchHlsCodec(i.codecs)); + } + + if (!selected) { + return { error: "youtube.no_matching_format" }; + } + + audio = selected.audio.find(i => i.isDefault); + + // some videos (mainly those with AI dubs) don't have any tracks marked as default + // why? god knows, but we assume that a default track is marked as such in the title + if (!audio) { + audio = selected.audio.find(i => i.name.endsWith("- original")); + } + + if (o.dubLang) { + const dubbedAudio = selected.audio.find(i => + i.language?.startsWith(o.dubLang) + ); + + if (dubbedAudio && !dubbedAudio.isDefault) { + dubbedLanguage = dubbedAudio.language; + audio = dubbedAudio; + } + } + + selected.audio = []; + selected.subtitles = []; + video = selected; + } else { + // i miss typescript so bad + const sorted_formats = { + h264: { + video: [], + audio: [], + bestVideo: undefined, + bestAudio: undefined, + }, + vp9: { + video: [], + audio: [], + bestVideo: undefined, + bestAudio: undefined, + }, + av1: { + video: [], + audio: [], + bestVideo: undefined, + bestAudio: undefined, + }, + } + + const checkFormat = (format, pCodec) => format.content_length && + (format.mime_type.includes(codecList[pCodec].videoCodec) + || format.mime_type.includes(codecList[pCodec].audioCodec)); + + // sort formats & weed out bad ones + info.streaming_data.adaptive_formats.sort((a, b) => + Number(b.bitrate) - Number(a.bitrate) + ).forEach(format => { + Object.keys(codecList).forEach(yCodec => { + const sorted = sorted_formats[yCodec]; + const goodFormat = checkFormat(format, yCodec); + if (!goodFormat) return; + + if (format.has_video) { + sorted.video.push(format); + if (!sorted.bestVideo) sorted.bestVideo = format; + } + if (format.has_audio) { + sorted.audio.push(format); + if (!sorted.bestAudio) sorted.bestAudio = format; + } + }) + }); + + const noBestMedia = () => { + const vid = sorted_formats[codec]?.bestVideo; + const aud = sorted_formats[codec]?.bestAudio; + return (!vid && !o.isAudioOnly) || (!aud && o.isAudioOnly) + }; + + if (noBestMedia()) { + if (codec === "av1") codec = "vp9"; + else if (codec === "vp9") codec = "av1"; + + // if there's no higher quality fallback, then use h264 + if (noBestMedia()) codec = "h264"; + } + + // if there's no proper combo of av1, vp9, or h264, then give up + if (noBestMedia()) { + return { error: "youtube.no_matching_format" }; + } + + audio = sorted_formats[codec].bestAudio; + + if (audio?.audio_track && !audio?.audio_track?.audio_is_default) { + audio = sorted_formats[codec].audio.find(i => + i?.audio_track?.audio_is_default + ); + } + + if (o.dubLang) { + const dubbedAudio = sorted_formats[codec].audio.find(i => + i.language?.startsWith(o.dubLang) && i.audio_track + ); + + if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) { + audio = dubbedAudio; + dubbedLanguage = dubbedAudio.language; + } + } + + if (!o.isAudioOnly) { + const qual = (i) => { + return normalizeQuality({ + width: i.width, + height: i.height, + }) + } + + const bestQuality = qual(sorted_formats[codec].bestVideo); + const useBestQuality = quality >= bestQuality; + + video = useBestQuality + ? sorted_formats[codec].bestVideo + : sorted_formats[codec].video.find(i => qual(i) === quality); + + if (!video) video = sorted_formats[codec].bestVideo; } } - if (!audio) { - audio = adaptive_formats.find(i => checkBestAudio(i)); - } - - let fileMetadata = { - title: cleanString(basicInfo.title.trim()), - artist: cleanString(basicInfo.author.replace("- Topic", "").trim()), + const fileMetadata = { + title: basicInfo.title.trim(), + artist: basicInfo.author.replace("- Topic", "").trim() } if (basicInfo?.short_description?.startsWith("Provided to YouTube by")) { - let descItems = basicInfo.short_description.split("\n\n", 5); + const descItems = basicInfo.short_description.split("\n\n", 5); + if (descItems.length === 5) { fileMetadata.album = descItems[2]; fileMetadata.copyright = descItems[3]; @@ -263,61 +418,70 @@ export default async function(o) { } } - let filenameAttributes = { + const filenameAttributes = { service: "youtube", id: o.id, title: fileMetadata.title, author: fileMetadata.artist, - youtubeDubName: isDubbed ? o.dubLang : false + youtubeDubName: dubbedLanguage || false, } - if (audio && o.isAudioOnly) return { - type: "audio", - isAudioOnly: true, - urls: audio.decipher(yt.session.player), - filenameAttributes: filenameAttributes, - fileMetadata: fileMetadata, - bestAudio: format === "h264" ? "m4a" : "opus" - } + if (audio && o.isAudioOnly) { + let bestAudio = codec === "h264" ? "m4a" : "opus"; + let urls = audio.url; - const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality, - checkSingle = i => - qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].videoCodec), - checkRender = i => - qual(i) === matchingQuality && i.has_video && !i.has_audio; + if (useHLS) { + bestAudio = "mp3"; + urls = audio.uri; + } - let match, type, urls; - - // prefer good premuxed videos if available - if (!o.isAudioOnly && !o.isAudioMuted && format === "h264" && bestVideo.fps <= 30) { - match = info.streaming_data.formats.find(checkSingle); - type = "proxy"; - urls = match?.decipher(yt.session.player); - } - - const video = adaptive_formats.find(checkRender); - - if (!match && video && audio) { - match = video; - type = "merge"; - urls = [ - video.decipher(yt.session.player), - audio.decipher(yt.session.player) - ] - } - - if (match) { - filenameAttributes.qualityLabel = match.quality_label; - filenameAttributes.resolution = `${match.width}x${match.height}`; - filenameAttributes.extension = codecMatch[format].container; - filenameAttributes.youtubeFormat = format; return { - type, + type: "audio", + isAudioOnly: true, urls, filenameAttributes, - fileMetadata + fileMetadata, + bestAudio, + isHLS: useHLS, } } - return { error: "fetch.fail" } + if (video && audio) { + let resolution; + + if (useHLS) { + resolution = normalizeQuality(video.resolution); + filenameAttributes.resolution = `${video.resolution.width}x${video.resolution.height}`; + filenameAttributes.extension = hlsCodecList[codec].container; + + video = video.uri; + audio = audio.uri; + } else { + resolution = normalizeQuality({ + width: video.width, + height: video.height, + }); + filenameAttributes.resolution = `${video.width}x${video.height}`; + filenameAttributes.extension = codecList[codec].container; + + video = video.url; + audio = audio.url; + } + + filenameAttributes.qualityLabel = `${resolution}p`; + filenameAttributes.youtubeFormat = codec; + + return { + type: "merge", + urls: [ + video, + audio, + ], + filenameAttributes, + fileMetadata, + isHLS: useHLS, + } + } + + return { error: "youtube.no_matching_format" }; } diff --git a/api/src/processing/url.js b/api/src/processing/url.js index 034a5d73..64517099 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -1,4 +1,4 @@ -import psl from "psl"; +import psl from "@imput/psl"; import { strict as assert } from "node:assert"; import { env } from "../config.js"; @@ -42,7 +42,7 @@ function aliasURL(url) { case "fixvx": case "x": if (services.twitter.altDomains.includes(url.hostname)) { - url.hostname = 'twitter.com' + url.hostname = 'twitter.com'; } break; @@ -85,6 +85,13 @@ function aliasURL(url) { url.hostname = 'instagram.com'; } break; + + case "vk": + case "vkvideo": + if (services.vk.altDomains.includes(url.hostname)) { + url.hostname = 'vk.com'; + } + break; } return url diff --git a/api/src/security/api-keys.js b/api/src/security/api-keys.js new file mode 100644 index 00000000..d534999c --- /dev/null +++ b/api/src/security/api-keys.js @@ -0,0 +1,227 @@ +import { env } from "../config.js"; +import { readFile } from "node:fs/promises"; +import { Green, Yellow } from "../misc/console-text.js"; +import ip from "ipaddr.js"; +import * as cluster from "../misc/cluster.js"; + +// this function is a modified variation of code +// from https://stackoverflow.com/a/32402438/14855621 +const generateWildcardRegex = rule => { + var escapeRegex = (str) => str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1"); + return new RegExp("^" + rule.split("*").map(escapeRegex).join(".*") + "$"); +} + +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +let keys = {}; + +const ALLOWED_KEYS = new Set(['name', 'ips', 'userAgents', 'limit']); + +/* Expected format pseudotype: +** type KeyFileContents = Record< +** UUIDv4String, +** { +** name?: string, +** limit?: number | "unlimited", +** ips?: CIDRString[], +** userAgents?: string[] +** } +** >; +*/ + +const validateKeys = (input) => { + if (typeof input !== 'object' || input === null) { + throw "input is not an object"; + } + + if (Object.keys(input).some(x => !UUID_REGEX.test(x))) { + throw "key file contains invalid key(s)"; + } + + Object.values(input).forEach(details => { + if (typeof details !== 'object' || details === null) { + throw "some key(s) are incorrectly configured"; + } + + const unexpected_key = Object.keys(details).find(k => !ALLOWED_KEYS.has(k)); + if (unexpected_key) { + throw "detail object contains unexpected key: " + unexpected_key; + } + + if (details.limit && details.limit !== 'unlimited') { + if (typeof details.limit !== 'number') + throw "detail object contains invalid limit (not a number)"; + else if (details.limit < 1) + throw "detail object contains invalid limit (not a positive number)"; + } + + if (details.ips) { + if (!Array.isArray(details.ips)) + throw "details object contains value for `ips` which is not an array"; + + const invalid_ip = details.ips.find( + addr => typeof addr !== 'string' || (!ip.isValidCIDR(addr) && !ip.isValid(addr)) + ); + + if (invalid_ip) { + throw "`ips` in details contains an invalid IP or CIDR range: " + invalid_ip; + } + } + + if (details.userAgents) { + if (!Array.isArray(details.userAgents)) + throw "details object contains value for `userAgents` which is not an array"; + + const invalid_ua = details.userAgents.find(ua => typeof ua !== 'string'); + if (invalid_ua) { + throw "`userAgents` in details contains an invalid user agent: " + invalid_ua; + } + } + }); +} + +const formatKeys = (keyData) => { + const formatted = {}; + + for (let key in keyData) { + const data = keyData[key]; + key = key.toLowerCase(); + + formatted[key] = {}; + + if (data.limit) { + if (data.limit === "unlimited") { + data.limit = Infinity; + } + + formatted[key].limit = data.limit; + } + + if (data.ips) { + formatted[key].ips = data.ips.map(addr => { + if (ip.isValid(addr)) { + const parsed = ip.parse(addr); + const range = parsed.kind() === 'ipv6' ? 128 : 32; + return [ parsed, range ]; + } + + return ip.parseCIDR(addr); + }); + } + + if (data.userAgents) { + formatted[key].userAgents = data.userAgents.map(generateWildcardRegex); + } + } + + return formatted; +} + +const updateKeys = (newKeys) => { + keys = formatKeys(newKeys); +} + +const loadKeys = async (source) => { + let updated; + if (source.protocol === 'file:') { + const pathname = source.pathname === '/' ? '' : source.pathname; + updated = JSON.parse( + await readFile( + decodeURIComponent(source.host + pathname), + 'utf8' + ) + ); + } else { + updated = await fetch(source).then(a => a.json()); + } + + validateKeys(updated); + + cluster.broadcast({ api_keys: updated }); + + updateKeys(updated); +} + +const wrapLoad = (url, initial = false) => { + loadKeys(url) + .then(() => { + if (initial) { + console.log(`${Green('[✓]')} api keys loaded successfully!`) + } + }) + .catch((e) => { + console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`); + console.error('Error:', e); + }) +} + +const err = (reason) => ({ success: false, error: reason }); + +export const validateAuthorization = (req) => { + const authHeader = req.get('Authorization'); + + if (typeof authHeader !== 'string') { + return err("missing"); + } + + const [ authType, keyString ] = authHeader.split(' ', 2); + if (authType.toLowerCase() !== 'api-key') { + return err("not_api_key"); + } + + if (!UUID_REGEX.test(keyString) || `${authType} ${keyString}` !== authHeader) { + return err("invalid"); + } + + const matchingKey = keys[keyString.toLowerCase()]; + if (!matchingKey) { + return err("not_found"); + } + + if (matchingKey.ips) { + let addr; + try { + addr = ip.parse(req.ip); + } catch { + return err("invalid_ip"); + } + + const ip_allowed = matchingKey.ips.some( + ([ allowed, size ]) => { + return addr.kind() === allowed.kind() + && addr.match(allowed, size); + } + ); + + if (!ip_allowed) { + return err("ip_not_allowed"); + } + } + + if (matchingKey.userAgents) { + const userAgent = req.get('User-Agent'); + if (!matchingKey.userAgents.some(regex => regex.test(userAgent))) { + return err("ua_not_allowed"); + } + } + + req.rateLimitKey = keyString.toLowerCase(); + req.rateLimitMax = matchingKey.limit; + + return { success: true }; +} + +export const setup = (url) => { + if (cluster.isPrimary) { + wrapLoad(url, true); + if (env.keyReloadInterval > 0) { + setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000); + } + } else if (cluster.isWorker) { + process.on('message', (message) => { + if ('api_keys' in message) { + updateKeys(message.api_keys); + } + }); + } +} diff --git a/api/src/security/secrets.js b/api/src/security/secrets.js new file mode 100644 index 00000000..fff24f84 --- /dev/null +++ b/api/src/security/secrets.js @@ -0,0 +1,62 @@ +import cluster from "node:cluster"; +import { createHmac, randomBytes } from "node:crypto"; + +const generateSalt = () => { + if (cluster.isPrimary) + return randomBytes(64); + + return null; +} + +let rateSalt = generateSalt(); +let streamSalt = generateSalt(); + +export const syncSecrets = () => { + return new Promise((resolve, reject) => { + if (cluster.isPrimary) { + let remaining = Object.values(cluster.workers).length; + const handleReady = (worker, m) => { + if (m.ready) + worker.send({ rateSalt, streamSalt }); + + if (!--remaining) + resolve(); + } + + for (const worker of Object.values(cluster.workers)) { + worker.once( + 'message', + (m) => handleReady(worker, m) + ); + } + } else if (cluster.isWorker) { + if (rateSalt || streamSalt) + return reject(); + + process.send({ ready: true }); + process.once('message', (message) => { + if (rateSalt || streamSalt) + return reject(); + + if (message.rateSalt && message.streamSalt) { + streamSalt = Buffer.from(message.streamSalt); + rateSalt = Buffer.from(message.rateSalt); + resolve(); + } + }); + } else reject(); + }); +} + + +export const hashHmac = (value, type) => { + let salt; + if (type === 'rate') + salt = rateSalt; + else if (type === 'stream') + salt = streamSalt; + else + throw "unknown salt"; + + return createHmac("sha256", salt).update(value).digest(); +} diff --git a/api/src/store/base-store.js b/api/src/store/base-store.js new file mode 100644 index 00000000..c2a59ff8 --- /dev/null +++ b/api/src/store/base-store.js @@ -0,0 +1,48 @@ +const _stores = new Set(); + +export class Store { + id; + + constructor(name) { + name = name.toUpperCase(); + + if (_stores.has(name)) + throw `${name} store already exists`; + _stores.add(name); + + this.id = name; + } + + async _has(_key) { await Promise.reject("needs implementation"); } + has(key) { + if (typeof key !== 'string') { + key = key.toString(); + } + + return this._has(key); + } + + async _get(_key) { await Promise.reject("needs implementation"); } + async get(key) { + if (typeof key !== 'string') { + key = key.toString(); + } + + const val = await this._get(key); + if (val === null) + return null; + + return val; + } + + async _set(_key, _val, _exp_sec = -1) { await Promise.reject("needs implementation") } + set(key, val, exp_sec = -1) { + if (typeof key !== 'string') { + key = key.toString(); + } + + exp_sec = Math.round(exp_sec); + + return this._set(key, val, exp_sec); + } +}; diff --git a/api/src/store/memory-store.js b/api/src/store/memory-store.js new file mode 100644 index 00000000..100a0e09 --- /dev/null +++ b/api/src/store/memory-store.js @@ -0,0 +1,77 @@ +import { MinPriorityQueue } from '@datastructures-js/priority-queue'; +import { Store } from './base-store.js'; + +// minimum delay between sweeps to avoid repeatedly +// sweeping entries close in proximity one by one. +const MIN_THRESHOLD_MS = 2500; + +export default class MemoryStore extends Store { + #store = new Map(); + #timeouts = new MinPriorityQueue/*<{ t: number, k: unknown }>*/((obj) => obj.t); + #nextSweep = { id: null, t: null }; + + constructor(name) { + super(name); + } + + _has(key) { + return this.#store.has(key); + } + + _get(key) { + const val = this.#store.get(key); + + return val === undefined ? null : val; + } + + _set(key, val, exp_sec = -1) { + if (this.#store.has(key)) { + this.#timeouts.remove(o => o.k === key); + } + + if (exp_sec > 0) { + const exp = 1000 * exp_sec; + const timeout_at = +new Date() + exp; + + this.#timeouts.enqueue({ k: key, t: timeout_at }); + } + + this.#store.set(key, val); + this.#reschedule(); + } + + #reschedule() { + const current_time = new Date().getTime(); + const time = this.#timeouts.front()?.t; + if (!time) { + return; + } else if (time < current_time) { + return this.#sweepNow(); + } + + const sweep = this.#nextSweep; + if (sweep.id === null || sweep.t > time) { + if (sweep.id) { + clearTimeout(sweep.id); + } + + sweep.t = time; + sweep.id = setTimeout( + () => this.#sweepNow(), + Math.max(MIN_THRESHOLD_MS, time - current_time) + ); + sweep.id.unref(); + } + } + + #sweepNow() { + while (this.#timeouts.front()?.t < new Date().getTime()) { + const item = this.#timeouts.dequeue(); + this.#store.delete(item.k); + } + + this.#nextSweep.id = null; + this.#nextSweep.t = null; + this.#reschedule(); + } +} diff --git a/api/src/store/redis-ratelimit.js b/api/src/store/redis-ratelimit.js new file mode 100644 index 00000000..64d11e5e --- /dev/null +++ b/api/src/store/redis-ratelimit.js @@ -0,0 +1,19 @@ +import { env } from "../config.js"; + +let client, redis, redisLimiter; + +export const createStore = async (name) => { + if (!env.redisURL) return; + + if (!client) { + redis = await import('redis'); + redisLimiter = await import('rate-limit-redis'); + client = redis.createClient({ url: env.redisURL }); + await client.connect(); + } + + return new redisLimiter.default({ + prefix: `RL${name}_`, + sendCommand: (...args) => client.sendCommand(args), + }); +} diff --git a/api/src/store/redis-store.js b/api/src/store/redis-store.js new file mode 100644 index 00000000..0b359526 --- /dev/null +++ b/api/src/store/redis-store.js @@ -0,0 +1,64 @@ +import { commandOptions, createClient } from "redis"; +import { env } from "../config.js"; +import { Store } from "./base-store.js"; + +export default class RedisStore extends Store { + #client = createClient({ + url: env.redisURL, + }); + #connected; + + constructor(name) { + super(name); + this.#connected = this.#client.connect(); + } + + #keyOf(key) { + return this.id + '_' + key; + } + + async _has(key) { + await this.#connected; + + return this.#client.hExists(key); + } + + async _get(key) { + await this.#connected; + + const valueType = await this.#client.get(this.#keyOf(key) + '_t'); + const value = await this.#client.get( + commandOptions({ returnBuffers: true }), + this.#keyOf(key) + ); + + if (!value) { + return null; + } + + if (valueType === 'b') + return value; + else + return JSON.parse(value); + } + + async _set(key, val, exp_sec = -1) { + await this.#connected; + + const options = exp_sec > 0 ? { EX: exp_sec } : undefined; + + if (val instanceof Buffer) { + await this.#client.set( + this.#keyOf(key) + '_t', + 'b', + options + ); + } + + await this.#client.set( + this.#keyOf(key), + val, + options + ); + } +} diff --git a/api/src/store/store.js b/api/src/store/store.js new file mode 100644 index 00000000..e268d88d --- /dev/null +++ b/api/src/store/store.js @@ -0,0 +1,10 @@ +import { env } from '../config.js'; + +let _export; +if (env.redisURL) { + _export = await import('./redis-store.js'); +} else { + _export = await import('./memory-store.js'); +} + +export default _export.default; diff --git a/api/src/stream/internal-hls.js b/api/src/stream/internal-hls.js index 07fcebde..83deb440 100644 --- a/api/src/stream/internal-hls.js +++ b/api/src/stream/internal-hls.js @@ -53,7 +53,7 @@ function transformMediaPlaylist(streamInfo, hlsPlaylist) { const HLS_MIME_TYPES = ["application/vnd.apple.mpegurl", "audio/mpegurl", "application/x-mpegURL"]; -export function isHlsRequest (req) { +export function isHlsResponse (req) { return HLS_MIME_TYPES.includes(req.headers['content-type']); } diff --git a/api/src/stream/internal.js b/api/src/stream/internal.js index 51552d4c..7d8bf4c9 100644 --- a/api/src/stream/internal.js +++ b/api/src/stream/internal.js @@ -1,7 +1,7 @@ import { request } from "undici"; import { Readable } from "node:stream"; import { closeRequest, getHeaders, pipe } from "./shared.js"; -import { handleHlsPlaylist, isHlsRequest } from "./internal-hls.js"; +import { handleHlsPlaylist, isHlsResponse } from "./internal-hls.js"; const CHUNK_SIZE = BigInt(8e6); // 8 MB const min = (a, b) => a < b ? a : b; @@ -83,7 +83,7 @@ async function handleGenericStream(streamInfo, res) { const cleanup = () => res.end(); try { - const req = await request(streamInfo.url, { + const fileResponse = await request(streamInfo.url, { headers: { ...Object.fromEntries(streamInfo.headers), host: undefined @@ -93,19 +93,28 @@ async function handleGenericStream(streamInfo, res) { maxRedirections: 16 }); - res.status(req.statusCode); - req.body.on('error', () => {}); + res.status(fileResponse.statusCode); + fileResponse.body.on('error', () => {}); - for (const [ name, value ] of Object.entries(req.headers)) - res.setHeader(name, value) + // bluesky's cdn responds with wrong content-type for the hls playlist, + // so we enforce it here until they fix it + const isHls = isHlsResponse(fileResponse) + || (streamInfo.service === "bsky" && streamInfo.url.endsWith('.m3u8')); - if (req.statusCode < 200 || req.statusCode > 299) + for (const [ name, value ] of Object.entries(fileResponse.headers)) { + if (!isHls || name.toLowerCase() !== 'content-length') { + res.setHeader(name, value); + } + } + + if (fileResponse.statusCode < 200 || fileResponse.statusCode > 299) { return cleanup(); + } - if (isHlsRequest(req)) { - await handleHlsPlaylist(streamInfo, req, res); + if (isHls) { + await handleHlsPlaylist(streamInfo, fileResponse, res); } else { - pipe(req.body, res, cleanup); + pipe(fileResponse.body, res, cleanup); } } catch { closeRequest(streamInfo.controller); @@ -114,7 +123,11 @@ async function handleGenericStream(streamInfo, res) { } export function internalStream(streamInfo, res) { - if (streamInfo.service === 'youtube') { + if (streamInfo.headers) { + streamInfo.headers.delete('icy-metadata'); + } + + if (streamInfo.service === 'youtube' && !streamInfo.isHLS) { return handleYoutubeStream(streamInfo, res); } diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index e25f4434..79b5c1db 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -1,4 +1,4 @@ -import NodeCache from "node-cache"; +import Store from "../store/store.js"; import { nanoid } from "nanoid"; import { randomBytes } from "crypto"; @@ -7,34 +7,26 @@ import { setMaxListeners } from "node:events"; import { env } from "../config.js"; import { closeRequest } from "./shared.js"; -import { decryptStream, encryptStream, generateHmac } from "../misc/crypto.js"; +import { decryptStream, encryptStream } from "../misc/crypto.js"; +import { hashHmac } from "../security/secrets.js"; // optional dependency const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); -const streamCache = new NodeCache({ - stdTTL: env.streamLifespan, - checkperiod: 10, - deleteOnExpire: true -}) - -streamCache.on("expired", (key) => { - streamCache.del(key); -}) +const streamCache = new Store('streams'); const internalStreamCache = new Map(); -const hmacSalt = randomBytes(64).toString('hex'); export function createStream(obj) { const streamID = nanoid(), iv = randomBytes(16).toString('base64url'), secret = randomBytes(32).toString('base64url'), exp = new Date().getTime() + env.streamLifespan * 1000, - hmac = generateHmac(`${streamID},${exp},${iv},${secret}`, hmacSalt), + hmac = hashHmac(`${streamID},${exp},${iv},${secret}`, 'stream').toString('base64url'), streamData = { exp: exp, type: obj.type, - urls: obj.u, + urls: obj.url, service: obj.service, filename: obj.filename, @@ -46,12 +38,18 @@ export function createStream(obj) { audioBitrate: obj.audioBitrate, audioCopy: !!obj.audioCopy, audioFormat: obj.audioFormat, + + isHLS: obj.isHLS || false, }; + // FIXME: this is now a Promise, but it is not awaited + // here. it may happen that the stream is not + // stored in the Store before it is requested. streamCache.set( streamID, - encryptStream(streamData, iv, secret) - ) + encryptStream(streamData, iv, secret), + env.streamLifespan + ); let streamLink = new URL('/tunnel', env.apiURL); @@ -77,7 +75,7 @@ export function getInternalStream(id) { export function createInternalStream(url, obj = {}) { assert(typeof url === 'string'); - let dispatcher; + let dispatcher = obj.dispatcher; if (obj.requestIP) { dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { strict: false }) } @@ -100,10 +98,11 @@ export function createInternalStream(url, obj = {}) { service: obj.service, headers, controller, - dispatcher + dispatcher, + isHLS: obj.isHLS, }); - let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.apiPort}`); + let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.tunnelPort}`); streamLink.searchParams.set('id', streamID); const cleanup = () => { @@ -146,10 +145,10 @@ function wrapStream(streamInfo) { return streamInfo; } -export function verifyStream(id, hmac, exp, secret, iv) { +export async function verifyStream(id, hmac, exp, secret, iv) { try { - const ghmac = generateHmac(`${id},${exp},${iv},${secret}`, hmacSalt); - const cache = streamCache.get(id.toString()); + const ghmac = hashHmac(`${id},${exp},${iv},${secret}`, 'stream').toString('base64url'); + const cache = await streamCache.get(id.toString()); if (ghmac !== String(hmac)) return { status: 401 }; if (!cache) return { status: 404 }; diff --git a/api/src/stream/shared.js b/api/src/stream/shared.js index 91e1ac2f..65af03f0 100644 --- a/api/src/stream/shared.js +++ b/api/src/stream/shared.js @@ -1,4 +1,5 @@ import { genericUserAgent } from "../config.js"; +import { vkClientAgent } from "../processing/services/vk.js"; const defaultHeaders = { 'user-agent': genericUserAgent @@ -13,6 +14,9 @@ const serviceHeaders = { origin: 'https://www.youtube.com', referer: 'https://www.youtube.com', DNT: '?1' + }, + vk: { + 'user-agent': vkClientAgent } } diff --git a/api/src/stream/types.js b/api/src/stream/types.js index 184af873..98c3b04e 100644 --- a/api/src/stream/types.js +++ b/api/src/stream/types.js @@ -4,7 +4,6 @@ import { spawn } from "child_process"; import { create as contentDisposition } from "content-disposition-header"; import { env } from "../config.js"; -import { metadataManager } from "../misc/utils.js"; import { destroyInternalStream } from "./manage.js"; import { hlsExceptions } from "../processing/service-config.js"; import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js"; @@ -16,6 +15,29 @@ const ffmpegArgs = { gif: ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"] } +const metadataTags = [ + "album", + "copyright", + "title", + "artist", + "track", + "date", +]; + +const convertMetadataToFFmpeg = (metadata) => { + let args = []; + + for (const [ name, value ] of Object.entries(metadata)) { + if (metadataTags.includes(name)) { + args.push('-metadata', `${name}=${value.replace(/[\u0000-\u0009]/g, "")}`); + } else { + throw `${name} metadata tag is not supported.`; + } + } + + return args; +} + const toRawHeaders = (headers) => { return Object.entries(headers) .map(([key, value]) => `${key}: ${value}\r\n`) @@ -101,12 +123,16 @@ const merge = (streamInfo, res) => { args = args.concat(ffmpegArgs[format]); - if (hlsExceptions.includes(streamInfo.service)) { - args.push('-bsf:a', 'aac_adtstoasc') + if (hlsExceptions.includes(streamInfo.service) && streamInfo.isHLS) { + if (streamInfo.service === "youtube" && format === "webm") { + args.push('-c:a', 'libopus'); + } else { + args.push('-c:a', 'aac', '-bsf:a', 'aac_adtstoasc'); + } } if (streamInfo.metadata) { - args = args.concat(metadataManager(streamInfo.metadata)) + args = args.concat(convertMetadataToFFmpeg(streamInfo.metadata)) } args.push('-f', format, 'pipe:3'); @@ -238,7 +264,7 @@ const convertAudio = (streamInfo, res) => { } if (streamInfo.metadata) { - args = args.concat(metadataManager(streamInfo.metadata)) + args = args.concat(convertMetadataToFFmpeg(streamInfo.metadata)) } args.push('-f', streamInfo.audioFormat === "m4a" ? "ipod" : streamInfo.audioFormat, 'pipe:3'); diff --git a/api/src/util/generate-jwt-secret.js b/api/src/util/generate-jwt-secret.js new file mode 100644 index 00000000..8db6e230 --- /dev/null +++ b/api/src/util/generate-jwt-secret.js @@ -0,0 +1,22 @@ +// run with `pnpm -r token:jwt` + +const makeSecureString = (length = 64) => { + const alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-'; + const out = []; + + while (out.length < length) { + for (const byte of crypto.getRandomValues(new Uint8Array(length))) { + if (byte < alphabet.length) { + out.push(alphabet[byte]); + } + + if (out.length === length) { + break; + } + } + } + + return out.join(''); +} + +console.log(`JWT_SECRET: ${JSON.stringify(makeSecureString(64))}`) diff --git a/api/src/util/test.js b/api/src/util/test.js index 34afde7e..2ba555ed 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -1,84 +1,129 @@ -import "dotenv/config"; +import path from "node:path"; + +import { env } from "../config.js"; +import { runTest } from "../misc/run-test.js"; +import { loadJSON } from "../misc/load-from-fs.js"; +import { Red, Bright } from "../misc/console-text.js"; +import { randomizeCiphers } from "../misc/randomize-ciphers.js"; import { services } from "../processing/service-config.js"; -import { extract } from "../processing/url.js"; -import match from "../processing/match.js"; -import { loadJSON } from "../misc/load-from-fs.js"; -import { normalizeRequest } from "../processing/request.js"; -import { env } from "../config.js"; -env.apiURL = 'http://localhost:9000' -let tests = loadJSON('./src/util/tests.json'); +const getTestPath = service => path.join('./src/util/tests/', `./${service}.json`); +const getTests = (service) => loadJSON(getTestPath(service)); -let noTest = []; -let failed = []; -let success = 0; +// services that are known to frequently fail due to external +// factors (e.g. rate limiting) +const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter']); -function addToFail(service, testName, url, status, response) { - failed.push({ - service: service, - name: testName, - url: url, - status: status, - response: response - }) -} -for (let i in services) { - if (tests[i]) { - console.log(`\nRunning tests for ${i}...\n`) - for (let k = 0; k < tests[i].length; k++) { - let test = tests[i][k]; +const runTestsFor = async (service) => { + const tests = getTests(service); + let softFails = 0, fails = 0; - console.log(`Running test ${k+1}: ${test.name}`); - console.log('params:'); - let params = {...{url: test.url}, ...test.params}; - console.log(params); - - let chck = await normalizeRequest(params); - if (chck.success) { - chck = chck.data; - - const parsed = extract(chck.url); - if (parsed === null) { - throw `Invalid URL: ${chck.url}` - } - - let j = await match({ - host: parsed.host, - patternMatch: parsed.patternMatch, - params: chck, - }); - console.log('\nReceived:'); - console.log(j) - if (j.status === test.expected.code && j.body.status === test.expected.status) { - console.log("\n✅ Success.\n"); - success++ - } else { - console.log(`\n❌ Fail. Expected: ${test.expected.code} & ${test.expected.status}, received: ${j.status} & ${j.body.status}\n`); - addToFail(i, test.name, test.url, j.body.status, j) - } - } else { - console.log("\n❌ couldn't validate the request JSON.\n"); - addToFail(i, test.name, test.url, "unknown", {}) - } - } - console.log("\n\n") - } else { - console.warn(`No tests found for ${i}.`); - noTest.push(i) + if (!tests) { + throw "no such service: " + service; } + + for (const test of tests) { + const { name, url, params, expected } = test; + const canFail = test.canFail || finnicky.has(service); + + try { + await runTest(url, params, expected); + console.log(`${service}/${name}: ok`); + + } catch (e) { + softFails += !canFail; + fails++; + + let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL')); + if (canFail && process.env.GITHUB_ACTION) { + console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`); + } + + console.error(`${service}/${name}: ${failText}`); + const errorString = e.toString().split('\n'); + let c = '┃'; + errorString.forEach((line, index) => { + line = line.replace('!=', Red('!=')); + + if (index === errorString.length - 1) { + c = '┗'; + } + + console.error(` ${c}`, line); + }); + } + } + + return { fails, softFails }; } -console.log(`✅ ${success} tests succeeded.`); -console.log(`❌ ${failed.length} tests failed.`); -console.log(`❔ ${noTest.length} services weren't tested.`); - -if (failed.length > 0) { - console.log(`\nFailed tests:`); - console.log(failed) +const printHeader = (service, padLen) => { + const padding = padLen - service.length; + service = service.padEnd(1 + service.length + padding, ' '); + console.log(service + '='.repeat(50)); } -if (noTest.length > 0) { - console.log(`\nMissing tests:`); - console.log(noTest) +const action = process.argv[2]; +switch (action) { + case "get-services": + const fromConfig = Object.keys(services); + + const missingTests = fromConfig.filter( + service => { + const tests = getTests(service); + return !tests || tests.length === 0 + } + ); + + if (missingTests.length) { + console.error('services have no tests:', missingTests); + process.exitCode = 1; + break; + } + + console.log(JSON.stringify(fromConfig)); + break; + + case "run-tests-for": + env.streamLifespan = 10000; + env.apiURL = 'http://x/'; + randomizeCiphers(); + + try { + const { softFails } = await runTestsFor(process.argv[3]); + process.exitCode = Number(!!softFails); + } catch (e) { + console.error(e); + process.exitCode = 1; + break; + } + + break; + default: + const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0); + const failCounters = {}; + + env.streamLifespan = 10000; + env.apiURL = 'http://x/'; + randomizeCiphers(); + + for (const service in services) { + printHeader(service, maxHeaderLen); + const { fails, softFails } = await runTestsFor(service); + failCounters[service] = fails; + console.log(); + + if (!process.exitCode && softFails) + process.exitCode = 1; + } + + console.log('='.repeat(50 + maxHeaderLen)); + console.log( + Bright('total fails:'), + Object.values(failCounters).reduce((a, b) => a + b) + ); + for (const [ service, fails ] of Object.entries(failCounters)) { + if (fails) console.log(`${Bright(service)} fails: ${fails}`); + } } diff --git a/api/src/util/tests/bilibili.json b/api/src/util/tests/bilibili.json new file mode 100644 index 00000000..61d60134 --- /dev/null +++ b/api/src/util/tests/bilibili.json @@ -0,0 +1,60 @@ +[ + { + "name": "1080p video", + "url": "https://www.bilibili.com/video/BV18i4y1m7xV/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "1080p video muted", + "url": "https://www.bilibili.com/video/BV18i4y1m7xV/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "1080p vertical video", + "url": "https://www.bilibili.com/video/BV1uu411z7VV/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "1080p vertical video muted", + "url": "https://www.bilibili.com/video/BV1uu411z7VV/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "b23.tv shortlink", + "url": "https://b23.tv/lbMyOI9", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "bilibili.tv link", + "url": "https://www.bilibili.tv/en/video/4789599404426256", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/bsky.json b/api/src/util/tests/bsky.json new file mode 100644 index 00000000..840f1169 --- /dev/null +++ b/api/src/util/tests/bsky.json @@ -0,0 +1,78 @@ +[ + { + "name": "horizontal video", + "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3giwtwp222m", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "horizontal video, recordWithMedia", + "url": "https://bsky.app/profile/did:plc:ywbm3iywnhzep3ckt6efhoh7/post/3l3wonhk23g2i", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video", + "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (muted)", + "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (audio)", + "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "single image", + "url": "https://bsky.app/profile/did:plc:k4a7d65fcyevbrnntjxh57go/post/3l33flpoygt26", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "several images", + "url": "https://bsky.app/profile/did:plc:rai7s6su2sy22ss7skouedl7/post/3kzxuxbiul626", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "deleted post/invalid user", + "url": "https://bsky.app/profile/notreal.bsky.team/post/3l2udah76ch2c", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/dailymotion.json b/api/src/util/tests/dailymotion.json new file mode 100644 index 00000000..4de9302c --- /dev/null +++ b/api/src/util/tests/dailymotion.json @@ -0,0 +1,29 @@ +[ + { + "name": "regular video", + "url": "https://www.dailymotion.com/video/x8t1eho", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private video", + "url": "https://www.dailymotion.com/video/k41fZWpx2TaAORA2nok", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "dai.ly shortened link", + "url": "https://dai.ly/k41fZWpx2TaAORA2nok", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/facebook.json b/api/src/util/tests/facebook.json new file mode 100644 index 00000000..876ac7fe --- /dev/null +++ b/api/src/util/tests/facebook.json @@ -0,0 +1,67 @@ +[ + { + "name": "direct video with username and id", + "url": "https://web.facebook.com/100048111287134/videos/1157798148685638/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "direct video with id as query param", + "url": "https://web.facebook.com/watch/?v=883839773514682&ref=sharing", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "direct video with caption", + "url": "https://web.facebook.com/wood57/videos/𝐒𝐞𝐛𝐚𝐬𝐤𝐨𝐦-𝐟𝐮𝐥𝐥/883839773514682", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "shortlink video", + "url": "https://fb.watch/r1K6XHMfGT/", + "canFail": true, + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "reel video", + "url": "https://web.facebook.com/reel/730293269054758", + "canFail": true, + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "shared video link", + "url": "https://www.facebook.com/share/v/NEf87jbPTvFE8LsL/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "shared video link v2", + "url": "https://web.facebook.com/share/r/JFZfPVgLkiJQmWrr/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/instagram.json b/api/src/util/tests/instagram.json new file mode 100644 index 00000000..2ee42194 --- /dev/null +++ b/api/src/util/tests/instagram.json @@ -0,0 +1,123 @@ +[ + { + "name": "single photo post", + "url": "https://www.instagram.com/p/CwIgW8Yu5-I/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "various picker (photos + video)", + "url": "https://www.instagram.com/p/CvYrSgnsKjv/", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "reel", + "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular video", + "url": "https://www.instagram.com/p/CmCVWoIr9OH/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "reel (isAudioOnly)", + "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "reel (isAudioMuted)", + "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent reel", + "url": "https://www.instagram.com/reel/XXXXXXXXXX/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "inexistent post", + "url": "https://www.instagram.com/p/XXXXXXXXXX/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "post info in an array (for whatever reason??)", + "url": "https://www.instagram.com/reel/CrVB9tatUDv/?igshid=blaBlABALALbLABULLSHIT==", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "prone to get rate limited", + "url": "https://www.instagram.com/reel/CrO-T7Qo6rq/?igshid=fuckYouNoTrackingIdForYou==", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "ddinstagram link", + "url": "https://ddinstagram.com/p/CmCVWoIr9OH/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "d.ddinstagram.com link", + "url": "https://d.ddinstagram.com/p/CmCVWoIr9OH/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "g.ddinstagram.com link", + "url": "https://g.ddinstagram.com/p/CmCVWoIr9OH/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/loom.json b/api/src/util/tests/loom.json new file mode 100644 index 00000000..1849a000 --- /dev/null +++ b/api/src/util/tests/loom.json @@ -0,0 +1,33 @@ +[ + { + "name": "1080p video", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "1080p video (muted)", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "1080p video (audio only)", + "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/ok.json b/api/src/util/tests/ok.json new file mode 100644 index 00000000..8eb103eb --- /dev/null +++ b/api/src/util/tests/ok.json @@ -0,0 +1,11 @@ +[ + { + "name": "regular video", + "url": "https://ok.ru/video/7204071410346", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/pinterest.json b/api/src/util/tests/pinterest.json new file mode 100644 index 00000000..2f15fb0b --- /dev/null +++ b/api/src/util/tests/pinterest.json @@ -0,0 +1,87 @@ +[ + { + "name": "regular video", + "url": "https://www.pinterest.com/pin/70437485604616/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular video (isAudioOnly)", + "url": "https://www.pinterest.com/pin/70437485604616/", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular video (isAudioMuted)", + "url": "https://www.pinterest.com/pin/70437485604616/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular video (.ca TLD)", + "url": "https://www.pinterest.ca/pin/70437485604616/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "story", + "url": "https://www.pinterest.com/pin/gadget-cool-products-amazon-product-technology-kitchen-gadgets--1084663891475263837/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular picture", + "url": "https://www.pinterest.com/pin/412994228343400946/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular picture (.ca TLD)", + "url": "https://www.pinterest.ca/pin/412994228343400946/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular gif", + "url": "https://www.pinterest.com/pin/814447913881127862/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular gif (.ca TLD)", + "url": "https://www.pinterest.ca/pin/814447913881127862/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/reddit.json b/api/src/util/tests/reddit.json new file mode 100644 index 00000000..3afc6126 --- /dev/null +++ b/api/src/util/tests/reddit.json @@ -0,0 +1,60 @@ +[ + { + "name": "video with audio", + "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "video with audio (isAudioOnly)", + "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "video with audio (isAudioMuted)", + "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "video without audio", + "url": "https://www.reddit.com/r/catvideos/comments/ftoeo7/luna_doesnt_want_to_be_bothered_while_shes_napping/?utm_source=share&utm_medium=web2x&context=3", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "actual gif, not looping video", + "url": "https://www.reddit.com/r/whenthe/comments/109wqy1/god_really_did_some_trolling/?utm_source=share&utm_medium=web2x&context=3", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "different audio link, live render", + "url": "https://www.reddit.com/r/TikTokCringe/comments/15hce91/asian_daddy_kink/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/rutube.json b/api/src/util/tests/rutube.json new file mode 100644 index 00000000..2eaf69bf --- /dev/null +++ b/api/src/util/tests/rutube.json @@ -0,0 +1,100 @@ +[ + { + "name": "regular video", + "url": "https://rutube.ru/video/b2f6c27649907c2fde0af411b03825eb/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (isAudioMuted)", + "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "russian region lock", + "url": "https://rutube.ru/video/b521653b4f71ece57b8ff54e57ca9b82/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "vertical video", + "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "yappy", + "url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/", + "canFail": true, + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "shorts", + "url": "https://rutube.ru/shorts/935c1afafd0e7d52836d671967d53dac/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (isAudioOnly)", + "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (isAudioMuted)", + "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private video", + "url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "region locked video, should fail", + "canFail": true, + "url": "https://rutube.ru/video/e7ac82708cc22bd068a3bf6a7004d1b1/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/snapchat.json b/api/src/util/tests/snapchat.json new file mode 100644 index 00000000..44f764ce --- /dev/null +++ b/api/src/util/tests/snapchat.json @@ -0,0 +1,29 @@ +[ + { + "name": "spotlight", + "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "shortlinked spotlight", + "url": "https://t.snapchat.com/4ZsiBLDi", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "story", + "url": "https://www.snapchat.com/add/bazerkmakane", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/soundcloud.json b/api/src/util/tests/soundcloud.json new file mode 100644 index 00000000..04ed8632 --- /dev/null +++ b/api/src/util/tests/soundcloud.json @@ -0,0 +1,106 @@ +[ + { + "name": "public song (best)", + "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing", + "params": { + "audioFormat": "best" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "public song (mp3, isAudioMuted)", + "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing", + "params": { + "downloadMode": "mute", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private song", + "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private song (wav, isAudioMuted)", + "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90", + "params": { + "downloadMode": "mute", + "audioFormat": "wav" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private song (ogg, isAudioMuted, isAudioOnly)", + "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90", + "params": { + "downloadMode": "audio", + "audioFormat": "ogg" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "on.soundcloud link", + "url": "https://on.soundcloud.com/wLZre", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "on.soundcloud link, different stream type", + "url": "https://on.soundcloud.com/AG4c", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "no opus audio, fallback to mp3", + "url": "https://soundcloud.com/frums/credits", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "go+ song, should fail", + "url": "https://soundcloud.com/dualipa/illusion-1", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "region locked song, should fail", + "canFail": true, + "url": "https://soundcloud.com/gotye/somebody-2024-feat-kimbra", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/streamable.json b/api/src/util/tests/streamable.json new file mode 100644 index 00000000..bf03c228 --- /dev/null +++ b/api/src/util/tests/streamable.json @@ -0,0 +1,51 @@ +[ + { + "name": "regular video", + "url": "https://streamable.com/p9cln4", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "embedded link", + "url": "https://streamable.com/e/rsmo56", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular video (isAudioOnly)", + "url": "https://streamable.com/p9cln4", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular video (isAudioMuted)", + "url": "https://streamable.com/p9cln4", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent video", + "url": "https://streamable.com/XXXXXX", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/tiktok.json b/api/src/util/tests/tiktok.json new file mode 100644 index 00000000..c8dbce8c --- /dev/null +++ b/api/src/util/tests/tiktok.json @@ -0,0 +1,47 @@ +[ + { + "name": "long link video", + "url": "https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "images", + "url": "https://www.tiktok.com/@matryoshk4/video/7231234675476532526", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "long link inexistent", + "url": "https://www.tiktok.com/@blablabla/video/7120851458451417478", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "short link inexistent", + "url": "https://vt.tiktok.com/2p4ewa7/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "age restricted video", + "url": "https://www.tiktok.com/@.kyle.films/video/7415757181145877793", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/tumblr.json b/api/src/util/tests/tumblr.json new file mode 100644 index 00000000..87352255 --- /dev/null +++ b/api/src/util/tests/tumblr.json @@ -0,0 +1,49 @@ +[ + { + "name": "at.tumblr link", + "url": "https://at.tumblr.com/music/704177038274281472/n7x7pr7x4w2b", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "user subdomain link", + "url": "https://garfield-69.tumblr.com/post/696499862852780032", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "web app link", + "url": "https://www.tumblr.com/rongzhi/707729381162958848/english-added-by-me?source=share", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "tumblr audio", + "url": "https://www.tumblr.com/zedneon/737815079301562368/zedneon-ft-mr-sauceman-tech-n9ne-speed-of?source=share", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "tumblr video converted to audio", + "url": "https://garfield-69.tumblr.com/post/696499862852780032", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/twitch.json b/api/src/util/tests/twitch.json new file mode 100644 index 00000000..fd6b84af --- /dev/null +++ b/api/src/util/tests/twitch.json @@ -0,0 +1,33 @@ +[ + { + "name": "clip", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "clip (isAudioOnly)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "clip (isAudioMuted)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json new file mode 100644 index 00000000..0024d097 --- /dev/null +++ b/api/src/util/tests/twitter.json @@ -0,0 +1,213 @@ +[ + { + "name": "regular video", + "url": "https://twitter.com/X/status/1697304622749086011", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "video with mobile web mediaviewer", + "url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011¤tTweetUser=X¤tTweet=1697304622749086011¤tTweetUser=X", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "embedded twitter video", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "mixed media (image + gif)", + "url": "https://twitter.com/sky_mj26/status/1807756010712428565", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "picker: mixed media (3 videos)", + "url": "https://twitter.com/DankGameAlert/status/1584726006094794774", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "audio from embedded twitter video (mp3, isAudioOnly)", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "downloadMode": "audio", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "audio from embedded twitter video (best, isAudioOnly)", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "downloadMode": "audio", + "audioFormat": "best" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "audio from embedded twitter video (ogg, isAudioOnly, isAudioMuted)", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "downloadMode": "audio", + "audioFormat": "best" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "muted embedded twitter video", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "downloadMode": "mute", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "retweeted video", + "url": "https://twitter.com/uwukko/status/1696901469633421344", + "params": {}, + "canFail": true, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "age restricted video", + "url": "https://x.com/XSpaces/status/1526955853743546372", + "params": {}, + "canFail": true, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "twitter voice + x.com link", + "url": "https://x.com/eggsaladscreams/status/1693089534886506756?s=46", + "params": {}, + "canFail": true, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "vxtwitter link", + "url": "https://vxtwitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "post with 1 image", + "url": "https://x.com/PopCrave/status/1815960083475423235", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "post with 4 images", + "url": "https://x.com/PopCrave/status/1816260887147114696", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "retweeted video, isAudioOnly", + "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg", + "params": { + "downloadMode": "mute", + "audioFormat": "mp3" + }, + "canFail": true, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent post", + "url": "https://twitter.com/test/status/9487653", + "params": { + "audioFormat": "best" + }, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "post with no media content", + "url": "https://twitter.com/elonmusk/status/1604617643973124097?s=20", + "params": { + "audioFormat": "best" + }, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "bookmarked video", + "url": "https://twitter.com/i/bookmarks?post_id=1828099210220294314", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "bookmarked photo", + "url": "https://twitter.com/i/bookmarks?post_id=1837430141179289876", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/vimeo.json b/api/src/util/tests/vimeo.json new file mode 100644 index 00000000..6c44a47d --- /dev/null +++ b/api/src/util/tests/vimeo.json @@ -0,0 +1,64 @@ +[ + { + "name": "4k progressive", + "url": "https://vimeo.com/288386543", + "params": { + "videoQuality": "2160" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "720p progressive", + "url": "https://vimeo.com/288386543", + "params": { + "videoQuality": "720" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "1080p dash parcel", + "url": "https://vimeo.com/967252742", + "params": { + "videoQuality": "1440" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "720p dash parcel", + "url": "https://vimeo.com/967252742", + "params": { + "videoQuality": "360" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "private video", + "url": "https://vimeo.com/903115595/f14d06da38", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "mature video", + "url": "https://vimeo.com/973212054", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/vk.json b/api/src/util/tests/vk.json new file mode 100644 index 00000000..71720af5 --- /dev/null +++ b/api/src/util/tests/vk.json @@ -0,0 +1,82 @@ +[ + { + "name": "clip, defaults", + "url": "https://vk.com/clip-57274055_456239788", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "clip, 360", + "url": "https://vk.com/clip-57274055_456239788", + "params": { + "videoQuality": "360" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "clip different link, max", + "url": "https://vk.com/clips-57274055?z=clip-57274055_456239788", + "params": { + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "video, defaults", + "url": "https://vk.com/video-57274055_456239399", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "big 4k video", + "url": "https://vk.com/video-1112285_456248465", + "params": { + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "short 4k video, 480p, vkvideo.ru domain", + "url": "https://vkvideo.ru/video-26006257_456245538", + "params": { + "videoQuality": "480" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "ancient video (fallback to 240p)", + "url": "https://vk.com/video-1959_28496479", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent video", + "url": "https://vk.com/video-53333333_456233333", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] diff --git a/api/src/util/tests/youtube.json b/api/src/util/tests/youtube.json new file mode 100644 index 00000000..0655e683 --- /dev/null +++ b/api/src/util/tests/youtube.json @@ -0,0 +1,240 @@ +[ + { + "name": "4k video (h264, 1440)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "h264", + "videoQuality": "1440" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (vp9, 720)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "vp9", + "videoQuality": "720" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (av1, max)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "av1", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (h264, 720)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "h264", + "videoQuality": "720" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (vp9, max, isAudioMuted)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "mute", + "youtubeVideoCodec": "vp9", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (h264, max, isAudioMuted)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "mute", + "youtubeVideoCodec": "h264", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (av1, max, isAudioMuted, isAudioOnly, mp3)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "audio", + "audioFormat": "mp3", + "youtubeVideoCodec": "av1", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (av1, max, isAudioMuted, isAudioOnly, best)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "audio", + "audioFormat": "best", + "youtubeVideoCodec": "av1", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "music (mp3, isAudioOnly, isAudioMuted)", + "url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share", + "params": { + "downloadMode": "audio", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "music (mp3)", + "url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "audio bitrate higher than video, no vp9 video in response (mp3, isAudioOnly)", + "url": "https://www.youtube.com/watch?v=t5nC_ucYBrc", + "params": { + "downloadMode": "audio", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "short, defaults", + "url": "https://www.youtube.com/shorts/r5FpeOJItbw", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vr 360, av1, max", + "url": "https://www.youtube.com/watch?v=hEdzv7D4CbQ", + "params": { + "youtubeVideoCodec": "vp9", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "live link, defaults", + "url": "https://www.youtube.com/live/ENxZS6PUDuI?feature=shared", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent video", + "url": "https://youtube.com/watch?v=gnjuHYWGEW", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "broken audioOnly download", + "url": "https://www.youtube.com/watch?v=ink80Al5nbw", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "hls video (h264, 1440p)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "h264", + "videoQuality": "1440", + "youtubeHLS": true + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "hls video (vp9, 360p)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "vp9", + "videoQuality": "360", + "youtubeHLS": true + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "hls video (audio mode)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "audio", + "youtubeHLS": true + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "hls video (audio mode, best format)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "audio", + "youtubeHLS": true, + "audioFormat": "best" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 220a2cf3..ec6b0095 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,6 +10,12 @@ importers: api: dependencies: + '@datastructures-js/priority-queue': + specifier: ^6.3.1 + version: 6.3.1 + '@imput/psl': + specifier: ^2.0.4 + version: 2.0.4 '@imput/version-info': specifier: workspace:^ version: link:../packages/version-info @@ -26,11 +32,11 @@ importers: specifier: ^0.14.51 version: 0.14.54 express: - specifier: ^4.18.1 - version: 4.19.2 + specifier: ^4.21.1 + version: 4.21.2 express-rate-limit: - specifier: ^6.3.0 - version: 6.11.2(express@4.19.2) + specifier: ^7.4.1 + version: 7.4.1(express@4.21.2) ffmpeg-static: specifier: ^5.1.0 version: 5.2.0 @@ -38,17 +44,14 @@ importers: specifier: ^0.10.7 version: 0.10.9 ipaddr.js: - specifier: 2.1.0 - version: 2.1.0 + specifier: 2.2.0 + version: 2.2.0 nanoid: specifier: ^4.0.2 version: 4.0.2 node-cache: specifier: ^5.1.2 version: 5.1.2 - psl: - specifier: 1.9.0 - version: 1.9.0 set-cookie-parser: specifier: 2.6.0 version: 2.6.0 @@ -59,8 +62,8 @@ importers: specifier: 1.0.3 version: 1.0.3 youtubei.js: - specifier: ^10.3.0 - version: 10.3.0 + specifier: ^11.0.1 + version: 11.0.1 zod: specifier: ^3.23.8 version: 3.23.8 @@ -68,6 +71,12 @@ importers: freebind: specifier: ^0.2.2 version: 0.2.2 + rate-limit-redis: + specifier: ^4.2.0 + version: 4.2.0(express-rate-limit@7.4.1(express@4.21.2)) + redis: + specifier: ^4.7.0 + version: 4.7.0 packages/api-client: devDependencies: @@ -182,6 +191,15 @@ packages: resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} + '@bufbuild/protobuf@2.2.3': + resolution: {integrity: sha512-tFQoXHJdkEOSwj5tRIZSPNUuXK3RaR7T1nUrPgbYX1pUbvqqaaZAsfo+NXBPsz5rZMSKVFrgK1WL8Q/MSLvprg==} + + '@datastructures-js/heap@4.3.3': + resolution: {integrity: sha512-UcUu/DLh/aM4W3C8zZfwxxm6/6FIZUlm3mcAXuNOCa6Aj4iizNvNXQyb8DjZQH2jKSQbMRyNlngP6TPimuGjpQ==} + + '@datastructures-js/priority-queue@6.3.1': + resolution: {integrity: sha512-eoxkWql/j0VJ0UFMFTpnyJz4KbEEVQ6aZ/JuJUgenu0Im4tYKylAycNGsYCHGXiVNEd7OKGVwfx1Ac3oYkuu7A==} + '@derhuerst/http-basic@8.2.4': resolution: {integrity: sha512-F9rL9k9Xjf5blCz8HsJRO4diy111cayL2vkY2XE4r4t3n0yPXVYy3KD3nJ1qbrSn9743UWSXH4IwuCa/HWlGFw==} engines: {node: '>=6.0.0'} @@ -525,6 +543,9 @@ packages: '@imput/libav.js-remux-cli@5.5.6': resolution: {integrity: sha512-XdAab90EZKf6ULtD/x9Y2bnlmNJodXSO6w8aWrn97+N2IRuOS8zv3tAFPRC69SWKa8Utjeu5YTYuTolnX3QprQ==} + '@imput/psl@2.0.4': + resolution: {integrity: sha512-vuy76JX78/DnJegLuJoLpMmw11JTA/9HvlIADg/f8dDVXyxbh0jnObL0q13h+WvlBO4Gk26Pu8sUa7/h0JGQig==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -566,6 +587,35 @@ packages: '@polka/url@1.0.0-next.25': resolution: {integrity: sha512-j7P6Rgr3mmtdkeDGTe0E/aYyWEWVtc5yFXtHCRHs28/jptDEWfaVOc5T7cblqy1XKPPfCxJc/8DwQ5YgLOZOVQ==} + '@redis/bloom@1.2.0': + resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/client@1.6.0': + resolution: {integrity: sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==} + engines: {node: '>=14'} + + '@redis/graph@1.1.1': + resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/json@1.0.7': + resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/search@1.2.0': + resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/time-series@1.1.0': + resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==} + peerDependencies: + '@redis/client': ^1.0.0 + '@rollup/rollup-android-arm-eabi@4.19.2': resolution: {integrity: sha512-OHflWINKtoCFSpm/WmuQaWW4jeX+3Qt3XQDepkkiFTsoxFc5BpF3Z5aDxFZgBqRjO6ATP5+b1iilp4kGIZVWlA==} cpu: [arm] @@ -856,8 +906,8 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} - body-parser@1.20.2: - resolution: {integrity: sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==} + body-parser@1.20.3: + resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} brace-expansion@1.1.11: @@ -914,6 +964,10 @@ packages: resolution: {integrity: sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==} engines: {node: '>=0.8'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + code-red@1.0.4: resolution: {integrity: sha512-7qJWqItLA8/VPVlKJlFXU+NBlo/qyfs39aJcuMT/2ere32ZqvF5OSxgdM5xOfJJ7O429gg2HM47y8v9P+9wrNw==} @@ -960,6 +1014,10 @@ packages: resolution: {integrity: sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==} engines: {node: '>= 0.6'} + cookie@0.7.1: + resolution: {integrity: sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==} + engines: {node: '>= 0.6'} + cors@2.8.5: resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==} engines: {node: '>= 0.10'} @@ -1047,6 +1105,10 @@ packages: resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} engines: {node: '>= 0.8'} + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} @@ -1251,14 +1313,14 @@ packages: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} - express-rate-limit@6.11.2: - resolution: {integrity: sha512-a7uwwfNTh1U60ssiIkuLFWHt4hAC5yxlLGU2VP0X4YNlyEDZAqF4tK3GD3NSitVBrCQmQ0++0uOyFOgC2y4DDw==} - engines: {node: '>= 14'} + express-rate-limit@7.4.1: + resolution: {integrity: sha512-KS3efpnpIDVIXopMc65EMbWbUht7qvTCdtCR2dD/IZmi9MIkopYESwyRqLgv8Pfu589+KqDqOdzJWW7AHoACeg==} + engines: {node: '>= 16'} peerDependencies: - express: ^4 || ^5 + express: 4 || 5 || ^5.0.0-beta.1 - express@4.19.2: - resolution: {integrity: sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==} + express@4.21.2: + resolution: {integrity: sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==} engines: {node: '>= 0.10.0'} fast-deep-equal@3.1.3: @@ -1289,8 +1351,8 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} - finalhandler@1.2.0: - resolution: {integrity: sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==} + finalhandler@1.3.1: + resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==} engines: {node: '>= 0.8'} find-up@5.0.0: @@ -1330,6 +1392,10 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + generic-pool@3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + get-intrinsic@1.2.4: resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} engines: {node: '>= 0.4'} @@ -1448,6 +1514,10 @@ packages: resolution: {integrity: sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==} engines: {node: '>= 10'} + ipaddr.js@2.2.0: + resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} + engines: {node: '>= 10'} + is-binary-path@2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} engines: {node: '>=8'} @@ -1485,8 +1555,8 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} - jintr@2.1.1: - resolution: {integrity: sha512-89cwX4ouogeDGOBsEVsVYsnWWvWjchmwXBB4kiBhmjOKw19FiOKhNhMhpxhTlK2ctl7DS+d/ethfmuBpzoNNgA==} + jintr@3.1.0: + resolution: {integrity: sha512-azhCHApkRfBH8INpiUCwKBYaNCdB5G+x3NApsI2MxQXSlgFAx7rap3YwE3JAkN08GO8f3ilZsGB0Yvc+412ntQ==} joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} @@ -1558,8 +1628,8 @@ packages: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} - merge-descriptors@1.0.1: - resolution: {integrity: sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==} + merge-descriptors@1.0.3: + resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} @@ -1730,8 +1800,8 @@ packages: resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} engines: {node: '>=16 || 14 >=14.18'} - path-to-regexp@0.1.7: - resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} + path-to-regexp@0.1.12: + resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} path-type@4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} @@ -1797,15 +1867,12 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} - psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} - punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} - qs@6.11.0: - resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==} + qs@6.13.0: + resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} engines: {node: '>=0.6'} queue-microtask@1.2.3: @@ -1815,6 +1882,12 @@ packages: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} engines: {node: '>= 0.6'} + rate-limit-redis@4.2.0: + resolution: {integrity: sha512-wV450NQyKC24NmPosJb2131RoczLdfIJdKCReNwtVpm5998U8SgKrAZrIHaN/NfQgqOHaan8Uq++B4sa5REwjA==} + engines: {node: '>= 16'} + peerDependencies: + express-rate-limit: '>= 6' + raw-body@2.5.2: resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} engines: {node: '>= 0.8'} @@ -1827,6 +1900,9 @@ packages: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} + redis@4.7.0: + resolution: {integrity: sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==} + resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -1875,12 +1951,12 @@ packages: engines: {node: '>=10'} hasBin: true - send@0.18.0: - resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==} + send@0.19.0: + resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} engines: {node: '>= 0.8.0'} - serve-static@1.15.0: - resolution: {integrity: sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==} + serve-static@1.16.2: + resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==} engines: {node: '>= 0.8.0'} set-cookie-parser@2.6.0: @@ -2271,12 +2347,15 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - youtubei.js@10.3.0: - resolution: {integrity: sha512-tLmeJCECK2xF2hZZtF2nEqirdKVNLFSDpa0LhTaXY3tngtL7doQXyy7M2CLueramDTlmCnFaW+rctHirTPFaRQ==} + youtubei.js@11.0.1: + resolution: {integrity: sha512-ZsbOd+5XF2Ofi3FrLMfYd+f9g9H8xswlouFhjhOqbwT68dMJtX6CRGsHNj5VTFCR/+L/865x1lnUlllB2dDDTA==} zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -2288,6 +2367,14 @@ snapshots: '@jridgewell/gen-mapping': 0.3.5 '@jridgewell/trace-mapping': 0.3.25 + '@bufbuild/protobuf@2.2.3': {} + + '@datastructures-js/heap@4.3.3': {} + + '@datastructures-js/priority-queue@6.3.1': + dependencies: + '@datastructures-js/heap': 4.3.3 + '@derhuerst/http-basic@8.2.4': dependencies: caseless: 0.12.0 @@ -2486,6 +2573,10 @@ snapshots: '@imput/libav.js-remux-cli@5.5.6': {} + '@imput/psl@2.0.4': + dependencies: + punycode: 2.3.1 + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -2529,6 +2620,38 @@ snapshots: '@polka/url@1.0.0-next.25': {} + '@redis/bloom@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + + '@redis/client@1.6.0': + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + optional: true + + '@redis/graph@1.1.1(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + + '@redis/json@1.0.7(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + + '@redis/search@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + + '@redis/time-series@1.1.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + '@rollup/rollup-android-arm-eabi@4.19.2': optional: true @@ -2808,7 +2931,7 @@ snapshots: binary-extensions@2.3.0: {} - body-parser@1.20.2: + body-parser@1.20.3: dependencies: bytes: 3.1.2 content-type: 1.0.5 @@ -2818,7 +2941,7 @@ snapshots: http-errors: 2.0.0 iconv-lite: 0.4.24 on-finished: 2.4.1 - qs: 6.11.0 + qs: 6.13.0 raw-body: 2.5.2 type-is: 1.6.18 unpipe: 1.0.0 @@ -2882,6 +3005,9 @@ snapshots: clone@2.1.2: {} + cluster-key-slot@1.1.2: + optional: true + code-red@1.0.4: dependencies: '@jridgewell/sourcemap-codec': 1.5.0 @@ -2923,6 +3049,8 @@ snapshots: cookie@0.6.0: {} + cookie@0.7.1: {} + cors@2.8.5: dependencies: object-assign: 4.1.1 @@ -2987,6 +3115,8 @@ snapshots: encodeurl@1.0.2: {} + encodeurl@2.0.0: {} + env-paths@2.2.1: {} es-define-property@1.0.0: @@ -3226,38 +3356,38 @@ snapshots: signal-exit: 3.0.7 strip-final-newline: 2.0.0 - express-rate-limit@6.11.2(express@4.19.2): + express-rate-limit@7.4.1(express@4.21.2): dependencies: - express: 4.19.2 + express: 4.21.2 - express@4.19.2: + express@4.21.2: dependencies: accepts: 1.3.8 array-flatten: 1.1.1 - body-parser: 1.20.2 + body-parser: 1.20.3 content-disposition: 0.5.4 content-type: 1.0.5 - cookie: 0.6.0 + cookie: 0.7.1 cookie-signature: 1.0.6 debug: 2.6.9 depd: 2.0.0 - encodeurl: 1.0.2 + encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 - finalhandler: 1.2.0 + finalhandler: 1.3.1 fresh: 0.5.2 http-errors: 2.0.0 - merge-descriptors: 1.0.1 + merge-descriptors: 1.0.3 methods: 1.1.2 on-finished: 2.4.1 parseurl: 1.3.3 - path-to-regexp: 0.1.7 + path-to-regexp: 0.1.12 proxy-addr: 2.0.7 - qs: 6.11.0 + qs: 6.13.0 range-parser: 1.2.1 safe-buffer: 5.2.1 - send: 0.18.0 - serve-static: 1.15.0 + send: 0.19.0 + serve-static: 1.16.2 setprototypeof: 1.2.0 statuses: 2.0.1 type-is: 1.6.18 @@ -3301,10 +3431,10 @@ snapshots: dependencies: to-regex-range: 5.0.1 - finalhandler@1.2.0: + finalhandler@1.3.1: dependencies: debug: 2.6.9 - encodeurl: 1.0.2 + encodeurl: 2.0.0 escape-html: 1.0.3 on-finished: 2.4.1 parseurl: 1.3.3 @@ -3349,6 +3479,9 @@ snapshots: function-bind@1.1.2: {} + generic-pool@3.9.0: + optional: true + get-intrinsic@1.2.4: dependencies: es-errors: 1.3.0 @@ -3471,7 +3604,10 @@ snapshots: ipaddr.js@1.9.1: {} - ipaddr.js@2.1.0: {} + ipaddr.js@2.1.0: + optional: true + + ipaddr.js@2.2.0: {} is-binary-path@2.1.0: dependencies: @@ -3503,7 +3639,7 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 - jintr@2.1.1: + jintr@3.1.0: dependencies: acorn: 8.12.1 @@ -3564,7 +3700,7 @@ snapshots: media-typer@0.3.0: {} - merge-descriptors@1.0.1: {} + merge-descriptors@1.0.3: {} merge-stream@2.0.0: {} @@ -3695,7 +3831,7 @@ snapshots: lru-cache: 10.4.3 minipass: 7.1.2 - path-to-regexp@0.1.7: {} + path-to-regexp@0.1.12: {} path-type@4.0.0: {} @@ -3738,11 +3874,9 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 - psl@1.9.0: {} - punycode@2.3.1: {} - qs@6.11.0: + qs@6.13.0: dependencies: side-channel: 1.0.6 @@ -3750,6 +3884,11 @@ snapshots: range-parser@1.2.1: {} + rate-limit-redis@4.2.0(express-rate-limit@7.4.1(express@4.21.2)): + dependencies: + express-rate-limit: 7.4.1(express@4.21.2) + optional: true + raw-body@2.5.2: dependencies: bytes: 3.1.2 @@ -3767,6 +3906,16 @@ snapshots: dependencies: picomatch: 2.3.1 + redis@4.7.0: + dependencies: + '@redis/bloom': 1.2.0(@redis/client@1.6.0) + '@redis/client': 1.6.0 + '@redis/graph': 1.1.1(@redis/client@1.6.0) + '@redis/json': 1.0.7(@redis/client@1.6.0) + '@redis/search': 1.2.0(@redis/client@1.6.0) + '@redis/time-series': 1.1.0(@redis/client@1.6.0) + optional: true + resolve-from@4.0.0: {} resolve-from@5.0.0: {} @@ -3824,7 +3973,7 @@ snapshots: semver@7.6.3: {} - send@0.18.0: + send@0.19.0: dependencies: debug: 2.6.9 depd: 2.0.0 @@ -3842,12 +3991,12 @@ snapshots: transitivePeerDependencies: - supports-color - serve-static@1.15.0: + serve-static@1.16.2: dependencies: - encodeurl: 1.0.2 + encodeurl: 2.0.0 escape-html: 1.0.3 parseurl: 1.3.3 - send: 0.18.0 + send: 0.19.0 transitivePeerDependencies: - supports-color @@ -4183,11 +4332,15 @@ snapshots: wrappy@1.0.2: {} + yallist@4.0.0: + optional: true + yocto-queue@0.1.0: {} - youtubei.js@10.3.0: + youtubei.js@11.0.1: dependencies: - jintr: 2.1.1 + '@bufbuild/protobuf': 2.2.3 + jintr: 3.1.0 tslib: 2.6.3 undici: 5.28.4 diff --git a/web/src/lib/env.ts b/web/src/lib/env.ts index c5fe8f74..3df8e68f 100644 --- a/web/src/lib/env.ts +++ b/web/src/lib/env.ts @@ -55,7 +55,8 @@ const docs = { apiLicense: "https://github.com/imputnet/cobalt/blob/main/api/LICENSE", }; -const apiURL = "https://api.freesavevideo.online/"; +// const apiURL = "https://api.freesavevideo.online/"; +const apiURL = "http://localhost:9000"; export { donate, apiURL, contacts, partners, siriShortcuts, docs }; export default variables;