From 4a53a4a51e4e1fc063ee2a6db8ddfaed6833afb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D1=81=D0=B5=D0=B2=D0=BE=D0=BB=D0=BE=D0=B4=20=D0=9E?= =?UTF-8?q?=2E?= Date: Sun, 8 Dec 2024 16:16:24 +0300 Subject: [PATCH] Revert "Merge commit 'refs/pull/613/head' of https://github.com/imputnet/cobalt" This reverts commit f03188fea40fe53d099bfb62b7122be602f213da, reversing changes made to e1b84e747211f334eae38283960a20a2bf48662c. --- api/src/processing/services/twitter.js | 93 ++++--- docs/api.md | 193 ++++++++++----- src/modules/processing/matchActionDecider.js | 200 --------------- src/modules/processing/request.js | 164 ------------- src/modules/processing/services/vk.js | 61 ----- src/modules/processing/services/youtube.js | 245 ------------------- 6 files changed, 200 insertions(+), 756 deletions(-) delete mode 100644 src/modules/processing/matchActionDecider.js delete mode 100644 src/modules/processing/request.js delete mode 100644 src/modules/processing/services/vk.js delete mode 100644 src/modules/processing/services/youtube.js diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index 5add4620..b4a1d557 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -36,17 +36,6 @@ function bestQuality(arr) { .url } -function buildMediaMetadata(tweetResult, media){ - return { - duration: Math.round(media.video_info.duration_millis / 1000) || 0, - likes: tweetResult.legacy.favorite_count || 0, - views: Number(tweetResult.views.count) || 0, - title: (tweetResult.legacy && tweetResult.legacy.full_text && Array.isArray(tweetResult.legacy.display_text_range) && tweetResult.legacy.display_text_range[0] !== undefined && tweetResult.legacy.display_text_range[1] !== undefined) - ? tweetResult.legacy.full_text.substr(tweetResult.legacy.display_text_range[0], tweetResult.legacy.display_text_range[1] - tweetResult.legacy.display_text_range[0]) - : undefined - } -} - let _cachedToken; const getGuestToken = async (dispatcher, forceReload = false) => { if (_cachedToken && !forceReload) { @@ -112,11 +101,11 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => { return result } -export default async function({ id, index, toGif, dispatcher }) { +export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { const cookie = await getCookie('twitter'); let guestToken = await getGuestToken(dispatcher); - if (!guestToken) return { error: 'ErrorCouldntFetch' }; + if (!guestToken) return { error: "fetch.fail" }; let tweet = await requestTweet(dispatcher, id, guestToken); @@ -130,22 +119,26 @@ export default async function({ id, index, toGif, dispatcher }) { let tweetTypename = tweet?.data?.tweetResult?.result?.__typename; + if (!tweetTypename) { + return { error: "fetch.empty" } + } + if (tweetTypename === "TweetUnavailable") { const reason = tweet?.data?.tweetResult?.result?.reason; switch(reason) { case "Protected": - return { error: 'ErrorTweetProtected' } + return { error: "content.post.private" } case "NsfwLoggedOut": if (cookie) { tweet = await requestTweet(dispatcher, id, guestToken, cookie); tweet = await tweet.json(); tweetTypename = tweet?.data?.tweetResult?.result?.__typename; - } else return { error: 'ErrorTweetNSFW' } + } else return { error: "content.post.age" } } } if (!["Tweet", "TweetWithVisibilityResults"].includes(tweetTypename)) { - return { error: 'ErrorTweetUnavailable' } + return { error: "content.post.unavailable" } } let tweetResult = tweet.data.tweetResult.result, @@ -158,45 +151,83 @@ export default async function({ id, index, toGif, dispatcher }) { } let media = (repostedTweet?.media || baseTweet?.extended_entities?.media); - media = media?.filter(m => m.video_info?.variants?.length); // check if there's a video at given index (/video/) if (index >= 0 && index < media?.length) { media = [media[index]] } + const getFileExt = (url) => new URL(url).pathname.split(".", 2)[1]; + + const proxyMedia = (url, filename) => createStream({ + service: "twitter", + type: "proxy", + url, filename, + }) + switch (media?.length) { case undefined: case 0: - return { error: 'ErrorNoVideosInTweet' }; - case 1: return { - type: needsFixing(media[0]) ? "remux" : "normal", + error: "fetch.empty" + } + case 1: + if (media[0].type === "photo") { + return { + type: "proxy", + isPhoto: true, + filename: `twitter_${id}.${getFileExt(media[0].media_url_https)}`, + urls: `${media[0].media_url_https}?name=4096x4096` + } + } + + return { + type: needsFixing(media[0]) ? "remux" : "proxy", urls: bestQuality(media[0].video_info.variants), filename: `twitter_${id}.mp4`, audioFilename: `twitter_${id}_audio`, - isGif: media[0].type === "animated_gif", - mediaMetadata: buildMediaMetadata(tweetResult, media[0]) - }; + isGif: media[0].type === "animated_gif" + } default: + const proxyThumb = (url, i) => + proxyMedia(url, `twitter_${id}_${i + 1}.${getFileExt(url)}`); + const picker = media.map((content, i) => { + if (content.type === "photo") { + let url = `${content.media_url_https}?name=4096x4096`; + let proxiedImage = proxyThumb(url, i); + + if (alwaysProxy) url = proxiedImage; + + return { + type: "photo", + url, + thumb: proxiedImage, + } + } + let url = bestQuality(content.video_info.variants); - const shouldRenderGif = content.type === 'animated_gif' && toGif; + const shouldRenderGif = content.type === "animated_gif" && toGif; + const videoFilename = `twitter_${id}_${i + 1}.${shouldRenderGif ? "gif" : "mp4"}`; + + let type = "video"; + if (shouldRenderGif) type = "gif"; if (needsFixing(content) || shouldRenderGif) { url = createStream({ - service: 'twitter', - type: shouldRenderGif ? 'gif' : 'remux', - u: url, - filename: `twitter_${id}_${i + 1}.mp4` + service: "twitter", + type: shouldRenderGif ? "gif" : "remux", + url, + filename: videoFilename, }) + } else if (alwaysProxy) { + url = proxyMedia(url, videoFilename); } return { - type: 'video', + type, url, - thumb: content.media_url_https, - mediaMetadata: buildMediaMetadata(tweetResult, content) + thumb: proxyThumb(content.media_url_https, i), } }); return { picker }; diff --git a/docs/api.md b/docs/api.md index 2b23e2c2..6da93a44 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,78 +1,161 @@ # cobalt api documentation -this document provides info about methods and acceptable variables for all cobalt api requests. +this document provides info about methods and acceptable variables for all cobalt api requests. +> [!IMPORTANT] +> hosted api instances (such as `api.cobalt.tools`) use bot protection and are **not** intended to be used in other projects without explicit permission. if you want to access the cobalt api reliably, you should [host your own instance](/docs/run-an-instance.md) or ask an instance owner for access. + +## authentication +an api instance may be configured to require you to authenticate yourself. +if this is the case, you will typically receive an [error response](#error-response) +with a **`api.auth..missing`** code, which tells you that a particular method +of authentication is required. + +authentication is done by passing the `Authorization` header, containing +the authentication scheme and the token: ``` -👍 you can use api.cobalt.tools in your projects for free, just don't be an asshole. +Authorization: ``` -## POST: `/api/json` -cobalt's main processing endpoint. +currently, cobalt supports two ways of authentication. an instance can +choose to configure both, or neither: +- [`Api-Key`](#api-key-authentication) +- [`Bearer`](#bearer-authentication) -request body type: `application/json` -response body type: `application/json` +### api-key authentication +the api key authentication is the most straightforward. the instance owner +will assign you an api key which you can then use to authenticate like so: +``` +Authorization: Api-Key aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee +``` + +if you are an instance owner and wish to configure api key authentication, +see the [instance](run-an-instance.md#api-key-file-format) documentation! + +### bearer authentication +the cobalt server may be configured to issue JWT bearers, which are short-lived +tokens intended for use by regular users (e.g. after passing a challenge). +currently, cobalt can issue tokens for successfully solved [turnstile](run-an-instance.md#list-of-all-environment-variables) +challenge, if the instance has turnstile configured. the resulting token is passed like so: +``` +Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9... +``` + +## POST: `/` +cobalt's main processing endpoint. + +request body type: `application/json` +response body type: `application/json` + +> [!IMPORTANT] +> you must include `Accept` and `Content-Type` headers with every `POST /` request. ``` -⚠️ you must include Accept and Content-Type headers with every POST /api/json request. - Accept: application/json Content-Type: application/json ``` -### request body variables -| key | type | variables | default | description | -|:------------------|:----------|:-----------------------------------|:----------|:--------------------------------------------------------------------------------| -| `url` | `string` | URL encoded as URI | `null` | **must** be included in every request. | -| `vCodec` | `string` | `h264 / av1 / vp9` | `h264` | applies only to youtube downloads. `h264` is recommended for phones. | -| `vQuality` | `string` | `144 / ... / 2160 / max` | `720` | `720` quality is recommended for phones. | -| `aFormat` | `string` | `best / mp3 / ogg / wav / opus` | `mp3` | | -| `filenamePattern` | `string` | `classic / pretty / basic / nerdy` | `classic` | changes the way files are named. previews can be seen in the web app. | -| `isAudioOnly` | `boolean` | `true / false` | `false` | | -| `isTTFullAudio` | `boolean` | `true / false` | `false` | enables download of original sound used in a tiktok video. | -| `isAudioMuted` | `boolean` | `true / false` | `false` | disables audio track in video downloads. | -| `dubLang` | `boolean` | `true / false` | `false` | backend uses Accept-Language header for youtube video audio tracks when `true`. | -| `disableMetadata` | `boolean` | `true / false` | `false` | disables file metadata when set to `true`. | -| `twitterGif` | `boolean` | `true / false` | `false` | changes whether twitter gifs are converted to .gif | -| `tiktokH265` | `boolean` | `true / false` | `false` | changes whether 1080p h265 videos are preferred or not. | +### request body +| key | type | expected value(s) | default | description | +|:-----------------------------|:----------|:-----------------------------------|:----------|:--------------------------------------------------------------------------------| +| `url` | `string` | URL to download | -- | **must** be included in every request. | +| `videoQuality` | `string` | `144 / ... / 2160 / 4320 / max` | `1080` | `720` quality is recommended for phones. | +| `audioFormat` | `string` | `best / mp3 / ogg / wav / opus` | `mp3` | | +| `audioBitrate` | `string` | `320 / 256 / 128 / 96 / 64 / 8` | `128` | specifies the bitrate to use for the audio. applies only to audio conversion. | +| `filenameStyle` | `string` | `classic / pretty / basic / nerdy` | `classic` | changes the way files are named. previews can be seen in the web app. | +| `downloadMode` | `string` | `auto / audio / mute` | `auto` | `audio` downloads only the audio, `mute` skips the audio track in videos. | +| `youtubeVideoCodec` | `string` | `h264 / av1 / vp9` | `h264` | `h264` is recommended for phones. | +| `youtubeDubLang` | `string` | `en / ru / cs / ja / es-US / ...` | -- | specifies the language of audio to download when a youtube video is dubbed. | +| `alwaysProxy` | `boolean` | `true / false` | `false` | tunnels all downloads through the processing server, even when not necessary. | +| `disableMetadata` | `boolean` | `true / false` | `false` | disables file metadata when set to `true`. | +| `tiktokFullAudio` | `boolean` | `true / false` | `false` | enables download of original sound used in a tiktok video. | +| `tiktokH265` | `boolean` | `true / false` | `false` | changes whether 1080p h265 videos are preferred or not. | +| `twitterGif` | `boolean` | `true / false` | `true` | changes whether twitter gifs are converted to .gif | +| `youtubeHLS` | `boolean` | `true / false` | `false` | specifies whether to use HLS for downloading video or audio from youtube. | -### response body variables -| key | type | variables | -|:----------------|:---------|:------------------------------------------------------------| -| `status` | `string` | `error / redirect / stream / success / rate-limit / picker` | -| `text` | `string` | various text, mostly used for errors | -| `url` | `string` | direct link to a file or a link to cobalt's live render | -| `pickerType` | `string` | `various / images` | -| `picker` | `array` | array of picker items | -| `audio` | `string` | direct link to a file or a link to cobalt's live render | -| `mediaMetadata` | `object` | Supported only on YouTube and Twitter videos. Object, that contains values `duration` (duration of the video in seconds), `likes`, `views`, `title` | +### response +the response will always be a JSON object containing the `status` key, which will be one of: +- `error` - something went wrong +- `picker` - we have multiple items to choose from +- `redirect` - you are being redirected to the direct service URL +- `tunnel` - cobalt is proxying the download for you -### picker item variables -item type: `object` +### tunnel/redirect response +| key | type | values | +|:-------------|:---------|:------------------------------------------------------------| +| `status` | `string` | `tunnel / redirect` | +| `url` | `string` | url for the cobalt tunnel, or redirect to an external link | +| `filename` | `string` | cobalt-generated filename for the file being downloaded | -| key | type | variables | description | -|:--------|:---------|:--------------------------------------------------------|:---------------------------------------| -| `type` | `string` | `video` | used only if `pickerType`is `various`. | -| `url` | `string` | direct link to a file or a link to cobalt's live render | | -| `thumb` | `string` | item thumbnail that's displayed in the picker | used only for `video` type. | +### picker response +| key | type | values | +|:----------------|:---------|:-------------------------------------------------------------------------------------------------| +| `status` | `string` | `picker` | +| `audio` | `string` | **optional** returned when an image slideshow (such as on tiktok) has a general background audio | +| `audioFilename` | `string` | **optional** cobalt-generated filename, returned if `audio` exists | +| `picker` | `array` | array of objects containing the individual media | -## GET: `/api/stream` -cobalt's live render (or stream) endpoint. usually, you will receive a url to this endpoint -from a successful call to `/api/json`. however, the parameters passed to it are **opaque** -and **unmodifiable** from your (the api client's) perspective, and can change between versions. +#### picker object +| key | type | values | +|:-------------|:----------|:------------------------------------------------------------| +| `type` | `string` | `photo` / `video` / `gif` | +| `url` | `string` | | +| `thumb` | `string` | **optional** thumbnail url | -therefore you don't need to worry about what they mean - but if you really want to know, you can -[read the source code](/src/modules/stream/manage.js). +### error response +| key | type | values | +|:-------------|:---------|:------------------------------------------------------------| +| `status` | `string` | `error` | +| `error` | `object` | contains more context about the error | -## GET: `/api/serverInfo` -returns current basic server info. +#### error object +| key | type | values | +|:-------------|:---------|:------------------------------------------------------------| +| `code` | `string` | machine-readable error code explaining the failure reason | +| `context` | `object` | **optional** container for providing more context | + +#### error.context object +| key | type | values | +|:-------------|:---------|:---------------------------------------------------------------------------------------------------------------| +| `service` | `string` | **optional**, stating which service was being downloaded from | +| `limit` | `number` | **optional** number providing the ratelimit maximum number of requests, or maximum downloadable video duration | + +## GET: `/` +returns current basic server info. response body type: `application/json` -### response body variables +### response body +| key | type | variables | +|:------------|:---------|:---------------------------------------------------------| +| `cobalt` | `object` | information about the cobalt instance | +| `git` | `object` | information about the codebase that is currently running | + +#### cobalt object +| key | type | description | +|:----------------|:-----------|:-----------------------------------------------| +| `version` | `string` | current version | +| `url` | `string` | server url | +| `startTime` | `string` | server start time in unix milliseconds | +| `durationLimit` | `number` | maximum downloadable video length in seconds | +| `services` | `string[]` | array of services which this instance supports | + +#### git object | key | type | variables | |:------------|:---------|:------------------| -| `version` | `string` | cobalt version | -| `commit` | `string` | git commit | +| `commit` | `string` | commit hash | | `branch` | `string` | git branch | -| `name` | `string` | server name | -| `url` | `string` | server url | -| `cors` | `number` | cors status | -| `startTime` | `string` | server start time | +| `remote` | `string` | git remote | + +## POST: `/session` + +used for generating JWT tokens, if enabled. currently, cobalt only supports +generating tokens when a [turnstile](run-an-instance.md#list-of-all-environment-variables) challenge solution +is submitted by the client. + +the turnstile challenge response is submitted via the `cf-turnstile-response` header. +### response body +| key | type | description | +|:----------------|:-----------|:-------------------------------------------------------| +| `token` | `string` | a `Bearer` token used for later request authentication | +| `exp` | `number` | number in seconds indicating the token lifetime | + +on failure, an [error response](#error-response) is returned. diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js deleted file mode 100644 index 2f6c7c8a..00000000 --- a/src/modules/processing/matchActionDecider.js +++ /dev/null @@ -1,200 +0,0 @@ -import { audioIgnore, services, supportedAudio } from "../config.js"; -import { createResponse } from "../processing/request.js"; -import loc from "../../localization/manager.js"; -import createFilename from "./createFilename.js"; -import { createStream } from "../stream/manage.js"; - -export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) { - let action, - responseType = "stream", - defaultParams = { - u: r.urls, - headers: r.headers, - service: host, - filename: r.filenameAttributes ? - createFilename(r.filenameAttributes, filenamePattern, isAudioOnly, isAudioMuted) : r.filename, - fileMetadata: !disableMetadata ? r.fileMetadata : false, - mediaMetadata: r.mediaMetadata, - requestIP - }, - params = {}, - audioFormat = String(userFormat); - - if (r.isPhoto) action = "photo"; - else if (r.picker) action = "picker" - else if (r.isGif && toGif) action = "gif"; - else if (isAudioMuted) action = "muteVideo"; - else if (isAudioOnly) action = "audio"; - else if (r.isM3U8) action = "m3u8"; - else action = "video"; - - if (action === "picker" || action === "audio") { - if (!r.filenameAttributes) defaultParams.filename = r.audioFilename; - defaultParams.isAudioOnly = true; - defaultParams.audioFormat = audioFormat; - } - if (isAudioMuted && !r.filenameAttributes) { - defaultParams.filename = r.filename.replace('.', '_mute.') - } - - switch (action) { - default: - return createResponse("error", { t: loc(lang, 'ErrorEmptyDownload') }); - - case "photo": - responseType = "redirect"; - break; - - case "gif": - params = { type: "gif" } - break; - - case "m3u8": - params = { - type: Array.isArray(r.urls) ? "render" : "remux" - } - break; - - case "muteVideo": - let muteType = "mute"; - if (Array.isArray(r.urls) && !r.isM3U8) { - muteType = "bridge"; - } - params = { - type: muteType, - u: Array.isArray(r.urls) ? r.urls[0] : r.urls, - mute: true - } - if (host === "reddit" && r.typeId === "redirect") - responseType = "redirect"; - break; - - case "picker": - responseType = "picker"; - switch (host) { - case "instagram": - case "twitter": - params = { picker: r.picker }; - break; - case "tiktok": - let audioStreamType = "render"; - if (r.bestAudio === "mp3" && (audioFormat === "mp3" || audioFormat === "best")) { - audioFormat = "mp3"; - audioStreamType = "bridge" - } - params = { - picker: r.picker, - u: createStream({ - service: "tiktok", - type: audioStreamType, - u: r.urls, - headers: r.headers, - filename: r.audioFilename, - isAudioOnly: true, - audioFormat, - }), - copy: audioFormat === "best" - } - } - break; - - case "video": - switch (host) { - case "bilibili": - params = { type: "render" }; - break; - case "youtube": - params = { type: r.type }; - break; - case "reddit": - responseType = r.typeId; - params = { type: r.type }; - break; - case "vimeo": - if (Array.isArray(r.urls)) { - params = { type: "render" } - } else { - responseType = "redirect"; - } - break; - - case "twitter": - if (r.type === "remux") { - params = { type: r.type }; - } else { - responseType = "redirect"; - } - break; - - case "vk": - case "tiktok": - params = { type: "bridge" }; - break; - - case "vine": - case "instagram": - case "tumblr": - case "pinterest": - case "streamable": - case "loom": - responseType = "redirect"; - break; - } - break; - - case "audio": - if (audioIgnore.includes(host) - || (host === "reddit" && r.typeId === "redirect")) { - return createResponse("error", { t: loc(lang, 'ErrorEmptyDownload') }) - } - - let processType = "render", - copy = false; - - if (!supportedAudio.includes(audioFormat)) { - audioFormat = "best" - } - - const serviceBestAudio = r.bestAudio || services[host]["bestAudio"]; - const isBestAudio = audioFormat === "best"; - const isBestOrMp3 = isBestAudio || audioFormat === "mp3"; - const isBestAudioDefined = isBestAudio && serviceBestAudio; - const isBestHostAudio = serviceBestAudio && (audioFormat === serviceBestAudio); - - const isTumblrAudio = host === "tumblr" && !r.filename; - const isSoundCloud = host === "soundcloud"; - const isTiktok = host === "tiktok"; - - if (isBestAudioDefined || isBestHostAudio) { - audioFormat = serviceBestAudio; - processType = "bridge"; - if (isSoundCloud || (isTiktok && audioFormat === "m4a")) { - processType = "render" - copy = true - } - } else if (isBestAudio && !isSoundCloud) { - audioFormat = "m4a"; - copy = true - } - - if (isTumblrAudio && isBestOrMp3) { - audioFormat = "mp3"; - processType = "bridge" - } - - if (r.isM3U8 || host === "vimeo") { - copy = false; - processType = "render" - } - - params = { - type: processType, - u: Array.isArray(r.urls) ? r.urls[1] : r.urls, - audioFormat: audioFormat, - copy: copy - } - break; - } - - return createResponse(responseType, {...defaultParams, ...params}) -} diff --git a/src/modules/processing/request.js b/src/modules/processing/request.js deleted file mode 100644 index 269ea504..00000000 --- a/src/modules/processing/request.js +++ /dev/null @@ -1,164 +0,0 @@ -import ipaddr from "ipaddr.js"; - -import { normalizeURL } from "../processing/url.js"; -import { createStream } from "../stream/manage.js"; -import { verifyLanguageCode } from "../sub/utils.js"; - -const apiVar = { - allowed: { - vCodec: ["h264", "av1", "vp9"], - vQuality: ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"], - aFormat: ["best", "mp3", "ogg", "wav", "opus"], - filenamePattern: ["classic", "pretty", "basic", "nerdy"] - }, - booleanOnly: [ - "isAudioOnly", - "isTTFullAudio", - "isAudioMuted", - "dubLang", - "disableMetadata", - "twitterGif", - "tiktokH265" - ] -} - -export function createResponse(responseType, responseData) { - const internalError = (text) => { - return { - status: 500, - body: { - status: "error", - text: text || "Internal Server Error", - critical: true - } - } - } - - try { - let status = 200, - response = {}; - - switch(responseType) { - case "error": - status = 400; - break; - - case "rate-limit": - status = 429; - break; - } - - switch (responseType) { - case "error": - case "success": - case "rate-limit": - response = { - text: responseData.t - } - break; - - case "redirect": - response = { - url: responseData.u, - mediaMetadata: responseData.mediaMetadata, - } - break; - - case "stream": - response = { - url: createStream(responseData), - mediaMetadata: responseData.mediaMetadata, - } - break; - - case "picker": - let pickerType = "various", - audio = false; - - if (responseData.service === "tiktok") { - audio = responseData.u - pickerType = "images" - } - - response = { - pickerType: pickerType, - picker: responseData.picker, - audio: audio - } - break; - case "critical": - return internalError(responseData.t) - default: - throw "unreachable" - } - return { - status, - body: { - status: responseType, - ...response - } - } - } catch { - return internalError() - } -} - -export function normalizeRequest(request) { - try { - let template = { - url: normalizeURL(decodeURIComponent(request.url)), - vCodec: "h264", - vQuality: "720", - aFormat: "mp3", - filenamePattern: "classic", - isAudioOnly: false, - isTTFullAudio: false, - isAudioMuted: false, - disableMetadata: false, - dubLang: false, - twitterGif: false, - tiktokH265: false - } - - const requestKeys = Object.keys(request); - const templateKeys = Object.keys(template); - - if (requestKeys.length > templateKeys.length + 1 || !request.url) { - return false; - } - - for (const i in requestKeys) { - const key = requestKeys[i]; - const item = request[key]; - - if (String(key) !== "url" && templateKeys.includes(key)) { - if (apiVar.booleanOnly.includes(key)) { - template[key] = !!item; - } else if (apiVar.allowed[key] && apiVar.allowed[key].includes(item)) { - template[key] = String(item) - } - } - } - - if (template.dubLang) - template.dubLang = verifyLanguageCode(request.dubLang); - - return template - } catch { - return false - } -} - -export function getIP(req) { - const strippedIP = req.ip.replace(/^::ffff:/, ''); - const ip = ipaddr.parse(strippedIP); - if (ip.kind() === 'ipv4') { - return strippedIP; - } - - const prefix = 56; - const v6Bytes = ip.toByteArray(); - v6Bytes.fill(0, prefix / 8); - - return ipaddr.fromByteArray(v6Bytes).toString(); -} diff --git a/src/modules/processing/services/vk.js b/src/modules/processing/services/vk.js deleted file mode 100644 index fbc6fef2..00000000 --- a/src/modules/processing/services/vk.js +++ /dev/null @@ -1,61 +0,0 @@ -import { genericUserAgent, env } from "../../config.js"; -import { cleanString } from "../../sub/utils.js"; - -const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"]; - -export default async function(o) { - let html, url, quality = o.quality === "max" ? 2160 : o.quality; - - html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, { - headers: { "user-agent": genericUserAgent } - }).then(r => r.arrayBuffer()).catch(() => {}); - - if (!html) return { error: 'ErrorCouldntFetch' }; - - // decode cyrillic from windows-1251 because vk still uses apis from prehistoric times - let decoder = new TextDecoder('windows-1251'); - html = decoder.decode(html); - - if (!html.includes(`{"lang":`)) return { error: 'ErrorEmptyDownload' }; - - let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]); - - if (Number(js.mvData.is_active_live) !== 0) return { error: 'ErrorLiveVideo' }; - if (js.mvData.duration > env.durationLimit) - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - - for (let i in resolutions) { - if (js.player.params[0][`url${resolutions[i]}`]) { - quality = resolutions[i]; - break - } - } - if (Number(quality) > Number(o.quality)) quality = o.quality; - - url = js.player.params[0][`url${quality}`]; - - let fileMetadata = { - title: cleanString(js.player.params[0].md_title.trim()), - author: cleanString(js.player.params[0].md_author.trim()), - } - - if (url) return { - urls: url, - filenameAttributes: { - service: "vk", - id: `${o.userId}_${o.videoId}`, - title: fileMetadata.title, - author: fileMetadata.author, - resolution: `${quality}p`, - qualityLabel: `${quality}p`, - extension: "mp4" - }, - mediaMetadata: { - duration: js.player.params[0].duration, - likes: js.mvData.likes, - views: js.videoModalInfoData.views, - title: js.mvData.title - } - } - return { error: 'ErrorEmptyDownload' } -} diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js deleted file mode 100644 index bb630dca..00000000 --- a/src/modules/processing/services/youtube.js +++ /dev/null @@ -1,245 +0,0 @@ -import { Innertube, Session } from 'youtubei.js'; -import { env } from '../../config.js'; -import { cleanString } from '../../sub/utils.js'; -import { fetch } from 'undici' -import { getCookie, updateCookieValues } from '../cookie/manager.js' - -const ytBase = Innertube.create().catch(e => e); - -const codecMatch = { - h264: { - codec: "avc1", - aCodec: "mp4a", - container: "mp4" - }, - av1: { - codec: "av01", - aCodec: "mp4a", - container: "mp4" - }, - vp9: { - codec: "vp9", - aCodec: "opus", - container: "webm" - } -} - -const transformSessionData = (cookie) => { - if (!cookie) - return; - - const values = cookie.values(); - const REQUIRED_VALUES = [ - 'access_token', 'refresh_token', - 'client_id', 'client_secret', - 'expires' - ]; - - if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) { - return; - } - return { - ...values, - expires: new Date(values.expires), - }; -} - -const cloneInnertube = async (customFetch) => { - const innertube = await ytBase; - if (innertube instanceof Error) { - throw innertube; - } - - const session = new Session( - innertube.session.context, - innertube.session.key, - innertube.session.api_version, - innertube.session.account_index, - innertube.session.player, - undefined, - customFetch ?? innertube.session.http.fetch, - innertube.session.cache - ); - - const cookie = getCookie('youtube_oauth'); - const oauthData = transformSessionData(cookie); - - if (!session.logged_in && oauthData) { - await session.oauth.init(oauthData); - session.logged_in = true; - } - - if (session.logged_in) { - await session.oauth.refreshIfRequired(); - const oldExpiry = new Date(cookie.values().expires); - const newExpiry = session.oauth.credentials.expires; - - if (oldExpiry.getTime() !== newExpiry.getTime()) { - updateCookieValues(cookie, { - ...session.oauth.credentials, - expires: session.oauth.credentials.expires.toISOString() - }); - } - } - - const yt = new Innertube(session); - return yt; -} - -export default async function(o) { - const yt = await cloneInnertube( - (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher }) - ); - - let info, isDubbed, format = o.format || "h264"; - let quality = o.quality === "max" ? "9000" : o.quality; // 9000(p) - max quality - - function qual(i) { - if (!i.quality_label) { - return; - } - - return i.quality_label.split('p')[0].split('s')[0] - } - - try { - info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS'); - } catch(e) { - if (e?.message === 'This video is unavailable') { - return { error: 'ErrorCouldntFetch' }; - } else { - return { error: 'ErrorCantConnectToServiceAPI' }; - } - } - - if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; - - const playability = info.playability_status; - - if (playability.status === 'LOGIN_REQUIRED') { - if (playability.reason.endsWith('bot')) { - return { error: 'ErrorYTLogin' } - } - if (playability.reason.endsWith('age')) { - return { error: 'ErrorYTAgeRestrict' } - } - } - if (playability.status === "UNPLAYABLE" && playability.reason.endsWith('request limit.')) { - return { error: 'ErrorYTRateLimit' } - } - - if (playability.status !== 'OK') return { error: 'ErrorYTUnavailable' }; - if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' }; - - // return a critical error if returned video is "Video Not Available" - // or a similar stub by youtube - if (info.basic_info.id !== o.id) { - return { - error: 'ErrorCantConnectToServiceAPI', - critical: true - } - } - - let bestQuality, hasAudio; - - const filterByCodec = (formats) => formats.filter(e => - e.mime_type.includes(codecMatch[format].codec) || e.mime_type.includes(codecMatch[format].aCodec) - ).sort((a, b) => Number(b.bitrate) - Number(a.bitrate)); - - let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats); - if (adaptive_formats.length === 0 && format === "vp9") { - format = "h264" - adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats) - } - - bestQuality = adaptive_formats.find(i => i.has_video && i.content_length); - hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length); - - if (bestQuality) bestQuality = qual(bestQuality); - if (!bestQuality && !o.isAudioOnly || !hasAudio) return { error: 'ErrorYTTryOtherCodec' }; - if (info.basic_info.duration > env.durationLimit) return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - - let checkBestAudio = (i) => (i.has_audio && !i.has_video), - audio = adaptive_formats.find(i => checkBestAudio(i) && !i.is_dubbed); - - if (o.dubLang) { - let dubbedAudio = adaptive_formats.find(i => - checkBestAudio(i) && i.language === o.dubLang && i.audio_track && !i.audio_track.audio_is_default - ); - if (dubbedAudio) { - audio = dubbedAudio; - isDubbed = true - } - } - let fileMetadata = { - title: cleanString(info.basic_info.title.trim()), - artist: cleanString(info.basic_info.author.replace("- Topic", "").trim()), - } - if (info.basic_info.short_description && info.basic_info.short_description.startsWith("Provided to YouTube by")) { - let descItems = info.basic_info.short_description.split("\n\n"); - fileMetadata.album = descItems[2]; - fileMetadata.copyright = descItems[3]; - if (descItems[4].startsWith("Released on:")) { - fileMetadata.date = descItems[4].replace("Released on: ", '').trim() - } - } - - let filenameAttributes = { - service: "youtube", - id: o.id, - title: fileMetadata.title, - author: fileMetadata.artist, - youtubeDubName: isDubbed ? o.dubLang : false - } - - if (hasAudio && o.isAudioOnly) return { - type: "render", - isAudioOnly: true, - urls: audio.decipher(yt.session.player), - filenameAttributes: filenameAttributes, - fileMetadata: fileMetadata, - bestAudio: format === "h264" ? 'm4a' : 'opus' - } - const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality, - checkSingle = i => qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].codec), - checkRender = i => qual(i) === matchingQuality && i.has_video && !i.has_audio; - - let match, type, urls; - if (!o.isAudioOnly && !o.isAudioMuted && format === 'h264') { - match = info.streaming_data.formats.find(checkSingle); - type = "bridge"; - urls = match?.decipher(yt.session.player); - } - - const video = adaptive_formats.find(checkRender); - if (!match && video) { - match = video; - type = "render"; - urls = [video.decipher(yt.session.player), audio.decipher(yt.session.player)]; - } - - const mediaMetadata = { - duration: info.basic_info.duration, - likes: info.basic_info.like_count, - views: info.basic_info.view_count, - title: info.basic_info.title, - }; - - if (match) { - filenameAttributes.qualityLabel = match.quality_label; - filenameAttributes.resolution = `${match.width}x${match.height}`; - filenameAttributes.extension = codecMatch[format].container; - filenameAttributes.youtubeFormat = format; - - - return { - type, - urls, - filenameAttributes, - fileMetadata, - mediaMetadata, - } - } - - return { error: 'ErrorYTTryOtherCodec' } -}