Merge branch 'current' into support/odysee

.github/ISSUE_TEMPLATE/bug-main-instance.md

@@ -0,0 +1,36 @@
+---
+name: main instance bug report
+about: report an issue with cobalt.tools or api.cobalt.tools
+title: '[short description of the bug]'
+labels: main instance issue
+assignees: ''
+
+---
+
+### bug description
+clear and concise description of what the issue is.
+
+### reproduction steps
+steps to reproduce the described behavior.  
+here's an example of what it could look like:
+1. go to '...'
+2. click on '....'
+3. download [media type] from [service]
+4. see error
+
+### screenshots
+if applicable, add screenshots or screen recordings to support your explanation.  
+if not, remove this section.
+
+### links
+if applicable, add links that cause the issue. more = better.  
+if not, remove this section.
+
+### platform information
+- OS [e.g. iOS, windows]
+- browser [e.g. chrome, safari, firefox]
+- version [e.g. 115]
+
+### additional context
+add any other context about the problem here if applicable.  
+if not, remove this section.

.github/ISSUE_TEMPLATE/bug-report.md

@@ -1,32 +1,36 @@
 ---
 name: bug report
-about: report an issue with downloads or something else
-title: ''
+about: report a global issue with the cobalt codebase
+title: '[short description of the bug]'
 labels: bug
 assignees: ''
 
 ---
 
-**bug description**
-a clear and concise description of what the bug is.
+### bug description
+clear and concise description of what the issue is.
 
-**reproduction steps**
-steps to reproduce the behavior:
+### reproduction steps
+steps to reproduce the described behavior.  
+here's an example of what it could look like:
 1. go to '...'
 2. click on '....'
-3. download this video: **[link here]**
+3. download [media type] from [service]
 4. see error
 
-**screenshots**
-if applicable, add screenshots or screen recordings to help explain your problem.
+### screenshots
+if applicable, add screenshots or screen recordings to support your explanation.  
+if not, remove this section.
 
-**links**
+### links
 if applicable, add links that cause the issue. more = better.  
+if not, remove this section.
 
-**platform**
+### platform information
 - OS [e.g. iOS, windows]
 - browser [e.g. chrome, safari, firefox]
 - version [e.g. 115]
 
-**additional context**
-add any other context about the problem here.
+### additional context
+add any other context about the problem here if applicable.  
+if not, remove this section.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -1,17 +1,15 @@
 ---
 name: feature request
 about: suggest a feature for cobalt
-title: ''
+title: '[short feature request description]'
 labels: feature request
 assignees: ''
 
 ---
 
-**describe the feature you'd like to see**
-a clear and concise description of what you want to happen.
+### describe the feature you'd like to see
+clear and concise description of the feature you want to see in cobalt.
 
-**describe alternatives you've considered**
-a clear and concise description of any alternative solutions or features you've considered.
-
-**additional context**
-add any other context or screenshots about the feature request here.
+### additional context
+if applicable, add any other context or screenshots related to the feature request here.  
+if not, remove this section.

.github/ISSUE_TEMPLATE/hosting-help.md

@@ -0,0 +1,12 @@
+---
+name: instance hosting help
+about: ask any question regarding cobalt instance hosting
+title: '[short description of the problem]'
+labels: instance hosting help
+assignees: ''
+
+---
+
+### problem description
+describe what issue you're having, clearly and concisely. 
+support your description with screenshots/links/etc when needed.

.github/ISSUE_TEMPLATE/service-request.md

@@ -0,0 +1,18 @@
+---
+name: service request
+about: request service support in cobalt
+title: 'add support for [service name]'
+labels: service request
+assignees: ''
+
+---
+
+### service name & description
+provide the service name and brief description of what it is.
+
+### link samples for the service you'd like cobalt to support
+list of links that cobalt should recognize. 
+could be regular video link, shared video link, mobile video link, shortened link, etc.
+
+### additional context
+any additional context or screenshots should go here. if there aren't any, just remove this part.

.github/workflows/fast-forward.yml

@@ -0,0 +1,22 @@
+name: fast-forward
+on:
+  issue_comment:
+    types: [created, edited]
+jobs:
+  fast-forward:
+    # Only run if the comment contains the /fast-forward command.
+    if: ${{ contains(github.event.comment.body, '/fast-forward')
+            && github.event.issue.pull_request }}
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+
+    steps:
+      - name: Fast forwarding
+        uses: sequoia-pgp/fast-forward@v1
+        with:
+          merge: true
+          comment: 'on-error'

CONTRIBUTING.md

@@ -0,0 +1,39 @@
+# contributing to cobalt
+if you're reading this, you are probably interested in contributing to cobalt, which we are very thankful for :3
+
+this document serves as a guide to help you make contributions that we can merge into the cobalt codebase.
+
+## translations
+currently, we are **not accepting** translations of cobalt. this is because we are making significant changes to the frontend, and the currently used localization structure is being completely reworked. if this changes, this document will be updated.
+
+## adding features or support for services
+before putting in the effort to implement a feature, it's worth considering whether it would be appropriate to add it to cobalt. the cobalt api is built to assist people **only with downloading freely accessible content**. other functionality, such as:
+- downloading paid / not publicly accessible content
+- downloading content protected by DRM
+- scraping unrelated information & exposing it outside of file metadata
+
+will not be reviewed or merged.
+
+if you plan on adding a feature or support for a service, but are unsure whether it would be appropriate, it's best to open an issue and discuss it beforehand.
+
+## git
+when contributing code to cobalt, there are a few guidelines in place to ensure that the code history is readable and comprehensible.
+
+### clean commit messages
+internally, we use a format similar to [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) - the first part signifies which part of the code you are changing (the *scope*), and the second part explains the change. for inspiration on how to write appropriate commit titles, you can take a look at the [commit history](https://github.com/imputnet/cobalt/commits/).
+
+the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `stream/internal: fix object not being handled properly`).
+
+if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
+
+if your contribution has uninformative commit messages, you may be asked to interactively rebase your branch and amend each commit to include a meaningful message.
+
+### clean commit history
+if your branch is out of date and/or has some merge conflicts with the `current` branch, you should **rebase** it instead of merging. this prevents meaningless merge commits from being included in your branch, which would then end up in the cobalt git history.
+
+if you find a mistake or bug in your code before it's merged or reviewed, instead of making a brand new commit to fix it, it would be preferable to amend that specific commit where the mistake was first introduced. this also helps us easily revert a commit if we discover that it introduced a bug or some unwanted behavior.
+- if the commit you are fixing is the latest one, you can add your files to staging and then use `git commit --amend` to apply the change.
+- if the commit is somewhere deeper in your branch, you can use `git commit --fixup=HASH`, where *`HASH`* is the commit you are fixing.
+    - afterward, you must interactively rebase your branch with `git rebase -i current --autosquash`.
+      this will open up an editor, but you don't need to do anything else except save the file and exit.
+- once you do either of these things, you will need to do a **force push** to your branch with `git push --force-with-lease`.

docs/api.md

@@ -73,5 +73,5 @@ response body type: `application/json`
 | `branch`    | `string` | git branch        |
 | `name`      | `string` | server name       |
 | `url`       | `string` | server url        |
-| `cors`      | `int`    | cors status       |
+| `cors`      | `number` | cors status       |
 | `startTime` | `string` | server start time |

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "cobalt",
-    "version": "7.14.3",
+    "version": "7.14.5",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "cobalt",
-            "version": "7.14.3",
+            "version": "7.14.5",
             "license": "AGPL-3.0",
             "dependencies": {
                 "content-disposition-header": "0.6.0",

package.json

@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "7.14.3",
+    "version": "7.14.5",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",
@@ -11,9 +11,9 @@
     "scripts": {
         "start": "node src/cobalt",
         "setup": "node src/modules/setup",
-        "test": "node src/test/test",
+        "test": "node src/util/test",
         "build": "node src/modules/buildStatic",
-        "testFilenames": "node src/test/testFilenamePresets"
+        "token:youtube": "node src/util/generate-youtube-tokens"
     },
     "repository": {
         "type": "git",

src/core/api.js

@@ -10,6 +10,7 @@ import loc from "../localization/manager.js";
 
 import { createResponse, normalizeRequest, getIP } from "../modules/processing/request.js";
 import { verifyStream, getInternalStream } from "../modules/stream/manage.js";
+import { randomizeCiphers } from '../modules/sub/randomize-ciphers.js';
 import { extract } from "../modules/processing/url.js";
 import match from "../modules/processing/match.js";
 import stream from "../modules/stream/stream.js";
@@ -195,10 +196,10 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
             return res.sendStatus(404);
         }
 
-        streamInfo.headers = {
-            ...streamInfo.headers,
-            ...req.headers
-        };
+        streamInfo.headers = new Map([
+            ...(streamInfo.headers || []),
+            ...Object.entries(req.headers)
+        ]);
 
         return stream(res, { type: 'internal', ...streamInfo });
     })
@@ -215,6 +216,9 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
         res.redirect('/api/serverInfo')
     })
 
+    randomizeCiphers();
+    setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes
+
     app.listen(env.apiPort, env.listenAddress, () => {
         console.log(`\n` +
             `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` +

src/core/web.js

@@ -26,7 +26,13 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
 
     app.get('/onDemand', (req, res) => {
         try {
-            if (req.query.blockId) {
+            if (typeof req.query.blockId !== 'string') {
+                return res.status(400).json({
+                    status: "error",
+                    text: "couldn't render this block, please try again!"
+                });
+            }
+
             let blockId = req.query.blockId.slice(0, 3);
             let blockData;
             switch(blockId) {
@@ -54,17 +60,12 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
                     })
                     break;
             }
+
             if (blockData?.body) {
                 return res.status(blockData.status).json(blockData.body);
             } else {
                 return res.status(204).end();
             }
-            } else {
-                return res.status(400).json({
-                    status: "error",
-                    text: "couldn't render this block, please try again!"
-                });
-            }
         } catch {
             return res.status(400).json({
                 status: "error",

src/localization/languages/en.json

@@ -158,6 +158,7 @@
         "ErrorInvalidContentType": "invalid content type header",
         "UpdateOneMillion": "1 million users and blazing speed",
         "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!",
-        "ErrorYTLogin": "couldn't get this youtube video because it requires sign in.\n\nthis limitation is an a/b test done by google to seemingly stop scraping, coincidentally affecting all 3rd party tools and even their own clients.\n\nyou can track the <a class=\"text-backdrop link\" href=\"{repo}/issues/551\" target=\"_blank\">issue on github</a>."
+        "ErrorYTLogin": "couldn't get this youtube video because it requires an account to view.\n\nthis limitation is done by google to seemingly stop scraping, affecting all 3rd party tools and even their own clients.\n\ntry again, but if issue persists, {ContactLink}.",
+        "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}."
     }
 }

src/localization/languages/ru.json

@@ -1,7 +1,7 @@
 {
     "name": "русский",
     "substrings": {
-        "ContactLink": "глянь <a class=\"text-backdrop link\" href=\"{statusPage}\" target=\"_blank\">статус серверов</a> или <a class=\"text-backdrop link\" href=\"{repo}\" target=\"_blank\">напиши о проблеме на github (можно на русском)</a>"
+        "ContactLink": "глянь <a class=\"text-backdrop link\" href=\"{statusPage}\" target=\"_blank\">статус серверов</a> или <a class=\"text-backdrop link\" href=\"{repo}\" target=\"_blank\">напиши о проблеме на github</a>"
     },
     "strings": {
         "AppTitleCobalt": "кобальт",
@@ -158,6 +158,8 @@
         "SettingsYoutubeDub": "использовать язык браузера",
         "SettingsYoutubeDubDescription": "использует главный язык браузера для аудиодорожек на youtube. работает даже если кобальт не переведён в твой язык.",
         "UpdateOneMillion": "миллион и невероятная скорость",
-        "ErrorYTAgeRestrict": "это видео ограничено по возрасту, поэтому я не могу его скачать. попробуй другое!"
+        "ErrorYTAgeRestrict": "это видео ограничено по возрасту, поэтому я не могу его скачать. попробуй другое!",
+        "ErrorYTLogin": "не удалось получить это видео с youtube, т. к. для его просмотра требуется учетная запись.\n\nтакое ограничение сделано google, чтобы, по-видимому, помешать скрапингу, но в итоге ломает сторонние программы и даже собственные клиенты.\n\nпопробуй ещё раз, но если проблема останется, то {ContactLink}.",
+        "ErrorYTRateLimit": "youtube ограничил мне частоту запросов. попробуй ещё раз через несколько секунд, но если проблема останется, то {ContactLink}."
     }
 }

src/modules/processing/matchActionDecider.js

@@ -24,7 +24,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
     else if (r.isGif && toGif) action = "gif";
     else if (isAudioMuted) action = "muteVideo";
     else if (isAudioOnly) action = "audio";
-    else if (r.isM3U8) action = "singleM3U8";
+    else if (r.isM3U8) action = "m3u8";
     else action = "video";
 
     if (action === "picker" || action === "audio") {
@@ -48,13 +48,19 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
             params = { type: "gif" }
             break;
 
-        case "singleM3U8":
-            params = { type: "remux" }
+        case "m3u8":
+            params = {
+                type: Array.isArray(r.urls) ? "render" : "remux"
+            }
             break;
 
         case "muteVideo":
+            let muteType = "mute";
+            if (Array.isArray(r.urls) && !r.isM3U8) {
+                muteType = "bridge";
+            }
             params = {
-                type: Array.isArray(r.urls) ? "bridge" : "mute",
+                type: muteType,
                 u: Array.isArray(r.urls) ? r.urls[0] : r.urls,
                 mute: true
             }

src/modules/processing/services/instagram.js

@@ -256,11 +256,11 @@ export default function(obj) {
             if (!media_id && cookie) media_id = await getMediaId(id, { cookie });
 
             // mobile api (bearer)
-            if (media_id && token) data = await requestMobileApi(id, { token });
+            if (media_id && token) data = await requestMobileApi(media_id, { token });
 
             // mobile api (no cookie, cookie)
-            if (!data && media_id) data = await requestMobileApi(id);
-            if (!data && media_id && cookie) data = await requestMobileApi(id, { cookie });
+            if (media_id && !data) data = await requestMobileApi(media_id);
+            if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie });
 
             // html embed (no cookie, cookie)
             if (!data) data = await requestHTML(id);

src/modules/processing/services/ok.js

@@ -20,14 +20,15 @@ export default async function(o) {
     }).then(r => r.text()).catch(() => {});
 
     if (!html) return { error: 'ErrorCouldntFetch' };
-    if (!html.includes(`<div data-module="OKVideo" data-options="{`)) {
+
+    let videoData = html.match(/<div data-module="OKVideo" .*? data-options="({.*?})"( .*?)>/)
+                        ?.[1]
+                        ?.replaceAll("&quot;", '"');
+
+    if (!videoData) {
         return { error: 'ErrorEmptyDownload' };
     }
 
-    let videoData = html.split(`<div data-module="OKVideo" data-options="`)[1]
-                        .split('" data-')[0]
-                        .replaceAll("&quot;", '"');
-
     videoData = JSON.parse(JSON.parse(videoData).flashvars.metadata);
 
     if (videoData.provider !== "UPLOADED_ODKL")

src/modules/processing/services/pinterest.js

@@ -1,6 +1,6 @@
 import { genericUserAgent } from "../../config.js";
 
-const videoRegex = /"url":"(https:\/\/v1.pinimg.com\/videos\/.*?)"/g;
+const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
 const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
 
 export default async function(o) {

src/modules/processing/services/soundcloud.js

@@ -18,7 +18,9 @@ async function findClientID() {
         for (let script of scripts) {
             let url = script[1];
 
-            if (url && !url.startsWith('https://a-v2.sndcdn.com')) return;
+            if (!url?.startsWith('https://a-v2.sndcdn.com/')) {
+                return;
+            }
 
             let scrf = await fetch(url).then(r => r.text()).catch(() => {});
             let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);

src/modules/processing/services/vimeo.js

@@ -1,109 +1,169 @@
 import { env } from "../../config.js";
-import { cleanString } from '../../sub/utils.js';
+import { cleanString, merge } from '../../sub/utils.js';
+
+import HLS from "hls-parser";
 
 const resolutionMatch = {
-    "3840": "2160",
-    "2732": "1440",
-    "2560": "1440",
-    "2048": "1080",
-    "1920": "1080",
-    "1366": "720",
-    "1280": "720",
-    "960": "480",
-    "640": "360",
-    "426": "240"
+    "3840": 2160,
+    "2732": 1440,
+    "2560": 1440,
+    "2048": 1080,
+    "1920": 1080,
+    "1366": 720,
+    "1280": 720,
+    "960": 480,
+    "640": 360,
+    "426": 240
 }
 
-const qualityMatch = {
-    "2160": "4K",
-    "1440": "2K",
-    "480": "540",
-
-    "4K": "2160",
-    "2K": "1440",
-    "540": "480"
-}
-
-export default async function(obj) {
-    let quality = obj.quality === "max" ? "9000" : obj.quality;
-    if (!quality || obj.isAudioOnly) quality = "9000";
-
-    const url = new URL(`https://player.vimeo.com/video/${obj.id}/config`);
-    if (obj.password) {
-        url.searchParams.set('h', obj.password);
+const requestApiInfo = (videoId, password) => {
+    if (password) {
+        videoId += `:${password}`
     }
 
-    let api = await fetch(url)
+    return fetch(
+        `https://api.vimeo.com/videos/${videoId}`,
+        {
+            headers: {
+                Accept: 'application/vnd.vimeo.*+json; version=3.4.2',
+                'User-Agent': 'Vimeo/10.19.0 (com.vimeo; build:101900.57.0; iOS 17.5.1) Alamofire/5.9.0 VimeoNetworking/5.0.0',
+                Authorization: 'Basic MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==',
+                'Accept-Language': 'en'
+            }
+        }
+    )
+    .then(a => a.json())
+    .catch(() => {});
+}
+
+const compareQuality = (rendition, requestedQuality) => {
+    const quality = parseInt(rendition);
+    return Math.abs(quality - requestedQuality);
+}
+
+const getDirectLink = (data, quality) => {
+    if (!data.files) return;
+
+    const match = data.files
+        .filter(f => f.rendition?.endsWith('p'))
+        .reduce((prev, next) => {
+            const delta = {
+                prev: compareQuality(prev.rendition, quality),
+                next: compareQuality(next.rendition, quality)
+            };
+
+            return delta.prev < delta.next ? prev : next;
+        });
+
+    if (!match) return;
+
+    return {
+        urls: match.link,
+        filenameAttributes: {
+            resolution: `${match.width}x${match.height}`,
+            qualityLabel: match.rendition,
+            extension: "mp4"
+        }
+    }
+}
+
+const getHLS = async (configURL, obj) => {
+    if (!configURL) return;
+
+    const api = await fetch(configURL)
                     .then(r => r.json())
                     .catch(() => {});
     if (!api) return { error: 'ErrorCouldntFetch' };
 
-    let downloadType = "dash";
-
-    if (!obj.isAudioOnly && JSON.stringify(api).includes('"progressive":[{'))
-        downloadType = "progressive";
-
-    let fileMetadata = {
-        title: cleanString(api.video.title.trim()),
-        artist: cleanString(api.video.owner.name.trim()),
-    }
-
-    if (downloadType !== "dash") {
-        if (qualityMatch[quality]) quality = qualityMatch[quality];
-        let all = api.request.files.progressive.sort((a, b) => Number(b.width) - Number(a.width));
-        let best = all[0];
-
-        let bestQuality = all[0].quality.split('p')[0];
-        if (qualityMatch[bestQuality]) {
-            bestQuality = qualityMatch[bestQuality]
-        }
-
-        if (Number(quality) < Number(bestQuality)) {
-            best = all.find(i => i.quality.split('p')[0] === quality);
-        }
-
-        if (!best) return { error: 'ErrorEmptyDownload' };
-
-        return {
-            urls: best.url,
-            audioFilename: `vimeo_${obj.id}_audio`,
-            filename: `vimeo_${obj.id}_${best.width}x${best.height}.mp4`
-        }
-    }
-
-    if (api.video.duration > env.durationLimit)
+    if (api.video?.duration > env.durationLimit) {
         return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
-
-    let masterJSONURL = api.request.files.dash.cdns.akfire_interconnect_quic.url;
-    let masterJSON = await fetch(masterJSONURL).then(r => r.json()).catch(() => {});
-
-    if (!masterJSON) return { error: 'ErrorCouldntFetch' };
-    if (!masterJSON.video) return { error: 'ErrorEmptyDownload' };
-
-    let masterJSON_Video = masterJSON.video
-                            .sort((a, b) => Number(b.width) - Number(a.width))
-                            .filter(a => ["dash", "mp42"].includes(a.format));
-
-    let bestVideo = masterJSON_Video[0];
-    if (Number(quality) < Number(resolutionMatch[bestVideo.width])) {
-        bestVideo = masterJSON_Video.find(i => resolutionMatch[i.width] === quality)
     }
 
-    let masterM3U8 = `${masterJSONURL.split("/sep/")[0]}/sep/video/${bestVideo.id}/master.m3u8`;
-    const fallbackResolution = bestVideo.height > bestVideo.width ? bestVideo.width : bestVideo.height;
+    const urlMasterHLS = api.request?.files?.hls?.cdns?.akfire_interconnect_quic?.url;
+    if (!urlMasterHLS) return { error: 'ErrorCouldntFetch' }
+
+    const masterHLS = await fetch(urlMasterHLS)
+                            .then(r => r.text())
+                            .catch(() => {});
+
+    if (!masterHLS) return { error: 'ErrorCouldntFetch' };
+
+    const variants = HLS.parse(masterHLS)?.variants?.sort(
+        (a, b) => Number(b.bandwidth) - Number(a.bandwidth)
+    );
+    if (!variants || variants.length === 0) return { error: 'ErrorEmptyDownload' };
+
+    let bestQuality;
+
+    if (obj.quality < resolutionMatch[variants[0]?.resolution?.width]) {
+        bestQuality = variants.find(v =>
+            (obj.quality === resolutionMatch[v.resolution.width])
+        );
+    }
+
+    if (!bestQuality) bestQuality = variants[0];
+
+    const expandLink = (path) => {
+        return new URL(path, urlMasterHLS).toString();
+    };
+
+    let urls = expandLink(bestQuality.uri);
+
+    const audioPath = bestQuality?.audio[0]?.uri;
+    if (audioPath) {
+        urls = [
+            urls,
+            expandLink(audioPath)
+        ]
+    } else if (obj.isAudioOnly) {
+        return { error: 'ErrorEmptyDownload' };
+    }
 
     return {
-        urls: masterM3U8,
+        urls,
         isM3U8: true,
-        fileMetadata: fileMetadata,
+        filenameAttributes: {
+            resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
+            qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
+            extension: "mp4"
+        }
+    }
+}
+
+export default async function(obj) {
+    let quality = obj.quality === "max" ? 9000 : Number(obj.quality);
+    if (quality < 240) quality = 240;
+    if (!quality || obj.isAudioOnly) quality = 9000;
+
+    const info = await requestApiInfo(obj.id, obj.password);
+    let response;
+
+    if (obj.isAudioOnly) {
+        response = await getHLS(info.config_url, { ...obj, quality });
+    }
+
+    if (!response) response = getDirectLink(info, quality);
+    if (!response) response = { error: 'ErrorEmptyDownload' };
+
+    if (response.error) {
+        return response;
+    }
+
+    const fileMetadata = {
+        title: cleanString(info.name),
+        artist: cleanString(info.user.name),
+    };
+
+    return merge(
+        {
+            fileMetadata,
             filenameAttributes: {
                 service: "vimeo",
                 id: obj.id,
                 title: fileMetadata.title,
                 author: fileMetadata.artist,
-            resolution: `${bestVideo.width}x${bestVideo.height}`,
-            qualityLabel: `${resolutionMatch[bestVideo.width] || fallbackResolution}p`,
-            extension: "mp4"
-        }
             }
+        },
+        response
+    );
 }

src/modules/processing/services/youtube.js

@@ -2,7 +2,7 @@ import { Innertube, Session } from 'youtubei.js';
 import { env } from '../../config.js';
 import { cleanString } from '../../sub/utils.js';
 import { fetch } from 'undici'
-import { getCookie } from '../cookie/manager.js'
+import { getCookie, updateCookieValues } from '../cookie/manager.js'
 
 const ytBase = Innertube.create().catch(e => e);
 
@@ -24,6 +24,26 @@ const codecMatch = {
     }
 }
 
+const transformSessionData = (cookie) => {
+    if (!cookie)
+        return;
+
+    const values = cookie.values();
+    const REQUIRED_VALUES = [
+        'access_token', 'refresh_token',
+        'client_id', 'client_secret',
+        'expires'
+    ];
+
+    if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
+        return;
+    }
+    return {
+        ...values,
+        expires: new Date(values.expires),
+    };
+}
+
 const cloneInnertube = async (customFetch) => {
     const innertube = await ytBase;
     if (innertube instanceof Error) {
@@ -36,11 +56,32 @@ const cloneInnertube = async (customFetch) => {
         innertube.session.api_version,
         innertube.session.account_index,
         innertube.session.player,
-        getCookie('youtube')?.toString(),
+        undefined,
         customFetch ?? innertube.session.http.fetch,
         innertube.session.cache
     );
 
+    const cookie = getCookie('youtube_oauth');
+    const oauthData = transformSessionData(cookie);
+
+    if (!session.logged_in && oauthData) {
+        await session.oauth.init(oauthData);
+        session.logged_in = true;
+    }
+
+    if (session.logged_in) {
+        await session.oauth.refreshIfRequired();
+        const oldExpiry = new Date(cookie.values().expires);
+        const newExpiry = session.oauth.credentials.expires;
+
+        if (oldExpiry.getTime() !== newExpiry.getTime()) {
+            updateCookieValues(cookie, {
+                ...session.oauth.credentials,
+                expires: session.oauth.credentials.expires.toISOString()
+            });
+        }
+    }
+
     const yt = new Innertube(session);
     return yt;
 }
@@ -62,7 +103,7 @@ export default async function(o) {
     }
 
     try {
-        info = await yt.getBasicInfo(o.id, 'IOS');
+        info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS');
     } catch(e) {
         if (e?.message === 'This video is unavailable') {
             return { error: 'ErrorCouldntFetch' };
@@ -83,6 +124,9 @@ export default async function(o) {
             return { error: 'ErrorYTAgeRestrict' }
         }
     }
+    if (playability.status === "UNPLAYABLE" && playability.reason.endsWith('request limit.')) {
+        return { error: 'ErrorYTRateLimit' }
+    }
 
     if (playability.status !== 'OK') return { error: 'ErrorYTUnavailable' };
     if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' };

src/modules/processing/servicesConfig.json

@@ -15,7 +15,7 @@
             "alias": "reddit videos & gifs",
             "patterns": ["r/:sub/comments/:id/:title", "user/:user/comments/:id/:title"],
             "subdomains": "*",
-            "enabled": false
+            "enabled": true
         },
         "twitter": {
             "alias": "twitter videos & voice",
@@ -33,6 +33,7 @@
         "vk": {
             "alias": "vk video & clips",
             "patterns": ["video:userId_:videoId", "clip:userId_:videoId", "clips:duplicate?z=clip:userId_:videoId"],
+            "subdomains": ["m"],
             "enabled": true
         },
         "ok": {

src/modules/stream/internal-hls.js

@@ -1,16 +1,31 @@
 import { createInternalStream } from './manage.js';
 import HLS from 'hls-parser';
-import path from "node:path";
+
+function getURL(url) {
+    try {
+        return new URL(url);
+    } catch {
+        return null;
+    }
+}
 
 function transformObject(streamInfo, hlsObject) {
     if (hlsObject === undefined) {
         return (object) => transformObject(streamInfo, object);
     }
 
-    const fullUrl = hlsObject.uri.startsWith("/")
-        ? new URL(hlsObject.uri, streamInfo.url).toString()
-        : new URL(path.join(streamInfo.url, "/../", hlsObject.uri)).toString();
-    hlsObject.uri = createInternalStream(fullUrl, streamInfo);
+    let fullUrl;
+    if (getURL(hlsObject.uri)) {
+        fullUrl = hlsObject.uri;
+    } else {
+        fullUrl = new URL(hlsObject.uri, streamInfo.url);
+    }
+
+    hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
+
+    if (hlsObject.map) {
+        hlsObject.map = transformObject(streamInfo, hlsObject.map);
+    }
 
     return hlsObject;
 }

src/modules/stream/internal.js

@@ -1,7 +1,6 @@
 import { request } from 'undici';
 import { Readable } from 'node:stream';
-import { assert } from 'console';
-import { getHeaders, pipe } from './shared.js';
+import { closeRequest, getHeaders, pipe } from './shared.js';
 import { handleHlsPlaylist, isHlsRequest } from './internal-hls.js';
 
 const CHUNK_SIZE = BigInt(8e6); // 8 MB
@@ -27,7 +26,7 @@ async function* readChunks(streamInfo, size) {
         const received = BigInt(chunk.headers['content-length']);
 
         if (received < expected / 2n) {
-            streamInfo.controller.abort();
+            closeRequest(streamInfo.controller);
         }
         
         for await (const data of chunk.body) {
@@ -38,71 +37,86 @@ async function* readChunks(streamInfo, size) {
     }
 }
 
-function chunkedStream(streamInfo, size) {
-    assert(streamInfo.controller instanceof AbortController);
-    const stream = Readable.from(readChunks(streamInfo, size));
-    return stream;
-}
-
 async function handleYoutubeStream(streamInfo, res) {
+    const { signal } = streamInfo.controller;
+    const cleanup = () => (res.end(), closeRequest(streamInfo.controller));
+
     try {
         const req = await fetch(streamInfo.url, {
             headers: getHeaders('youtube'),
             method: 'HEAD',
             dispatcher: streamInfo.dispatcher,
-            signal: streamInfo.controller.signal
+            signal
         });
 
         streamInfo.url = req.url;
         const size = BigInt(req.headers.get('content-length'));
 
         if (req.status !== 200 || !size) {
-            return res.end();
+            return cleanup();
         }
 
-        const stream = chunkedStream(streamInfo, size);
+        const generator = readChunks(streamInfo, size);
+
+        const abortGenerator = () => {
+            generator.return();
+            signal.removeEventListener('abort', abortGenerator);
+        }
+
+        signal.addEventListener('abort', abortGenerator);
+    
+        const stream = Readable.from(generator);
 
         for (const headerName of ['content-type', 'content-length']) {
             const headerValue = req.headers.get(headerName);
             if (headerValue) res.setHeader(headerName, headerValue);
         }
 
-        pipe(stream, res, () => res.end());
+        pipe(stream, res, cleanup);
     } catch {
-        res.end();
+        cleanup();
     }
 }
 
-export async function internalStream(streamInfo, res) {
-    if (streamInfo.service === 'youtube') {
-        return handleYoutubeStream(streamInfo, res);
-    }
+async function handleGenericStream(streamInfo, res) {
+    const { signal } = streamInfo.controller;
+    const cleanup = () => res.end();
 
     try {
         const req = await request(streamInfo.url, {
             headers: {
-                ...streamInfo.headers,
+                ...Object.fromEntries(streamInfo.headers),
                 host: undefined
             },
             dispatcher: streamInfo.dispatcher,
-            signal: streamInfo.controller.signal,
+            signal,
             maxRedirections: 16
         });
 
         res.status(req.statusCode);
+        req.body.on('error', () => {});
 
         for (const [ name, value ] of Object.entries(req.headers))
             res.setHeader(name, value)
 
         if (req.statusCode < 200 || req.statusCode > 299)
-            return res.end();
+            return cleanup();
 
         if (isHlsRequest(req)) {
             await handleHlsPlaylist(streamInfo, req, res);
         } else {
-            pipe(req.body, res, () => res.end());
+            pipe(req.body, res, cleanup);
         }
     } catch {
-        streamInfo.controller.abort();
+        closeRequest(streamInfo.controller);
+        cleanup();
     }
 }
+
+export function internalStream(streamInfo, res) {
+    if (streamInfo.service === 'youtube') {
+        return handleYoutubeStream(streamInfo, res);
+    }
+
+    return handleGenericStream(streamInfo, res);
+}

src/modules/stream/manage.js

@@ -1,10 +1,12 @@
 import NodeCache from "node-cache";
 import { randomBytes } from "crypto";
 import { nanoid } from "nanoid";
+import { setMaxListeners } from "node:events";
 
 import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
 import { env } from "../config.js";
 import { strict as assert } from "assert";
+import { closeRequest } from "./shared.js";
 
 // optional dependency
 const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
@@ -78,16 +80,36 @@ export function createInternalStream(url, obj = {}) {
     }
 
     const streamID = nanoid();
+    let controller = obj.controller;
+
+    if (!controller) {
+        controller = new AbortController(); 
+        setMaxListeners(Infinity, controller.signal);
+    }
+
+    let headers;
+    if (obj.headers) {
+        headers = new Map(Object.entries(obj.headers));
+    }
+
     internalStreamCache[streamID] = {
         url,
         service: obj.service,
-        headers: obj.headers,
-        controller: new AbortController(),
+        headers,
+        controller,
         dispatcher
     };
 
     let streamLink = new URL('/api/istream', `http://127.0.0.1:${env.apiPort}`);
     streamLink.searchParams.set('id', streamID);
+
+    const cleanup = () => {
+        destroyInternalStream(streamLink);
+        controller.signal.removeEventListener('abort', cleanup);
+    }
+
+    controller.signal.addEventListener('abort', cleanup);
+
     return streamLink.toString();
 }
 
@@ -100,7 +122,7 @@ export function destroyInternalStream(url) {
     const id = url.searchParams.get('id');
 
     if (internalStreamCache[id]) {
-        internalStreamCache[id].controller.abort();
+        closeRequest(internalStreamCache[id].controller);
         delete internalStreamCache[id];
     }
 }

src/modules/stream/shared.js

@@ -16,6 +16,10 @@ const serviceHeaders = {
     }
 }
 
+export function closeRequest(controller) {
+    try { controller.abort() } catch {}
+}
+
 export function closeResponse(res) {
     if (!res.headersSent) {
         res.sendStatus(500);

src/modules/stream/types.js

@@ -6,7 +6,7 @@ import { create as contentDisposition } from "content-disposition-header";
 import { metadataManager } from "../sub/utils.js";
 import { destroyInternalStream } from "./manage.js";
 import { env, ffmpegArgs, hlsExceptions } from "../config.js";
-import { getHeaders, closeResponse, pipe } from "./shared.js";
+import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js";
 
 function toRawHeaders(headers) {
     return Object.entries(headers)
@@ -14,10 +14,6 @@ function toRawHeaders(headers) {
                  .join('');
 }
 
-function closeRequest(controller) {
-    try { controller.abort() } catch {}
-}
-
 function killProcess(p) {
     // ask the process to terminate itself gracefully
     p?.kill('SIGTERM');
@@ -96,6 +92,10 @@ export function streamLiveRender(streamInfo, res) {
 
         args = args.concat(ffmpegArgs[format]);
 
+        if (hlsExceptions.includes(streamInfo.service)) {
+            args.push('-bsf:a', 'aac_adtstoasc')
+        }
+
         if (streamInfo.metadata) {
             args = args.concat(metadataManager(streamInfo.metadata))
         }

src/modules/sub/randomize-ciphers.js

@@ -0,0 +1,28 @@
+import tls from 'node:tls';
+import { randomBytes } from 'node:crypto';
+
+const ORIGINAL_CIPHERS = tls.DEFAULT_CIPHERS;
+
+// How many ciphers from the top of the list to shuffle.
+// The remaining ciphers are left in the original order.
+const TOP_N_SHUFFLE = 8;
+
+// Modified variation of https://stackoverflow.com/a/12646864
+const shuffleArray = (array) => {
+    for (let i = array.length - 1; i > 0; i--) {
+        const j = randomBytes(4).readUint32LE() % array.length;
+        [array[i], array[j]] = [array[j], array[i]];
+    }
+
+    return array;
+}
+
+export const randomizeCiphers = () => {
+    do {
+        const cipherList = ORIGINAL_CIPHERS.split(':');
+        const shuffled = shuffleArray(cipherList.slice(0, TOP_N_SHUFFLE));
+        const retained = cipherList.slice(TOP_N_SHUFFLE);
+
+        tls.DEFAULT_CIPHERS = [ ...shuffled, ...retained ].join(':');
+    } while (tls.DEFAULT_CIPHERS === ORIGINAL_CIPHERS);
+}

src/modules/sub/utils.js

@@ -44,3 +44,17 @@ export function cleanHTML(html) {
     clean = clean.replace(/\n/g, '');
     return clean
 }
+
+export function merge(a, b) {
+    for (const k of Object.keys(b)) {
+        if (Array.isArray(b[k])) {
+            a[k] = [...(a[k] ?? []), ...b[k]];
+        } else if (typeof b[k] === 'object') {
+            a[k] = merge(a[k], b[k]);
+        } else {
+            a[k] = b[k];
+        }
+    }
+
+    return a;
+}

src/util/generate-youtube-tokens.js

@@ -0,0 +1,38 @@
+import { Innertube } from 'youtubei.js';
+import { Red } from '../modules/sub/consoleText.js'
+
+const bail = (...msg) => {
+    console.error(...msg);
+    throw new Error(msg);
+};
+
+const tube = await Innertube.create();
+
+tube.session.once(
+    'auth-pending',
+    ({ verification_url, user_code }) => {
+        console.log(`${Red('[!]')} The token generated by this script is sensitive and you should not share it with anyone!`);
+        console.log(`    By using this token, you are risking your Google account getting terminated.`);
+        console.log(`    You should ${Red('NOT')} use your personal account!`);
+        console.log();
+        console.log(`Open ${verification_url} in a browser and enter ${user_code} when asked for the code.`);
+    }
+);
+
+tube.session.once('auth-error', (err) => bail('An error occurred:', err));
+tube.session.once('auth', ({ status, credentials, ...rest }) => {
+    if (status !== 'SUCCESS') {
+        bail('something went wrong', rest);
+    }
+
+    console.log(
+        'add this cookie to the youtube_oauth array in your cookies file:',
+        JSON.stringify(
+            Object.entries(credentials)
+                .map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`)
+                .join('; ')
+        )
+    );
+});
+
+await tube.session.signIn();

src/util/test.js

@@ -9,7 +9,7 @@ import { normalizeRequest } from "../modules/processing/request.js";
 import { env } from "../modules/config.js";
 
 env.apiURL = 'http://localhost:9000'
-let tests = loadJSON('./src/test/tests.json');
+let tests = loadJSON('./src/util/tests.json');
 
 let noTest = [];
 let failed = [];

src/util/tests.json

@@ -674,7 +674,7 @@
         "params": {},
         "expected": {
             "code": 200,
-            "status": "stream"
+            "status": "redirect"
         }
     }],
     "reddit": [{