Merge branch 'imputnet:current' into fix-twitter-links

.github/workflows/fast-forward.yml

@@ -0,0 +1,22 @@
+name: fast-forward
+on:
+  issue_comment:
+    types: [created, edited]
+jobs:
+  fast-forward:
+    # Only run if the comment contains the /fast-forward command.
+    if: ${{ contains(github.event.comment.body, '/fast-forward')
+            && github.event.issue.pull_request }}
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+
+    steps:
+      - name: Fast forwarding
+        uses: sequoia-pgp/fast-forward@v1
+        with:
+          merge: true
+          comment: 'on-error'

CONTRIBUTING.md

@@ -0,0 +1,39 @@
+# contributing to cobalt
+if you're reading this, you are probably interested in contributing to cobalt, which we are very thankful for :3
+
+this document serves as a guide to help you make contributions that we can merge into the cobalt codebase.
+
+## translations
+currently, we are **not accepting** translations of cobalt. this is because we are making significant changes to the frontend, and the currently used localization structure is being completely reworked. if this changes, this document will be updated.
+
+## adding features or support for services
+before putting in the effort to implement a feature, it's worth considering whether it would be appropriate to add it to cobalt. the cobalt api is built to assist people **only with downloading freely accessible content**. other functionality, such as:
+- downloading paid / not publicly accessible content
+- downloading content protected by DRM
+- scraping unrelated information & exposing it outside of file metadata
+
+will not be reviewed or merged.
+
+if you plan on adding a feature or support for a service, but are unsure whether it would be appropriate, it's best to open an issue and discuss it beforehand.
+
+## git
+when contributing code to cobalt, there are a few guidelines in place to ensure that the code history is readable and comprehensible.
+
+### clean commit messages
+internally, we use a format similar to [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) - the first part signifies which part of the code you are changing (the *scope*), and the second part explains the change. for inspiration on how to write appropriate commit titles, you can take a look at the [commit history](https://github.com/imputnet/cobalt/commits/).
+
+the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `stream/internal: fix object not being handled properly`).
+
+if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
+
+if your contribution has uninformative commit titles, you may be asked to interactively rebase your branch and amend each commit to include a meaningful title.
+
+### clean commit history
+if your branch is out of date and/or has some merge conflicts with the `current` branch, you should **rebase** it instead of merging. this prevents meaningless merge commits from being included in your branch, which would then end up in the cobalt git history.
+
+if you find a mistake or bug in your code before it's merged or reviewed, instead of making a brand new commit to fix it, it would be preferable to amend that specific commit where the mistake was first introduced. this also helps us easily revert a commit if we discover that it introduced a bug or some unwanted behavior.
+- if the commit you are fixing is the latest one, you can add your files to staging and then use `git commit --amend` to apply the change.
+- if the commit is somewhere deeper in your branch, you can use `git commit --fixup=HASH`, where *`HASH`* is the commit you are fixing.
+    - afterward, you must interactively rebase your branch with `git rebase -i current --autosquash`.
+      this will open up an editor, but you don't need to do anything else except save the file and exit.
+- once you do either of these things, you will need to do a **force push** to your branch with `git push --force-with-lease`.

docs/api.md

@@ -73,5 +73,5 @@ response body type: `application/json`
 | `branch`    | `string` | git branch        |
 | `name`      | `string` | server name       |
 | `url`       | `string` | server url        |
-| `cors`      | `int`    | cors status       |
+| `cors`      | `number` | cors status       |
 | `startTime` | `string` | server start time |

src/core/api.js

@@ -196,10 +196,10 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
             return res.sendStatus(404);
         }
 
-        streamInfo.headers = {
+        streamInfo.headers = new Map([
-            ...streamInfo.headers,
+            ...(streamInfo.headers || []),
-            ...req.headers
+            ...Object.entries(req.headers)
-        };
+        ]);
 
         return stream(res, { type: 'internal', ...streamInfo });
     })

src/core/web.js

@@ -26,45 +26,46 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
 
     app.get('/onDemand', (req, res) => {
         try {
-            if (req.query.blockId) {
+            if (typeof req.query.blockId !== 'string') {
-                let blockId = req.query.blockId.slice(0, 3);
-                let blockData;
-                switch(blockId) {
-                    // changelog history
-                    case "0":
-                        let history = changelogHistory();
-                        if (history) {
-                            blockData = createResponse("success", { t: history })
-                        } else {
-                            blockData = createResponse("error", {
-                                t: "couldn't render this block, please try again!"
-                            })
-                        }
-                        break;
-                    // celebrations emoji
-                    case "1":
-                        let celebration = celebrationsEmoji();
-                        if (celebration) {
-                            blockData = createResponse("success", { t: celebration })
-                        }
-                        break;
-                    default:
-                        blockData = createResponse("error", {
-                            t: "couldn't find a block with this id"
-                        })
-                        break;
-                }
-                if (blockData?.body) {
-                    return res.status(blockData.status).json(blockData.body);
-                } else {
-                    return res.status(204).end();
-                }
-            } else {
                 return res.status(400).json({
                     status: "error",
                     text: "couldn't render this block, please try again!"
                 });
             }
+
+            let blockId = req.query.blockId.slice(0, 3);
+            let blockData;
+            switch(blockId) {
+                // changelog history
+                case "0":
+                    let history = changelogHistory();
+                    if (history) {
+                        blockData = createResponse("success", { t: history })
+                    } else {
+                        blockData = createResponse("error", {
+                            t: "couldn't render this block, please try again!"
+                        })
+                    }
+                    break;
+                // celebrations emoji
+                case "1":
+                    let celebration = celebrationsEmoji();
+                    if (celebration) {
+                        blockData = createResponse("success", { t: celebration })
+                    }
+                    break;
+                default:
+                    blockData = createResponse("error", {
+                        t: "couldn't find a block with this id"
+                    })
+                    break;
+            }
+
+            if (blockData?.body) {
+                return res.status(blockData.status).json(blockData.body);
+            } else {
+                return res.status(204).end();
+            }
         } catch {
             return res.status(400).json({
                 status: "error",

src/modules/processing/services/ok.js

@@ -20,14 +20,15 @@ export default async function(o) {
     }).then(r => r.text()).catch(() => {});
 
     if (!html) return { error: 'ErrorCouldntFetch' };
-    if (!html.includes(`<div data-module="OKVideo" data-options="{`)) {
+
+    let videoData = html.match(/<div data-module="OKVideo" .*? data-options="({.*?})"( .*?)>/)
+                        ?.[1]
+                        ?.replaceAll("&quot;", '"');
+
+    if (!videoData) {
         return { error: 'ErrorEmptyDownload' };
     }
 
-    let videoData = html.split(`<div data-module="OKVideo" data-options="`)[1]
-                        .split('" data-')[0]
-                        .replaceAll("&quot;", '"');
-
     videoData = JSON.parse(JSON.parse(videoData).flashvars.metadata);
 
     if (videoData.provider !== "UPLOADED_ODKL")

src/modules/processing/services/pinterest.js

@@ -1,6 +1,6 @@
 import { genericUserAgent } from "../../config.js";
 
-const videoRegex = /"url":"(https:\/\/v1.pinimg.com\/videos\/.*?)"/g;
+const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
 const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
 
 export default async function(o) {

src/modules/processing/services/rutube.js

@@ -10,6 +10,8 @@ async function requestJSON(url) {
     } catch {}
 }
 
+const delta = (a, b) => Math.abs(a - b);
+
 export default async function(obj) {
     if (obj.yappyId) {
         const yappy = await requestJSON(
@@ -25,7 +27,7 @@ export default async function(obj) {
         }
     }
 
-    const quality = obj.quality === "max" ? "9000" : obj.quality;
+    const quality = Number(obj.quality) || 9000;
 
     const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`);
     if (obj.key) requestURL.searchParams.set('p', obj.key);
@@ -45,12 +47,16 @@ export default async function(obj) {
 
     if (!m3u8) return { error: 'ErrorCouldntFetch' };
 
-    m3u8 = HLS.parse(m3u8).variants.sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
+    m3u8 = HLS.parse(m3u8).variants;
 
-    let bestQuality = m3u8[0];
+    const matchingQuality = m3u8.reduce((prev, next) => {
-    if (Number(quality) < bestQuality.resolution.height) {
+        const diff = {
-        bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height));
+            prev: delta(quality, prev.resolution.height),
-    }
+            next: delta(quality, next.resolution.height)
+        };
+
+        return diff.prev < diff.next ? prev : next;
+    });
 
     const fileMetadata = {
         title: cleanString(play.title.trim()),
@@ -58,15 +64,15 @@ export default async function(obj) {
     }
 
     return {
-        urls: bestQuality.uri,
+        urls: matchingQuality.uri,
         isM3U8: true,
         filenameAttributes: {
             service: "rutube",
             id: obj.id,
             title: fileMetadata.title,
             author: fileMetadata.artist,
-            resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
+            resolution: `${matchingQuality.resolution.width}x${matchingQuality.resolution.height}`,
-            qualityLabel: `${bestQuality.resolution.height}p`,
+            qualityLabel: `${matchingQuality.resolution.height}p`,
             extension: "mp4"
         },
         fileMetadata: fileMetadata

src/modules/processing/services/soundcloud.js

@@ -12,17 +12,19 @@ async function findClientID() {
         let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
 
         if (cachedID.version === scVersion) return cachedID.id;
-        
+
         let scripts = sc.matchAll(/<script.+src="(.+)">/g);
         let clientid;
         for (let script of scripts) {
             let url = script[1];
-    
+
-            if (url && !url.startsWith('https://a-v2.sndcdn.com')) return;
+            if (!url?.startsWith('https://a-v2.sndcdn.com/')) {
-    
+                return;
+            }
+
             let scrf = await fetch(url).then(r => r.text()).catch(() => {});
             let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
-    
+
             if (id && typeof id[0] === 'string') {
                 clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
                 break;

src/modules/processing/services/vimeo.js

@@ -1,5 +1,5 @@
 import { env } from "../../config.js";
-import { cleanString } from '../../sub/utils.js';
+import { cleanString, merge } from '../../sub/utils.js';
 
 import HLS from "hls-parser";
 
@@ -16,32 +16,70 @@ const resolutionMatch = {
     "426": 240
 }
 
-export default async function(obj) {
+const requestApiInfo = (videoId, password) => {
-    let quality = obj.quality === "max" ? 9000 : Number(obj.quality);
+    if (password) {
-    if (quality < 240) quality = 240;
+        videoId += `:${password}`
-    if (!quality || obj.isAudioOnly) quality = 9000;
-
-    const url = new URL(`https://player.vimeo.com/video/${obj.id}/config`);
-    if (obj.password) {
-        url.searchParams.set('h', obj.password);
     }
 
-    const api = await fetch(url)
+    return fetch(
+        `https://api.vimeo.com/videos/${videoId}`,
+        {
+            headers: {
+                Accept: 'application/vnd.vimeo.*+json; version=3.4.2',
+                'User-Agent': 'Vimeo/10.19.0 (com.vimeo; build:101900.57.0; iOS 17.5.1) Alamofire/5.9.0 VimeoNetworking/5.0.0',
+                Authorization: 'Basic MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==',
+                'Accept-Language': 'en'
+            }
+        }
+    )
+    .then(a => a.json())
+    .catch(() => {});
+}
+
+const compareQuality = (rendition, requestedQuality) => {
+    const quality = parseInt(rendition);
+    return Math.abs(quality - requestedQuality);
+}
+
+const getDirectLink = (data, quality) => {
+    if (!data.files) return;
+
+    const match = data.files
+        .filter(f => f.rendition?.endsWith('p'))
+        .reduce((prev, next) => {
+            const delta = {
+                prev: compareQuality(prev.rendition, quality),
+                next: compareQuality(next.rendition, quality)
+            };
+
+            return delta.prev < delta.next ? prev : next;
+        });
+
+    if (!match) return;
+
+    return {
+        urls: match.link,
+        filenameAttributes: {
+            resolution: `${match.width}x${match.height}`,
+            qualityLabel: match.rendition,
+            extension: "mp4"
+        }
+    }
+}
+
+const getHLS = async (configURL, obj) => {
+    if (!configURL) return;
+
+    const api = await fetch(configURL)
                     .then(r => r.json())
                     .catch(() => {});
-
     if (!api) return { error: 'ErrorCouldntFetch' };
 
-    const fileMetadata = {
+    if (api.video?.duration > env.durationLimit) {
-        title: cleanString(api.video.title.trim()),
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
-        artist: cleanString(api.video.owner.name.trim()),
     }
 
-    if (api.video?.duration > env.durationLimit)
-        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
-
     const urlMasterHLS = api.request?.files?.hls?.cdns?.akfire_interconnect_quic?.url;
-
     if (!urlMasterHLS) return { error: 'ErrorCouldntFetch' }
 
     const masterHLS = await fetch(urlMasterHLS)
@@ -57,9 +95,9 @@ export default async function(obj) {
 
     let bestQuality;
 
-    if (quality < resolutionMatch[variants[0]?.resolution?.width]) {
+    if (obj.quality < resolutionMatch[variants[0]?.resolution?.width]) {
         bestQuality = variants.find(v =>
-            (quality === resolutionMatch[v.resolution.width])
+            (obj.quality === resolutionMatch[v.resolution.width])
         );
     }
 
@@ -84,15 +122,48 @@ export default async function(obj) {
     return {
         urls,
         isM3U8: true,
-        fileMetadata: fileMetadata,
         filenameAttributes: {
-            service: "vimeo",
-            id: obj.id,
-            title: fileMetadata.title,
-            author: fileMetadata.artist,
             resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
             qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
             extension: "mp4"
         }
     }
 }
+
+export default async function(obj) {
+    let quality = obj.quality === "max" ? 9000 : Number(obj.quality);
+    if (quality < 240) quality = 240;
+    if (!quality || obj.isAudioOnly) quality = 9000;
+
+    const info = await requestApiInfo(obj.id, obj.password);
+    let response;
+
+    if (obj.isAudioOnly) {
+        response = await getHLS(info.config_url, { ...obj, quality });
+    }
+
+    if (!response) response = getDirectLink(info, quality);
+    if (!response) response = { error: 'ErrorEmptyDownload' };
+
+    if (response.error) {
+        return response;
+    }
+
+    const fileMetadata = {
+        title: cleanString(info.name),
+        artist: cleanString(info.user.name),
+    };
+
+    return merge(
+        {
+            fileMetadata,
+            filenameAttributes: {
+                service: "vimeo",
+                id: obj.id,
+                title: fileMetadata.title,
+                author: fileMetadata.artist,
+            }
+        },
+        response
+    );
+}

src/modules/stream/internal.js

@@ -85,7 +85,7 @@ async function handleGenericStream(streamInfo, res) {
     try {
         const req = await request(streamInfo.url, {
             headers: {
-                ...streamInfo.headers,
+                ...Object.fromEntries(streamInfo.headers),
                 host: undefined
             },
             dispatcher: streamInfo.dispatcher,

src/modules/stream/manage.js

@@ -87,10 +87,15 @@ export function createInternalStream(url, obj = {}) {
         setMaxListeners(Infinity, controller.signal);
     }
 
+    let headers;
+    if (obj.headers) {
+        headers = new Map(Object.entries(obj.headers));
+    }
+
     internalStreamCache[streamID] = {
         url,
         service: obj.service,
-        headers: obj.headers,
+        headers,
         controller,
         dispatcher
     };

src/modules/sub/utils.js

@@ -44,3 +44,17 @@ export function cleanHTML(html) {
     clean = clean.replace(/\n/g, '');
     return clean
 }
+
+export function merge(a, b) {
+    for (const k of Object.keys(b)) {
+        if (Array.isArray(b[k])) {
+            a[k] = [...(a[k] ?? []), ...b[k]];
+        } else if (typeof b[k] === 'object') {
+            a[k] = merge(a[k], b[k]);
+        } else {
+            a[k] = b[k];
+        }
+    }
+
+    return a;
+}

src/util/tests.json

@@ -674,7 +674,7 @@
         "params": {},
         "expected": {
             "code": 200,
-            "status": "stream"
+            "status": "redirect"
         }
     }],
     "reddit": [{