Merge branch 'current' into feature/snapchat

Signed-off-by: Snazzah <7025343+Snazzah@users.noreply.github.com>

.github/test.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+set -e
+
+# thx: https://stackoverflow.com/a/27601038
+waitport() {
+    ATTEMPTS=50
+    while [ $((ATTEMPTS-=1)) -gt 0 ] && ! nc -z localhost $1; do   
+        sleep 0.1
+    done
+
+    [ "$ATTEMPTS" != 0 ] || exit 1
+}
+
+test_api() {
+    waitport 3000
+    curl -m 3 http://localhost:3000/api/serverInfo
+    API_RESPONSE=$(curl -m 3 http://localhost:3000/api/json \
+         -X POST \
+         -H "Accept: application/json" \
+         -H "Content-Type: application/json" \
+         -d '{"url":"https://www.youtube.com/watch?v=jNQXAC9IVRw"}')
+
+    echo "$API_RESPONSE"
+    STATUS=$(echo "$API_RESPONSE" | jq -r .status)
+    STREAM_URL=$(echo "$API_RESPONSE" | jq -r .url)
+    [ "$STATUS" = stream ] || exit 1;
+
+    CONTENT_LENGTH=$(curl -I -m 3 "$STREAM_URL" \
+                        | grep -i content-length \
+                        | cut -d' ' -f2 \
+                        | tr -d '\r')
+
+    echo "$CONTENT_LENGTH"
+    [ "$CONTENT_LENGTH" = 0 ] && exit 1
+    if [ "$CONTENT_LENGTH" -lt 512 ]; then
+        exit 1
+    fi
+}
+
+test_web() {
+    waitport 3001
+    curl -m 3 http://127.0.0.1:3001/onDemand?blockId=0 \
+        | grep -q '"status":"success"' 
+}
+
+setup_api() {
+    export API_PORT=3000
+    export API_URL=http://localhost:3000
+    timeout 10 npm run start
+}
+
+setup_web() {
+    export WEB_PORT=3001
+    export WEB_URL=http://localhost:3001
+    export API_URL=http://localhost:3000
+    timeout 5 npm run start
+}
+
+cd "$(git rev-parse --show-toplevel)"
+npm i
+
+if [ "$1" = "api" ]; then
+    setup_api &
+    test_api
+elif [ "$1" = "web" ]; then
+    setup_web &
+    test_web
+else
+    echo "usage: $0 <api/web>" >&2
+    exit 1
+fi
+
+wait || exit $?

.github/workflows/docker.yml

@@ -55,3 +55,5 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/test.yml

@@ -0,0 +1,25 @@
+name: Run tests
+
+on:
+  pull_request:
+  push:
+    branches: [ current ]
+
+jobs:
+  test-web:
+    name: web sanity check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Run test script
+        run: .github/test.sh web
+
+  test-api:
+    name: api sanity check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Run test script
+        run: .github/test.sh api

.gitignore

@@ -4,18 +4,10 @@ desktop.ini
 
 # npm
 node_modules
-package-lock.json
 
 # secrets
 .env
 
-# page build
-min
-build
-
-# stuff i already made but delayed
-future
-
 # docker
 docker-compose.yml
 
@@ -24,3 +16,6 @@ docker-compose.yml
 
 # cookie file
 cookies.json
+
+# page build
+build

Dockerfile

@@ -3,9 +3,10 @@ WORKDIR /app
 
 COPY package*.json ./
 
-RUN apt-get update && \
+RUN apt-get update                                    && \
-    apt-get install -y git python3 build-essential && \
+    apt-get install -y git python3 build-essential    && \
-    npm install && \
+    npm ci                                            && \
+    npm cache clean --force                           && \
     apt purge --autoremove -y python3 build-essential && \
     rm -rf ~/.cache/ /var/lib/apt/lists/*
 

LICENSE

@@ -629,8 +629,8 @@ to attach them to the start of each source file to most effectively
 state the exclusion of warranty; and each file should have at least
 the "copyright" line and a pointer to where the full notice is found.
 
-    cobalt is possibly the nicest social media downloader out there.
+    save what you love with cobalt.
-    Copyright (C) 2022 wukko
+    Copyright (C) 2024 imput
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU Affero General Public License as published by

README.md

@@ -3,6 +3,9 @@ best way to save what you love: [cobalt.tools](https://cobalt.tools/)
 
 ![cobalt logo with repeated logo (double arrow) pattern background](/src/front/icons/pattern.png "cobalt logo with repeated logo (double arrow) pattern background")  
 
+[💬 community discord server](https://discord.gg/pQPt8HBUPu)  
+[🐦 twitter/x](https://x.com/justusecobalt)  
+
 ## what's cobalt?
 cobalt is a media downloader that doesn't piss you off. it's fast, friendly, and doesn't have any bullshit that modern web is filled with: ***no ads, trackers, or invasive analytics***.  
 
@@ -15,8 +18,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | :--------                      | :-----------: | :--------: | :--------: | :------: | :-------------: |
 | bilibili.com & bilibili.tv     | ✅            | ✅         | ✅         | ➖         | ➖              |
 | dailymotion                    | ✅            | ✅         | ✅         | ✅         | ✅              |
-| instagram posts & stories      | ✅            | ✅         | ✅         | ➖         | ➖              |
+| instagram posts & reels        | ✅            | ✅         | ✅         | ➖         | ➖              |
-| instagram reels                | ✅            | ✅         | ✅         | ➖         | ➖              |
 | ok video                       | ✅            | ❌         | ❌         | ✅         | ✅              |
 | pinterest                      | ✅            | ✅         | ✅         | ➖         | ➖              |
 | reddit                         | ✅            | ✅         | ✅         | ❌         | ❌              |
@@ -42,7 +44,7 @@ this list is not final and keeps expanding over time. if support for a service y
 ### additional notes or features (per service)
 | service    | notes or features                                                                                                    |
 | :--------  | :-----                                                                                                               |
-| instagram  | supports photos, videos, and stories. lets you pick what to save from multi-media posts.                             |
+| instagram  | supports reels, photos, and videos. lets you pick what to save from multi-media posts.                               |
 | pinterest  | supports photos, gifs, videos and stories.                                                                           |
 | reddit     | supports gifs and videos.                                                                                            |
 | snapchat   | supports spotlights and stories. lets you pick what to save from stories.                                            |
@@ -55,7 +57,7 @@ this list is not final and keeps expanding over time. if support for a service y
 ## cobalt api
 cobalt has an open api that you can use in your projects *for free~*. it's easy and straightforward to use, [check out the docs](/docs/api.md) to learn how to use it. 
 
-✅ you can use the main api instance ([co.wuk.sh](https://co.wuk.sh/)) in your **personal** projects.  
+✅ you can use the main api instance ([api.cobalt.tools](https://api.cobalt.tools/)) in your **personal** projects.  
 ❌ you cannot use the free api commercially (anywhere that's gated behind paywalls or ads). host your own instance for this.
 
 we reserve the right to restrict abusive/excessive access to the main instance api.
@@ -64,8 +66,8 @@ we reserve the right to restrict abusive/excessive access to the main instance a
 if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md).  
 it's *highly* recommended to use a docker compose method unless you run for developing/debugging purposes.
 
-## sponsors 
+## partners 
-cobalt is sponsored by [royalehosting.net](https://royalehosting.net/), all main instances are currently hosted on their network :)  
+cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), all main instances are currently hosted on their network :)  
 
 ## ethics and disclaimer
 cobalt is a tool for easing content downloads from internet and takes ***zero liability***. you are responsible for what you download, how you use and distribute that content. please be mindful when using content of others and always credit original creators. fair use and credits benefit everyone.

docs/api.md

@@ -2,7 +2,7 @@
 this document provides info about methods and acceptable variables for all cobalt api requests.  
 
 ```
-👍 you can use co.wuk.sh instance in your projects for free, just don't be an asshole.
+👍 you can use api.cobalt.tools in your projects for free, just don't be an asshole.
 ```
 
 ## POST: `/api/json`

docs/examples/docker-compose.example.yml

@@ -17,8 +17,8 @@ services:
             #- 127.0.0.1:9000:9000
 
         environment:
-            # replace https://co.wuk.sh/ with your instance's target url in same format
+            # replace https://api.cobalt.tools/ with your instance's target url in same format
-            API_URL: "https://co.wuk.sh/"
+            API_URL: "https://api.cobalt.tools/"
             # replace eu-nl with your instance's distinctive name
             API_NAME: "eu-nl"
             # if you want to use cookies when fetching data from services, uncomment the next line and the lines under volume
@@ -49,8 +49,8 @@ services:
         environment:
             # replace https://cobalt.tools/ with your instance's target url in same format
             WEB_URL: "https://cobalt.tools/"
-            # replace https://co.wuk.sh/ with preferred api instance url
+            # replace https://api.cobalt.tools/ with preferred api instance url
-            API_URL: "https://co.wuk.sh/"
+            API_URL: "https://api.cobalt.tools/"
 
         labels:
             - com.centurylinklabs.watchtower.scope=cobalt
@@ -59,6 +59,6 @@ services:
     watchtower:
         image: ghcr.io/containrrr/watchtower
         restart: unless-stopped
-        command: --cleanup --scope cobalt --interval 900
+        command: --cleanup --scope cobalt --interval 900 --include-restarting
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock

docs/run-an-instance.md

@@ -35,13 +35,13 @@ it's highly recommended to use a reverse proxy (such as nginx) if you want your
 ## using regular node.js (useful for local development)
 setup script installs all needed `npm` dependencies, but you have to install `node.js` *(version 18 or above)* and `git` yourself.
 
-1. clone the repo: `git clone https://github.com/wukko/cobalt`.
+1. clone the repo: `git clone https://github.com/imputnet/cobalt`.
 2. run setup script and follow instructions: `npm run setup`. you need to host api and web instances separately, so pick whichever applies.
 3. run cobalt via `npm start`.
 4. done.
 
 ### ubuntu 22.04 workaround
-`nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/wukko/cobalt/issues/101#issuecomment-1494822258)):
+`nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/imputnet/cobalt/issues/101#issuecomment-1494822258)):
 
 ```bash
 sudo apt install nscd
@@ -52,41 +52,21 @@ sudo service nscd start
 ### variables for api
 | variable name         | default   | example                 | description |
 |:----------------------|:----------|:------------------------|:------------|
-| `API_PORT`            | `9000`    |  `9000`                 | changes port from which api server is accessible. |
+| `API_PORT`            | `9000`    | `9000`                  | changes port from which api server is accessible. |
-| `API_LISTEN_ADDRESS`  | `0.0.0.0` |  `127.0.0.1`            | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** |
+| `API_LISTEN_ADDRESS`  | `0.0.0.0` | `127.0.0.1`             | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** |
-| `API_URL`             | ➖        | `https://co.wuk.sh/`    | changes url from which api server is accessible. <br> ***REQUIRED TO RUN API***. |
+| `API_URL`             | ➖        | `https://api.cobalt.tools/` | changes url from which api server is accessible. <br> ***REQUIRED TO RUN THE API***. |
 | `API_NAME`            | `unknown` | `ams-1`                 | api server name that is shown in `/api/serverInfo`. |
 | `CORS_WILDCARD`       | `1`       | `0`                     | toggles cross-origin resource sharing. <br> `0`: disabled. `1`: enabled. |
 | `CORS_URL`            | not used  | `https://cobalt.tools/` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. |
 | `COOKIE_PATH`         | not used  | `/cookies.json`         | path for cookie file relative to main folder. |
 | `PROCESSING_PRIORITY` | not used  | `10`                    | changes `nice` value* for ffmpeg subprocess. available only on unix systems. |
-| `TIKTOK_DEVICE_INFO`  | ➖        | *see below*             | device info (including `iid` and `device_id`) for tiktok functionality. required for tiktok to work. |
+| `FREEBIND_CIDR`       | ➖        | `2001:db8::/32`         | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. see below for more info. |
-| `FREEBIND_CIDR`       | ➖        | `2001:db8::/32`         | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. for more info, see below. |
+| `RATELIMIT_WINDOW`    | `60`      | `120`                   | rate limit time window in **seconds**. |
+| `RATELIMIT_MAX`       | `20`      | `30`                    | max requests per time window. requests above this amount will be blocked for the rate limit window duration. |
+| `DURATION_LIMIT`      | `10800`   | `18000`                 | max allowed video duration in **seconds**. |
 
 \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
 
-#### TIKTOK_DEVICE_INFO
-you need to get your own device info for tiktok functionality to work. this can be done by proxying the app through any request-intercepting proxy (such as [mitmproxy](https://mitmproxy.org)). you need to disable ssl pinning to see requests. there will be no assistance provided by cobalt for this.
-
-example config (replace **ALL** values with ones you got from mitm):
-```
-'{
-    "iid": "<install_id here>",
-    "device_id": "<device_id here>",
-    "channel": "googleplay",
-    "app_name": "musical_ly",
-    "version_code": "310503",
-    "device_platform": "android",
-    "device_type": "Redmi+7",
-    "os_version": "13"
-}'
-```
-
-you can compress the json to save space. if you're using a `.env` file then the line would would look like this (***note the quotes***):
-```
-TIKTOK_DEVICE_INFO='{"iid":"<install_id here>","device_id":"<device_id here>","channel":"googleplay","app_name":"musical_ly","version_code":"310503","device_platform":"android","device_type":"Redmi+7","os_version":"13"}'
-```
-
 #### FREEBIND_CIDR
 setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all
 requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt
@@ -94,13 +74,13 @@ in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127
 `network_mode` for the container to `host`.
 
 ### variables for web
-| variable name        | default              | example                 | description                                                                           |
+| variable name        | default                     | example                     | description                                                                           |
-|:---------------------|:---------------------|:------------------------|:--------------------------------------------------------------------------------------|
+|:---------------------|:----------------------------|:----------------------------|:--------------------------------------------------------------------------------------|
-| `WEB_PORT`           | `9001`               |  `9001`                 | changes port from which frontend server is accessible.                                |
+| `WEB_PORT`           | `9001`                      |  `9001`                     | changes port from which frontend server is accessible.                                |
-| `WEB_URL`            | ➖                   | `https://cobalt.tools/` | changes url from which frontend server is accessible. <br> ***REQUIRED TO RUN WEB***. |
+| `WEB_URL`            | ➖                          | `https://cobalt.tools/`     | changes url from which frontend server is accessible. <br> ***REQUIRED TO RUN WEB***. |
-| `API_URL`            | `https://co.wuk.sh/` | `https://co.wuk.sh/`    | changes url which is used for api requests by frontend clients.                       |
+| `API_URL`            | `https://api.cobalt.tools/` | `https://api.cobalt.tools/` | changes url which is used for api requests by frontend clients.                       |
-| `SHOW_SPONSORS`      | `0`                  | `1`                     | toggles sponsor list in about popup. <br> `0`: disabled. `1`: enabled.                |
+| `SHOW_SPONSORS`      | `0`                         | `1`                         | toggles sponsor list in about popup. <br> `0`: disabled. `1`: enabled.                |
-| `IS_BETA`            | `0`                  | `1`                     | toggles beta tag next to cobalt logo. <br> `0`: disabled. `1`: enabled.               |
+| `IS_BETA`            | `0`                         | `1`                         | toggles beta tag next to cobalt logo. <br> `0`: disabled. `1`: enabled.               |
-| `PLAUSIBLE_HOSTNAME` | ➖                   | `plausible.io`*         | enables plausible analytics with provided hostname as receiver backend.               |
+| `PLAUSIBLE_HOSTNAME` | ➖                          | `plausible.io`*             | enables plausible analytics with provided hostname as receiver backend.               |
 
 \* don't use plausible.io as receiver backend unless you paid for their cloud service. use your own domain when hosting community edition of plausible. refer to their [docs](https://plausible.io/docs) when needed.

package.json

@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "7.13.3",
+    "version": "7.14.1",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",

src/config.json

@@ -1,6 +1,4 @@
 {
-    "streamLifespan": 90000,
-    "maxVideoDuration": 10800000,
     "genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
     "authorInfo": {
         "support": {
@@ -54,7 +52,7 @@
         "saveToGalleryShortcut": "https://www.icloud.com/shortcuts/14e9aebf04b24156acc34ceccf7e6fcd",
         "saveToFilesShortcut": "https://www.icloud.com/shortcuts/2134cd9d4d6b41448b2201f933542b2e",
         "statusPage": "https://status.cobalt.tools/",
-        "troubleshootingGuide": "https://github.com/wukko/cobalt/blob/current/docs/troubleshooting.md"
+        "troubleshootingGuide": "https://github.com/imputnet/cobalt/blob/current/docs/troubleshooting.md"
     },
     "celebrations": {
         "01-01": "🎄",
@@ -95,7 +93,7 @@
     "sponsors": [{
         "name": "royale",
         "fullName": "RoyaleHosting",
-        "url": "https://royalehosting.net/",
+        "url": "https://royalehosting.net/?partner=cobalt",
         "logo": {
             "width": 605,
             "height": 136,

src/core/api.js

@@ -1,89 +1,109 @@
 import cors from "cors";
 import rateLimit from "express-rate-limit";
-import { randomBytes } from "crypto";
-
-const ipSalt = randomBytes(64).toString('hex');
 
 import { env, version } from "../modules/config.js";
-import { getJSON } from "../modules/api.js";
+
-import { apiJSON, checkJSONPost, getIP, languageCode } from "../modules/sub/utils.js";
+import { generateHmac, generateSalt } from "../modules/sub/crypto.js";
 import { Bright, Cyan } from "../modules/sub/consoleText.js";
-import stream from "../modules/stream/stream.js";
+import { languageCode } from "../modules/sub/utils.js";
 import loc from "../localization/manager.js";
-import { generateHmac } from "../modules/sub/crypto.js";
+
+import { createResponse, normalizeRequest, getIP } from "../modules/processing/request.js";
 import { verifyStream, getInternalStream } from "../modules/stream/manage.js";
+import { extract } from "../modules/processing/url.js";
+import match from "../modules/processing/match.js";
+import stream from "../modules/stream/stream.js";
+
+const acceptRegex = /^application\/json(; charset=utf-8)?$/;
+
+const ipSalt = generateSalt();
+const corsConfig = env.corsWildcard ? {} : {
+    origin: env.corsURL,
+    optionsSuccessStatus: 200
+}
 
 export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
-    const corsConfig = !env.corsWildcard ? {
-        origin: env.corsURL,
-        optionsSuccessStatus: 200
-    } : {};
-
-    const apiLimiter = rateLimit({
-        windowMs: 60000,
-        max: 20,
-        standardHeaders: true,
-        legacyHeaders: false,
-        keyGenerator: req => generateHmac(getIP(req), ipSalt),
-        handler: (req, res) => {
-            return res.status(429).json({
-                "status": "rate-limit",
-                "text": loc(languageCode(req), 'ErrorRateLimit')
-            });
-        }
-    });
-    const apiLimiterStream = rateLimit({
-        windowMs: 60000,
-        max: 25,
-        standardHeaders: true,
-        legacyHeaders: false,
-        keyGenerator: req => generateHmac(getIP(req), ipSalt),
-        handler: (req, res) => {
-            return res.status(429).json({
-                "status": "rate-limit",
-                "text": loc(languageCode(req), 'ErrorRateLimit')
-            });
-        }
-    });
-    
     const startTime = new Date();
     const startTimestamp = startTime.getTime();
     
+    const serverInfo = {
+        version: version,
+        commit: gitCommit,
+        branch: gitBranch,
+        name: env.apiName,
+        url: env.apiURL,
+        cors: Number(env.corsWildcard),
+        startTime: `${startTimestamp}`
+    }
+
+    const apiLimiter = rateLimit({
+        windowMs: env.rateLimitWindow * 1000,
+        max: env.rateLimitMax,
+        standardHeaders: true,
+        legacyHeaders: false,
+        keyGenerator: req => generateHmac(getIP(req), ipSalt),
+        handler: (req, res) => {
+            return res.status(429).json({
+                "status": "rate-limit",
+                "text": loc(languageCode(req), 'ErrorRateLimit', env.rateLimitWindow)
+            });
+        }
+    })
+
+    const apiLimiterStream = rateLimit({
+        windowMs: env.rateLimitWindow * 1000,
+        max: env.rateLimitMax,
+        standardHeaders: true,
+        legacyHeaders: false,
+        keyGenerator: req => generateHmac(getIP(req), ipSalt),
+        handler: (req, res) => {
+            return res.sendStatus(429)
+        }
+    })
+
     app.set('trust proxy', ['loopback', 'uniquelocal']);
 
-    app.use('/api/:type', cors({
+    app.use('/api', cors({
         methods: ['GET', 'POST'],
-        ...corsConfig
+        exposedHeaders: [
-    }));
+            'Ratelimit-Limit',
+            'Ratelimit-Policy',
+            'Ratelimit-Remaining',
+            'Ratelimit-Reset'
+        ],
+        ...corsConfig,
+    }))
 
     app.use('/api/json', apiLimiter);
     app.use('/api/stream', apiLimiterStream);
-    app.use('/api/onDemand', apiLimiter);
 
     app.use((req, res, next) => {
-        try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') }
+        try {
+            decodeURIComponent(req.path)
+        } catch { 
+            return res.redirect('/')
+        }
         next();
-    });
+    })
 
     app.use('/api/json', express.json({
         verify: (req, res, buf) => {
-            let acceptCon = String(req.header('Accept')) === "application/json";
+            if (String(req.header('Accept')) === "application/json") {
-            if (acceptCon) {
                 if (buf.length > 720) throw new Error();
                 JSON.parse(buf);
             } else {
                 throw new Error();
             }
         }
-    }));
+    }))
 
     // handle express.json errors properly (https://github.com/expressjs/express/issues/4065)
     app.use('/api/json', (err, req, res, next) => {
         let errorText = "invalid json body";
-        let acceptCon = String(req.header('Accept')) !== "application/json";
+        const acceptHeader = String(req.header('Accept')) !== "application/json";
 
-        if (err || acceptCon) {
+        if (err || acceptHeader) {
-            if (acceptCon) errorText = "invalid accept header";
+            if (acceptHeader) errorText = "invalid accept header";
             return res.status(400).json({
                 status: "error",
                 text: errorText
@@ -91,108 +111,110 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
         } else {
             next();
         }
-    });
+    })
 
     app.post('/api/json', async (req, res) => {
+        const request = req.body;
+        const lang = languageCode(req);
+
+        const fail = (t) => {
+            const { status, body } = createResponse("error", { t: loc(lang, t) });
+            res.status(status).json(body);
+        }
+
+        if (!acceptRegex.test(req.header('Content-Type'))) {
+            return fail('ErrorInvalidContentType');
+        }
+
+        if (!request.url) {
+            return fail('ErrorNoLink');
+        }
+
+        request.dubLang = request.dubLang ? lang : false;
+        const normalizedRequest = normalizeRequest(request);
+        if (!normalizedRequest) {
+            return fail('ErrorCantProcess');
+        }
+
+        const parsed = extract(normalizedRequest.url);
+        if (parsed === null) {
+            return fail('ErrorUnsupported');
+        }
+
         try {
-            let lang = languageCode(req);
+            const result = await match(
-            let j = apiJSON(0, { t: "bad request" });
+                parsed.host, parsed.patternMatch, lang, normalizedRequest
-            try {
+            );
-                let contentCon = String(req.header('Content-Type')) === "application/json";
-                let request = req.body;
-                if (contentCon && request.url) {
-                    request.dubLang = request.dubLang ? lang : false;
 
-                    let chck = checkJSONPost(request);
+            res.status(result.status).json(result.body);
-                    if (!chck) throw new Error();
+        } catch {
+            fail('ErrorSomethingWentWrong');
+        }
+    })
 
-                    j = await getJSON(chck.url, lang, chck);
+    app.get('/api/stream', (req, res) => {
-                } else {
+        const id = String(req.query.id);
-                    j = apiJSON(0, {
+        const exp = String(req.query.exp);
-                        t: !contentCon ? "invalid content type header" : loc(lang, 'ErrorNoLink')
+        const sig = String(req.query.sig);
-                    });
+        const sec = String(req.query.sec);
-                }
+        const iv = String(req.query.iv);
-            } catch (e) {
+
-                j = apiJSON(0, { t: loc(lang, 'ErrorCantProcess') });
+        const checkQueries = id && exp && sig && sec && iv;
+        const checkBaseLength = id.length === 21 && exp.length === 13;
+        const checkSafeLength = sig.length === 43 && sec.length === 43 && iv.length === 22;
+
+        if (checkQueries && checkBaseLength && checkSafeLength) {
+            // rate limit probe, will not return json after 8.0
+            if (req.query.p) {
+                return res.status(200).json({
+                    status: "continue"
+                })
             }
-            return res.status(j.status).json(j.body);
+            try {
-        } catch (e) {
+                const streamInfo = verifyStream(id, sig, exp, sec, iv);
+                if (!streamInfo?.service) {
+                    return res.sendStatus(streamInfo.status);
+                }
+                return stream(res, streamInfo);
+            } catch {
+                return res.destroy();
+            }
+        }
+        return res.sendStatus(400);
+    })
+
+    app.get('/api/istream', (req, res) => {
+        try {
+            if (!req.ip.endsWith('127.0.0.1'))
+                return res.sendStatus(403);
+            if (String(req.query.id).length !== 21)
+                return res.sendStatus(400);
+
+            const streamInfo = getInternalStream(req.query.id);
+            if (!streamInfo) return res.sendStatus(404);
+            streamInfo.headers = req.headers;
+
+            return stream(res, { type: 'internal', ...streamInfo });
+        } catch {
             return res.destroy();
         }
-    });
+    })
 
-    app.get('/api/:type', (req, res) => {
+    app.get('/api/serverInfo', (req, res) => {
         try {
-            let j;
+            return res.status(200).json(serverInfo);
-            switch (req.params.type) {
+        } catch {
-                case 'stream':
+            return res.destroy();
-                    const q = req.query;
-                    const checkQueries = q.t && q.e && q.h && q.s && q.i;
-                    const checkBaseLength = q.t.length === 21 && q.e.length === 13;
-                    const checkSafeLength = q.h.length === 43 && q.s.length === 43 && q.i.length === 22;
-                    if (checkQueries && checkBaseLength && checkSafeLength) {
-                        if (q.p) {
-                            return res.status(200).json({
-                                status: "continue"
-                            })
-                        }
-                        let streamInfo = verifyStream(q.t, q.h, q.e, q.s, q.i);
-                        if (streamInfo.error) {
-                            return res.status(streamInfo.status).json(apiJSON(0, { t: streamInfo.error }).body);
-                        }
-                        return stream(res, streamInfo);
-                    } 
-
-                    j = apiJSON(0, {
-                        t: "bad request. stream link may be incomplete or corrupted."
-                    })
-                    return res.status(j.status).json(j.body);
-                case 'istream':
-                    if (!req.ip.endsWith('127.0.0.1'))
-                        return res.sendStatus(403);
-                    if (('' + req.query.t).length !== 21)
-                        return res.sendStatus(400);
-    
-                    let streamInfo = getInternalStream(req.query.t);
-                    if (!streamInfo) return res.sendStatus(404);
-                    streamInfo.headers = req.headers;
-    
-                    return stream(res, { type: 'internal', ...streamInfo });
-                case 'serverInfo':
-                    return res.status(200).json({
-                        version: version,
-                        commit: gitCommit,
-                        branch: gitBranch,
-                        name: env.apiName,
-                        url: env.apiURL,
-                        cors: Number(env.corsWildcard),
-                        startTime: `${startTimestamp}`
-                    });
-                default:
-                    j = apiJSON(0, {
-                        t: "unknown response type"
-                    })
-                    return res.status(j.status).json(j.body);
-            }
-        } catch (e) {
-            return res.status(500).json({
-                status: "error",
-                text: loc(languageCode(req), 'ErrorCantProcess')
-            });
         }
-    });
+    })
-
-    app.get('/api/status', (req, res) => {
-        res.status(200).end()
-    });
 
     app.get('/favicon.ico', (req, res) => {
         res.sendFile(`${__dirname}/src/front/icons/favicon.ico`)
-    });
+    })
 
     app.get('/*', (req, res) => {
-        res.redirect('/api/json')
+        res.redirect('/api/serverInfo')
-    });
+    })
 
     app.listen(env.apiPort, env.listenAddress, () => {
         console.log(`\n` +
@@ -201,5 +223,5 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
             `URL: ${Cyan(`${env.apiURL}`)}\n` +
             `Port: ${env.apiPort}\n`
         )
-    });
+    })
 }

src/core/web.js

@@ -1,12 +1,14 @@
-import { version, env } from "../modules/config.js";
+
-import { apiJSON, languageCode } from "../modules/sub/utils.js";
 import { Bright, Cyan } from "../modules/sub/consoleText.js";
+import { languageCode } from "../modules/sub/utils.js";
+import { version, env } from "../modules/config.js";
 
 import { buildFront } from "../modules/build.js";
 import findRendered from "../modules/pageRender/findRendered.js";
 
 import { celebrationsEmoji } from "../modules/pageRender/elements.js";
 import { changelogHistory } from "../modules/pageRender/onDemand.js";
+import { createResponse } from "../modules/processing/request.js";
 
 export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
     const startTime = new Date();
@@ -20,33 +22,40 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
     app.use((req, res, next) => {
         try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') }
         next();
-    });
+    })
+
     app.get('/onDemand', (req, res) => {
         try {
             if (req.query.blockId) {
                 let blockId = req.query.blockId.slice(0, 3);
-                let r, j;
+                let blockData;
                 switch(blockId) {
                     // changelog history
                     case "0":
-                        r = changelogHistory();
+                        let history = changelogHistory();
-                        j = r ? apiJSON(3, { t: r }) : apiJSON(0, {
+                        if (history) {
-                            t: "couldn't render this block, please try again!"
+                            blockData = createResponse("success", { t: history })
-                        })
+                        } else {
+                            blockData = createResponse("error", {
+                                t: "couldn't render this block, please try again!"
+                            })
+                        }
                         break;
                     // celebrations emoji
                     case "1":
-                        r = celebrationsEmoji();
+                        let celebration = celebrationsEmoji();
-                        j = r ? apiJSON(3, { t: r }) : false
+                        if (celebration) {
+                            blockData = createResponse("success", { t: celebration })
+                        }
                         break;
                     default:
-                        j = apiJSON(0, {
+                        blockData = createResponse("error", {
                             t: "couldn't find a block with this id"
                         })
                         break;
                 }
-                if (j.body) {
+                if (blockData?.body) {
-                    return res.status(j.status).json(j.body);
+                    return res.status(blockData.status).json(blockData.body);
                 } else {
                     return res.status(204).end();
                 }
@@ -56,25 +65,25 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
                     text: "couldn't render this block, please try again!"
                 });
             }
-        } catch (e) {
+        } catch {
             return res.status(400).json({
                 status: "error",
                 text: "couldn't render this block, please try again!"
             })
         }
-    });
+    })
-    app.get("/status", (req, res) => {
+
-        return res.status(200).end()
-    });
     app.get("/", (req, res) => {
         return res.sendFile(`${__dirname}/${findRendered(languageCode(req))}`)
-    });
+    })
+
     app.get("/favicon.ico", (req, res) => {
         return res.sendFile(`${__dirname}/src/front/icons/favicon.ico`)
-    });
+    })
+
     app.get("/*", (req, res) => {
         return res.redirect('/')
-    });
+    })
 
     app.listen(env.webPort, () => {
         console.log(`\n` +

src/front/cobalt.css

@@ -252,7 +252,7 @@ button:active,
     flex-direction: row;
 }
 #cobalt-main-box #bottom {
-    padding-top: 1rem;
+    padding-top: calc(1rem - var(--padding-small));
     display: flex;
     flex-direction: row;
     justify-content: space-between;
@@ -610,7 +610,7 @@ button:active,
     margin-top: 0.5rem;
 }
 .explanation {
-    margin-top: 0.8rem;
+    margin-top: var(--padding);
     width: 100%;
     font-size: 0.8rem;
     text-align: left;

src/front/cobalt.js

@@ -563,16 +563,9 @@ const loadCelebrationsEmoji = async() => {
     let aboutButtonBackup = eid("about-footer").innerHTML;
     try {
         let j = await fetch(`/onDemand?blockId=1`).then(r => r.json()).catch(() => {});
-
         if (j && j.status === "success" && j.text) {
             eid("about-footer").innerHTML = eid("about-footer").innerHTML.replace(
-                `<img class="emoji"
+                `${aboutButtonBackup.split('> ')[0]}>`,
-                    draggable="false"
-                    height="22"
-                    width="22
-                    alt="🐲"
-                    src="emoji/dragon_face.svg"
-                    loading="lazy">`,
                 j.text
             )
         }

src/localization/languages/en.json

@@ -24,7 +24,7 @@
         "ErrorBrokenLink": "{s} is supported, but something is wrong with your link. maybe you didn't copy it fully?",
         "ErrorNoLink": "i can't guess what you want to download! please give me a link :(",
         "ErrorPageRenderFail": "if you're reading this, then there's something wrong with the page renderer. please {ContactLink}. make sure to provide the domain this error is present on and current commit hash ({s}). thank you in advance :D",
-        "ErrorRateLimit": "you're making too many requests. try again in a minute!",
+        "ErrorRateLimit": "you're making too many requests. try again in {s} seconds!",
         "ErrorCouldntFetch": "i couldn't find anything about this link. check if it works and try again! some content may be region restricted, so keep that in mind.",
         "ErrorLengthLimit": "i can't process videos longer than {s} minutes, so pick something shorter instead!",
         "ErrorBadFetch": "something went wrong when i tried getting info about your link. are you sure it works? check if it does, and try again.",
@@ -155,6 +155,7 @@
         "SettingsTikTokH265Description": "download 1080p videos from tiktok in h265/hevc format when available.",
         "SettingsYoutubeDub": "use browser language",
         "SettingsYoutubeDubDescription": "uses your browser's default language for youtube dubbed audio tracks. works even if cobalt ui isn't translated to your language.",
-        "UpdateIstream": "better service support and ux"
+        "ErrorInvalidContentType": "invalid content type header",
+        "UpdateOneMillion": "1 million users and blazing speed"
     }
 }

src/localization/languages/ru.json

@@ -24,7 +24,7 @@
         "ErrorBrokenLink": "{s} поддерживается, но с твоей ссылкой что-то не так. может быть, ты её не полностью скопировал?",
         "ErrorNoLink": "пока что я не умею угадывать, что ты хочешь скачать. дай мне, пожалуйста, ссылку :(",
         "ErrorPageRenderFail": "если ты видишь этот текст, значит что-то не так с рендером страницы. пожалуйста, {ContactLink}. также приложи домен, на котором присутсвует эта ошибка, и хэш коммита ({s}). спасибо :)",
-        "ErrorRateLimit": "ты делаешь слишком много запросов. попробуй ещё раз через минуту!",
+        "ErrorRateLimit": "ты делаешь слишком много запросов. попробуй ещё раз через {s} секунд!",
         "ErrorCouldntFetch": "у меня не получилось ничего найти по этой ссылке. убедись, что она работает, и попробуй ещё раз. некоторый контент может быть залочен на регион.",
         "ErrorLengthLimit": "я не могу обрабатывать видео длиннее чем {s} минут(ы), так что скачай что-нибудь покороче!",
         "ErrorBadFetch": "произошла какая-то ошибка при получении данных по твоей ссылке. убедись, что она работает, и попробуй ещё раз.",
@@ -157,6 +157,6 @@
         "SettingsTikTokH265Description": "скачивает видео с tiktok в 1080p и h265/hevc, когда это возможно.",
         "SettingsYoutubeDub": "использовать язык браузера",
         "SettingsYoutubeDubDescription": "использует главный язык браузера для аудиодорожек на youtube. работает даже если кобальт не переведён в твой язык.",
-        "UpdateIstream": "быстрые загрузки и приятный интерфейс"
+        "UpdateOneMillion": "миллион и невероятная скорость"
     }
 }

src/modules/api.js

@@ -1,33 +0,0 @@
-import { services } from "./config.js";
-
-import { apiJSON } from "./sub/utils.js";
-import { errorUnsupported } from "./sub/errors.js";
-import loc from "../localization/manager.js";
-import match from "./processing/match.js";
-import { getHostIfValid } from "./processing/url.js";
-
-export async function getJSON(url, lang, obj) {
-    try {
-        const host = getHostIfValid(url);
-
-        if (!host || !services[host].enabled) {
-            return apiJSON(0, { t: errorUnsupported(lang) });
-        }
-
-        let patternMatch;
-        for (const pattern of services[host].patterns) {
-            patternMatch = pattern.match(
-                url.pathname.substring(1) + url.search
-            );
-            if (patternMatch) break;
-        }
-
-        if (!patternMatch) {
-            return apiJSON(0, { t: errorUnsupported(lang) });
-        }
-
-        return await match(host, patternMatch, url, lang, obj)
-    } catch (e) {
-        return apiJSON(0, { t: loc(lang, 'ErrorSomethingWentWrong') })
-    }
-}

src/modules/changelog/changelog.json

@@ -1,5 +1,17 @@
 {
     "current": {
+        "version": "7.14",
+        "date": "May 17, 2024",
+        "title": "now helping over 1 million people monthly",
+        "banner": {
+            "file": "millionusers.webp",
+            "alt": "collage of two photos, side by side. left photo: brown cake with 7 lit candles forming 1000000 and one ferrero rocher candy in the middle with cobalt (double greater than symbol) logo on it. right photo: chocolate cake with 7 lit candles forming 1000000 and cobalt logo formed with whipped cream on the cake. two plushes of meowth and pompompurin in party hats are seen behind the cake.",
+            "width": 1736,
+            "height": 1440
+        },
+        "content": "yesterday, cobalt hit 1 million users around the world! it's an absolutely insane milestone for us and we're incredibly grateful to everyone saving and creating what they love with help of cobalt. thank you for being our friends.\n\nin anticipation of 7 figure user count, we've revamped the cobalt codebase and infrastructure to be faster and more reliable than ever. a combination of many changes has resulted into incredible download speeds (up to 30 MB/s, as tested by both developers in europe).\n\nnote: there's no backend instance in asia just yet, so if you're there, you might experience average speeds *for now*. you can help us afford a dedicated server in asia by donating to cobalt in the \"donate\" menu.\n\n<span class=\"text-backdrop\">changes since the last major update</span>\n\nservice improvements:\n*; youtube music support on the main instance is back!\n*; added support for pinterest images and gifs.\n*; cobalt will now use original soundcloud mp3 file when available.\n*; fixed a youtube bug that prevented some videos from downloading.\n\nui/ux improvements:\n*; cobalt web app is now fully optimized for ipad. you can add it to home screen from share menu to make it act like a native app!\n*; majorly reduced vertical padding when viewing cobalt in mobile web browser, allowing for more content at once. most noticeable on smaller screens.\n*; status bar color is now dynamic in the web browser on ios and web app on android.\n*; web app on android feels way more native than before.\n*; filename style icons are no longer blurry in safari.\n*; changelog notification no longer overlaps with dynamic island on newer iphones when cobalt is installed as a web app.\n*; fixed safe area padding.\n\nother changes:\n*; added support for <a class=\"text-backdrop link\" href=\"https://github.com/imputnet/freebind.js\" target=\"_blank\">freebind</a>, made by one of the cobalt developers.\n*; rate limit and max video length limits are now customizable through <a class=\"text-backdrop link\" href=\"{repo}/blob/current/docs/run-an-instance.md#variables-for-api\" target=\"_blank\">environment variables</a>.\n*; cobalt api now returns rate limit headers at all times.\n*; majorly cleaned up the codebase: removed unnecessary functions, rewrote those that were cryptic and confusing. it's way more comprehensible and contribution-friendly than ever before.\n*; moved the <a class=\"text-backdrop link\" href=\"{repo}\" target=\"_blank\">cobalt repo</a> to our organization on github. everything stayed the same and all old links link back to it.\n\nnote for instance hosters:\nalong with cobalt repo, the docker image also moved! please update the url for it in your config along with watchtower args to include restarting containers (just in case) as seen in <a class=\"text-backdrop link\" href=\"{repo}/blob/current/docs/examples/docker-compose.example.yml\" target=\"_blank\">updated docker compose example</a>. we're mirroring packages to old url for now, but it won't last forever.\n\nthat's it for now! hope you have an amazing day and share the 1 million celebration with us :)\n\njoin our <a class=\"text-backdrop link\" href=\"https://discord.gg/pQPt8HBUPu\" target=\"_blank\">discord server</a> to discuss everything cobalt there"
+    },
+    "history": [{
         "version": "7.13",
         "date": "May 5, 2024",
         "title": "better ux, improvements for youtube, twitter, tiktok, instagram, and more!",
@@ -10,8 +22,7 @@
             "height": 960
         },
         "content": "long time no see! well, actually, you've been using the latest version for some time now. we've moved to a rolling release scheme, allowing for speedy update rollouts :)\n\nsince 7.11, there has been a ton of changes. here are the most notable of them:\n*; youtube downloads are now faster and more reliable than ever.\n*; all posts from twitter are now downloadable, including sensitive ones.\n*; you now can download tiktok videos in 1080p h265! just enable h265 support in settings > video.\n*; added support for sharing links directly to the cobalt web app on android.\n*; added 240p and 144p quality options to the quality picker in settings (for some reason, many of you wanted this).\n*; pasting a link with additional text around it will now work; cobalt will extract the link for you (works only via the paste button).\n*; added anonymous traffic analytics by plausible. we're using a selfhosted instance and don't collect any identifiable information about you. you can learn more in about > privacy policy. you can also opt out of anonymous analytics in settings > other.\n\nservice support improvements:\n*; implemented internal streams functionality, allowing for more fine-grained file streaming and therefore proper youtube support.\n*; added fallback to m4a if opus isn't available for youtube.\n*; added a total of 7 ways to get instagram post info, including mobile api, embed, and graphql api. absolute torture.\n*; added support for reddit user posts.\n*; updated the way tiktok downloads are handled for better reliability and 1080p support.\n*; added tiktok author's username to filename.\n*; added support for rutube shorts and yappy videos.\n*; added support for m.soundcloud.com links.\n*; added support for new post and reel links from instagram.\n*; added support for photo twitter links, only used for gifs.\n*; added support for m.bilibili.com links.\n*; added support for new type of vimeo links.\n*; added support for ddinstagram.com links.\n*; updated youtube codec info in settings to display the fact that av1 is a better choice now.\n*; updated best audio picking for tiktok and soundcloud.\n*; changed the youtube client to web, since android client no longer works.\n*; removed the vimeo download type switcher, as it should've always been automatic instead.\n*; removed an ability to enable the tiktok watermark, as it no longer includes the author's username.\n\nui & ux improvements:\n*; youtube audio dub switcher is now a toggle with a much easier to understand description.\n*; meowbalt now sticks out on the left side of download popup on desktop.\n*; updated \"made with love\" text to include the research & dev team behind cobalt, imput.\n*; fixed grammar of russian localization.\n*; rounded corners are now correctly rendered across all browsers.\n*; various minor improvements, including smaller button padding.\n*; removed the notification (red dot) functionality as the most recent changelog is already always on screen.\n*; removed settings migration from the old domain.\n\nother changes:\n*; various docs updates in github repo, making sure they're functional across branches and forks.\n*; major codebase cleanup.\n\nthank you for using cobalt, and thank you for being one of our 900k friends! i hope you like this update as much as we liked making it.\n\nwe're committed to keeping cobalt the best way to save what you love without ads or invasion of your privacy. there's a ton of cool stuff to come soon; stay tuned and have an amazing rest of your day <3\n\nif you want to help our goal of a better internet for everyone, just share cobalt with a friend!\n\n(original photo of a man in a suit by benzoix on freepik)"
-    },
+    }, {
-    "history": [{
         "version": "7.11",
         "date": "March 6, 2024",
         "title": "cache encryption, meowbalt, dailymotion, bilibili, and much more!",

src/modules/config.js

@@ -28,26 +28,33 @@ const
 
     // API mode related environment variables
     apiEnvs = {
+        apiURL,
         apiPort: process.env.API_PORT || 9000,
         apiName: process.env.API_NAME || 'unknown',
+
         listenAddress: process.env.API_LISTEN_ADDRESS,
+        freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
+
         corsWildcard: process.env.CORS_WILDCARD !== '0',
         corsURL: process.env.CORS_URL,
+
         cookiePath: process.env.COOKIE_PATH,
+
+        rateLimitWindow: (process.env.RATELIMIT_WINDOW  && parseInt(process.env.RATELIMIT_WINDOW)) || 60,
+        rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20,
+
+        durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800,
+        streamLifespan: 90,
+
         processingPriority: process.platform !== 'win32'
                                 && process.env.PROCESSING_PRIORITY
-                                && parseInt(process.env.PROCESSING_PRIORITY),
+                                && parseInt(process.env.PROCESSING_PRIORITY)
-        tiktokDeviceInfo: process.env.TIKTOK_DEVICE_INFO && JSON.parse(process.env.TIKTOK_DEVICE_INFO),
-        freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
-        apiURL
     }
 
 export const
     services = servicesConfigJson.config,
     audioIgnore = servicesConfigJson.audioIgnore,
     version = packageJson.version,
-    streamLifespan = config.streamLifespan,
-    maxVideoDuration = config.maxVideoDuration,
     genericUserAgent = config.genericUserAgent,
     repo = packageJson.bugs.url.replace('/issues', ''),
     authorInfo = config.authorInfo,

src/modules/pageRender/page.js

@@ -584,8 +584,8 @@ export default function(obj) {
         <div id="popup-backdrop" onclick="hideAllPopups()"></div>
         <div id="home" style="visibility:hidden">
             ${urgentNotice({
-                emoji: "🫧",
+                emoji: "🎉",
-                text: t("UpdateIstream"),
+                text: t("UpdateOneMillion"),
                 visible: true,
                 action: "popup('about', 1, 'changelog')"
             })}

src/modules/processing/match.js

@@ -1,8 +1,7 @@
 import { strict as assert } from "node:assert";
 
-import { apiJSON } from "../sub/utils.js";
+import { env } from '../config.js';
-import { errorUnsupported, genericError, brokenLink } from "../sub/errors.js";
+import { createResponse } from "../processing/request.js";
-
 import loc from "../../localization/manager.js";
 
 import { testers } from "./servicesPatternTesters.js";
@@ -29,7 +28,9 @@ import snapchat from "./services/snapchat.js";
 import { env } from '../config.js';
 
 let freebind;
-export default async function(host, patternMatch, url, lang, obj) {
+
+export default async function(host, patternMatch, lang, obj) {
+    const { url } = obj;
     assert(url instanceof URL);
     let dispatcher, requestIP;
 
@@ -43,10 +44,20 @@ export default async function(host, patternMatch, url, lang, obj) {
     }
 
     try {
-        let r, isAudioOnly = !!obj.isAudioOnly, disableMetadata = !!obj.disableMetadata;
+        let r,
+            isAudioOnly = !!obj.isAudioOnly,
+            disableMetadata = !!obj.disableMetadata;
 
-        if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
+        if (!testers[host]) {
-        if (!(testers[host](patternMatch))) return apiJSON(0, { t: brokenLink(lang, host) });
+            return createResponse("error", {
+                t: loc(lang, 'ErrorUnsupported')
+            });
+        }
+        if (!(testers[host](patternMatch))) {
+            return createResponse("error", {
+                t: loc(lang, 'ErrorBrokenLink', host)
+            });
+        }
 
         switch (host) {
             case "twitter":
@@ -186,21 +197,26 @@ export default async function(host, patternMatch, url, lang, obj) {
                 });
                 break;
             default:
-                return apiJSON(0, { t: errorUnsupported(lang) });
+                return createResponse("error", {
+                    t: loc(lang, 'ErrorUnsupported')
+                });
         }
 
         if (r.isAudioOnly) isAudioOnly = true;
         let isAudioMuted = isAudioOnly ? false : obj.isAudioMuted;
 
-        if (r.error && r.critical)
+        if (r.error && r.critical) {
-            return apiJSON(6, { t: loc(lang, r.error) })
+            return createResponse("critical", {
-
+                t: loc(lang, r.error)
-        if (r.error)
+            })
-            return apiJSON(0, {
+        }
+        if (r.error) {
+            return createResponse("error", {
                 t: Array.isArray(r.error)
                     ? loc(lang, r.error[0], r.error[1])
                     : loc(lang, r.error)
             })
+        }
 
         return matchActionDecider(
             r, host, obj.aFormat, isAudioOnly,
@@ -208,7 +224,9 @@ export default async function(host, patternMatch, url, lang, obj) {
             obj.filenamePattern, obj.twitterGif,
             requestIP
         )
-    } catch (e) {
+    } catch {
-        return apiJSON(0, { t: genericError(lang, host) })
+        return createResponse("error", {
+            t: loc(lang, 'ErrorBadFetch', host)
+        })
     }
 }

src/modules/processing/matchActionDecider.js

@@ -1,11 +1,12 @@
 import { audioIgnore, services, supportedAudio } from "../config.js";
-import { apiJSON } from "../sub/utils.js";
+import { createResponse } from "../processing/request.js";
 import loc from "../../localization/manager.js";
 import createFilename from "./createFilename.js";
+import { createStream } from "../stream/manage.js";
 
 export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) {
     let action,
-        responseType = 2,
+        responseType = "stream",
         defaultParams = {
             u: r.urls,
             service: host,
@@ -36,10 +37,10 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
 
     switch (action) {
         default:
-            return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
+            return createResponse("error", { t: loc(lang, 'ErrorEmptyDownload') });
 
         case "photo":
-            responseType = 1;
+            responseType = "redirect";
             break;
 
         case "gif":
@@ -56,29 +57,35 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                 u: Array.isArray(r.urls) ? r.urls[0] : r.urls,
                 mute: true
             }
-            if (host === "reddit" && r.typeId === 1) responseType = 1;
+            if (host === "reddit" && r.typeId === "redirect")
+                responseType = "redirect";
             break;
 
         case "picker":
-            responseType = 5;
+            responseType = "picker";
             switch (host) {
                 case "instagram":
                 case "twitter":
                 case "snapchat":
                     params = { picker: r.picker };
                     break;
-                case "douyin":
                 case "tiktok":
-                    let pickerType = "render";
+                    let audioStreamType = "render";
-                    if (audioFormat === "mp3" || audioFormat === "best") {
+                    if (r.bestAudio === "mp3" && (audioFormat === "mp3" || audioFormat === "best")) {
                         audioFormat = "mp3";
-                        pickerType = "bridge"
+                        audioStreamType = "bridge"
                     }
                     params = {
-                        type: pickerType,
                         picker: r.picker,
-                        u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
+                        u: createStream({
-                        copy: audioFormat === "best" ? true : false
+                            service: "tiktok",
+                            type: audioStreamType,
+                            u: r.urls,
+                            filename: r.audioFilename,
+                            isAudioOnly: true,
+                            audioFormat,
+                        }),
+                        copy: audioFormat === "best"
                     }
             }
             break;
@@ -99,7 +106,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                     if (Array.isArray(r.urls)) {
                         params = { type: "render" }
                     } else {
-                        responseType = 1;
+                        responseType = "redirect";
                     }
                     break;
 
@@ -107,12 +114,11 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                     if (r.type === "remux") {
                         params = { type: r.type };
                     } else {
-                        responseType = 1;
+                        responseType = "redirect";
                     }
                     break;
 
                 case "vk":
-                case "douyin":
                 case "tiktok":
                     params = { type: "bridge" };
                     break;
@@ -123,14 +129,15 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                 case "pinterest":
                 case "streamable":
                 case "snapchat":
-                    responseType = 1;
+                    responseType = "redirect";
                     break;
             }
             break;
 
         case "audio":
-            if ((host === "reddit" && r.typeId === 1) || audioIgnore.includes(host)) {
+            if (audioIgnore.includes(host)
-                return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') })
+                || (host === "reddit" && r.typeId === "redirect")) {
+                return createResponse("error", { t: loc(lang, 'ErrorEmptyDownload') })
             }
 
             let processType = "render",
@@ -148,11 +155,12 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
 
             const isTumblrAudio = host === "tumblr" && !r.filename;
             const isSoundCloud = host === "soundcloud";
+            const isTiktok = host === "tiktok";
 
             if (isBestAudioDefined || isBestHostAudio) {
                 audioFormat = serviceBestAudio;
                 processType = "bridge";
-                if (isSoundCloud) {
+                if (isSoundCloud || (isTiktok && audioFormat === "m4a")) {
                     processType = "render"
                     copy = true
                 }
@@ -180,5 +188,5 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
             break;
     }
 
-    return apiJSON(responseType, {...defaultParams, ...params})
+    return createResponse(responseType, {...defaultParams, ...params})
 }

src/modules/processing/request.js

@@ -0,0 +1,162 @@
+import ipaddr from "ipaddr.js";
+
+import { normalizeURL } from "../processing/url.js";
+import { createStream } from "../stream/manage.js";
+import { verifyLanguageCode } from "../sub/utils.js";
+
+const apiVar = {
+    allowed: {
+        vCodec: ["h264", "av1", "vp9"],
+        vQuality: ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"],
+        aFormat: ["best", "mp3", "ogg", "wav", "opus"],
+        filenamePattern: ["classic", "pretty", "basic", "nerdy"]
+    },
+    booleanOnly: [
+        "isAudioOnly",
+        "isTTFullAudio",
+        "isAudioMuted",
+        "dubLang",
+        "disableMetadata",
+        "twitterGif",
+        "tiktokH265"
+    ]
+}
+
+export function createResponse(responseType, responseData) {
+    const internalError = (text) => {
+        return {
+            status: 500,
+            body: {
+                status: "error",
+                text: text || "Internal Server Error",
+                critical: true
+            }
+        }
+    }
+
+    try {
+        let status = 200,
+            response = {};
+        
+        switch(responseType) {
+            case "error":
+                status = 400;
+                break;
+
+            case "rate-limit":
+                status = 429;
+                break;
+        }
+
+        switch (responseType) {
+            case "error":
+            case "success":
+            case "rate-limit":
+                response = {
+                    text: responseData.t
+                }
+                break;
+
+            case "redirect":
+                response = {
+                    url: responseData.u
+                }
+                break;
+
+            case "stream":
+                response = {
+                    url: createStream(responseData)
+                }
+                break;
+
+            case "picker":
+                let pickerType = "various",
+                    audio = false;
+
+                if (responseData.service === "tiktok") {
+                    audio = responseData.u
+                    pickerType = "images"
+                }
+
+                response = {
+                    pickerType: pickerType,
+                    picker: responseData.picker,
+                    audio: audio
+                }
+                break;
+            case "critical": 
+                return internalError(responseData.t)
+            default: 
+                throw "unreachable"
+        }
+        return {
+            status,
+            body: {
+                status: responseType,
+                ...response
+            }
+        }
+    } catch {
+        return internalError()
+    }
+}
+
+export function normalizeRequest(request) {
+    try {
+        let template = {
+            url: normalizeURL(decodeURIComponent(request.url)),
+            vCodec: "h264",
+            vQuality: "720",
+            aFormat: "mp3",
+            filenamePattern: "classic",
+            isAudioOnly: false,
+            isTTFullAudio: false,
+            isAudioMuted: false,
+            disableMetadata: false,
+            dubLang: false,
+            twitterGif: false,
+            tiktokH265: false
+        }
+    
+        const requestKeys = Object.keys(request);
+        const templateKeys = Object.keys(template);
+
+        if (requestKeys.length > templateKeys.length + 1 || !request.url) {
+            return false;
+        }
+
+        for (const i in requestKeys) {
+            const key = requestKeys[i];
+            const item = request[key];
+
+            if (String(key) !== "url" && templateKeys.includes(key)) {
+                if (apiVar.booleanOnly.includes(key)) {
+                    template[key] = !!item;
+                } else if (apiVar.allowed[key] && apiVar.allowed[key].includes(item)) {
+                    template[key] = String(item)
+                }
+            }
+        }
+
+        if (template.dubLang)
+            template.dubLang = verifyLanguageCode(request.dubLang);
+
+        return template
+    } catch {
+        return false
+    }
+}
+
+export function getIP(req) {
+    const strippedIP = req.ip.replace(/^::ffff:/, '');
+    const ip = ipaddr.parse(strippedIP);
+    if (ip.kind() === 'ipv4') {
+        return strippedIP;
+    }
+
+    const prefix = 56;
+    const v6Bytes = ip.toByteArray();
+          v6Bytes.fill(0, prefix / 8);
+
+    return ipaddr.fromByteArray(v6Bytes).toString();
+}

src/modules/processing/services/bilibili.js

@@ -1,4 +1,4 @@
-import { genericUserAgent, maxVideoDuration } from "../../config.js";
+import { genericUserAgent, env } from "../../config.js";
 
 // TO-DO: higher quality downloads (currently requires an account)
 
@@ -31,7 +31,7 @@ function extractBestQuality(dashData) {
 async function com_download(id) {
     let html = await fetch(`https://bilibili.com/video/${id}`, {
         headers: { "user-agent": genericUserAgent }
-    }).then((r) => { return r.text() }).catch(() => { return false });
+    }).then(r => r.text()).catch(() => {});
     if (!html) return { error: 'ErrorCouldntFetch' };
 
     if (!(html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"'))) {
@@ -39,8 +39,8 @@ async function com_download(id) {
     }
 
     let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
-    if (streamData.data.timelength > maxVideoDuration) {
+    if (streamData.data.timelength > env.durationLimit * 1000) {
-        return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
     }
 
     const [ video, audio ] = extractBestQuality(streamData.data.dash);
@@ -79,8 +79,8 @@ async function tv_download(id) {
         return { error: 'ErrorEmptyDownload' };
     }
 
-    if (video.duration > maxVideoDuration) {
+    if (video.duration > env.durationLimit * 1000) {
-        return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
     }
 
     return {

src/modules/processing/services/dailymotion.js

@@ -1,5 +1,5 @@
 import HLSParser from 'hls-parser';
-import { maxVideoDuration } from '../../config.js';
+import { env } from '../../config.js';
 
 let _token;
 
@@ -73,8 +73,8 @@ export default async function({ id }) {
         return { error: 'ErrorEmptyDownload' }
     }
 
-    if (media.duration * 1000 > maxVideoDuration) {
+    if (media.duration > env.durationLimit) {
-        return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
     }
 
     const manifest = await fetch(media.hlsURL).then(r => r.text()).catch(() => {});

src/modules/processing/services/instagram.js

@@ -338,7 +338,7 @@ export default function(obj) {
             }
         }
 
-        return { error: 'ErrorCouldntFetch' };
+        return { error: 'ErrorUnsupported' };
     }
 
     const { postId, storyId, username } = obj;

src/modules/processing/services/ok.js

@@ -1,4 +1,4 @@
-import { genericUserAgent, maxVideoDuration } from "../../config.js";
+import { genericUserAgent, env } from "../../config.js";
 import { cleanString } from "../../sub/utils.js";
 
 const resolutions = {
@@ -17,19 +17,27 @@ export default async function(o) {
 
     let html = await fetch(`https://ok.ru/video/${o.id}`, {
         headers: { "user-agent": genericUserAgent }
-    }).then((r) => { return r.text() }).catch(() => { return false });
+    }).then(r => r.text()).catch(() => {});
 
     if (!html) return { error: 'ErrorCouldntFetch' };
     if (!html.includes(`<div data-module="OKVideo" data-options="{`)) {
         return { error: 'ErrorEmptyDownload' };
     }
 
-    let videoData = html.split(`<div data-module="OKVideo" data-options="`)[1].split('" data-')[0].replaceAll("&quot;", '"');
+    let videoData = html.split(`<div data-module="OKVideo" data-options="`)[1]
+                        .split('" data-')[0]
+                        .replaceAll("&quot;", '"');
+
     videoData = JSON.parse(JSON.parse(videoData).flashvars.metadata);
 
-    if (videoData.provider !== "UPLOADED_ODKL") return { error: 'ErrorUnsupported' };
+    if (videoData.provider !== "UPLOADED_ODKL")
-    if (videoData.movie.is_live) return { error: 'ErrorLiveVideo' };
+        return { error: 'ErrorUnsupported' };
-    if (videoData.movie.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+
+    if (videoData.movie.is_live)
+        return { error: 'ErrorLiveVideo' };
+
+    if (videoData.movie.duration > env.durationLimit)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 
     let videos = videoData.videos.filter(v => !v.disallowed);
     let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1];

src/modules/processing/services/pinterest.js

@@ -7,16 +7,16 @@ export default async function(o) {
     let id = o.id;
 
     if (!o.id && o.shortLink) {
-        id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" }).then((r) => {
+        id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" })
-            return r.headers.get("location").split('pin/')[1].split('/')[0]
+                   .then(r => r.headers.get("location").split('pin/')[1].split('/')[0])
-        }).catch(() => {});
+                   .catch(() => {});
     }
     if (id.includes("--")) id = id.split("--")[1];
     if (!id) return { error: 'ErrorCouldntFetch' };
 
     let html = await fetch(`https://www.pinterest.com/pin/${id}/`, {
         headers: { "user-agent": genericUserAgent }
-    }).then((r) => { return r.text() }).catch(() => { return false });
+    }).then(r => r.text()).catch(() => {});
 
     if (!html) return { error: 'ErrorCouldntFetch' };
 

src/modules/processing/services/reddit.js

@@ -1,4 +1,4 @@
-import { genericUserAgent, maxVideoDuration } from "../../config.js";
+import { genericUserAgent, env } from "../../config.js";
 import { getCookie, updateCookieValues } from "../cookie/manager.js";
 
 async function getAccessToken() {
@@ -32,7 +32,7 @@ async function getAccessToken() {
             'accept': 'application/json'
         },
         body: `grant_type=refresh_token&refresh_token=${encodeURIComponent(values.refresh_token)}`
-    }).then(r => r.json()).catch(_ => {});
+    }).then(r => r.json()).catch(() => {});
     if (!data) return;
 
     const { access_token, refresh_token, expires_in } = data;
@@ -59,24 +59,28 @@ export default async function(obj) {
 
     let data = await fetch(
         url, {
-            headers: accessToken && { authorization: `Bearer ${accessToken}` }
+            headers: {
+                'User-Agent': genericUserAgent,
+                accept: 'application/json',
+                authorization: accessToken && `Bearer ${accessToken}`
+            }
         }
-    ).then(r => r.json() ).catch(() => {});
+    ).then(r => r.json()).catch(() => {});
 
     if (!data || !Array.isArray(data)) return { error: 'ErrorCouldntFetch' };
 
     data = data[0]?.data?.children[0]?.data;
 
     if (data?.url?.endsWith('.gif')) return {
-        typeId: 1,
+        typeId: "redirect",
         urls: data.url
     }
 
     if (!data.secure_media?.reddit_video)
         return { error: 'ErrorEmptyDownload' };
 
-    if (data.secure_media?.reddit_video?.duration * 1000 > maxVideoDuration)
+    if (data.secure_media?.reddit_video?.duration > env.durationLimit)
-        return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 
     let audio = false,
         video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0],
@@ -87,7 +91,7 @@ export default async function(obj) {
     }
 
     // test the existence of audio
-    await fetch(audioFileLink, { method: "HEAD" }).then((r) => {
+    await fetch(audioFileLink, { method: "HEAD" }).then(r => {
         if (Number(r.status) === 200) {
             audio = true
         }
@@ -96,7 +100,7 @@ export default async function(obj) {
     // fallback for videos with variable audio quality
     if (!audio) {
         audioFileLink = `${video.split('_')[0]}_AUDIO_128.mp4`
-        await fetch(audioFileLink, { method: "HEAD" }).then((r) => {
+        await fetch(audioFileLink, { method: "HEAD" }).then(r => {
             if (Number(r.status) === 200) {
                 audio = true
             }
@@ -106,12 +110,12 @@ export default async function(obj) {
     let id = video.split('/')[3];
 
     if (!audio) return {
-        typeId: 1,
+        typeId: "redirect",
         urls: video
     }
 
     return {
-        typeId: 2,
+        typeId: "stream",
         type: "render",
         urls: [video, audioFileLink],
         audioFilename: `reddit_${id}_audio`,

src/modules/processing/services/rutube.js

@@ -1,6 +1,6 @@
 import HLS from 'hls-parser';
 
-import { maxVideoDuration } from "../../config.js";
+import { env } from "../../config.js";
 import { cleanString } from '../../sub/utils.js';
 
 async function requestJSON(url) {
@@ -35,8 +35,8 @@ export default async function(obj) {
     if (play.detail || !play.video_balancer) return { error: 'ErrorEmptyDownload' };
     if (play.live_streams?.hls) return { error: 'ErrorLiveVideo' };
 
-    if (play.duration > maxVideoDuration)
+    if (play.duration > env.durationLimit * 1000)
-        return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 
     let m3u8 = await fetch(play.video_balancer.m3u8)
                      .then(r => r.text())
@@ -48,7 +48,7 @@ export default async function(obj) {
 
     let bestQuality = m3u8[0];
     if (Number(quality) < bestQuality.resolution.height) {
-        bestQuality = m3u8.find((i) => (Number(quality) === i["resolution"].height));
+        bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height));
     }
 
     let fileMetadata = {

src/modules/processing/services/soundcloud.js

@@ -1,4 +1,4 @@
-import { maxVideoDuration } from "../../config.js";
+import { env } from "../../config.js";
 import { cleanString } from "../../sub/utils.js";
 
 const cachedID = {
@@ -8,7 +8,7 @@ const cachedID = {
 
 async function findClientID() {
     try {
-        let sc = await fetch('https://soundcloud.com/').then((r) => { return r.text() }).catch(() => { return false });
+        let sc = await fetch('https://soundcloud.com/').then(r => r.text()).catch(() => {});
         let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
 
         if (cachedID.version === scVersion) return cachedID.id;
@@ -20,7 +20,7 @@ async function findClientID() {
     
             if (url && !url.startsWith('https://a-v2.sndcdn.com')) return;
     
-            let scrf = await fetch(url).then((r) => {return r.text()}).catch(() => { return false });
+            let scrf = await fetch(url).then(r => r.text()).catch(() => {});
             let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
     
             if (id && typeof id[0] === 'string') {
@@ -41,7 +41,7 @@ export default async function(obj) {
 
     let link;
     if (obj.url.hostname === 'on.soundcloud.com' && obj.shortLink) {
-        link = await fetch(`https://on.soundcloud.com/${obj.shortLink}/`, { redirect: "manual" }).then((r) => {
+        link = await fetch(`https://on.soundcloud.com/${obj.shortLink}/`, { redirect: "manual" }).then(r => {
             if (r.status === 302 && r.headers.get("location").startsWith("https://soundcloud.com/")) {
                 return r.headers.get("location").split('?', 1)[0]
             }
@@ -54,13 +54,13 @@ export default async function(obj) {
 
     if (!link) return { error: 'ErrorCouldntFetch' };
 
-    let json = await fetch(`https://api-v2.soundcloud.com/resolve?url=${link}&client_id=${clientId}`).then((r) => {
+    let json = await fetch(`https://api-v2.soundcloud.com/resolve?url=${link}&client_id=${clientId}`)
-        return r.status === 200 ? r.json() : false
+                     .then(r => r.status === 200 ? r.json() : false)
-    }).catch(() => {});
+                     .catch(() => {});
 
     if (!json) return { error: 'ErrorCouldntFetch' };
 
-    if (!json["media"]["transcodings"]) return { error: 'ErrorEmptyDownload' };
+    if (!json.media.transcodings) return { error: 'ErrorEmptyDownload' };
 
     let bestAudio = "opus",
         selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"),
@@ -75,12 +75,15 @@ export default async function(obj) {
     let fileUrlBase = selectedStream.url;
     let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
 
-    if (fileUrl.substring(0, 54) !== "https://api-v2.soundcloud.com/media/soundcloud:tracks:") return { error: 'ErrorEmptyDownload' };
+    if (!fileUrl.startsWith("https://api-v2.soundcloud.com/media/soundcloud:tracks:"))
+        return { error: 'ErrorEmptyDownload' };
 
-    if (json.duration > maxVideoDuration)
+    if (json.duration > env.durationLimit * 1000)
-        return { error: ['ErrorLengthAudioConvert', maxVideoDuration / 60000] };
+        return { error: ['ErrorLengthAudioConvert', env.durationLimit / 60] };
 
-    let file = await fetch(fileUrl).then(async (r) => { return (await r.json()).url }).catch(() => {});
+    let file = await fetch(fileUrl)
+                     .then(async r => (await r.json()).url)
+                     .catch(() => {});
     if (!file) return { error: 'ErrorCouldntFetch' };
 
     let fileMetadata = {

src/modules/processing/services/streamable.js

@@ -1,5 +1,8 @@
 export default async function(obj) {
-    let video = await fetch(`https://api.streamable.com/videos/${obj.id}`).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
+    let video = await fetch(`https://api.streamable.com/videos/${obj.id}`)
+                      .then(r => r.status === 200 ? r.json() : false)
+                      .catch(() => {});
+
     if (!video) return { error: 'ErrorEmptyDownload' };
 
     let best = video.files['mp4-mobile'];

src/modules/processing/services/tiktok.js

@@ -1,13 +1,13 @@
-import { genericUserAgent, env } from "../../config.js";
+import { genericUserAgent } from "../../config.js";
+import { updateCookie } from "../cookie/manager.js";
+import { extract } from "../url.js";
+import Cookie from "../cookie/cookie.js";
 
 const shortDomain = "https://vt.tiktok.com/";
-const apiPath = "https://api22-normal-c-alisg.tiktokv.com/aweme/v1/feed/?region=US&carrier_region=US";
+export const cookie = new Cookie({});
-const apiUserAgent = "TikTok/338014 CFNetwork/1410.1 Darwin/22.6.0";
 
 export default async function(obj) {
-    let postId = obj.postId ? obj.postId : false;
+    let postId = obj.postId;
-
-    if (!env.tiktokDeviceInfo) return { error: 'ErrorCouldntFetch' };
 
     if (!postId) {
         let html = await fetch(`${shortDomain}${obj.id}`, {
@@ -19,54 +19,60 @@ export default async function(obj) {
 
         if (!html) return { error: 'ErrorCouldntFetch' };
 
-        if (html.slice(0, 17) === '<a href="https://') {
+        if (html.startsWith('<a href="https://')) {
-            postId = html.split('<a href="https://')[1].split('?')[0].split('/')[3]
+            const extractedURL = html.split('<a href="')[1].split('?')[0];
-        } else if (html.slice(0, 32) === '<a href="https://m.tiktok.com/v/' && html.includes('/v/')) {
+            const { patternMatch } = extract(extractedURL);
-            postId = html.split('/v/')[1].split('.html')[0].replace("/", '')
+            postId = patternMatch.postId
         }
     }
     if (!postId) return { error: 'ErrorCantGetID' };
 
-    let deviceInfo = new URLSearchParams(env.tiktokDeviceInfo).toString();
+    // should always be /video/, even for photos
-
+    const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
-    let apiURL = new URL(apiPath);
-        apiURL.searchParams.append("aweme_id", postId);
-
-    let detail = await fetch(`${apiURL.href}&${deviceInfo}`, {
         headers: {
-            "user-agent": apiUserAgent
+            "user-agent": genericUserAgent,
+            cookie,
         }
-    }).then(r => r.json()).catch(() => {});
+    })
+    updateCookie(cookie, res.headers);
 
-    detail = detail?.aweme_list?.find(v => v.aweme_id === postId);
+    const html = await res.text();
-    if (!detail) return { error: 'ErrorCouldntFetch' };
+
+    let detail;
+    try {
+        const json = html
+            .split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1]
+            .split('</script>')[0]
+        const data = JSON.parse(json)
+        detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
+    } catch {
+        return { error: 'ErrorCouldntFetch' };
+    }
 
     let video, videoFilename, audioFilename, audio, images,
-        filenameBase = `tiktok_${detail.author.unique_id}_${postId}`,
+        filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`,
         bestAudio = 'm4a';
 
-    images = detail.image_post_info?.images;
+    images = detail.imagePost?.images;
 
-    let playAddr = detail.video.play_addr_h264;
+    let playAddr = detail.video.playAddr;
     if (obj.h265) {
-        playAddr = detail.video.bit_rate[0].play_addr
+        const h265PlayAddr = detail.video.bitrateInfo.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
-    }
+        playAddr = h265PlayAddr || playAddr
-    if (!playAddr && detail.video.play_addr) {
-        playAddr = detail.video.play_addr
     }
 
     if (!obj.isAudioOnly && !images) {
-        video = playAddr.url_list[0];
+        video = playAddr;
         videoFilename = `${filenameBase}.mp4`;
     } else {
-        let fallback = playAddr.url_list[0];
+        audio = playAddr;
-        audio = fallback;
         audioFilename = `${filenameBase}_audio`;
-        if (obj.fullAudio || fallback.includes("music")) {
+
-            audio = detail.music.play_url.url_list[0]
+        if (obj.fullAudio || !audio) {
-            audioFilename = `${filenameBase}_audio_original`
+            audio = detail.music.playUrl;
+            audioFilename += `_original`
         }
-        if (audio.slice(-4) === ".mp3") bestAudio = 'mp3';
+        if (audio.includes("mime_type=audio_mpeg")) bestAudio = 'mp3';
     }
 
     if (video) return {
@@ -80,12 +86,9 @@ export default async function(obj) {
         bestAudio
     }
     if (images) {
-        let imageLinks = [];
+        let imageLinks = images
-        for (let i in images) {
+            .map(i => i.imageURL.urlList.find(p => p.includes(".jpeg?")))
-            let sel = images[i].display_image.url_list;
+            .map(url => ({ url }))
-            sel = sel.filter(p => p.includes(".jpeg?"))
-            imageLinks.push({url: sel[0]})
-        }
         return {
             picker: imageLinks,
             urls: audio,

src/modules/processing/services/twitch.js

@@ -1,4 +1,4 @@
-import { maxVideoDuration } from "../../config.js";
+import { env } from "../../config.js";
 import { cleanString } from '../../sub/utils.js';
 
 const gqlURL = "https://gql.twitch.tv/gql";
@@ -29,13 +29,15 @@ export default async function (obj) {
             }
         }`
         })
-    }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false });
+    }).then(r => r.status === 200 ? r.json() : false).catch(() => {});
     if (!req_metadata) return { error: 'ErrorCouldntFetch' };
 
     let clipMetadata = req_metadata.data.clip;
 
-    if (clipMetadata.durationSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+    if (clipMetadata.durationSeconds > env.durationLimit)
-    if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) return { error: 'ErrorEmptyDownload' };
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+    if (!clipMetadata.videoQualities || !clipMetadata.broadcaster)
+        return { error: 'ErrorEmptyDownload' };
 
     let req_token = await fetch(gqlURL, {
         method: "POST",
@@ -54,7 +56,7 @@ export default async function (obj) {
                 }
             }
         ])
-    }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false });
+    }).then(r => r.status === 200 ? r.json() : false).catch(() => {});
 
     if (!req_token) return { error: 'ErrorCouldntFetch' };
 

src/modules/processing/services/vimeo.js

@@ -1,4 +1,4 @@
-import { maxVideoDuration } from "../../config.js";
+import { env } from "../../config.js";
 import { cleanString } from '../../sub/utils.js';
 
 const resolutionMatch = {
@@ -39,7 +39,9 @@ export default async function(obj) {
     if (!api) return { error: 'ErrorCouldntFetch' };
 
     let downloadType = "dash";
-    if (!obj.isAudioOnly && JSON.stringify(api).includes('"progressive":[{')) downloadType = "progressive";
+
+    if (!obj.isAudioOnly && JSON.stringify(api).includes('"progressive":[{'))
+        downloadType = "progressive";
 
     let fileMetadata = {
         title: cleanString(api.video.title.trim()),
@@ -48,33 +50,43 @@ export default async function(obj) {
 
     if (downloadType !== "dash") {
         if (qualityMatch[quality]) quality = qualityMatch[quality];
-        let all = api["request"]["files"]["progressive"].sort((a, b) => Number(b.width) - Number(a.width));
+        let all = api.request.files.progressive.sort((a, b) => Number(b.width) - Number(a.width));
         let best = all[0];
 
-        let bestQuality = all[0]["quality"].split('p')[0];
+        let bestQuality = all[0].quality.split('p')[0];
-        bestQuality = qualityMatch[bestQuality] ? qualityMatch[bestQuality] : bestQuality;
+        if (qualityMatch[bestQuality]) {
-        if (Number(quality) < Number(bestQuality)) best = all.find(i => i["quality"].split('p')[0] === quality);
+            bestQuality = qualityMatch[bestQuality]
+        }
+
+        if (Number(quality) < Number(bestQuality)) {
+            best = all.find(i => i.quality.split('p')[0] === quality);
+        }
 
         if (!best) return { error: 'ErrorEmptyDownload' };
+
         return {
-            urls: best["url"],
+            urls: best.url,
             audioFilename: `vimeo_${obj.id}_audio`,
-            filename: `vimeo_${obj.id}_${best["width"]}x${best["height"]}.mp4`
+            filename: `vimeo_${obj.id}_${best.width}x${best.height}.mp4`
         }
     }
 
-    if (api.video.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+    if (api.video.duration > env.durationLimit)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 
-    let masterJSONURL = api["request"]["files"]["dash"]["cdns"]["akfire_interconnect_quic"]["url"];
+    let masterJSONURL = api.request.files.dash.cdns.akfire_interconnect_quic.url;
-    let masterJSON = await fetch(masterJSONURL).then((r) => { return r.json() }).catch(() => { return false });
+    let masterJSON = await fetch(masterJSONURL).then(r => r.json()).catch(() => {});
 
     if (!masterJSON) return { error: 'ErrorCouldntFetch' };
     if (!masterJSON.video) return { error: 'ErrorEmptyDownload' };
 
-    let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width)).filter(a => ["dash", "mp42"].includes(a['format'])),
+    let masterJSON_Video = masterJSON.video
-        bestVideo = masterJSON_Video[0];
+                            .sort((a, b) => Number(b.width) - Number(a.width))
-    if (Number(quality) < Number(resolutionMatch[bestVideo["width"]])) {
+                            .filter(a => ["dash", "mp42"].includes(a.format));
-        bestVideo = masterJSON_Video.find(i => resolutionMatch[i["width"]] === quality)
+
+    let bestVideo = masterJSON_Video[0];
+    if (Number(quality) < Number(resolutionMatch[bestVideo.width])) {
+        bestVideo = masterJSON_Video.find(i => resolutionMatch[i.width] === quality)
     }
 
     let masterM3U8 = `${masterJSONURL.split("/sep/")[0]}/sep/video/${bestVideo.id}/master.m3u8`;

src/modules/processing/services/vine.js

@@ -1,5 +1,8 @@
 export default async function(obj) {
-    let post = await fetch(`https://archive.vine.co/posts/${obj.id}.json`).then((r) => { return r.json() }).catch(() => { return false });
+    let post = await fetch(`https://archive.vine.co/posts/${obj.id}.json`)
+                    .then(r => r.json())
+                    .catch(() => {});
+
     if (!post) return { error: 'ErrorEmptyDownload' };
 
     if (post.videoUrl) return {

src/modules/processing/services/vk.js

@@ -1,4 +1,4 @@
-import { genericUserAgent, maxVideoDuration } from "../../config.js";
+import { genericUserAgent, env } from "../../config.js";
 import { cleanString } from "../../sub/utils.js";
 
 const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"];
@@ -8,7 +8,7 @@ export default async function(o) {
 
     html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, {
         headers: { "user-agent": genericUserAgent }
-    }).then((r) => { return r.arrayBuffer() }).catch(() => { return false });
+    }).then(r => r.arrayBuffer()).catch(() => {});
 
     if (!html) return { error: 'ErrorCouldntFetch' };
 
@@ -21,7 +21,8 @@ export default async function(o) {
     let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
 
     if (Number(js.mvData.is_active_live) !== 0) return { error: 'ErrorLiveVideo' };
-    if (js.mvData.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+    if (js.mvData.duration > env.durationLimit)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 
     for (let i in resolutions) {
         if (js.player.params[0][`url${resolutions[i]}`]) {

src/modules/processing/services/youtube.js

@@ -1,5 +1,5 @@
 import { Innertube, Session } from 'youtubei.js';
-import { maxVideoDuration } from '../../config.js';
+import { env } from '../../config.js';
 import { cleanString } from '../../sub/utils.js';
 import { fetch } from 'undici'
 
@@ -57,7 +57,7 @@ export default async function(o) {
 
     try {
         info = await yt.getBasicInfo(o.id, 'WEB');
-    } catch (e) {
+    } catch {
         return { error: 'ErrorCantConnectToServiceAPI' };
     }
 
@@ -92,7 +92,7 @@ export default async function(o) {
 
     if (bestQuality) bestQuality = qual(bestQuality);
     if (!bestQuality && !o.isAudioOnly || !hasAudio) return { error: 'ErrorYTTryOtherCodec' };
-    if (info.basic_info.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+    if (info.basic_info.duration > env.durationLimit) return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 
     let checkBestAudio = (i) => (i.has_audio && !i.has_video),
         audio = adaptive_formats.find(i => checkBestAudio(i) && !i.is_dubbed);

src/modules/processing/servicesConfig.json

@@ -55,16 +55,10 @@
         },
         "tiktok": {
             "alias": "tiktok videos, photos & audio",
-            "patterns": [":user/video/:postId", ":id", "t/:id", ":user/photo/:postId"],
+            "patterns": [":user/video/:postId", ":id", "t/:id", ":user/photo/:postId", "v/:id.html"],
-            "subdomains": ["vt", "vm"],
+            "subdomains": ["vt", "vm", "m"],
             "enabled": true
         },
-        "douyin": {
-            "alias": "douyin videos & audio",
-            "patterns": ["video/:postId", ":id"],
-            "subdomains": ["v"],
-            "enabled": false
-        },
         "vimeo": {
             "patterns": [":id", "video/:id", ":id/:password", "/channels/:user/:id"],
             "enabled": true,
@@ -76,7 +70,7 @@
             "enabled": true
         },
         "instagram": {
-            "alias": "instagram reels, posts & stories",
+            "alias": "instagram posts & reels",
             "altDomains": ["ddinstagram.com"],
             "patterns": [
                 "reels/:postId", ":username/reel/:postId", "reel/:postId", "p/:postId", ":username/p/:postId",

src/modules/processing/url.js

@@ -2,7 +2,7 @@ import { services } from "../config.js";
 import { strict as assert } from "node:assert";
 import psl from "psl";
 
-export function aliasURL(url) {
+function aliasURL(url) {
     assert(url instanceof URL);
 
     const host = psl.parse(url.hostname);
@@ -75,7 +75,7 @@ export function aliasURL(url) {
     return url
 }
 
-export function cleanURL(url) {
+function cleanURL(url) {
     assert(url instanceof URL);
     const host = psl.parse(url.hostname).sld;
     let stripQuery = true;
@@ -103,15 +103,7 @@ export function cleanURL(url) {
     return url
 }
 
-export function normalizeURL(url) {
+function getHostIfValid(url) {
-    return cleanURL(
-        aliasURL(
-            new URL(url.replace(/^https\/\//, 'https://'))
-        )
-    );
-}
-
-export function getHostIfValid(url) {
     const host = psl.parse(url.hostname);
     if (host.error) return;
 
@@ -125,3 +117,40 @@ export function getHostIfValid(url) {
 
     return host.sld;
 }
+
+export function normalizeURL(url) {
+    return cleanURL(
+        aliasURL(
+            new URL(url.replace(/^https\/\//, 'https://'))
+        )
+    );
+}
+
+export function extract(url) {
+    if (!(url instanceof URL)) {
+        url = new URL(url);
+    }
+
+    const host = getHostIfValid(url);
+
+    if (!host || !services[host].enabled) {
+        return null;
+    }
+
+    let patternMatch;
+    for (const pattern of services[host].patterns) {
+        patternMatch = pattern.match(
+            url.pathname.substring(1) + url.search
+        );
+
+        if (patternMatch) {
+            break;
+        }
+    }
+
+    if (!patternMatch) {
+        return null;
+    }
+
+    return { host, patternMatch };
+}

src/modules/setup.js

@@ -36,7 +36,7 @@ function setup() {
     rl.question(q, r1 => {
         switch (r1.toLowerCase()) {
             case 'api':
-                console.log(Bright("\nCool! What's the domain this API instance will be running on? (localhost)\nExample: co.wuk.sh"));
+                console.log(Bright("\nCool! What's the domain this API instance will be running on? (localhost)\nExample: api.cobalt.tools"));
 
                 rl.question(q, apiURL => {
                     ob.API_URL = `http://localhost:9000/`;
@@ -83,13 +83,13 @@ function setup() {
                         if (webPort && (webURL === "localhost" || !webURL)) ob.WEB_URL = `http://localhost:${webPort}/`;
 
                         console.log(
-                            Bright("\nOne last thing: what default API domain should be used? (co.wuk.sh)\nIf it's hosted locally, make sure to include the port:") + Cyan(" localhost:9000")
+                            Bright("\nOne last thing: what default API domain should be used? (api.cobalt.tools)\nIf it's hosted locally, make sure to include the port:") + Cyan(" localhost:9000")
                         );
 
                         rl.question(q, apiURL => {
                             ob.API_URL = `https://${apiURL.toLowerCase()}/`;
                             if (apiURL.includes(':')) ob.API_URL = `http://${apiURL.toLowerCase()}/`;
-                            if (!apiURL) ob.API_URL = "https://co.wuk.sh/";
+                            if (!apiURL) ob.API_URL = "https://api.cobalt.tools/";
                             final()
                         })
                     });

src/modules/stream/manage.js

@@ -1,9 +1,9 @@
 import NodeCache from "node-cache";
 import { randomBytes } from "crypto";
-import { nanoid } from 'nanoid';
+import { nanoid } from "nanoid";
 
 import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
-import { streamLifespan, env } from "../config.js";
+import { env } from "../config.js";
 import { strict as assert } from "assert";
 
 // optional dependency
@@ -11,17 +11,8 @@ const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
 
 const M3U_SERVICES = ['dailymotion', 'vimeo', 'rutube'];
 
-const streamNoAccess = {
-    error: "i couldn't verify if you have access to this stream. go back and try again!",
-    status: 401
-}
-const streamNoExist = {
-    error: "this download link has expired or doesn't exist. go back and try again!",
-    status: 400
-}
-
 const streamCache = new NodeCache({
-    stdTTL: streamLifespan/1000,
+    stdTTL: env.streamLifespan,
     checkperiod: 10,
     deleteOnExpire: true
 })
@@ -37,7 +28,7 @@ export function createStream(obj) {
     const streamID = nanoid(),
         iv = randomBytes(16).toString('base64url'),
         secret = randomBytes(32).toString('base64url'),
-        exp = new Date().getTime() + streamLifespan,
+        exp = new Date().getTime() + env.streamLifespan * 1000,
         hmac = generateHmac(`${streamID},${exp},${iv},${secret}`, hmacSalt),
         streamData = {
             exp: exp,
@@ -61,11 +52,11 @@ export function createStream(obj) {
     let streamLink = new URL('/api/stream', env.apiURL);
 
     const params = {
-        't': streamID,
+        'id': streamID,
-        'e': exp,
+        'exp': exp,
-        'h': hmac,
+        'sig': hmac,
-        's': secret,
+        'sec': secret,
-        'i': iv
+        'iv': iv
     }
 
     for (const [key, value] of Object.entries(params)) {
@@ -96,7 +87,7 @@ export function createInternalStream(url, obj = {}) {
     };
 
     let streamLink = new URL('/api/istream', `http://127.0.0.1:${env.apiPort}`);
-    streamLink.searchParams.set('t', streamID);
+    streamLink.searchParams.set('id', streamID);
     return streamLink.toString();
 }
 
@@ -106,7 +97,7 @@ export function destroyInternalStream(url) {
         return;
     }
 
-    const id = url.searchParams.get('t');
+    const id = url.searchParams.get('id');
 
     if (internalStreamCache[id]) {
         internalStreamCache[id].controller.abort();
@@ -141,22 +132,19 @@ export function verifyStream(id, hmac, exp, secret, iv) {
         const ghmac = generateHmac(`${id},${exp},${iv},${secret}`, hmacSalt);
         const cache = streamCache.get(id.toString());
 
-        if (ghmac !== String(hmac)) return streamNoAccess;
+        if (ghmac !== String(hmac)) return { status: 401 };
-        if (!cache) return streamNoExist;
+        if (!cache) return { status: 404 };
 
         const streamInfo = JSON.parse(decryptStream(cache, iv, secret));
 
-        if (!streamInfo) return streamNoExist;
+        if (!streamInfo) return { status: 404 };
 
         if (Number(exp) <= new Date().getTime())
-            return streamNoExist;
+            return { status: 404 };
 
         return wrapStream(streamInfo);
     }
     catch {
-        return {
+        return { status: 500 };
-            error: "something went wrong and i couldn't verify this stream. go back and try again!",
-            status: 500
-        }
     }
 }

src/modules/stream/shared.js

@@ -1,4 +1,5 @@
 import { genericUserAgent } from "../config.js";
+import { cookie as tiktokCookie } from "../processing/services/tiktok.js";
 
 const defaultHeaders = {
     'user-agent': genericUserAgent
@@ -13,9 +14,19 @@ const serviceHeaders = {
         origin: 'https://www.youtube.com',
         referer: 'https://www.youtube.com',
         DNT: '?1'
+    },
+    tiktok: {
+        cookie: tiktokCookie
     }
 }
 
-export function getHeaders(service) {
+export function closeResponse(res) {
-    return { ...defaultHeaders, ...serviceHeaders[service] }
+    if (!res.headersSent) res.sendStatus(500);
+    return res.destroy();
+}
+
+export function getHeaders(service) {
+    // Converting all header values to strings
+    return Object.entries({ ...defaultHeaders, ...serviceHeaders[service] })
+        .reduce((p, [key, val]) => ({ ...p, [key]: String(val) }), {})
 }

src/modules/stream/stream.js

@@ -1,5 +1,6 @@
 import { streamAudioOnly, streamDefault, streamLiveRender, streamVideoOnly, convertToGif } from "./types.js";
 import { internalStream } from './internal.js';
+import { closeResponse } from "./shared.js";
 
 export default async function(res, streamInfo) {
     try {
@@ -25,6 +26,6 @@ export default async function(res, streamInfo) {
                 break;
         }
     } catch {
-        res.status(500).json({ status: "error", text: "Internal Server Error" });
+        closeResponse(res)
     }
 }

src/modules/stream/types.js

@@ -6,7 +6,7 @@ import { create as contentDisposition } from "content-disposition-header";
 import { metadataManager } from "../sub/utils.js";
 import { destroyInternalStream } from "./manage.js";
 import { env, ffmpegArgs } from "../config.js";
-import { getHeaders } from "./shared.js";
+import { getHeaders, closeResponse } from "./shared.js";
 
 function toRawHeaders(headers) {
     return Object.entries(headers)
@@ -18,11 +18,6 @@ function closeRequest(controller) {
     try { controller.abort() } catch {}
 }
 
-function closeResponse(res) {
-    if (!res.headersSent) res.sendStatus(500);
-    return res.destroy();
-}
-
 function killProcess(p) {
     // ask the process to terminate itself gracefully
     p?.kill('SIGTERM');

src/modules/sub/consoleText.js

@@ -1,49 +1,16 @@
 function t(color, tt) {
     return color + tt + "\x1b[0m"
 }
-export function Reset(tt) {
+
-    return "\x1b[0m" + tt
-}
 export function Bright(tt) {
     return t("\x1b[1m", tt)
 }
-export function Dim(tt) {
-    return t("\x1b[2m", tt)
-}
-export function Underscore(tt) {
-    return t("\x1b[4m", tt)
-}
-export function Blink(tt) {
-    return t("\x1b[5m", tt)
-}
-export function Reverse(tt) {
-    return t("\x1b[7m", tt)
-}
-export function Hidden(tt) {
-    return t("\x1b[8m", tt)
-}
-
-export function Black(tt) {
-    return t("\x1b[30m", tt)
-}
 export function Red(tt) {
     return t("\x1b[31m", tt)
 }
 export function Green(tt) {
     return t("\x1b[32m", tt)
 }
-export function Yellow(tt) {
-    return t("\x1b[33m", tt)
-}
-export function Blue(tt) {
-    return t("\x1b[34m", tt)
-}
-export function Magenta(tt) {
-    return t("\x1b[35m", tt)
-}
 export function Cyan(tt) {
     return t("\x1b[36m", tt)
 }
-export function White(tt) {
-    return t("\x1b[37m", tt)
-}

src/modules/sub/crypto.js

@@ -1,6 +1,10 @@
-import { createHmac, createCipheriv, createDecipheriv } from "crypto";
+import { createHmac, createCipheriv, createDecipheriv, randomBytes } from "crypto";
 
-const algorithm = "aes256"
+const algorithm = "aes256";
+
+export function generateSalt() {
+    return randomBytes(64).toString('hex');
+}
 
 export function generateHmac(str, salt) {
     return createHmac("sha256", salt).update(str).digest("base64url");

src/modules/sub/errors.js

@@ -1,11 +0,0 @@
-import loc from "../../localization/manager.js";
-
-export function errorUnsupported(lang) {
-    return loc(lang, 'ErrorUnsupported');
-}
-export function brokenLink(lang, host) {
-    return loc(lang, 'ErrorBrokenLink', host);
-}
-export function genericError(lang, host) {
-    return loc(lang, 'ErrorBadFetch', host);
-}

src/modules/sub/loadFromFs.js

@@ -3,14 +3,14 @@ import * as fs from "fs";
 export function loadJSON(path) {
     try {
         return JSON.parse(fs.readFileSync(path, 'utf-8'))
-    } catch(e) {
+    } catch {
         return false
     }
 }
 export function loadFile(path) {
     try {
         return fs.readFileSync(path, 'utf-8')
-    } catch(e) {
+    } catch {
         return false
     }
 }

src/modules/sub/utils.js

@@ -1,134 +1,43 @@
-import { normalizeURL } from "../processing/url.js";
-import { createStream } from "../stream/manage.js";
-import ipaddr from "ipaddr.js";
-
-const apiVar = {
-    allowed: {
-        vCodec: ["h264", "av1", "vp9"],
-        vQuality: ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"],
-        aFormat: ["best", "mp3", "ogg", "wav", "opus"],
-        filenamePattern: ["classic", "pretty", "basic", "nerdy"]
-    },
-    booleanOnly: [
-        "isAudioOnly",
-        "isTTFullAudio",
-        "isAudioMuted",
-        "dubLang",
-        "disableMetadata",
-        "twitterGif",
-        "tiktokH265"
-    ]
-}
 const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '='];
 
-export function apiJSON(type, obj) {
-    try {
-        switch (type) {
-            case 0:
-                return { status: 400, body: { status: "error", text: obj.t } };
-            case 1:
-                return { status: 200, body: { status: "redirect", url: obj.u } };
-            case 2:
-                return { status: 200, body: { status: "stream", url: createStream(obj) } };
-            case 3:
-                return { status: 200, body: { status: "success", text: obj.t } };
-            case 4:
-                return { status: 429, body: { status: "rate-limit", text: obj.t } };
-            case 5:
-                let pickerType = "various", audio = false
-                switch (obj.service) {
-                    case "douyin":
-                    case "tiktok":
-                        audio = obj.u
-                        pickerType = "images"
-                        break;
-                }
-                return { status: 200, body: { status: "picker", pickerType: pickerType, picker: obj.picker, audio: audio } };
-            case 6: // critical error, action should be taken by balancer/other server software
-                return { status: 500, body: { status: "error", text: obj.t, critical: true } };
-            default:
-                return { status: 400, body: { status: "error", text: "Bad Request" } };
-        }
-    } catch (e) {
-        return { status: 500, body: { status: "error", text: "Internal Server Error", critical: true } };
-    }
-}
 export function metadataManager(obj) {
-    let keys = Object.keys(obj);
+    const keys = Object.keys(obj);
-    let tags = ["album", "composer", "genre", "copyright", "encoded_by", "title", "language", "artist", "album_artist", "performer", "disc", "publisher", "track", "encoder", "compilation", "date", "creation_time", "comment"]
+    const tags = [
+        "album",
+        "copyright",
+        "title",
+        "artist",
+        "track",
+        "date"
+    ]
     let commands = []
 
-    for (let i in keys) { if (tags.includes(keys[i])) commands.push('-metadata', `${keys[i]}=${obj[keys[i]]}`) }
+    for (const i in keys) {
+        if (tags.includes(keys[i]))
+            commands.push('-metadata', `${keys[i]}=${obj[keys[i]]}`)
+        }
     return commands;
 }
 
 export function cleanString(string) {
-    for (let i in forbiddenCharsString) {
+    for (const i in forbiddenCharsString) {
-        string = string.replaceAll("/", "_").replaceAll(forbiddenCharsString[i], '')
+        string = string.replaceAll("/", "_")
+                       .replaceAll(forbiddenCharsString[i], '')
     }
     return string;
 }
 export function verifyLanguageCode(code) {
-    return RegExp(/[a-z]{2}/).test(String(code.slice(0, 2).toLowerCase())) ? String(code.slice(0, 2).toLowerCase()) : "en"
+    const langCode = String(code.slice(0, 2).toLowerCase());
+    if (RegExp(/[a-z]{2}/).test(code)) {
+        return langCode
+    }
+    return "en"
 }
 export function languageCode(req) {
-    return req.header('Accept-Language') ? verifyLanguageCode(req.header('Accept-Language')) : "en"
+    if (req.header('Accept-Language')) {
-}
+        return verifyLanguageCode(req.header('Accept-Language'))
-export function unicodeDecode(str) {
-    return str.replace(/\\u[\dA-F]{4}/gi, (unicode) => {
-        return String.fromCharCode(parseInt(unicode.replace(/\\u/g, ""), 16));
-    });
-}
-export function checkJSONPost(obj) {
-    let def = {
-        url: normalizeURL(decodeURIComponent(obj.url)),
-        vCodec: "h264",
-        vQuality: "720",
-        aFormat: "mp3",
-        filenamePattern: "classic",
-        isAudioOnly: false,
-        isTTFullAudio: false,
-        isAudioMuted: false,
-        disableMetadata: false,
-        dubLang: false,
-        twitterGif: false,
-        tiktokH265: false
     }
-    try {
+    return "en"
-        let objKeys = Object.keys(obj);
-        let defKeys = Object.keys(def);
-        if (objKeys.length > defKeys.length + 1 || !obj.url) return false;
-
-        for (let i in objKeys) {
-            if (String(objKeys[i]) !== "url" && defKeys.includes(objKeys[i])) {
-                if (apiVar.booleanOnly.includes(objKeys[i])) {
-                    def[objKeys[i]] = obj[objKeys[i]] ? true : false;
-                } else {
-                    if (apiVar.allowed[objKeys[i]] && apiVar.allowed[objKeys[i]].includes(obj[objKeys[i]])) def[objKeys[i]] = String(obj[objKeys[i]])
-                }
-            }
-        }
-
-        if (def.dubLang)
-            def.dubLang = verifyLanguageCode(obj.dubLang);
-
-        return def
-    } catch (e) {
-        return false
-    }
-}
-export function getIP(req) {
-    const strippedIP = req.ip.replace(/^::ffff:/, '');
-    const ip = ipaddr.parse(strippedIP);
-    if (ip.kind() === 'ipv4') {
-        return strippedIP;
-    }
-
-    const prefix = 56;
-    const v6Bytes = ip.toByteArray();
-          v6Bytes.fill(0, prefix / 8);
-
-    return ipaddr.fromByteArray(v6Bytes).toString();
 }
 export function cleanHTML(html) {
     let clean = html.replace(/ {4}/g, '');

src/test/test.js

@@ -1,11 +1,14 @@
 import "dotenv/config";
 import "../modules/sub/alias-envs.js";
 
-import { getJSON } from "../modules/api.js";
 import { services } from "../modules/config.js";
+import { extract } from "../modules/processing/url.js";
+import match from "../modules/processing/match.js";
 import { loadJSON } from "../modules/sub/loadFromFs.js";
-import { checkJSONPost } from "../modules/sub/utils.js";
+import { normalizeRequest } from "../modules/processing/request.js";
+import { env } from "../modules/config.js";
 
+env.apiURL = 'http://localhost:9000'
 let tests = loadJSON('./src/test/tests.json');
 
 let noTest = [];
@@ -32,10 +35,14 @@ for (let i in services) {
             let params = {...{url: test.url}, ...test.params};
             console.log(params);
 
-            let chck = checkJSONPost(params);
+            let chck = normalizeRequest(params);
             if (chck) {
-                chck["ip"] = "d21ec524bc2ade41bef569c0361ac57728c69e2764b5cb3cb310fe36568ca53f"; // random sha256
+                const parsed = extract(chck.url);
-                let j = await getJSON(chck["url"], "en", chck);
+                if (parsed === null) {
+                    throw `Invalid URL: ${chck.url}`
+                }
+
+                let j = await match(parsed.host, parsed.patternMatch, "en", chck);
                 console.log('\nReceived:');
                 console.log(j)
                 if (j.status === test.expected.code && j.body.status === test.expected.status) {

src/test/tests.json

@@ -172,70 +172,6 @@
             "code": 400,
             "status": "error"
         }
-    }, {
-        "name": "recorded space by nyc (best)",
-        "url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB",
-        "params": {
-            "aFormat": "best",
-            "isAudioOnly": false,
-            "isAudioMuted": false
-        },
-        "expected": {
-            "code": 200,
-            "status": "stream"
-        }
-    }, {
-        "name": "recorded space by nyc (mp3)",
-        "url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB",
-        "params": {
-            "aFormat": "mp3",
-            "isAudioOnly": false,
-            "isAudioMuted": false
-        },
-        "expected": {
-            "code": 200,
-            "status": "stream"
-        }
-    }, {
-        "name": "recorded space by nyc (wav, isAudioMuted)",
-        "url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB",
-        "params": {
-            "aFormat": "wav",
-            "isAudioOnly": false,
-            "isAudioMuted": true
-        },
-        "expected": {
-            "code": 200,
-            "status": "stream"
-        }
-    }, {
-        "name": "recorded space by service95 & dualipa (mp3, isAudioMuted, isAudioOnly)",
-        "url": "https://twitter.com/i/spaces/1nAJErvvVXgxL",
-        "params": {
-            "aFormat": "mp3",
-            "isAudioOnly": true,
-            "isAudioMuted": true
-        },
-        "expected": {
-            "code": 200,
-            "status": "stream"
-        }
-    }, {
-        "name": "unavailable space",
-        "url": "https://twitter.com/i/spaces/1OwGWwjRjVVGQ?s=20",
-        "params": {},
-        "expected": {
-            "code": 400,
-            "status": "error"
-        }
-    }, {
-        "name": "inexistent space",
-        "url": "https://twitter.com/i/spaces/10Wkie2j29iiI",
-        "params": {},
-        "expected": {
-            "code": 400,
-            "status": "error"
-        }
     }],
     "soundcloud": [{
         "name": "public song (best)",