start work on Tor support

package.json

@@ -31,6 +31,7 @@
         "esbuild": "^0.14.51",
         "express": "^4.18.1",
         "express-rate-limit": "^6.3.0",
+        "fetch-socks": "^1.2.0",
         "ffmpeg-static": "^5.1.0",
         "hls-parser": "^0.10.7",
         "nanoid": "^4.0.2",

src/cobalt.js

@@ -1,7 +1,8 @@
 import "dotenv/config";
 
 import express from "express";
-
+import { setGlobalDispatcher } from "undici";
+import { socksDispatcher } from "fetch-socks";
 import { Bright, Green, Red } from "./modules/sub/consoleText.js";
 import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js";
 import { loadLoc } from "./localization/manager.js";
@@ -21,8 +22,31 @@ app.disable('x-powered-by');
 
 await loadLoc();
 
-const apiMode = process.env.apiURL && process.env.apiPort && !((process.env.webURL && process.env.webPort) || (process.env.selfURL && process.env.port));
+const torEnabled = (process.env.torHost && process.env.torPort && true) ? true : false;
-const webMode = process.env.webURL && process.env.webPort && !((process.env.apiURL && process.env.apiPort) || (process.env.selfURL && process.env.port));
+const torGlobal = (process.env.torGlobal && process.env.torGlobal == "true") ? true : false;
+global.torEnabled = torEnabled;
+
+if (torEnabled) {
+    let torProxy = {
+        type: 5,
+        host: process.env.torHost,
+        port: Number(process.env.torPort)
+    }
+    let torOptions = {
+        connect: {
+            timeout: 30000
+        }
+    }
+    let twitterTorOptions = torOptions;
+    twitterTorOptions['connect']['rejectUnauthorized'] = false;
+
+    global.torDispatcher = socksDispatcher(torProxy, torOptions)
+    global.twitterTorDispatcher = socksDispatcher(torProxy, twitterTorOptions)
+    if (torGlobal) setGlobalDispatcher(global.torDispatcher)
+}
+
+const apiMode = (process.env.apiURL && process.env.apiPort && !((process.env.webURL && process.env.webPort) || (process.env.selfURL && process.env.port))) ? true : false;
+const webMode = (process.env.webURL && process.env.webPort && !((process.env.apiURL && process.env.apiPort) || (process.env.selfURL && process.env.port))) ? true : false;
 
 if (apiMode) {
     const { runAPI } = await import('./core/api.js');

src/modules/api.js

@@ -14,7 +14,11 @@ export async function getJSON(originalURL, lang, obj) {
             hostname = new URL(url).hostname.split('.'),
             host = hostname[hostname.length - 2];
 
+        if (url.startsWith('http://')) url = url.replace('http://', 'https://');
         if (!url.startsWith('https://')) return apiJSON(0, { t: errorUnsupported(lang) });
+        if(url.startsWith("https://www.")) {
+            url = url.replace("https://www.", "https://")
+        }
 
         let overrides = hostOverrides(host, url);
         host = overrides.host;

src/modules/pageRender/page.js

@@ -571,7 +571,7 @@ export default function(obj) {
                 <div id="logo">${t("AppTitleCobalt")}${betaTag()}</div>
                 <div id="download-area">
                     <div id="top">
-                        <input id="url-input-area" class="mono" type="text" autocorrect="off" maxlength="128" autocapitalize="off" placeholder="${t('LinkInput')}" aria-label="${t('AccessibilityInputArea')}" oninput="button()"></input>
+                        <input id="url-input-area" class="mono" type="text" autocorrect="off" maxlength="256" autocapitalize="off" placeholder="${t('LinkInput')}" aria-label="${t('AccessibilityInputArea')}" oninput="button()"></input>
                         <button id="url-clear" onclick="clearInput()" style="display:none;">x</button>
                         <input id="download-button" class="mono dontRead" onclick="download(document.getElementById('url-input-area').value)" type="submit" value="" disabled=true aria-label="${t('AccessibilityDownloadButton')}">
                     </div>

src/modules/processing/hostOverrides.js

@@ -3,30 +3,10 @@ export default function (inHost, inURL) {
     let url = String(inURL);
 
     switch(host) {
-        case "youtube":
+        case "reddittorjg6rue252oqsxryoxengawnmo46qy4kyii5wtqnwfj4ooad":
-            if (url.startsWith("https://youtube.com/live/") || url.startsWith("https://www.youtube.com/live/")) {
+            if (url.startsWith("https://reddittorjg6rue252oqsxryoxengawnmo46qy4kyii5wtqnwfj4ooad.onion/")) {
-                url = url.split("?")[0].replace("www.", "");
+                host = "reddit";
-                url = `https://youtube.com/watch?v=${url.replace("https://youtube.com/live/", "")}`
+                url = url.replace("https://reddittorjg6rue252oqsxryoxengawnmo46qy4kyii5wtqnwfj4ooad.onion/", "https://reddit.com/")
-            }
-            if (url.includes('youtube.com/shorts/')) {
-                url = url.split('?')[0].replace('shorts/', 'watch?v=');
-            }
-            break;
-        case "youtu":
-            if (url.startsWith("https://youtu.be/")) {
-                host = "youtube";
-                url = `https://youtube.com/watch?v=${url.replace("https://youtu.be/", "")}`
-            }
-            break;
-        case "vxtwitter":
-        case "x":
-            if (url.startsWith("https://x.com/")) {
-                host = "twitter";
-                url = url.replace("https://x.com/", "https://twitter.com/")
-            }
-            if (url.startsWith("https://vxtwitter.com/")) {
-                host = "twitter";
-                url = url.replace("https://vxtwitter.com/", "https://twitter.com/")
             }
             break;
         case "tumblr":
@@ -36,10 +16,72 @@ export default function (inHost, inURL) {
             }
             break;
         case "twitch":
-            if (url.includes('clips.twitch.tv')) {
+            if (url.startsWith("https://clips.twitch.tv")) {
                 url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/');
             }
             break;
+        case "vxtwitter":
+        case "fixvx":
+        case "fxtwitter":
+        case "twittpr":
+        case "fixupx":
+        case "x":
+        case "twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid":
+            if (url.startsWith("https://vxtwitter.com/")) {
+                host = "twitter";
+                url = url.replace("https://vxtwitter.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://fixvx.com/")) {
+                host = "twitter";
+                url = url.replace("https://fixvx.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://fxtwitter.com/")) {
+                host = "twitter";
+                url = url.replace("https://fxtwitter.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://d.fxtwitter.com/")) {
+                host = "twitter";
+                url = url.replace("https://d.fxtwitter.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://twittpr.com/")) {
+                host = "twitter";
+                url = url.replace("https://twittpr.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://d.twittpr.com/")) {
+                host = "twitter";
+                url = url.replace("https://d.twittpr.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://fixupx.com/")) {
+                host = "twitter";
+                url = url.replace("https://fixupx.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://d.fixupx.com/")) {
+                host = "twitter";
+                url = url.replace("https://d.fixupx.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://x.com/")) {
+                host = "twitter";
+                url = url.replace("https://x.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/")) {
+                host = "twitter";
+                url = url.replace("https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/", "https://twitter.com/")
+            }
+            break;
+        case "youtube":
+            if (url.startsWith("https://youtube.com/live/")) {
+                url = `https://youtube.com/watch?v=${url.split("?")[0].replace("https://youtube.com/live/", "")}`
+            }
+            if (url.startsWith("https://youtube.com/shorts/")) {
+                url = url.split('?')[0].replace('shorts/', 'watch?v=');
+            }
+            break;
+        case "youtu":
+            if (url.startsWith("https://youtu.be/")) {
+                host = "youtube";
+                url = `https://youtube.com/watch?v=${url.replace("https://youtu.be/", "")}`
+            }
+            break;
     }
     return {
         host: host,

src/modules/processing/matchActionDecider.js

@@ -44,6 +44,10 @@ export default function(r, host, audioFormat, isAudioOnly, lang, isAudioMuted, d
                     params = { type: r.type };
                     break;
                 case "reddit":
+                    // ffmpeg doesn't support SOCKS5 proxies, which is necessary to use tor.
+                    // ffmpeg won't be able to resolve the .onion address, so we need to
+                    // change it back to the clearnet counterpart
+                    r.urls.forEach((url, index) => { r.urls[index] = url.replace('redditdotzhmh3mao6r5i2j7speppwqkizwo7vksy3mbz5iz7rlhocyd.onion', 'redd.it') });
                     responseType = r.typeId;
                     params = { type: r.type };
                     break;

src/modules/processing/services/reddit.js

@@ -1,3 +1,4 @@
+import { fetch, getGlobalDispatcher } from "undici";
 import { genericUserAgent, maxVideoDuration } from "../../config.js";
 import { getCookie, updateCookieValues } from "../cookie/manager.js";
 
@@ -21,7 +22,7 @@ async function getAccessToken() {
                         || Number(values.expiry) > new Date().getTime();
     if (!needRefresh) return values.access_token;
 
-    const data = await fetch('https://www.reddit.com/api/v1/access_token', {
+    const data = await fetch(`https://www.${redditURL}/api/v1/access_token`, {
         method: 'POST',
         headers: {
             'authorization': `Basic ${Buffer.from(
@@ -48,13 +49,24 @@ async function getAccessToken() {
 }
 
 export default async function(obj) {
-    const url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}/${obj.title}.json`);
+    let redditURL;
+    let regularURL = "reddit.com";
+    let torURL = "reddittorjg6rue252oqsxryoxengawnmo46qy4kyii5wtqnwfj4ooad.onion";
+    if (torEnabled) redditURL = torURL
+    else redditURL = regularURL;
+
+    let twitterDispatcher;
+    twitterDispatcher = false;
+    let redditDispatcher = global.torDispatcher ? global.torDispatcher : getGlobalDispatcher();
+    
+    const url = new URL(`https://www.${redditURL}/r/${obj.sub}/comments/${obj.id}/${obj.title}.json`);
+    console.log(url);
 
     const accessToken = await getAccessToken();
-    if (accessToken) url.hostname = 'oauth.reddit.com';
+    if (accessToken) url.hostname = `oauth.${redditURL}`;
 
     let data = await fetch(
-        url, { headers: accessToken && { authorization: `Bearer ${accessToken}` } }
+        url, { dispatcher: redditDispatcher, headers: accessToken && { authorization: `Bearer ${accessToken}` } }
     ).then((r) => { return r.json() }).catch(() => { return false });
     if (!data) return { error: 'ErrorCouldntFetch' };
 
@@ -69,12 +81,12 @@ export default async function(obj) {
         video = data["secure_media"]["reddit_video"]["fallback_url"].split('?')[0],
         audioFileLink = video.match('.mp4') ? `${video.split('_')[0]}_audio.mp4` : `${data["secure_media"]["reddit_video"]["fallback_url"].split('DASH')[0]}audio`;
 
-    await fetch(audioFileLink, { method: "HEAD" }).then((r) => { if (Number(r.status) === 200) audio = true }).catch(() => { audio = false });
+    await fetch(audioFileLink, { dispatcher: redditDispatcher, method: "HEAD" }).then((r) => { if (Number(r.status) === 200) audio = true }).catch(() => { audio = false });
 
     // fallback for videos with variable audio quality
     if (!audio) {
         audioFileLink = `${video.split('_')[0]}_AUDIO_128.mp4`
-        await fetch(audioFileLink, { method: "HEAD" }).then((r) => { if (Number(r.status) === 200) audio = true }).catch(() => { audio = false });
+        await fetch(audioFileLink, { dispatcher: redditDispatcher, method: "HEAD" }).then((r) => { if (Number(r.status) === 200) audio = true }).catch(() => { audio = false });
     }
 
     let id = video.split('/')[3];

src/modules/processing/services/twitter.js

@@ -1,10 +1,22 @@
+import { fetch, getGlobalDispatcher } from "undici";
 import { genericUserAgent } from "../../config.js";
+import { socksDispatcher } from "fetch-socks";
 
 function bestQuality(arr) {
     return arr.filter(v => v["content_type"] === "video/mp4").sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"]
 }
 
 export default async function(obj) {
+    let twitterURL;
+    let regularURL = "twitter.com";
+    let torURL = "twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion";
+    if (torEnabled) twitterURL = torURL
+    else twitterURL = regularURL;
+
+    let twitterDispatcher;
+    twitterDispatcher = false;
+    twitterDispatcher = global.twitterTorDispatcher ? global.twitterTorDispatcher : getGlobalDispatcher();
+    
     let _headers = {
         "user-agent": genericUserAgent,
         "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
@@ -14,17 +26,20 @@ export default async function(obj) {
         "accept-language": "en"
     };
 
+    // guest accounts cannot be created through the onion website (rate limited)
     let activateURL = `https://api.twitter.com/1.1/guest/activate.json`;
-    let graphqlTweetURL = `https://twitter.com/i/api/graphql/0hWvDhmW8YQ-S_ib3azIrw/TweetResultByRestId`;
+
-    let graphqlSpaceURL = `https://twitter.com/i/api/graphql/Gdz2uCtmIGMmhjhHG3V7nA/AudioSpaceById`;
+    let graphqlTweetURL = `https://${twitterURL}/i/api/graphql/0hWvDhmW8YQ-S_ib3azIrw/TweetResultByRestId`;
+    let graphqlSpaceURL = `https://${twitterURL}/i/api/graphql/Gdz2uCtmIGMmhjhHG3V7nA/AudioSpaceById`;
 
     let req_act = await fetch(activateURL, {
+        dispatcher: twitterDispatcher,
         method: "POST",
         headers: _headers
-    }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
+    }).then((r) => { return r.status === 200 ? r.json() : false }).catch((err) => { return false });
     if (!req_act) return { error: 'ErrorCouldntFetch' };
 
-    _headers["host"] = "twitter.com";
+    _headers["host"] = twitterURL;
     _headers["content-type"] = "application/json";
 
     _headers["x-guest-token"] = req_act["guest_token"];
@@ -39,7 +54,7 @@ export default async function(obj) {
         query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1);
         query = `${graphqlTweetURL}?variables=${query.variables}&features=${query.features}`;
 
-        let TweetResultByRestId = await fetch(query, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false });
+        let TweetResultByRestId = await fetch(query, { dispatcher: twitterDispatcher, headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false });
 
         // {"data":{"tweetResult":{"result":{"__typename":"TweetUnavailable","reason":"Protected"}}}}
         if (!TweetResultByRestId || TweetResultByRestId.data.tweetResult.result.__typename !== "Tweet") return { error: 'ErrorTweetUnavailable' };
@@ -79,7 +94,7 @@ export default async function(obj) {
     }
     // spaces no longer work with guest authorization
     if (obj.spaceId) {
-        _headers["host"] = "twitter.com";
+        _headers["host"] = twitterURL;
         _headers["content-type"] = "application/json";
 
         let query = {
@@ -90,14 +105,14 @@ export default async function(obj) {
         query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1);
         query = `${graphqlSpaceURL}?variables=${query.variables}&features=${query.features}`;
 
-        let AudioSpaceById = await fetch(query, { headers: _headers }).then((r) => {return r.status === 200 ? r.json() : false}).catch((e) => { return false });
+        let AudioSpaceById = await fetch(query, { dispatcher: twitterDispatcher, headers: _headers }).then((r) => {return r.status === 200 ? r.json() : false}).catch((e) => { return false });
         if (!AudioSpaceById) return { error: 'ErrorEmptyDownload' };
 
         if (!AudioSpaceById.data.audioSpace.metadata) return { error: 'ErrorEmptyDownload' };
         if (AudioSpaceById.data.audioSpace.metadata.is_space_available_for_replay !== true) return { error: 'TwitterSpaceWasntRecorded' };
 
         let streamStatus = await fetch(
-            `https://twitter.com/i/api/1.1/live_video_stream/status/${AudioSpaceById.data.audioSpace.metadata.media_key}`, { headers: _headers }
+            `https://${twitterURL}/i/api/1.1/live_video_stream/status/${AudioSpaceById.data.audioSpace.metadata.media_key}`, { dispatcher: twitterDispatcher, headers: _headers }
         ).then((r) =>{ return r.status === 200 ? r.json() : false }).catch(() => { return false });
         if (!streamStatus) return { error: 'ErrorCouldntFetch' };
 

src/modules/setup.js

@@ -44,6 +44,7 @@ function setup() {
                     ob['apiURL'] = `http://localhost:9000/`;
                     ob['apiPort'] = 9000;
                     if (apiURL && apiURL !== "localhost") ob['apiURL'] = `https://${apiURL.toLowerCase()}/`;
+                    if (apiURL && apiURL.endsWith('.onion/')) ob['apiURL'] = apiURL.replace('https://', 'http://');
 
                     console.log(Bright("\nGreat! Now, what port will it be running on? (9000)"));