[MOVED] [WIP] Tor support + FixTweet support

.gitignore

@@ -1,6 +1,6 @@
 # os stuff
-.DS_Store
-desktop.ini
+**/.DS_Store
+**/desktop.ini
 
 # npm
 node_modules
@@ -24,3 +24,6 @@ docker-compose.yml
 
 # cookie file
 cookies.json
+
+# tor
+tor

package.json

@@ -31,6 +31,7 @@
         "esbuild": "^0.14.51",
         "express": "^4.18.1",
         "express-rate-limit": "^6.3.0",
+        "fetch-socks": "^1.2.0",
         "ffmpeg-static": "^5.1.0",
         "hls-parser": "^0.10.7",
         "nanoid": "^4.0.2",

src/cobalt.js

@@ -1,7 +1,8 @@
 import "dotenv/config";
 
 import express from "express";
-
+import { setGlobalDispatcher } from "undici";
+import { socksDispatcher } from "fetch-socks";
 import { Bright, Green, Red } from "./modules/sub/consoleText.js";
 import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js";
 import { loadLoc } from "./localization/manager.js";
@@ -21,8 +22,31 @@ app.disable('x-powered-by');
 
 await loadLoc();
 
-const apiMode = process.env.apiURL && process.env.apiPort && !((process.env.webURL && process.env.webPort) || (process.env.selfURL && process.env.port));
-const webMode = process.env.webURL && process.env.webPort && !((process.env.apiURL && process.env.apiPort) || (process.env.selfURL && process.env.port));
+const torEnabled = (process.env.torHost && process.env.torPort && true) ? true : false;
+const torGlobal = (process.env.torGlobal && process.env.torGlobal == "true") ? true : false;
+global.torEnabled = torEnabled;
+
+if (torEnabled) {
+    let torProxy = {
+        type: 5,
+        host: process.env.torHost,
+        port: Number(process.env.torPort)
+    }
+    let torOptions = {
+        connect: {
+            timeout: 30000
+        }
+    }
+    let twitterTorOptions = torOptions;
+    twitterTorOptions['connect']['rejectUnauthorized'] = false;
+
+    global.torDispatcher = socksDispatcher(torProxy, torOptions)
+    global.twitterTorDispatcher = socksDispatcher(torProxy, twitterTorOptions)
+    if (torGlobal) setGlobalDispatcher(global.torDispatcher)
+}
+
+const apiMode = (process.env.apiURL && process.env.apiPort && !((process.env.webURL && process.env.webPort) || (process.env.selfURL && process.env.port))) ? true : false;
+const webMode = (process.env.webURL && process.env.webPort && !((process.env.apiURL && process.env.apiPort) || (process.env.selfURL && process.env.port))) ? true : false;
 
 if (apiMode) {
     const { runAPI } = await import('./core/api.js');

src/core/api.js

@@ -15,7 +15,7 @@ import { verifyStream } from "../modules/stream/manage.js";
 
 export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
     const corsConfig = process.env.cors === '0' ? {
-        origin: process.env.webURL,
+        origin: [process.env.webURL, process.env.webOnion],
         optionsSuccessStatus: 200
     } : {};
 
@@ -49,6 +49,11 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
     const startTime = new Date();
     const startTimestamp = Math.floor(startTime.getTime());
 
+    app.use((req, res, next) => {
+        if (global.torEnabled && process.env.apiOnion && !(req.hostname == process.env.apiOnion)) res.setHeader('Onion-Location', `${process.env.apiOnion}${req.path}`)
+        next();
+    });
+
     app.use('/api/:type', cors(corsConfig));
     app.use('/api/json', apiLimiter);
     app.use('/api/stream', apiLimiterStream);

src/core/web.js

@@ -14,6 +14,11 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
 
     await buildFront(gitCommit, gitBranch);
 
+    app.use((req, res, next) => {
+        if (global.torEnabled && process.env.webOnion && !(req.hostname == process.env.webOnion)) res.setHeader('Onion-Location', `${process.env.webOnion}${req.path}`)
+        next();
+    });
+
     app.use('/', express.static('./build/min'));
     app.use('/', express.static('./src/front'));
 

src/front/cobalt.js

@@ -18,7 +18,8 @@ const switchers = {
     "dubLang": ["original", "auto"],
     "vimeoDash": ["false", "true"],
     "audioMode": ["false", "true"],
-    "filenamePattern": ["classic", "pretty", "basic", "nerdy"]
+    "filenamePattern": ["classic", "pretty", "basic", "nerdy"],
+    "onionPreference": ["noOnions", "onionStrict", "clearnetFallback"]
 };
 const checkboxes = [
     "alwaysVisibleButton",
@@ -59,6 +60,14 @@ function sGet(id) {
 function sSet(id, value) {
     localStorage.setItem(id, value)
 }
+if (sGet('onionPreference') == null) {
+    if (window.location.hostname.endsWith(".onion")) sSet("onionPreference", "clearnetFallback")
+    else sSet('onionPreference', 'noOnions');
+}
+if (!(window.location.hostname.endsWith(".onion"))) {
+    document.getElementById('settings-tor').style = 'display:none;';
+    if (!(sGet('onionPreference') === 'noOnions')) sSet("onionPreference", "noOnions");
+}
 function enable(id) {
     eid(id).dataset.enabled = "true";
 }
@@ -131,6 +140,7 @@ function detectColorScheme() {
     } else if (!window.matchMedia) {
         theme = "dark"
     }
+    if (window.location.hostname.endsWith(".onion") && theme == "auto") theme = "dark";
     document.documentElement.setAttribute("data-theme", theme);
 }
 function changeTab(evnt, tabId, tabClass) {
@@ -334,7 +344,9 @@ function resetSettings() {
     localStorage.clear();
     window.location.reload();
 }
+if (window.location.hostname.endsWith(".onion") && !(navigator.getBattery())) document.getElementById('paste').style = "pointer-events:none;visibility:hidden;";
 async function pasteClipboard() {
+    if (window.location.hostname.endsWith(".onion") && !(navigator.getBattery())) return
     try {
         let t = await navigator.clipboard.readText();
         if (regex.test(t)) {
@@ -379,6 +391,7 @@ async function download(url) {
         if (url.includes("youtube.com/") || url.includes("/youtu.be/")) req.vCodec = sGet("vCodec").slice(0, 4);
         if ((url.includes("tiktok.com/") || url.includes("douyin.com/")) && sGet("disableTikTokWatermark") === "true") req.isNoTTWatermark = true;
     }
+    if (window.location.hostname.endsWith(".onion") && !(sGet("onionPreference") == "noOnions")) req.onionPreference = sGet("onionPreference");
 
     if (sGet("disableMetadata") === "true") req.disableMetadata = true;
 

src/localization/languages/en.json

@@ -156,6 +156,11 @@
         "FilenamePreviewVideoTitle": "Video Title",
         "FilenamePreviewAudioTitle": "Audio Title",
         "FilenamePreviewAudioAuthor": "Audio Author",
+        "Tor": "tor",
+        "SettingsOnionStrict": "onions only",
+        "SettingsClearnetFallback": "prefer onions",
+        "SettingsNoOnions": "no onions",
+        "SettingsTorDescription": "onions only: strictly uses .onions throughout entire process except where already known to be required.\nprefer onions: uses .onions throughout entire process but will fallback to clearnet if cobalt can't connect.\nno onions: will only use clearnet throughout entire process.\n\nthis setting affects twitter and reddit.",
         "UrgentFilenameUpdate": "customizable file names!"
     }
 }

src/modules/api.js

@@ -14,7 +14,11 @@ export async function getJSON(originalURL, lang, obj) {
             hostname = new URL(url).hostname.split('.'),
             host = hostname[hostname.length - 2];
 
+        if (url.startsWith('http://')) url = url.replace('http://', 'https://');
         if (!url.startsWith('https://')) return apiJSON(0, { t: errorUnsupported(lang) });
+        if(url.startsWith("https://www.")) {
+            url = url.replace("https://www.", "https://")
+        }
 
         let overrides = hostOverrides(host, url);
         host = overrides.host;

src/modules/pageRender/page.js

@@ -38,6 +38,8 @@ export default function(obj) {
 
     audioFormats[0]["text"] = t('SettingsAudioFormatBest');
 
+    let onionLocation = (process.env.torHost && process.env.torPort && true) && process.env.webOnion ? `<meta http-equiv="onion-location" content="${process.env.webOnion}" />` : "";
+
     try {
         return `
 <!DOCTYPE html>
@@ -49,6 +51,7 @@ export default function(obj) {
         <title>${t("AppTitleCobalt")}</title>
 
         <meta property="og:url" content="${process.env.webURL || process.env.selfURL}" />
+        ${onionLocation}
         <meta property="og:title" content="${t("AppTitleCobalt")}" />
         <meta property="og:description" content="${t('EmbedBriefDescription')}" />
         <meta property="og:image" content="${process.env.webURL || process.env.selfURL}icons/generic.png" />
@@ -497,6 +500,24 @@ export default function(obj) {
                         padding: "no-margin"
                     }])
                 })
+                + settingsCategory({
+                    name: "tor",
+                    title: t('Tor'),
+                    body: switcher({
+                        name: "onionPreference",
+                        items: [{
+                            action: "onionStrict",
+                            text: t('SettingsOnionStrict')
+                        }, {
+                            action: "clearnetFallback",
+                            text: t('SettingsClearnetFallback')
+                        }, {
+                            action: "noOnions",
+                            text: t('SettingsNoOnions')
+                        }]
+                    })
+                    + explanation(t('SettingsTorDescription'))
+                })
             }]
         })}
         ${popupWithBottomButtons({
@@ -571,7 +592,7 @@ export default function(obj) {
                 <div id="logo">${t("AppTitleCobalt")}${betaTag()}</div>
                 <div id="download-area">
                     <div id="top">
-                        <input id="url-input-area" class="mono" type="text" autocorrect="off" maxlength="128" autocapitalize="off" placeholder="${t('LinkInput')}" aria-label="${t('AccessibilityInputArea')}" oninput="button()"></input>
+                        <input id="url-input-area" class="mono" type="text" autocorrect="off" maxlength="256" autocapitalize="off" placeholder="${t('LinkInput')}" aria-label="${t('AccessibilityInputArea')}" oninput="button()"></input>
                         <button id="url-clear" onclick="clearInput()" style="display:none;">x</button>
                         <input id="download-button" class="mono dontRead" onclick="download(document.getElementById('url-input-area').value)" type="submit" value="" disabled=true aria-label="${t('AccessibilityDownloadButton')}">
                     </div>
@@ -613,7 +634,11 @@ export default function(obj) {
         </div>
     </body>
     <script type="text/javascript">
-        let defaultApiUrl = '${process.env.apiURL ? process.env.apiURL : ''}';
+        let defaultApiUrl;
+        let regularApiUrl = '${process.env.apiURL ? process.env.apiURL : ''}';
+        let onionApiUrl = '${process.env.apiOnion ? process.env.apiOnion : ''}';
+        defaultApiUrl = regularApiUrl;
+        if (window.location.hostname.endsWith('.onion')) defaultApiUrl = onionApiUrl;
         const loc = ${webLoc(t,
         [
             'ErrorNoInternet',

src/modules/processing/hostOverrides.js

@@ -3,41 +3,83 @@ export default function (inHost, inURL) {
     let url = String(inURL);
 
     switch(host) {
-        case "youtube":
-            if (url.startsWith("https://youtube.com/live/") || url.startsWith("https://www.youtube.com/live/")) {
-                url = url.split("?")[0].replace("www.", "");
-                url = `https://youtube.com/watch?v=${url.replace("https://youtube.com/live/", "")}`
+        case "reddittorjg6rue252oqsxryoxengawnmo46qy4kyii5wtqnwfj4ooad":
+            if (url.startsWith("https://reddittorjg6rue252oqsxryoxengawnmo46qy4kyii5wtqnwfj4ooad.onion/")) {
+                host = "reddit";
+                url = url.replace("https://reddittorjg6rue252oqsxryoxengawnmo46qy4kyii5wtqnwfj4ooad.onion/", "https://reddit.com/")
             }
-            if (url.includes('youtube.com/shorts/')) {
-                url = url.split('?')[0].replace('shorts/', 'watch?v=');
+            break;
+        case "tumblr":
+            if (!url.includes("blog/view")) {
+                if (url.slice(-1) === '/') url = url.slice(0, -1);
+                url = url.replace(url.split('/')[5], '')
             }
             break;
-        case "youtu":
-            if (url.startsWith("https://youtu.be/")) {
-                host = "youtube";
-                url = `https://youtube.com/watch?v=${url.replace("https://youtu.be/", "")}`
+        case "twitch":
+            if (url.startsWith("https://clips.twitch.tv")) {
+                url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/');
             }
             break;
         case "vxtwitter":
+        case "fixvx":
+        case "fxtwitter":
+        case "twittpr":
+        case "fixupx":
         case "x":
+        case "twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid":
+            if (url.startsWith("https://vxtwitter.com/")) {
+                host = "twitter";
+                url = url.replace("https://vxtwitter.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://fixvx.com/")) {
+                host = "twitter";
+                url = url.replace("https://fixvx.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://fxtwitter.com/")) {
+                host = "twitter";
+                url = url.replace("https://fxtwitter.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://d.fxtwitter.com/")) {
+                host = "twitter";
+                url = url.replace("https://d.fxtwitter.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://twittpr.com/")) {
+                host = "twitter";
+                url = url.replace("https://twittpr.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://d.twittpr.com/")) {
+                host = "twitter";
+                url = url.replace("https://d.twittpr.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://fixupx.com/")) {
+                host = "twitter";
+                url = url.replace("https://fixupx.com/", "https://twitter.com/")
+            }
+            if (url.startsWith("https://d.fixupx.com/")) {
+                host = "twitter";
+                url = url.replace("https://d.fixupx.com/", "https://twitter.com/")
+            }
             if (url.startsWith("https://x.com/")) {
                 host = "twitter";
                 url = url.replace("https://x.com/", "https://twitter.com/")
             }
-            if (url.startsWith("https://vxtwitter.com/")) {
+            if (url.startsWith("https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/")) {
                 host = "twitter";
-                url = url.replace("https://vxtwitter.com/", "https://twitter.com/")
+                url = url.replace("https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/", "https://twitter.com/")
             }
             break;
-        case "tumblr":
-            if (!url.includes("blog/view")) {
-                if (url.slice(-1) === '/') url = url.slice(0, -1);
-                url = url.replace(url.split('/')[5], '')
+        case "youtube":
+            if (url.startsWith("https://youtube.com/live/")) {
+                url = `https://youtube.com/watch?v=${url.split("?")[0].replace("https://youtube.com/live/", "")}`
+            }
+            if (url.startsWith("https://youtube.com/shorts/")) {
+                url = url.split('?')[0].replace('shorts/', 'watch?v=');
             }
             break;
-        case "twitch":
-            if (url.includes('clips.twitch.tv')) {
-                url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/');
+        case "youtu":
+            if (url.startsWith("https://youtu.be/")) {
+                host = "youtube";
+                url = `https://youtube.com/watch?v=${url.replace("https://youtu.be/", "")}`
             }
             break;
     }

src/modules/processing/matchActionDecider.js

@@ -44,6 +44,10 @@ export default function(r, host, audioFormat, isAudioOnly, lang, isAudioMuted, d
                     params = { type: r.type };
                     break;
                 case "reddit":
+                    // ffmpeg doesn't support SOCKS5 proxies, which is necessary to use tor.
+                    // ffmpeg won't be able to resolve the .onion address, so we need to
+                    // change it back to the clearnet counterpart
+                    r.urls.forEach((url, index) => { r.urls[index] = url.replace('redditdotzhmh3mao6r5i2j7speppwqkizwo7vksy3mbz5iz7rlhocyd.onion', 'redd.it') });
                     responseType = r.typeId;
                     params = { type: r.type };
                     break;