api: dynamic env reloading from path/url

api/src/config.js

@@ -1,5 +1,6 @@
 import { getVersion } from "@imput/version-info";
-import { loadEnvs, validateEnvs } from "./core/env.js";
+import { loadEnvs, validateEnvs, setupEnvWatcher } from "./core/env.js";
+import * as cluster from "./misc/cluster.js";
 
 const version = await getVersion();
 
@@ -8,11 +9,22 @@ let env = loadEnvs();
 const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";
 const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
 
+export const canonicalEnv = Object.freeze(structuredClone(process.env));
 export const setTunnelPort = (port) => env.tunnelPort = port;
 export const isCluster = env.instanceCount > 1;
+export const updateEnv = (newEnv) => {
+    // tunnelPort is special and needs to get carried over here
+    newEnv.tunnelPort = env.tunnelPort;
+    env = newEnv;
+    cluster.broadcast({ env_update: newEnv });
+}
 
 await validateEnvs(env);
 
+if (env.envFile) {
+    setupEnvWatcher();
+}
+
 export {
     env,
     genericUserAgent,

api/src/core/api.js

@@ -48,7 +48,8 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
     const startTime = new Date();
     const startTimestamp = startTime.getTime();
 
-    const serverInfo = JSON.stringify({
+    const getServerInfo = () => {
+        return JSON.stringify({
             cobalt: {
                 version: version,
                 url: env.apiURL,
@@ -60,7 +61,10 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
                 }),
             },
             git,
-    })
+        });
+    }
+
+    const serverInfo = getServerInfo();
 
     const handleRateExceeded = (_, res) => {
         const { body } = createResponse("error", {
@@ -311,7 +315,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
 
     app.get('/', (_, res) => {
         res.type('json');
-        res.status(200).send(serverInfo);
+        res.status(200).send(env.envFile ? getServerInfo() : serverInfo);
     })
 
     app.get('/favicon.ico', (req, res) => {
@@ -331,10 +335,6 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
     setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes
 
     if (env.externalProxy) {
-        if (env.freebindCIDR) {
-            throw new Error('Freebind is not available when external proxy is enabled')
-        }
-
         setGlobalDispatcher(new ProxyAgent(env.externalProxy))
     }
 

api/src/core/env.js

@@ -1,7 +1,12 @@
 
 import { Constants } from "youtubei.js";
-import { supportsReusePort } from "../misc/cluster.js";
 import { services } from "../processing/service-config.js";
+import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js";
+
+import { FileWatcher } from "../misc/file-watcher.js";
+import { isURL } from "../misc/utils.js";
+import * as cluster from "../misc/cluster.js";
+import { Yellow } from "../misc/console-text.js";
 
 const forceLocalProcessingOptions = ["never", "session", "always"];
 
@@ -54,6 +59,7 @@ export const loadEnvs = (env = process.env) => {
                             && env.JWT_SECRET,
 
         apiKeyURL: env.API_KEY_URL && new URL(env.API_KEY_URL),
+        envFile: env.API_ENV_FILE,
         authRequired: env.API_AUTH_REQUIRED === '1',
         redisURL: env.API_REDIS_URL,
         instanceCount: (env.API_INSTANCE_COUNT && parseInt(env.API_INSTANCE_COUNT)) || 1,
@@ -64,6 +70,7 @@ export const loadEnvs = (env = process.env) => {
         customInnertubeClient: env.CUSTOM_INNERTUBE_CLIENT,
         ytSessionServer: env.YOUTUBE_SESSION_SERVER,
         ytSessionReloadInterval: 300,
+        envReloadInterval: 300,
         ytSessionInnertubeClient: env.YOUTUBE_SESSION_INNERTUBE_CLIENT,
         ytAllowBetterAudio: env.YOUTUBE_ALLOW_BETTER_AUDIO !== "0",
 
@@ -79,7 +86,7 @@ export const validateEnvs = async (env) => {
 
     if (env.instanceCount > 1 && !env.redisURL) {
         throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2");
-    } else if (env.instanceCount > 1 && !await supportsReusePort()) {
+    } else if (env.instanceCount > 1 && !await cluster.supportsReusePort()) {
         console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js');
         console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux');
         console.error('(or other OS that supports it). for more info, see `reusePort` option on');
@@ -98,4 +105,81 @@ export const validateEnvs = async (env) => {
         console.error(`Supported options are are: ${forceLocalProcessingOptions.join(', ')}\n`);
         throw new Error("Invalid FORCE_LOCAL_PROCESSING");
     }
+
+    if (env.externalProxy && env.freebindCIDR) {
+        throw new Error('freebind is not available when external proxy is enabled')
+    }
+}
+
+const reloadEnvs = async (contents) => {
+    const newEnvs = {};
+
+    for (let line of (await contents).split('\n')) {
+        line = line.trim();
+        if (line === '') {
+            continue;
+        }
+
+        const [ key, value ] = line.split(/=(.+)?/);
+        if (key) {
+            newEnvs[key] = value || '';
+        }
+    }
+
+    const candidate = {
+        ...canonicalEnv,
+        ...newEnvs,
+    };
+
+    const parsed = loadEnvs(candidate);
+    await validateEnvs(parsed);
+    updateEnv(parsed);
+}
+
+const wrapReload = (contents) => {
+    reloadEnvs(contents)
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed reloading environment variables at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    });
+}
+
+let watcher;
+const setupWatcherFromFile = (path) => {
+    const load = () => wrapReload(watcher.read());
+
+    if (isURL(path)) {
+        watcher = FileWatcher.fromFileProtocol(path);
+    } else {
+        watcher = new FileWatcher({ path });
+    }
+
+    watcher.on('file-updated', load);
+    load();
+}
+
+const setupWatcherFromFetch = (url) => {
+    const load = () => wrapReload(fetch(url).then(r => r.text()));
+    setInterval(load, currentEnv.envReloadInterval);
+    load();
+}
+
+export const setupEnvWatcher = () => {
+    if (cluster.isPrimary) {
+        const envFile = currentEnv.envFile;
+        const isFile = !isURL(envFile)
+                       || new URL(envFile).protocol === 'file:';
+
+        if (isFile) {
+            setupWatcherFromFile(envFile);
+        } else {
+            setupWatcherFromFetch(envFile);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('env_update' in message) {
+                updateEnv(message.env_update);
+            }
+        });
+    }
 }

api/src/misc/utils.js

@@ -52,3 +52,12 @@ export function splitFilenameExtension(filename) {
 export function zip(a, b) {
     return a.map((value, i) => [ value, b[i] ]);
 }
+
+export function isURL(input) {
+    try {
+        new URL(input);
+        return true;
+    } catch {
+        return false;
+    }
+}

docs/api-env-variables.md

@@ -13,6 +13,7 @@ this document is not final and will expand over time. feel free to improve it!
 | API_REDIS_URL          |         | `redis://localhost:6379`              |
 | DISABLED_SERVICES      |         | `bilibili,youtube`                    |
 | FORCE_LOCAL_PROCESSING | `never` | `always`                              |
+| API_ENV_FILE           |         | `/.env`                               |
 
 [*view details*](#general)
 
@@ -111,6 +112,9 @@ when set to `session`, only requests from session (Bearer token) clients will be
 
 when set to `always`, all requests will be forced to use on-device processing, no matter the preference.
 
+### API_ENV_FILE
+the URL or local path to a `key=value`-style environment variable file. this is used for dynamically reloading environment variables. **not all environment variables are able to be updated by this.** (e.g. the ratelimiters are instantiated when starting cobalt, and cannot be changed)
+
 ## networking
 [*jump to the table*](#networking-vars)