Merge db7930eb86 into 35530459b6

api/package.json

@@ -36,6 +36,7 @@
         "ipaddr.js": "2.2.0",
         "mime": "^4.0.4",
         "nanoid": "^5.0.9",
+        "prom-client": "^15.1.3",
         "set-cookie-parser": "2.6.0",
         "undici": "^5.19.1",
         "url-pattern": "1.0.3",

api/src/core/api.js

@@ -3,6 +3,7 @@ import http from "node:http";
 import rateLimit from "express-rate-limit";
 import { setGlobalDispatcher, ProxyAgent } from "undici";
 import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info";
+import { httpRequests, httpRequestDuration, WORKER_ID, aggregatorRegistry } from "../misc/metrics.js";
 
 import jwt from "../security/jwt.js";
 import stream from "../stream/stream.js";
@@ -10,7 +11,7 @@ import match from "../processing/match.js";
 
 import { env } from "../config.js";
 import { extract } from "../processing/url.js";
-import { Bright, Cyan } from "../misc/console-text.js";
+import { Bright, Cyan, Green } from "../misc/console-text.js";
 import { hashHmac } from "../security/secrets.js";
 import { createStore } from "../store/redis-ratelimit.js";
 import { randomizeCiphers } from "../misc/randomize-ciphers.js";
@@ -39,6 +40,8 @@ const corsConfig = env.corsWildcard ? {} : {
     optionsSuccessStatus: 200
 }
 
+const metrics = Boolean(env.metrics && env.metricsPort);
+
 const fail = (res, code, context) => {
     const { status, body } = createResponse("error", { code, context });
     res.status(status).json(body);
@@ -54,6 +57,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
                 version: version,
                 url: env.apiURL,
                 startTime: `${startTimestamp}`,
+                metrics: metrics,
                 turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,
                 services: [...env.enabledServices].map(e => {
                     return friendlyServiceName(e);
@@ -111,6 +115,19 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
 
     app.set('trust proxy', ['loopback', 'uniquelocal']);
 
+    if (metrics) {
+        app.use((req, res, next) => {
+            const end = httpRequestDuration.startTimer({ method: req.method, worker_id: WORKER_ID });
+
+            res.on('finish', () => {
+                httpRequests.labels(req.method, res.statusCode.toString(), WORKER_ID).inc();
+                end();
+            });
+          
+            next();
+        });
+    }
+
     app.use('/', cors({
         methods: ['GET', 'POST'],
         exposedHeaders: [
@@ -370,7 +387,29 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
         if (env.ytSessionServer) {
             YouTubeSession.setup();
         }
+
+        if (metrics && isPrimary) {
+            const metricsApp = express();
+        
+            metricsApp.get('/metrics', async (req, res) => {
+                try {
+                    const data = await aggregatorRegistry.clusterMetrics();
+                    res.set('Content-Type', 'text/plain');
+                    res.end(data);
+                } catch (err) {
+                    res.status(500).end(err.message);
+                }
+            });
+
+            metricsApp.get('/*', (req, res) => {
+                res.redirect('/metrics');
+            });
+        
+            metricsApp.listen(env.metricsPort, '0.0.0.0', () => {
+                console.log(`${Green('[✓]')} prometheus metrics running on 0.0.0.0:${env.metricsPort}/metrics`);
+            });
+        }
     });
 
     setupTunnelHandler();
-}
+}

api/src/core/env.js

@@ -65,6 +65,9 @@ export const loadEnvs = (env = process.env) => {
 
         enabledServices,
 
+        metrics: env.METRICS_ENABLED,
+        metricsPort: env.METRICS_PORT,
+
         customInnertubeClient: env.CUSTOM_INNERTUBE_CLIENT,
         ytSessionServer: env.YOUTUBE_SESSION_SERVER,
         ytSessionReloadInterval: 300,

api/src/misc/metrics.js

@@ -0,0 +1,50 @@
+import { collectDefaultMetrics, Counter, Histogram, Registry, AggregatorRegistry } from 'prom-client';
+  
+import cluster from 'node:cluster';
+
+export const WORKER_ID = `worker_${cluster.worker?.id ?? process.pid}`;
+
+export const registry = new Registry();
+export const aggregatorRegistry = new AggregatorRegistry();
+
+collectDefaultMetrics({
+    register: registry
+});
+
+export const failedRequests = new Counter({
+    name: 'cobalt_fail_request_count',
+    help: 'Total number of failed requests',
+    labelNames: ['service', 'worker_id'],
+});
+
+export const successfulRequests = new Counter({
+    name: 'cobalt_success_request_count',
+    help: 'Total number of successful requests',
+    labelNames: ['service', 'worker_id'],
+});
+
+export const httpRequests = new Counter({
+    name: 'http_requests_total',
+    help: 'Total number of HTTP requests',
+    labelNames: ['method', 'status', 'worker_id'],
+});
+
+export const httpRequestDuration = new Histogram({
+    name: 'http_request_duration_seconds',
+    help: 'Duration of HTTP requests in seconds',
+    labelNames: ['method', 'worker_id'],
+    buckets: [0.1, 0.5, 1, 1.5, 2, 5],
+});
+
+registry.registerMetric(failedRequests);
+registry.registerMetric(successfulRequests);
+registry.registerMetric(httpRequests);
+registry.registerMetric(httpRequestDuration);
+
+export function incrementFailed(service) {
+    failedRequests.labels(service, WORKER_ID).inc();
+}
+
+export function incrementSuccessful(service) {
+    successfulRequests.labels(service, WORKER_ID).inc();
+}

api/src/processing/match.js

@@ -8,6 +8,8 @@ import matchAction from "./match-action.js";
 
 import { friendlyServiceName } from "./service-alias.js";
 
+import { incrementFailed, incrementSuccessful } from "../misc/metrics.js"
+
 import bilibili from "./services/bilibili.js";
 import reddit from "./services/reddit.js";
 import twitter from "./services/twitter.js";
@@ -286,12 +288,16 @@ export default async function({ host, patternMatch, params, isSession }) {
                     break;
             }
 
+            if (env.metrics && env.metricsPort) incrementFailed(host);
+
             return createResponse("error", {
                 code: `error.api.${r.error}`,
                 context,
             })
         }
 
+        if (env.metrics && env.metricsPort) incrementSuccessful(host);
+
         let localProcessing = params.localProcessing;
 
         const lpEnv = env.forceLocalProcessing;

docs/api-env-variables.md

@@ -64,6 +64,14 @@ this document is not final and will expand over time. feel free to improve it!
 
 [*view details*](#service-specific)
 
+### prometheus metrics vars
+| name                             | value example            |
+|:---------------------------------|:-------------------------|
+| METRICS                          | `1`                      |
+| METRICS_PORT                     | `9100`                   |
+
+[*view details*](#metrics)
+
 ## general
 [*jump to the table*](#general-vars)
 
@@ -256,3 +264,16 @@ the value is a string.
 when set to `1`, cobalt will try to use higher quality audio if user requests it via `youtubeBetterAudio`. will negatively impact the rate limit of a secondary youtube client with a session.
 
 the value is a number, either `0` or `1`.
+
+## prometheus metrics
+[*jump to the table*](#prometheus-metrics-vars)
+
+### METRICS
+enable prometheus compatible metrics. metrics include: successful/failed requests to services, http requests, and http requests duration.
+
+the value is a number, either `0` or `1`.
+
+### METRICS_PORT
+port from which the metrics will be server under. these are local to the container/host.
+
+the value is a number from 1024 to 65535.

pnpm-lock.yaml

@@ -49,6 +49,9 @@ importers:
       nanoid:
         specifier: ^5.0.9
         version: 5.0.9
+      prom-client:
+        specifier: ^15.1.3
+        version: 15.1.3
       set-cookie-parser:
         specifier: 2.6.0
         version: 2.6.0
@@ -613,6 +616,10 @@ packages:
     resolution: {integrity: sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==}
     engines: {node: '>=8.0'}
 
+  '@opentelemetry/api@1.9.0':
+    resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==}
+    engines: {node: '>=8.0.0'}
+
   '@pkgjs/parseargs@0.11.0':
     resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
     engines: {node: '>=14'}
@@ -930,6 +937,9 @@ packages:
     resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
     engines: {node: '>=8'}
 
+  bintrees@1.0.2:
+    resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
+
   body-parser@1.20.3:
     resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==}
     engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@@ -1749,6 +1759,10 @@ packages:
     resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}
     engines: {node: '>=0.4.0'}
 
+  prom-client@15.1.3:
+    resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
+    engines: {node: ^16 || ^18 || >=20}
+
   proxy-addr@2.0.7:
     resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
     engines: {node: '>= 0.10'}
@@ -1982,6 +1996,9 @@ packages:
   syscall-napi@0.0.6:
     resolution: {integrity: sha512-qHbwjyFXAAekKUXxl70lhDiBYJ3e7XM7kQwu7LV3F0pHMenKox+VcZPZkRkhdmL/wNJD3NmrMGnL7161kdecUQ==}
 
+  tdigest@0.1.2:
+    resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
+
   thenify-all@1.6.0:
     resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==}
     engines: {node: '>=0.8'}
@@ -2363,7 +2380,7 @@ snapshots:
   '@eslint/config-array@0.19.1':
     dependencies:
       '@eslint/object-schema': 2.1.5
-      debug: 4.3.6
+      debug: 4.4.0
       minimatch: 3.1.2
     transitivePeerDependencies:
       - supports-color
@@ -2375,7 +2392,7 @@ snapshots:
   '@eslint/eslintrc@3.2.0':
     dependencies:
       ajv: 6.12.6
-      debug: 4.3.6
+      debug: 4.4.0
       espree: 10.3.0
       globals: 14.0.0
       ignore: 5.3.1
@@ -2478,6 +2495,8 @@ snapshots:
 
   '@oozcitak/util@8.3.8': {}
 
+  '@opentelemetry/api@1.9.0': {}
+
   '@pkgjs/parseargs@0.11.0':
     optional: true
 
@@ -2675,7 +2694,7 @@ snapshots:
       '@typescript-eslint/types': 8.18.0
       '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4)
       '@typescript-eslint/visitor-keys': 8.18.0
-      debug: 4.3.6
+      debug: 4.4.0
       eslint: 9.16.0
       typescript: 5.5.4
     transitivePeerDependencies:
@@ -2690,7 +2709,7 @@ snapshots:
     dependencies:
       '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4)
       '@typescript-eslint/utils': 8.18.0(eslint@9.16.0)(typescript@5.5.4)
-      debug: 4.3.6
+      debug: 4.4.0
       eslint: 9.16.0
       ts-api-utils: 1.3.0(typescript@5.5.4)
       typescript: 5.5.4
@@ -2703,7 +2722,7 @@ snapshots:
     dependencies:
       '@typescript-eslint/types': 8.18.0
       '@typescript-eslint/visitor-keys': 8.18.0
-      debug: 4.3.6
+      debug: 4.4.0
       fast-glob: 3.3.2
       is-glob: 4.0.3
       minimatch: 9.0.5
@@ -2746,7 +2765,7 @@ snapshots:
 
   agent-base@6.0.2:
     dependencies:
-      debug: 4.3.6
+      debug: 4.4.0
     transitivePeerDependencies:
       - supports-color
 
@@ -2790,6 +2809,8 @@ snapshots:
 
   binary-extensions@2.3.0: {}
 
+  bintrees@1.0.2: {}
+
   body-parser@1.20.3:
     dependencies:
       bytes: 3.1.2
@@ -3320,7 +3341,7 @@ snapshots:
   https-proxy-agent@5.0.1:
     dependencies:
       agent-base: 6.0.2
-      debug: 4.3.6
+      debug: 4.4.0
     transitivePeerDependencies:
       - supports-color
 
@@ -3608,6 +3629,11 @@ snapshots:
 
   progress@2.0.3: {}
 
+  prom-client@15.1.3:
+    dependencies:
+      '@opentelemetry/api': 1.9.0
+      tdigest: 0.1.2
+
   proxy-addr@2.0.7:
     dependencies:
       forwarded: 0.2.0
@@ -3866,6 +3892,10 @@ snapshots:
   syscall-napi@0.0.6:
     optional: true
 
+  tdigest@0.1.2:
+    dependencies:
+      bintrees: 1.0.2
+
   thenify-all@1.6.0:
     dependencies:
       thenify: 3.3.1