diff --git a/api/package.json b/api/package.json index de1e170f..bdc4a135 100644 --- a/api/package.json +++ b/api/package.json @@ -36,6 +36,7 @@ "ipaddr.js": "2.2.0", "mime": "^4.0.4", "nanoid": "^5.0.9", + "prom-client": "^15.1.3", "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", diff --git a/api/src/core/api.js b/api/src/core/api.js index eb5cf4ff..02f2950f 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -3,6 +3,7 @@ import http from "node:http"; import rateLimit from "express-rate-limit"; import { setGlobalDispatcher, ProxyAgent } from "undici"; import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info"; +import { httpRequests, httpRequestDuration, WORKER_ID, aggregatorRegistry } from "../misc/metrics.js"; import jwt from "../security/jwt.js"; import stream from "../stream/stream.js"; @@ -10,7 +11,7 @@ import match from "../processing/match.js"; import { env } from "../config.js"; import { extract } from "../processing/url.js"; -import { Bright, Cyan } from "../misc/console-text.js"; +import { Bright, Cyan, Green } from "../misc/console-text.js"; import { hashHmac } from "../security/secrets.js"; import { createStore } from "../store/redis-ratelimit.js"; import { randomizeCiphers } from "../misc/randomize-ciphers.js"; @@ -39,6 +40,8 @@ const corsConfig = env.corsWildcard ? {} : { optionsSuccessStatus: 200 } +const metrics = Boolean(env.metrics && env.metricsPort); + const fail = (res, code, context) => { const { status, body } = createResponse("error", { code, context }); res.status(status).json(body); @@ -54,6 +57,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { version: version, url: env.apiURL, startTime: `${startTimestamp}`, + metrics: metrics, turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined, services: [...env.enabledServices].map(e => { return friendlyServiceName(e); @@ -111,6 +115,19 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { app.set('trust proxy', ['loopback', 'uniquelocal']); + if (metrics) { + app.use((req, res, next) => { + const end = httpRequestDuration.startTimer({ method: req.method, worker_id: WORKER_ID }); + + res.on('finish', () => { + httpRequests.labels(req.method, res.statusCode.toString(), WORKER_ID).inc(); + end(); + }); + + next(); + }); + } + app.use('/', cors({ methods: ['GET', 'POST'], exposedHeaders: [ @@ -370,7 +387,29 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { if (env.ytSessionServer) { YouTubeSession.setup(); } + + if (metrics && isPrimary) { + const metricsApp = express(); + + metricsApp.get('/metrics', async (req, res) => { + try { + const data = await aggregatorRegistry.clusterMetrics(); + res.set('Content-Type', 'text/plain'); + res.end(data); + } catch (err) { + res.status(500).end(err.message); + } + }); + + metricsApp.get('/*', (req, res) => { + res.redirect('/metrics'); + }); + + metricsApp.listen(env.metricsPort, '0.0.0.0', () => { + console.log(`${Green('[✓]')} prometheus metrics running on 0.0.0.0:${env.metricsPort}/metrics`); + }); + } }); setupTunnelHandler(); -} +} \ No newline at end of file diff --git a/api/src/core/env.js b/api/src/core/env.js index 37ab36c1..53ee5964 100644 --- a/api/src/core/env.js +++ b/api/src/core/env.js @@ -65,6 +65,9 @@ export const loadEnvs = (env = process.env) => { enabledServices, + metrics: env.METRICS_ENABLED, + metricsPort: env.METRICS_PORT, + customInnertubeClient: env.CUSTOM_INNERTUBE_CLIENT, ytSessionServer: env.YOUTUBE_SESSION_SERVER, ytSessionReloadInterval: 300, diff --git a/api/src/misc/metrics.js b/api/src/misc/metrics.js new file mode 100644 index 00000000..02a87278 --- /dev/null +++ b/api/src/misc/metrics.js @@ -0,0 +1,50 @@ +import { collectDefaultMetrics, Counter, Histogram, Registry, AggregatorRegistry } from 'prom-client'; + +import cluster from 'node:cluster'; + +export const WORKER_ID = `worker_${cluster.worker?.id ?? process.pid}`; + +export const registry = new Registry(); +export const aggregatorRegistry = new AggregatorRegistry(); + +collectDefaultMetrics({ + register: registry +}); + +export const failedRequests = new Counter({ + name: 'cobalt_fail_request_count', + help: 'Total number of failed requests', + labelNames: ['service', 'worker_id'], +}); + +export const successfulRequests = new Counter({ + name: 'cobalt_success_request_count', + help: 'Total number of successful requests', + labelNames: ['service', 'worker_id'], +}); + +export const httpRequests = new Counter({ + name: 'http_requests_total', + help: 'Total number of HTTP requests', + labelNames: ['method', 'status', 'worker_id'], +}); + +export const httpRequestDuration = new Histogram({ + name: 'http_request_duration_seconds', + help: 'Duration of HTTP requests in seconds', + labelNames: ['method', 'worker_id'], + buckets: [0.1, 0.5, 1, 1.5, 2, 5], +}); + +registry.registerMetric(failedRequests); +registry.registerMetric(successfulRequests); +registry.registerMetric(httpRequests); +registry.registerMetric(httpRequestDuration); + +export function incrementFailed(service) { + failedRequests.labels(service, WORKER_ID).inc(); +} + +export function incrementSuccessful(service) { + successfulRequests.labels(service, WORKER_ID).inc(); +} \ No newline at end of file diff --git a/api/src/processing/match.js b/api/src/processing/match.js index 65a021b0..15812c63 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -8,6 +8,8 @@ import matchAction from "./match-action.js"; import { friendlyServiceName } from "./service-alias.js"; +import { incrementFailed, incrementSuccessful } from "../misc/metrics.js" + import bilibili from "./services/bilibili.js"; import reddit from "./services/reddit.js"; import twitter from "./services/twitter.js"; @@ -286,12 +288,16 @@ export default async function({ host, patternMatch, params, isSession }) { break; } + if (env.metrics && env.metricsPort) incrementFailed(host); + return createResponse("error", { code: `error.api.${r.error}`, context, }) } + if (env.metrics && env.metricsPort) incrementSuccessful(host); + let localProcessing = params.localProcessing; const lpEnv = env.forceLocalProcessing; diff --git a/docs/api-env-variables.md b/docs/api-env-variables.md index 92e7a6cf..a84601aa 100644 --- a/docs/api-env-variables.md +++ b/docs/api-env-variables.md @@ -64,6 +64,14 @@ this document is not final and will expand over time. feel free to improve it! [*view details*](#service-specific) +### prometheus metrics vars +| name | value example | +|:---------------------------------|:-------------------------| +| METRICS | `1` | +| METRICS_PORT | `9100` | + +[*view details*](#metrics) + ## general [*jump to the table*](#general-vars) @@ -256,3 +264,16 @@ the value is a string. when set to `1`, cobalt will try to use higher quality audio if user requests it via `youtubeBetterAudio`. will negatively impact the rate limit of a secondary youtube client with a session. the value is a number, either `0` or `1`. + +## prometheus metrics +[*jump to the table*](#prometheus-metrics-vars) + +### METRICS +enable prometheus compatible metrics. metrics include: successful/failed requests to services, http requests, and http requests duration. + +the value is a number, either `0` or `1`. + +### METRICS_PORT +port from which the metrics will be server under. these are local to the container/host. + +the value is a number from 1024 to 65535. \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32364730..16594603 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -49,6 +49,9 @@ importers: nanoid: specifier: ^5.0.9 version: 5.0.9 + prom-client: + specifier: ^15.1.3 + version: 15.1.3 set-cookie-parser: specifier: 2.6.0 version: 2.6.0 @@ -613,6 +616,10 @@ packages: resolution: {integrity: sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==} engines: {node: '>=8.0'} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -930,6 +937,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + body-parser@1.20.3: resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -1749,6 +1759,10 @@ packages: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -1982,6 +1996,9 @@ packages: syscall-napi@0.0.6: resolution: {integrity: sha512-qHbwjyFXAAekKUXxl70lhDiBYJ3e7XM7kQwu7LV3F0pHMenKox+VcZPZkRkhdmL/wNJD3NmrMGnL7161kdecUQ==} + tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -2363,7 +2380,7 @@ snapshots: '@eslint/config-array@0.19.1': dependencies: '@eslint/object-schema': 2.1.5 - debug: 4.3.6 + debug: 4.4.0 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -2375,7 +2392,7 @@ snapshots: '@eslint/eslintrc@3.2.0': dependencies: ajv: 6.12.6 - debug: 4.3.6 + debug: 4.4.0 espree: 10.3.0 globals: 14.0.0 ignore: 5.3.1 @@ -2478,6 +2495,8 @@ snapshots: '@oozcitak/util@8.3.8': {} + '@opentelemetry/api@1.9.0': {} + '@pkgjs/parseargs@0.11.0': optional: true @@ -2675,7 +2694,7 @@ snapshots: '@typescript-eslint/types': 8.18.0 '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4) '@typescript-eslint/visitor-keys': 8.18.0 - debug: 4.3.6 + debug: 4.4.0 eslint: 9.16.0 typescript: 5.5.4 transitivePeerDependencies: @@ -2690,7 +2709,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4) '@typescript-eslint/utils': 8.18.0(eslint@9.16.0)(typescript@5.5.4) - debug: 4.3.6 + debug: 4.4.0 eslint: 9.16.0 ts-api-utils: 1.3.0(typescript@5.5.4) typescript: 5.5.4 @@ -2703,7 +2722,7 @@ snapshots: dependencies: '@typescript-eslint/types': 8.18.0 '@typescript-eslint/visitor-keys': 8.18.0 - debug: 4.3.6 + debug: 4.4.0 fast-glob: 3.3.2 is-glob: 4.0.3 minimatch: 9.0.5 @@ -2746,7 +2765,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.6 + debug: 4.4.0 transitivePeerDependencies: - supports-color @@ -2790,6 +2809,8 @@ snapshots: binary-extensions@2.3.0: {} + bintrees@1.0.2: {} + body-parser@1.20.3: dependencies: bytes: 3.1.2 @@ -3320,7 +3341,7 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.6 + debug: 4.4.0 transitivePeerDependencies: - supports-color @@ -3608,6 +3629,11 @@ snapshots: progress@2.0.3: {} + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.0 + tdigest: 0.1.2 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 @@ -3866,6 +3892,10 @@ snapshots: syscall-napi@0.0.6: optional: true + tdigest@0.1.2: + dependencies: + bintrees: 1.0.2 + thenify-all@1.6.0: dependencies: thenify: 3.3.1