From df2dcbc69dbfef4a0a994bc2de8c5e1c3baaeaa3 Mon Sep 17 00:00:00 2001 From: hyperdefined Date: Sun, 1 Jun 2025 19:45:54 -0400 Subject: [PATCH 1/8] feat: add prometheus metrics --- api/package.json | 1 + api/src/core/api.js | 18 +++++++++++++++ api/src/misc/metrics.js | 45 +++++++++++++++++++++++++++++++++++++ api/src/processing/match.js | 6 +++++ pnpm-lock.yaml | 44 ++++++++++++++++++++++++++++++------ 5 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 api/src/misc/metrics.js diff --git a/api/package.json b/api/package.json index f71d0a5c..9f63265b 100644 --- a/api/package.json +++ b/api/package.json @@ -36,6 +36,7 @@ "ipaddr.js": "2.2.0", "mime": "^4.0.4", "nanoid": "^5.0.9", + "prom-client": "^15.1.3", "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", diff --git a/api/src/core/api.js b/api/src/core/api.js index eb5cf4ff..27173a5a 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -3,6 +3,8 @@ import http from "node:http"; import rateLimit from "express-rate-limit"; import { setGlobalDispatcher, ProxyAgent } from "undici"; import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info"; +import registry from "../misc/metrics.js" +import { httpRequests, httpRequestDuration } from "../misc/metrics.js" import jwt from "../security/jwt.js"; import stream from "../stream/stream.js"; @@ -111,6 +113,17 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { app.set('trust proxy', ['loopback', 'uniquelocal']); + app.use((req, res, next) => { + const end = httpRequestDuration.startTimer({ method: req.method }); + + res.on('finish', () => { + httpRequests.labels(req.method, res.statusCode.toString()).inc(); + end(); + }); + + next(); + }); + app.use('/', cors({ methods: ['GET', 'POST'], exposedHeaders: [ @@ -321,6 +334,11 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { res.status(404).end(); }) + app.get('/metrics', async (req, res) => { + res.set('Content-Type', registry.contentType); + res.send(await registry.metrics()); + }); + app.get('/*', (req, res) => { res.redirect('/'); }) diff --git a/api/src/misc/metrics.js b/api/src/misc/metrics.js new file mode 100644 index 00000000..29af31e6 --- /dev/null +++ b/api/src/misc/metrics.js @@ -0,0 +1,45 @@ +import { Registry } from 'prom-client'; +import { collectDefaultMetrics, Counter, Histogram } from "prom-client"; + +const registry = new Registry(); +export default registry; + +collectDefaultMetrics({ register: registry }); + +export const failedRequests = new Counter({ + name: 'cobalt_fail_request_count', + help: 'Total number of failed requests', + labelNames: ['service'], +}); + +export const successfulRequests = new Counter({ + name: 'cobalt_success_request_count', + help: 'Total number of successful requests', + labelNames: ['service'], +}); + +export const httpRequests = new Counter({ + name: 'http_requests_total', + help: 'Total number of HTTP requests', + labelNames: ['method', 'status'], +}); + +export const httpRequestDuration = new Histogram({ + name: 'http_request_duration_seconds', + help: 'Duration of HTTP requests in seconds', + labelNames: ['method'], + buckets: [0.1, 0.5, 1, 1.5, 2, 5], +}); + +registry.registerMetric(failedRequests); +registry.registerMetric(successfulRequests); +registry.registerMetric(httpRequests); +registry.registerMetric(httpRequestDuration); + +export function incrementFailed(type) { + failedRequests.labels(type).inc(); +} + +export function incrementSuccessful(type) { + successfulRequests.labels(type).inc(); +} \ No newline at end of file diff --git a/api/src/processing/match.js b/api/src/processing/match.js index 65a021b0..63005073 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -8,6 +8,8 @@ import matchAction from "./match-action.js"; import { friendlyServiceName } from "./service-alias.js"; +import { incrementFailed, incrementSuccessful } from "../misc/metrics.js" + import bilibili from "./services/bilibili.js"; import reddit from "./services/reddit.js"; import twitter from "./services/twitter.js"; @@ -286,12 +288,16 @@ export default async function({ host, patternMatch, params, isSession }) { break; } + incrementFailed(host); + return createResponse("error", { code: `error.api.${r.error}`, context, }) } + incrementSuccessful(host); + let localProcessing = params.localProcessing; const lpEnv = env.forceLocalProcessing; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32364730..16594603 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -49,6 +49,9 @@ importers: nanoid: specifier: ^5.0.9 version: 5.0.9 + prom-client: + specifier: ^15.1.3 + version: 15.1.3 set-cookie-parser: specifier: 2.6.0 version: 2.6.0 @@ -613,6 +616,10 @@ packages: resolution: {integrity: sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==} engines: {node: '>=8.0'} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -930,6 +937,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + body-parser@1.20.3: resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -1749,6 +1759,10 @@ packages: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -1982,6 +1996,9 @@ packages: syscall-napi@0.0.6: resolution: {integrity: sha512-qHbwjyFXAAekKUXxl70lhDiBYJ3e7XM7kQwu7LV3F0pHMenKox+VcZPZkRkhdmL/wNJD3NmrMGnL7161kdecUQ==} + tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -2363,7 +2380,7 @@ snapshots: '@eslint/config-array@0.19.1': dependencies: '@eslint/object-schema': 2.1.5 - debug: 4.3.6 + debug: 4.4.0 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -2375,7 +2392,7 @@ snapshots: '@eslint/eslintrc@3.2.0': dependencies: ajv: 6.12.6 - debug: 4.3.6 + debug: 4.4.0 espree: 10.3.0 globals: 14.0.0 ignore: 5.3.1 @@ -2478,6 +2495,8 @@ snapshots: '@oozcitak/util@8.3.8': {} + '@opentelemetry/api@1.9.0': {} + '@pkgjs/parseargs@0.11.0': optional: true @@ -2675,7 +2694,7 @@ snapshots: '@typescript-eslint/types': 8.18.0 '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4) '@typescript-eslint/visitor-keys': 8.18.0 - debug: 4.3.6 + debug: 4.4.0 eslint: 9.16.0 typescript: 5.5.4 transitivePeerDependencies: @@ -2690,7 +2709,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4) '@typescript-eslint/utils': 8.18.0(eslint@9.16.0)(typescript@5.5.4) - debug: 4.3.6 + debug: 4.4.0 eslint: 9.16.0 ts-api-utils: 1.3.0(typescript@5.5.4) typescript: 5.5.4 @@ -2703,7 +2722,7 @@ snapshots: dependencies: '@typescript-eslint/types': 8.18.0 '@typescript-eslint/visitor-keys': 8.18.0 - debug: 4.3.6 + debug: 4.4.0 fast-glob: 3.3.2 is-glob: 4.0.3 minimatch: 9.0.5 @@ -2746,7 +2765,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.6 + debug: 4.4.0 transitivePeerDependencies: - supports-color @@ -2790,6 +2809,8 @@ snapshots: binary-extensions@2.3.0: {} + bintrees@1.0.2: {} + body-parser@1.20.3: dependencies: bytes: 3.1.2 @@ -3320,7 +3341,7 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.6 + debug: 4.4.0 transitivePeerDependencies: - supports-color @@ -3608,6 +3629,11 @@ snapshots: progress@2.0.3: {} + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.0 + tdigest: 0.1.2 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 @@ -3866,6 +3892,10 @@ snapshots: syscall-napi@0.0.6: optional: true + tdigest@0.1.2: + dependencies: + bintrees: 1.0.2 + thenify-all@1.6.0: dependencies: thenify: 3.3.1 From ad7d78a9f16ac499de1d35dd93d67854ca0a006f Mon Sep 17 00:00:00 2001 From: hyperdefined Date: Sun, 1 Jun 2025 20:02:35 -0400 Subject: [PATCH 2/8] feat: allow port change of metrics --- api/src/core/api.js | 41 +++++++++++++++++++++++-------------- api/src/core/env.js | 3 +++ api/src/processing/match.js | 4 ++-- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/api/src/core/api.js b/api/src/core/api.js index 27173a5a..99360e73 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -12,7 +12,7 @@ import match from "../processing/match.js"; import { env } from "../config.js"; import { extract } from "../processing/url.js"; -import { Bright, Cyan } from "../misc/console-text.js"; +import { Bright, Cyan, Green } from "../misc/console-text.js"; import { hashHmac } from "../security/secrets.js"; import { createStore } from "../store/redis-ratelimit.js"; import { randomizeCiphers } from "../misc/randomize-ciphers.js"; @@ -41,6 +41,8 @@ const corsConfig = env.corsWildcard ? {} : { optionsSuccessStatus: 200 } +const metrics = env.metrics && env.metricsPort; + const fail = (res, code, context) => { const { status, body } = createResponse("error", { code, context }); res.status(status).json(body); @@ -113,16 +115,18 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { app.set('trust proxy', ['loopback', 'uniquelocal']); - app.use((req, res, next) => { - const end = httpRequestDuration.startTimer({ method: req.method }); - - res.on('finish', () => { - httpRequests.labels(req.method, res.statusCode.toString()).inc(); - end(); + if (metrics) { + app.use((req, res, next) => { + const end = httpRequestDuration.startTimer({ method: req.method }); + + res.on('finish', () => { + httpRequests.labels(req.method, res.statusCode.toString()).inc(); + end(); + }); + + next(); }); - - next(); - }); + } app.use('/', cors({ methods: ['GET', 'POST'], @@ -334,11 +338,6 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { res.status(404).end(); }) - app.get('/metrics', async (req, res) => { - res.set('Content-Type', registry.contentType); - res.send(await registry.metrics()); - }); - app.get('/*', (req, res) => { res.redirect('/'); }) @@ -388,6 +387,18 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { if (env.ytSessionServer) { YouTubeSession.setup(); } + + if (metrics) { + const metricsApp = express(); + + metricsApp.get('/metrics', async (req, res) => { + res.set('Content-Type', registry.contentType); + res.send(await registry.metrics()); + }); + metricsApp.listen(env.metricsPort, () => { + console.log(`${Green('[✓]')} prometheus metrics running on 127.0.0.1:${env.metricsPort}/metrics`); + }); + } }); setupTunnelHandler(); diff --git a/api/src/core/env.js b/api/src/core/env.js index 35c892f9..06458af7 100644 --- a/api/src/core/env.js +++ b/api/src/core/env.js @@ -65,6 +65,9 @@ export const loadEnvs = (env = process.env) => { enabledServices, + metrics: env.METRICS_ENABLED, + metricsPort: env.METRICS_PORT, + customInnertubeClient: env.CUSTOM_INNERTUBE_CLIENT, ytSessionServer: env.YOUTUBE_SESSION_SERVER, ytSessionReloadInterval: 300, diff --git a/api/src/processing/match.js b/api/src/processing/match.js index 63005073..15812c63 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -288,7 +288,7 @@ export default async function({ host, patternMatch, params, isSession }) { break; } - incrementFailed(host); + if (env.metrics && env.metricsPort) incrementFailed(host); return createResponse("error", { code: `error.api.${r.error}`, @@ -296,7 +296,7 @@ export default async function({ host, patternMatch, params, isSession }) { }) } - incrementSuccessful(host); + if (env.metrics && env.metricsPort) incrementSuccessful(host); let localProcessing = params.localProcessing; From 69ca909e7dbc6726e007457adc8189e5ba94e353 Mon Sep 17 00:00:00 2001 From: hyperdefined Date: Sun, 1 Jun 2025 20:05:49 -0400 Subject: [PATCH 3/8] api/core: expose metrics status --- api/src/core/api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/core/api.js b/api/src/core/api.js index 99360e73..762b71e4 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -41,7 +41,7 @@ const corsConfig = env.corsWildcard ? {} : { optionsSuccessStatus: 200 } -const metrics = env.metrics && env.metricsPort; +const metrics = Boolean(env.metrics && env.metricsPort); const fail = (res, code, context) => { const { status, body } = createResponse("error", { code, context }); @@ -58,6 +58,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { version: version, url: env.apiURL, startTime: `${startTimestamp}`, + metrics: metrics, turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined, services: [...env.enabledServices].map(e => { return friendlyServiceName(e); From c9dc352f626cf14563511822936ba012364aff98 Mon Sep 17 00:00:00 2001 From: hyperdefined Date: Sun, 1 Jun 2025 20:09:33 -0400 Subject: [PATCH 4/8] api/core: redirect all other requests to metrics --- api/src/core/api.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/core/api.js b/api/src/core/api.js index 762b71e4..91370fe6 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -399,6 +399,10 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { metricsApp.listen(env.metricsPort, () => { console.log(`${Green('[✓]')} prometheus metrics running on 127.0.0.1:${env.metricsPort}/metrics`); }); + + metricsApp.get('/*', (req, res) => { + res.redirect('/metrics'); + }) } }); From 0e8c527982f245ed9192711ab2ed1792715dcd39 Mon Sep 17 00:00:00 2001 From: hyperdefined Date: Sun, 1 Jun 2025 20:21:04 -0400 Subject: [PATCH 5/8] docs/api-env-variables: add metrics vars --- docs/api-env-variables.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/api-env-variables.md b/docs/api-env-variables.md index 92e7a6cf..a84601aa 100644 --- a/docs/api-env-variables.md +++ b/docs/api-env-variables.md @@ -64,6 +64,14 @@ this document is not final and will expand over time. feel free to improve it! [*view details*](#service-specific) +### prometheus metrics vars +| name | value example | +|:---------------------------------|:-------------------------| +| METRICS | `1` | +| METRICS_PORT | `9100` | + +[*view details*](#metrics) + ## general [*jump to the table*](#general-vars) @@ -256,3 +264,16 @@ the value is a string. when set to `1`, cobalt will try to use higher quality audio if user requests it via `youtubeBetterAudio`. will negatively impact the rate limit of a secondary youtube client with a session. the value is a number, either `0` or `1`. + +## prometheus metrics +[*jump to the table*](#prometheus-metrics-vars) + +### METRICS +enable prometheus compatible metrics. metrics include: successful/failed requests to services, http requests, and http requests duration. + +the value is a number, either `0` or `1`. + +### METRICS_PORT +port from which the metrics will be server under. these are local to the container/host. + +the value is a number from 1024 to 65535. \ No newline at end of file From 77c4e134c7222226c702b98308c7ade27a3af30b Mon Sep 17 00:00:00 2001 From: hyperdefined Date: Sun, 1 Jun 2025 21:29:52 -0400 Subject: [PATCH 6/8] api/core: bind on localhost --- api/src/core/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/core/api.js b/api/src/core/api.js index 91370fe6..eb93563a 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -396,7 +396,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { res.set('Content-Type', registry.contentType); res.send(await registry.metrics()); }); - metricsApp.listen(env.metricsPort, () => { + metricsApp.listen(env.metricsPort, '127.0.0.1', () => { console.log(`${Green('[✓]')} prometheus metrics running on 127.0.0.1:${env.metricsPort}/metrics`); }); From 87b92a7a624c3d8401d98c9218a00c03d4ddcac4 Mon Sep 17 00:00:00 2001 From: hyperdefined Date: Wed, 4 Jun 2025 00:43:34 -0400 Subject: [PATCH 7/8] metrics: support clusters --- api/src/core/api.js | 35 ++++++++++++++++++++--------------- api/src/misc/metrics.js | 33 +++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/api/src/core/api.js b/api/src/core/api.js index eb93563a..aeb856ee 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -3,8 +3,7 @@ import http from "node:http"; import rateLimit from "express-rate-limit"; import { setGlobalDispatcher, ProxyAgent } from "undici"; import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info"; -import registry from "../misc/metrics.js" -import { httpRequests, httpRequestDuration } from "../misc/metrics.js" +import { httpRequests, httpRequestDuration, WORKER_ID, aggregatorRegistry } from "../misc/metrics.js"; import jwt from "../security/jwt.js"; import stream from "../stream/stream.js"; @@ -118,11 +117,11 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { if (metrics) { app.use((req, res, next) => { - const end = httpRequestDuration.startTimer({ method: req.method }); - + const end = httpRequestDuration.startTimer({ method: req.method, worker_id: WORKER_ID }); + res.on('finish', () => { - httpRequests.labels(req.method, res.statusCode.toString()).inc(); - end(); + httpRequests.labels(req.method, res.statusCode.toString(), WORKER_ID).inc(); + end(); }); next(); @@ -389,22 +388,28 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { YouTubeSession.setup(); } - if (metrics) { + if (metrics && isPrimary) { const metricsApp = express(); - + metricsApp.get('/metrics', async (req, res) => { - res.set('Content-Type', registry.contentType); - res.send(await registry.metrics()); - }); - metricsApp.listen(env.metricsPort, '127.0.0.1', () => { - console.log(`${Green('[✓]')} prometheus metrics running on 127.0.0.1:${env.metricsPort}/metrics`); + try { + const data = await aggregatorRegistry.clusterMetrics(); + res.set('Content-Type', 'text/plain'); + res.end(data); + } catch (err) { + res.status(500).end(err.message); + } }); metricsApp.get('/*', (req, res) => { res.redirect('/metrics'); - }) + }); + + metricsApp.listen(env.metricsPort, '127.0.0.1', () => { + console.log(`${Green('[✓]')} prometheus metrics running on 127.0.0.1:${env.metricsPort}/metrics`); + }); } }); setupTunnelHandler(); -} +} \ No newline at end of file diff --git a/api/src/misc/metrics.js b/api/src/misc/metrics.js index 29af31e6..02a87278 100644 --- a/api/src/misc/metrics.js +++ b/api/src/misc/metrics.js @@ -1,33 +1,38 @@ -import { Registry } from 'prom-client'; -import { collectDefaultMetrics, Counter, Histogram } from "prom-client"; +import { collectDefaultMetrics, Counter, Histogram, Registry, AggregatorRegistry } from 'prom-client'; + +import cluster from 'node:cluster'; -const registry = new Registry(); -export default registry; +export const WORKER_ID = `worker_${cluster.worker?.id ?? process.pid}`; -collectDefaultMetrics({ register: registry }); +export const registry = new Registry(); +export const aggregatorRegistry = new AggregatorRegistry(); + +collectDefaultMetrics({ + register: registry +}); export const failedRequests = new Counter({ name: 'cobalt_fail_request_count', help: 'Total number of failed requests', - labelNames: ['service'], + labelNames: ['service', 'worker_id'], }); export const successfulRequests = new Counter({ name: 'cobalt_success_request_count', help: 'Total number of successful requests', - labelNames: ['service'], + labelNames: ['service', 'worker_id'], }); export const httpRequests = new Counter({ name: 'http_requests_total', help: 'Total number of HTTP requests', - labelNames: ['method', 'status'], + labelNames: ['method', 'status', 'worker_id'], }); - + export const httpRequestDuration = new Histogram({ name: 'http_request_duration_seconds', help: 'Duration of HTTP requests in seconds', - labelNames: ['method'], + labelNames: ['method', 'worker_id'], buckets: [0.1, 0.5, 1, 1.5, 2, 5], }); @@ -36,10 +41,10 @@ registry.registerMetric(successfulRequests); registry.registerMetric(httpRequests); registry.registerMetric(httpRequestDuration); -export function incrementFailed(type) { - failedRequests.labels(type).inc(); +export function incrementFailed(service) { + failedRequests.labels(service, WORKER_ID).inc(); } -export function incrementSuccessful(type) { - successfulRequests.labels(type).inc(); +export function incrementSuccessful(service) { + successfulRequests.labels(service, WORKER_ID).inc(); } \ No newline at end of file From db7930eb8677224e1361a5348d35fb3c8214d844 Mon Sep 17 00:00:00 2001 From: hyperdefined Date: Wed, 4 Jun 2025 01:12:47 -0400 Subject: [PATCH 8/8] metrics: make sure we listen correctly --- api/src/core/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/core/api.js b/api/src/core/api.js index aeb856ee..02f2950f 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -405,8 +405,8 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { res.redirect('/metrics'); }); - metricsApp.listen(env.metricsPort, '127.0.0.1', () => { - console.log(`${Green('[✓]')} prometheus metrics running on 127.0.0.1:${env.metricsPort}/metrics`); + metricsApp.listen(env.metricsPort, '0.0.0.0', () => { + console.log(`${Green('[✓]')} prometheus metrics running on 0.0.0.0:${env.metricsPort}/metrics`); }); } });