mirrors/cobalt
1
0
Fork 0
mirror of https://github.com/imputnet/cobalt.git synced 2025-06-28 09:28:29 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge db7930eb86 into 35530459b6

Browse Source
This commit is contained in:
hyperdefined 2025-06-11 20:18:52 +02:00 committed by GitHub
commit e9ec0899b3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 159 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/package.json
View File

@ -36,6 +36,7 @@
"ipaddr.js": "2.2.0",
"mime": "^4.0.4",
"nanoid": "^5.0.9",
"prom-client": "^15.1.3",
"set-cookie-parser": "2.6.0",
"undici": "^5.19.1",
"url-pattern": "1.0.3",

41
api/src/core/api.js
View File

@ -3,6 +3,7 @@ import http from "node:http";
import rateLimit from "express-rate-limit";
import { setGlobalDispatcher, ProxyAgent } from "undici";
import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info";
import { httpRequests, httpRequestDuration, WORKER_ID, aggregatorRegistry } from "../misc/metrics.js";
import jwt from "../security/jwt.js";
import stream from "../stream/stream.js";
@ -10,7 +11,7 @@ import match from "../processing/match.js";
import { env } from "../config.js";
import { extract } from "../processing/url.js";
import { Bright, Cyan } from "../misc/console-text.js";
import { Bright, Cyan, Green } from "../misc/console-text.js";
import { hashHmac } from "../security/secrets.js";
import { createStore } from "../store/redis-ratelimit.js";
import { randomizeCiphers } from "../misc/randomize-ciphers.js";
@ -39,6 +40,8 @@ const corsConfig = env.corsWildcard ? {} : {
optionsSuccessStatus: 200
}
const metrics = Boolean(env.metrics && env.metricsPort);
const fail = (res, code, context) => {
const { status, body } = createResponse("error", { code, context });
res.status(status).json(body);
@ -54,6 +57,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
version: version,
url: env.apiURL,
startTime: `${startTimestamp}`,
metrics: metrics,
turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,
services: [...env.enabledServices].map(e => {
return friendlyServiceName(e);
@ -111,6 +115,19 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
app.set('trust proxy', ['loopback', 'uniquelocal']);
if (metrics) {
app.use((req, res, next) => {
const end = httpRequestDuration.startTimer({ method: req.method, worker_id: WORKER_ID });
res.on('finish', () => {
httpRequests.labels(req.method, res.statusCode.toString(), WORKER_ID).inc();
end();
});
next();
});
}
app.use('/', cors({
methods: ['GET', 'POST'],
exposedHeaders: [
@ -370,6 +387,28 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
if (env.ytSessionServer) {
YouTubeSession.setup();
}
if (metrics && isPrimary) {
const metricsApp = express();
metricsApp.get('/metrics', async (req, res) => {
try {
const data = await aggregatorRegistry.clusterMetrics();
res.set('Content-Type', 'text/plain');
res.end(data);
} catch (err) {
res.status(500).end(err.message);
}
});
metricsApp.get('/*', (req, res) => {
res.redirect('/metrics');
});
metricsApp.listen(env.metricsPort, '0.0.0.0', () => {
console.log(`${Green('[✓]')} prometheus metrics running on 0.0.0.0:${env.metricsPort}/metrics`);
});
}
});
setupTunnelHandler();

3
api/src/core/env.js
View File

@ -65,6 +65,9 @@ export const loadEnvs = (env = process.env) => {
enabledServices,
metrics: env.METRICS_ENABLED,
metricsPort: env.METRICS_PORT,
customInnertubeClient: env.CUSTOM_INNERTUBE_CLIENT,
ytSessionServer: env.YOUTUBE_SESSION_SERVER,
ytSessionReloadInterval: 300,

50
api/src/misc/metrics.js Normal file
View File

@ -0,0 +1,50 @@
import { collectDefaultMetrics, Counter, Histogram, Registry, AggregatorRegistry } from 'prom-client';
import cluster from 'node:cluster';
export const WORKER_ID = `worker_${cluster.worker?.id ?? process.pid}`;
export const registry = new Registry();
export const aggregatorRegistry = new AggregatorRegistry();
collectDefaultMetrics({
register: registry
});
export const failedRequests = new Counter({
name: 'cobalt_fail_request_count',
help: 'Total number of failed requests',
labelNames: ['service', 'worker_id'],
});
export const successfulRequests = new Counter({
name: 'cobalt_success_request_count',
help: 'Total number of successful requests',
labelNames: ['service', 'worker_id'],
});
export const httpRequests = new Counter({
name: 'http_requests_total',
help: 'Total number of HTTP requests',
labelNames: ['method', 'status', 'worker_id'],
});
export const httpRequestDuration = new Histogram({
name: 'http_request_duration_seconds',
help: 'Duration of HTTP requests in seconds',
labelNames: ['method', 'worker_id'],
buckets: [0.1, 0.5, 1, 1.5, 2, 5],
});
registry.registerMetric(failedRequests);
registry.registerMetric(successfulRequests);
registry.registerMetric(httpRequests);
registry.registerMetric(httpRequestDuration);
export function incrementFailed(service) {
failedRequests.labels(service, WORKER_ID).inc();
}
export function incrementSuccessful(service) {
successfulRequests.labels(service, WORKER_ID).inc();
}

6
api/src/processing/match.js
View File

@ -8,6 +8,8 @@ import matchAction from "./match-action.js";
import { friendlyServiceName } from "./service-alias.js";
import { incrementFailed, incrementSuccessful } from "../misc/metrics.js"
import bilibili from "./services/bilibili.js";
import reddit from "./services/reddit.js";
import twitter from "./services/twitter.js";
@ -286,12 +288,16 @@ export default async function({ host, patternMatch, params, isSession }) {
break;
}
if (env.metrics && env.metricsPort) incrementFailed(host);
return createResponse("error", {
code: `error.api.${r.error}`,
context,
})
}
if (env.metrics && env.metricsPort) incrementSuccessful(host);
let localProcessing = params.localProcessing;
const lpEnv = env.forceLocalProcessing;

21
docs/api-env-variables.md
View File

@ -64,6 +64,14 @@ this document is not final and will expand over time. feel free to improve it!
[*view details*](#service-specific)
### prometheus metrics vars
| name | value example |
|:---------------------------------|:-------------------------|
| METRICS | `1` |
| METRICS_PORT | `9100` |
[*view details*](#metrics)
## general
[*jump to the table*](#general-vars)
@ -256,3 +264,16 @@ the value is a string.
when set to `1`, cobalt will try to use higher quality audio if user requests it via `youtubeBetterAudio`. will negatively impact the rate limit of a secondary youtube client with a session.
the value is a number, either `0` or `1`.
## prometheus metrics
[*jump to the table*](#prometheus-metrics-vars)
### METRICS
enable prometheus compatible metrics. metrics include: successful/failed requests to services, http requests, and http requests duration.
the value is a number, either `0` or `1`.
### METRICS_PORT
port from which the metrics will be server under. these are local to the container/host.
the value is a number from 1024 to 65535.

44
pnpm-lock.yaml
View File

@ -49,6 +49,9 @@ importers:
nanoid:
specifier: ^5.0.9
version: 5.0.9
prom-client:
specifier: ^15.1.3
version: 15.1.3
set-cookie-parser:
specifier: 2.6.0
version: 2.6.0
@ -613,6 +616,10 @@ packages:
resolution: {integrity: sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==}
engines: {node: '>=8.0'}
'@opentelemetry/api@1.9.0':
resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==}
engines: {node: '>=8.0.0'}
'@pkgjs/parseargs@0.11.0':
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
@ -930,6 +937,9 @@ packages:
resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
engines: {node: '>=8'}
bintrees@1.0.2:
resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
body-parser@1.20.3:
resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==}
engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@ -1749,6 +1759,10 @@ packages:
resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}
engines: {node: '>=0.4.0'}
prom-client@15.1.3:
resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
engines: {node: ^16 || ^18 || >=20}
proxy-addr@2.0.7:
resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
engines: {node: '>= 0.10'}
@ -1982,6 +1996,9 @@ packages:
syscall-napi@0.0.6:
resolution: {integrity: sha512-qHbwjyFXAAekKUXxl70lhDiBYJ3e7XM7kQwu7LV3F0pHMenKox+VcZPZkRkhdmL/wNJD3NmrMGnL7161kdecUQ==}
tdigest@0.1.2:
resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
thenify-all@1.6.0:
resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==}
engines: {node: '>=0.8'}
@ -2363,7 +2380,7 @@ snapshots:
'@eslint/config-array@0.19.1':
dependencies:
'@eslint/object-schema': 2.1.5
debug: 4.3.6
debug: 4.4.0
minimatch: 3.1.2
transitivePeerDependencies:
- supports-color
@ -2375,7 +2392,7 @@ snapshots:
'@eslint/eslintrc@3.2.0':
dependencies:
ajv: 6.12.6
debug: 4.3.6
debug: 4.4.0
espree: 10.3.0
globals: 14.0.0
ignore: 5.3.1
@ -2478,6 +2495,8 @@ snapshots:
'@oozcitak/util@8.3.8': {}
'@opentelemetry/api@1.9.0': {}
'@pkgjs/parseargs@0.11.0':
optional: true
@ -2675,7 +2694,7 @@ snapshots:
'@typescript-eslint/types': 8.18.0
'@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4)
'@typescript-eslint/visitor-keys': 8.18.0
debug: 4.3.6
debug: 4.4.0
eslint: 9.16.0
typescript: 5.5.4
transitivePeerDependencies:
@ -2690,7 +2709,7 @@ snapshots:
dependencies:
'@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4)
'@typescript-eslint/utils': 8.18.0(eslint@9.16.0)(typescript@5.5.4)
debug: 4.3.6
debug: 4.4.0
eslint: 9.16.0
ts-api-utils: 1.3.0(typescript@5.5.4)
typescript: 5.5.4
@ -2703,7 +2722,7 @@ snapshots:
dependencies:
'@typescript-eslint/types': 8.18.0
'@typescript-eslint/visitor-keys': 8.18.0
debug: 4.3.6
debug: 4.4.0
fast-glob: 3.3.2
is-glob: 4.0.3
minimatch: 9.0.5
@ -2746,7 +2765,7 @@ snapshots:
agent-base@6.0.2:
dependencies:
debug: 4.3.6
debug: 4.4.0
transitivePeerDependencies:
- supports-color
@ -2790,6 +2809,8 @@ snapshots:
binary-extensions@2.3.0: {}
bintrees@1.0.2: {}
body-parser@1.20.3:
dependencies:
bytes: 3.1.2
@ -3320,7 +3341,7 @@ snapshots:
https-proxy-agent@5.0.1:
dependencies:
agent-base: 6.0.2
debug: 4.3.6
debug: 4.4.0
transitivePeerDependencies:
- supports-color
@ -3608,6 +3629,11 @@ snapshots:
progress@2.0.3: {}
prom-client@15.1.3:
dependencies:
'@opentelemetry/api': 1.9.0
tdigest: 0.1.2
proxy-addr@2.0.7:
dependencies:
forwarded: 0.2.0
@ -3866,6 +3892,10 @@ snapshots:
syscall-napi@0.0.6:
optional: true
tdigest@0.1.2:
dependencies:
bintrees: 1.0.2
thenify-all@1.6.0:
dependencies:
thenify: 3.3.1