diff --git a/api/package.json b/api/package.json index 5864da24..de1e170f 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "11.0.3", + "version": "11.1", "author": "imput", "exports": "./src/cobalt.js", "type": "module", diff --git a/api/src/cobalt.js b/api/src/cobalt.js index 5cac208d..cf531297 100644 --- a/api/src/cobalt.js +++ b/api/src/cobalt.js @@ -9,6 +9,7 @@ import { fileURLToPath } from "url"; import { env, isCluster } from "./config.js" import { Red } from "./misc/console-text.js"; import { initCluster } from "./misc/cluster.js"; +import { setupEnvWatcher } from "./core/env.js"; const app = express(); @@ -24,6 +25,10 @@ if (env.apiURL) { await initCluster(); } + if (env.envFile) { + setupEnvWatcher(); + } + runAPI(express, app, __dirname, cluster.isPrimary); } else { console.log( diff --git a/api/src/config.js b/api/src/config.js index 095780ee..8f1579b8 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,5 +1,5 @@ import { getVersion } from "@imput/version-info"; -import { loadEnvs, validateEnvs, setupEnvWatcher } from "./core/env.js"; +import { loadEnvs, validateEnvs } from "./core/env.js"; const version = await getVersion(); @@ -12,20 +12,23 @@ export const canonicalEnv = Object.freeze(structuredClone(process.env)); export const setTunnelPort = (port) => env.tunnelPort = port; export const isCluster = env.instanceCount > 1; export const updateEnv = (newEnv) => { + const changes = []; + // tunnelPort is special and needs to get carried over here newEnv.tunnelPort = env.tunnelPort; for (const key in env) { + if (String(env[key]) !== String(newEnv[key])) { + changes.push(key); + } env[key] = newEnv[key]; } + + return changes; } await validateEnvs(env); -if (env.envFile) { - setupEnvWatcher(); -} - export { env, genericUserAgent, diff --git a/api/src/core/env.js b/api/src/core/env.js index 35c892f9..37ab36c1 100644 --- a/api/src/core/env.js +++ b/api/src/core/env.js @@ -5,7 +5,7 @@ import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js"; import { FileWatcher } from "../misc/file-watcher.js"; import { isURL } from "../misc/utils.js"; import * as cluster from "../misc/cluster.js"; -import { Yellow } from "../misc/console-text.js"; +import { Green, Yellow } from "../misc/console-text.js"; const forceLocalProcessingOptions = ["never", "session", "always"]; @@ -109,6 +109,8 @@ export const validateEnvs = async (env) => { if (env.externalProxy && env.freebindCIDR) { throw new Error('freebind is not available when external proxy is enabled') } + + return env; } const reloadEnvs = async (contents) => { @@ -136,14 +138,34 @@ const reloadEnvs = async (contents) => { ...newEnvs, }; - const parsed = loadEnvs(candidate); - await validateEnvs(parsed); - updateEnv(parsed); + const parsed = await validateEnvs( + loadEnvs(candidate) + ); + cluster.broadcast({ env_update: resolvedContents }); + return updateEnv(parsed); } const wrapReload = (contents) => { reloadEnvs(contents) + .then(changes => { + if (changes.length === 0) { + return; + } + + console.log(`${Green('[✓]')} envs reloaded successfully!`); + for (const key of changes) { + const value = currentEnv[key]; + const isSecret = key.toLowerCase().includes('apikey') + || key.toLowerCase().includes('secret'); + + if (!value) { + console.log(` removed: ${key}`); + } else { + console.log(` changed: ${key} -> ${isSecret ? '***' : value}`); + } + } + }) .catch((e) => { console.error(`${Yellow('[!]')} Failed reloading environment variables at ${new Date().toISOString()}.`); console.error('Error:', e); diff --git a/api/src/misc/cluster.js b/api/src/misc/cluster.js index 56664d15..1320bb4c 100644 --- a/api/src/misc/cluster.js +++ b/api/src/misc/cluster.js @@ -13,7 +13,8 @@ export const supportsReusePort = async () => { server.on('error', (err) => (server.close(), reject(err))); }); - return true; + const [major, minor] = process.versions.node.split('.').map(Number); + return major > 23 || (major === 23 && minor >= 1); } catch { return false; } diff --git a/api/src/processing/services/pinterest.js b/api/src/processing/services/pinterest.js index 15566cc4..c21400e1 100644 --- a/api/src/processing/services/pinterest.js +++ b/api/src/processing/services/pinterest.js @@ -3,6 +3,7 @@ import { resolveRedirectingURL } from "../url.js"; const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g; const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g; +const notFoundRegex = /"__typename"\s*:\s*"PinNotFound"/; export default async function(o) { let id = o.id; @@ -19,6 +20,10 @@ export default async function(o) { headers: { "user-agent": genericUserAgent } }).then(r => r.text()).catch(() => {}); + const invalidPin = html.match(notFoundRegex); + + if (invalidPin) return { error: "fetch.empty" }; + if (!html) return { error: "fetch.fail" }; const videoLink = [...html.matchAll(videoRegex)] diff --git a/api/src/processing/services/soundcloud.js b/api/src/processing/services/soundcloud.js index 890af8db..046ebd79 100644 --- a/api/src/processing/services/soundcloud.js +++ b/api/src/processing/services/soundcloud.js @@ -40,6 +40,27 @@ async function findClientID() { } catch {} } +const findBestForPreset = (transcodings, preset) => { + let inferior; + for (const entry of transcodings) { + const protocol = entry?.format?.protocol; + + if (entry.snipped || protocol?.includes('encrypted')) { + continue; + } + + if (entry?.preset?.startsWith(`${preset}_`)) { + if (protocol === 'progressive') { + return entry; + } + + inferior = entry; + } + } + + return inferior; +} + export default async function(obj) { const clientId = await findClientID(); if (!clientId) return { error: "fetch.fail" }; @@ -89,9 +110,9 @@ export default async function(obj) { } let bestAudio = "opus", - selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"); + selectedStream = findBestForPreset(json.media.transcodings, "opus"); - const mp3Media = json.media.transcodings.find(v => v.preset === "mp3_0_0"); + const mp3Media = findBestForPreset(json.media.transcodings, "mp3"); // use mp3 if present if user prefers it or if opus isn't available if (mp3Media && (obj.format === "mp3" || !selectedStream)) { @@ -113,9 +134,16 @@ export default async function(obj) { if (!file) return { error: "fetch.empty" }; + const artist = json.user?.username?.trim(); const fileMetadata = { - title: json.title.trim(), - artist: json.user.username.trim(), + title: json.title?.trim(), + album: json.publisher_metadata?.album_title?.trim(), + artist, + album_artist: artist, + composer: json.publisher_metadata?.writer_composer?.trim(), + genre: json.genre?.trim(), + date: json.display_date?.trim().slice(0, 10), + copyright: json.license?.trim(), } return { diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index e184ff1f..74959063 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -280,7 +280,7 @@ export default async function (o) { // some videos (mainly those with AI dubs) don't have any tracks marked as default // why? god knows, but we assume that a default track is marked as such in the title if (!audio) { - audio = selected.audio.find(i => i.name.endsWith("- original")); + audio = selected.audio.find(i => i.name.endsWith("original")); } if (o.dubLang) { @@ -369,9 +369,9 @@ export default async function (o) { audio = sorted_formats[codec].bestAudio; - if (audio?.audio_track && !audio?.audio_track?.audio_is_default) { + if (audio?.audio_track && !audio?.is_original) { audio = sorted_formats[codec].audio.find(i => - i?.audio_track?.audio_is_default + i?.is_original ); } @@ -380,7 +380,7 @@ export default async function (o) { i.language?.startsWith(o.dubLang) && i.audio_track ); - if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) { + if (dubbedAudio && !dubbedAudio?.is_original) { audio = dubbedAudio; dubbedLanguage = dubbedAudio.language; } diff --git a/api/src/stream/types.js b/api/src/stream/types.js index 1e270df8..6b493efa 100644 --- a/api/src/stream/types.js +++ b/api/src/stream/types.js @@ -17,9 +17,12 @@ const ffmpegArgs = { const metadataTags = [ "album", + "composer", + "genre", "copyright", "title", "artist", + "album_artist", "track", "date", ]; diff --git a/api/src/util/tests/pinterest.json b/api/src/util/tests/pinterest.json index 4760dd36..adf502c0 100644 --- a/api/src/util/tests/pinterest.json +++ b/api/src/util/tests/pinterest.json @@ -8,6 +8,16 @@ "status": "redirect" } }, + { + "name": "invalid link", + "url": "https://www.pinterest.com/pin/eeeeeee/", + "params": {}, + "expected": { + "code": 400, + "status": "error", + "errorCode": "error.api.fetch.empty" + } + }, { "name": "regular video (isAudioOnly)", "url": "https://www.pinterest.com/pin/70437485604616/", diff --git a/api/src/util/tests/soundcloud.json b/api/src/util/tests/soundcloud.json index 725c8ebf..e7f54a3b 100644 --- a/api/src/util/tests/soundcloud.json +++ b/api/src/util/tests/soundcloud.json @@ -86,7 +86,7 @@ }, { "name": "go+ song, should fail", - "url": "https://soundcloud.com/dualipa/illusion-1", + "url": "https://soundcloud.com/dualipa/physical-feat-troye-sivan", "params": {}, "expected": { "code": 400, diff --git a/docs/api.md b/docs/api.md index ac536380..f093e148 100644 --- a/docs/api.md +++ b/docs/api.md @@ -129,14 +129,17 @@ the response will always be a JSON object containing the `status` key, which is #### output.metadata object all keys in this table are optional. -| key | type | description | -|:------------|:---------|:-------------------------------------------| -| `album` | `string` | album name or collection title | -| `copyright` | `string` | copyright information or ownership details | -| `title` | `string` | title of the track or media file | -| `artist` | `string` | artist or creator name | -| `track` | `string` | track number or position in album | -| `date` | `string` | release date or creation date | +| key | type | description | +|:---------------|:---------|:-------------------------------------------| +| `album` | `string` | album name or collection title | +| `composer` | `string` | composer of the track | +| `genre` | `string` | track's genre(s) | +| `copyright` | `string` | copyright information or ownership details | +| `title` | `string` | title of the track or media file | +| `artist` | `string` | artist or creator name | +| `album_artist` | `string` | album's artist or creator name | +| `track` | `string` | track number or position in album | +| `date` | `string` | release date or creation date | #### audio object | key | type | value | diff --git a/web/src/lib/types/api.ts b/web/src/lib/types/api.ts index 6e030856..13d2b8b7 100644 --- a/web/src/lib/types/api.ts +++ b/web/src/lib/types/api.ts @@ -45,9 +45,12 @@ type CobaltTunnelResponse = { export const CobaltFileMetadataKeys = [ 'album', + 'composer', + 'genre', 'copyright', 'title', 'artist', + 'album_artist', 'track', 'date' ];