merge: 11.1 api update

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "11.0.3",
+    "version": "11.1",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",

api/src/cobalt.js

@@ -9,6 +9,7 @@ import { fileURLToPath } from "url";
 import { env, isCluster } from "./config.js"
 import { Red } from "./misc/console-text.js";
 import { initCluster } from "./misc/cluster.js";
+import { setupEnvWatcher } from "./core/env.js";
 
 const app = express();
 
@@ -24,6 +25,10 @@ if (env.apiURL) {
        await initCluster();
     }
 
+    if (env.envFile) {
+        setupEnvWatcher();
+    }
+
     runAPI(express, app, __dirname, cluster.isPrimary);
 } else {
     console.log(

api/src/config.js

@@ -1,5 +1,5 @@
 import { getVersion } from "@imput/version-info";
-import { loadEnvs, validateEnvs, setupEnvWatcher } from "./core/env.js";
+import { loadEnvs, validateEnvs } from "./core/env.js";
 
 const version = await getVersion();
 
@@ -12,20 +12,23 @@ export const canonicalEnv = Object.freeze(structuredClone(process.env));
 export const setTunnelPort = (port) => env.tunnelPort = port;
 export const isCluster = env.instanceCount > 1;
 export const updateEnv = (newEnv) => {
+    const changes = [];
+
     // tunnelPort is special and needs to get carried over here
     newEnv.tunnelPort = env.tunnelPort;
 
     for (const key in env) {
+        if (String(env[key]) !== String(newEnv[key])) {
+            changes.push(key);
+        }
         env[key] = newEnv[key];
     }
+
+    return changes;
 }
 
 await validateEnvs(env);
 
-if (env.envFile) {
-    setupEnvWatcher();
-}
-
 export {
     env,
     genericUserAgent,

api/src/core/env.js

@@ -5,7 +5,7 @@ import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js";
 import { FileWatcher } from "../misc/file-watcher.js";
 import { isURL } from "../misc/utils.js";
 import * as cluster from "../misc/cluster.js";
-import { Yellow } from "../misc/console-text.js";
+import { Green, Yellow } from "../misc/console-text.js";
 
 const forceLocalProcessingOptions = ["never", "session", "always"];
 
@@ -109,6 +109,8 @@ export const validateEnvs = async (env) => {
     if (env.externalProxy && env.freebindCIDR) {
         throw new Error('freebind is not available when external proxy is enabled')
     }
+
+    return env;
 }
 
 const reloadEnvs = async (contents) => {
@@ -136,14 +138,34 @@ const reloadEnvs = async (contents) => {
         ...newEnvs,
     };
 
-    const parsed = loadEnvs(candidate);
+    const parsed = await validateEnvs(
-    await validateEnvs(parsed);
+        loadEnvs(candidate)
-    updateEnv(parsed);
+    );
+
     cluster.broadcast({ env_update: resolvedContents });
+    return updateEnv(parsed);
 }
 
 const wrapReload = (contents) => {
     reloadEnvs(contents)
+    .then(changes => {
+        if (changes.length === 0) {
+            return;
+        }
+
+        console.log(`${Green('[✓]')} envs reloaded successfully!`);
+        for (const key of changes) {
+            const value = currentEnv[key];
+            const isSecret = key.toLowerCase().includes('apikey')
+                          || key.toLowerCase().includes('secret');
+
+            if (!value) {
+                console.log(`    removed: ${key}`);
+            } else {
+                console.log(`    changed: ${key} -> ${isSecret ? '***' : value}`);
+            }
+        }
+    })
     .catch((e) => {
         console.error(`${Yellow('[!]')} Failed reloading environment variables at ${new Date().toISOString()}.`);
         console.error('Error:', e);

api/src/misc/cluster.js

@@ -13,7 +13,8 @@ export const supportsReusePort = async () => {
             server.on('error', (err) => (server.close(), reject(err)));
         });
 
-        return true;
+        const [major, minor] = process.versions.node.split('.').map(Number);
+        return major > 23 || (major === 23 && minor >= 1);
     } catch {
         return false;
     }

api/src/processing/services/pinterest.js

@@ -3,6 +3,7 @@ import { resolveRedirectingURL } from "../url.js";
 
 const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
 const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
+const notFoundRegex = /"__typename"\s*:\s*"PinNotFound"/;
 
 export default async function(o) {
     let id = o.id;
@@ -19,6 +20,10 @@ export default async function(o) {
         headers: { "user-agent": genericUserAgent }
     }).then(r => r.text()).catch(() => {});
 
+    const invalidPin = html.match(notFoundRegex);
+
+    if (invalidPin) return { error: "fetch.empty" };
+
     if (!html) return { error: "fetch.fail" };
 
     const videoLink = [...html.matchAll(videoRegex)]

api/src/processing/services/soundcloud.js

@@ -40,6 +40,27 @@ async function findClientID() {
     } catch {}
 }
 
+const findBestForPreset = (transcodings, preset) => {
+    let inferior;
+    for (const entry of transcodings) {
+        const protocol = entry?.format?.protocol;
+
+        if (entry.snipped || protocol?.includes('encrypted')) {
+            continue;
+        }
+
+        if (entry?.preset?.startsWith(`${preset}_`)) {
+            if (protocol === 'progressive') {
+                return entry;
+            }
+
+            inferior = entry;
+        }
+    }
+
+    return inferior;
+}
+
 export default async function(obj) {
     const clientId = await findClientID();
     if (!clientId) return { error: "fetch.fail" };
@@ -89,9 +110,9 @@ export default async function(obj) {
     }
 
     let bestAudio = "opus",
-        selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0");
+        selectedStream = findBestForPreset(json.media.transcodings, "opus");
 
-    const mp3Media = json.media.transcodings.find(v => v.preset === "mp3_0_0");
+    const mp3Media = findBestForPreset(json.media.transcodings, "mp3");
 
     // use mp3 if present if user prefers it or if opus isn't available
     if (mp3Media && (obj.format === "mp3" || !selectedStream)) {
@@ -113,9 +134,16 @@ export default async function(obj) {
 
     if (!file) return { error: "fetch.empty" };
 
+    const artist = json.user?.username?.trim();
     const fileMetadata = {
-        title: json.title.trim(),
+        title: json.title?.trim(),
-        artist: json.user.username.trim(),
+        album: json.publisher_metadata?.album_title?.trim(),
+        artist,
+        album_artist: artist,
+        composer: json.publisher_metadata?.writer_composer?.trim(),
+        genre: json.genre?.trim(),
+        date: json.display_date?.trim().slice(0, 10),
+        copyright: json.license?.trim(),
     }
 
     return {

api/src/processing/services/youtube.js

@@ -280,7 +280,7 @@ export default async function (o) {
         // some videos (mainly those with AI dubs) don't have any tracks marked as default
         // why? god knows, but we assume that a default track is marked as such in the title
         if (!audio) {
-            audio = selected.audio.find(i => i.name.endsWith("- original"));
+            audio = selected.audio.find(i => i.name.endsWith("original"));
         }
 
         if (o.dubLang) {
@@ -369,9 +369,9 @@ export default async function (o) {
 
         audio = sorted_formats[codec].bestAudio;
 
-        if (audio?.audio_track && !audio?.audio_track?.audio_is_default) {
+        if (audio?.audio_track && !audio?.is_original) {
             audio = sorted_formats[codec].audio.find(i =>
-                i?.audio_track?.audio_is_default
+                i?.is_original
             );
         }
 
@@ -380,7 +380,7 @@ export default async function (o) {
                 i.language?.startsWith(o.dubLang) && i.audio_track
             );
 
-            if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) {
+            if (dubbedAudio && !dubbedAudio?.is_original) {
                 audio = dubbedAudio;
                 dubbedLanguage = dubbedAudio.language;
             }

api/src/stream/types.js

@@ -17,9 +17,12 @@ const ffmpegArgs = {
 
 const metadataTags = [
     "album",
+    "composer",
+    "genre",
     "copyright",
     "title",
     "artist",
+    "album_artist",
     "track",
     "date",
 ];

api/src/util/tests/pinterest.json

@@ -8,6 +8,16 @@
             "status": "redirect"
         }
     },
+    {
+        "name": "invalid link",
+        "url": "https://www.pinterest.com/pin/eeeeeee/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error",
+            "errorCode": "error.api.fetch.empty"
+        }
+    },
     {
         "name": "regular video (isAudioOnly)",
         "url": "https://www.pinterest.com/pin/70437485604616/",

api/src/util/tests/soundcloud.json

@@ -86,7 +86,7 @@
     },
     {
         "name": "go+ song, should fail",
-        "url": "https://soundcloud.com/dualipa/illusion-1",
+        "url": "https://soundcloud.com/dualipa/physical-feat-troye-sivan",
         "params": {},
         "expected": {
             "code": 400,

docs/api.md

@@ -129,14 +129,17 @@ the response will always be a JSON object containing the `status` key, which is
 #### output.metadata object
 all keys in this table are optional.
 
-| key         | type     | description                                |
+| key            | type     | description                                |
-|:------------|:---------|:-------------------------------------------|
+|:---------------|:---------|:-------------------------------------------|
-| `album`     | `string` | album name or collection title             |
+| `album`        | `string` | album name or collection title             |
-| `copyright` | `string` | copyright information or ownership details |
+| `composer`     | `string` | composer of the track                      |
-| `title`     | `string` | title of the track or media file           |
+| `genre`        | `string` | track's genre(s)                           |
-| `artist`    | `string` | artist or creator name                     |
+| `copyright`    | `string` | copyright information or ownership details |
-| `track`     | `string` | track number or position in album          |
+| `title`        | `string` | title of the track or media file           |
-| `date`      | `string` | release date or creation date              |
+| `artist`       | `string` | artist or creator name                     |
+| `album_artist` | `string` | album's artist or creator name             |
+| `track`        | `string` | track number or position in album          |
+| `date`         | `string` | release date or creation date              |
 
 #### audio object
 | key       | type      | value                                      |

web/src/lib/types/api.ts

@@ -45,9 +45,12 @@ type CobaltTunnelResponse = {
 
 export const CobaltFileMetadataKeys = [
     'album',
+    'composer',
+    'genre',
     'copyright',
     'title',
     'artist',
+    'album_artist',
     'track',
     'date'
 ];