Merge 9bf943362d into a940eb13fd

api/src/processing/cookie/manager.js

@@ -13,7 +13,8 @@ const VALID_SERVICES = new Set([
     'reddit',
     'twitter',
     'youtube',
-    'youtube_oauth'
+    'youtube_oauth',
+    'threads'
 ]);
 
 const invalidCookies = {};

api/src/processing/match-action.js

@@ -82,6 +82,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
             switch (host) {
                 case "instagram":
                 case "twitter":
+                case "threads":
                 case "snapchat":
                 case "bsky":
                 case "xiaohongshu":
@@ -156,6 +157,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                 case "streamable":
                 case "snapchat":
                 case "loom":
+                case "threads":
                 case "twitch":
                     responseType = "redirect";
                     break;

api/src/processing/match.js

@@ -26,6 +26,7 @@ import rutube from "./services/rutube.js";
 import dailymotion from "./services/dailymotion.js";
 import snapchat from "./services/snapchat.js";
 import loom from "./services/loom.js";
+import threads from "./services/threads.js";
 import facebook from "./services/facebook.js";
 import bluesky from "./services/bluesky.js";
 import xiaohongshu from "./services/xiaohongshu.js";
@@ -225,6 +226,15 @@ export default async function({ host, patternMatch, params }) {
                 });
                 break;
 
+            case "threads":
+                r = await threads({
+                    ...patternMatch,
+                    quality: params.videoQuality,
+                    alwaysProxy: params.alwaysProxy,
+                    dispatcher
+                });
+                break;
+
             case "facebook":
                 r = await facebook({
                     ...patternMatch,

api/src/processing/service-config.js

@@ -133,6 +133,10 @@ export const services = {
             "s/:id"
         ],
     },
+    threads: {
+        patterns: [":user/post/:id"],
+        tld: "net",
+    },
     tiktok: {
         patterns: [
             ":user/video/:postId",

api/src/processing/service-patterns.js

@@ -41,6 +41,9 @@ export const testers = {
     "streamable": pattern =>
         pattern.id?.length <= 6,
 
+    "threads": pattern =>
+        pattern.user?.length <= 33 && pattern.id?.length <= 32,
+
     "tiktok": pattern =>
         pattern.postId?.length <= 21 || pattern.shortLink?.length <= 13,
 

api/src/processing/services/threads.js

@@ -0,0 +1,142 @@
+import { createStream } from "../../stream/manage.js";
+import { getCookie, updateCookie } from "../cookie/manager.js";
+import { genericUserAgent } from "../../config.js";
+
+const commonHeaders = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.7",
+    "Cache-Control": "no-cache",
+    "Dnt": "1",
+    "Priority": "u=0, i",
+    "Sec-Ch-Ua": '"Not/A)Brand";v="8", "Chromium";v="126", "Brave";v="126"',
+    "Sec-Ch-Ua-Mobile": "?0",
+    "Sec-Ch-Ua-Model": '""',
+    "Sec-Ch-Ua-Platform": '"Windows"',
+    "Sec-Ch-Ua-Platform-Version": '"15.0.0"',
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "same-origin",
+    "Sec-Gpc": "1",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": genericUserAgent,
+};
+
+const DATA_REGEX = /<script type="application\/json" {2}data-content-len="\d+" data-sjs>({"require":\[\["ScheduledServerJS","handle",null,\[{"__bbox":{"require":\[\["RelayPrefetchedStreamCache(?:(?:@|\\u0040)[0-9a-f]{32})?","next",\[],\["adp_BarcelonaPostPage(?:Direct)?QueryRelayPreloader_[0-9a-f]{23}",[^\n]+})<\/script>\n/;
+
+export default async function({ user, id, quality, alwaysProxy, dispatcher }) {
+    const cookie = getCookie("threads");
+    const response = await fetch(`https://www.threads.net/${user}/post/${id}`, {
+        headers: {
+            ...commonHeaders,
+            cookie
+        },
+        dispatcher
+    });
+    if (cookie) updateCookie(cookie, response.headers);
+
+    if (response.status !== 200) {
+        return { error: "fetch.fail" };
+    }
+    const html = await response.text();
+    const dataString = html.match(DATA_REGEX)?.[1];
+    if (!dataString) {
+        return { error: "fetch.fail" };
+    }
+
+    const data = JSON.parse(dataString);
+    const post = data?.require?.[0]?.[3]?.[0]?.__bbox?.require?.[0]?.[3]?.[1]?.__bbox?.result?.data?.data?.edges[0]?.node?.thread_items[0]?.post;
+    if (!post) {
+        return { error: "fetch.fail" };
+    }
+
+    const filenameBase = `threads_${post.user.username}_${post.code}`;
+
+    // Video
+    if (post.media_type === 2) {
+        if (!post.video_versions) {
+            return { error: "fetch.empty" };
+        }
+
+        // types: 640p = 101, 480p = 102, 480p-low = 103
+        const selectedQualityType = quality === "max" ? 101 : quality && parseInt(quality) <= 480 ? 102 : 101;
+        const video = post.video_versions.find((v) => v.type === selectedQualityType) || post.video_versions.sort((a, b) => a.type - b.type)[0];
+        if (!video) {
+            return { error: "fetch.empty" };
+        }
+
+        return {
+            urls: video.url,
+            filename: `${filenameBase}.mp4`,
+            audioFilename: `${filenameBase}_audio`
+        }
+    }
+
+    // Photo
+    if (post.media_type === 1) {
+        if (!post.image_versions2?.candidates) {
+            return { error: "fetch.empty" };
+        }
+
+        return {
+            urls: post.image_versions2.candidates[0].url,
+            filename: `${filenameBase}.jpg`,
+            isPhoto: true
+        }
+    }
+
+    // Mixed
+    if (post.media_type === 8) {
+        if (!post.carousel_media) {
+            return { error: "fetch.empty" };
+        }
+
+        return {
+            picker: post.carousel_media.map((media, i) => {
+                const type = media.video_versions ? "video" : "photo";
+                let url = media.video_versions ? media.video_versions[0].url : media.image_versions2.candidates[0].url;
+                const thumbProxy = createStream({
+                    service: "threads",
+                    type: "proxy",
+                    u: media.image_versions2.candidates[0].url,
+                    filename: `${filenameBase}_${i}.jpg`,
+                });
+                if (alwaysProxy) {
+                    url = type === 'photo' ? thumbProxy : createStream({
+                        service: "threads",
+                        type: "proxy",
+                        u: media.video_versions[0].url,
+                        filename: `${filenameBase}_${i}.mp4`,
+                    });
+                }
+
+                return {
+                    type,
+                    url,
+                    thumb: thumbProxy
+                }
+            })
+        }
+    }
+
+    // GIPHY GIF
+    if (post.media_type === 19) {
+        if (!post.giphy_media_info?.images?.fixed_height?.webp) {
+            return { error: "fetch.empty" };
+        }
+
+        let giphyUrl = post.giphy_media_info.images.fixed_height.webp;
+
+        // In a regular browser (probably through the Accept header), the GIPHY link shows an HTML page with an ad that shows the GIF
+        // I'd rather have the GIF directly to save bandwidth and avoid ads.
+        let giphyId = giphyUrl.split("/")?.[5];
+        if (giphyId && !giphyId.includes(".")) giphyUrl = `https://i.giphy.com/${giphyId}.webp`;
+
+        return {
+            urls: giphyUrl,
+            filename: `${filenameBase}.webp`,
+            isPhoto: true
+        }
+    }
+
+    return { error: "fetch.fail" };
+}

api/src/util/tests/threads.json

@@ -0,0 +1,38 @@
+[
+    {
+        "name": "video",
+        "url": "https://www.threads.net/@zuck/post/CzecNnZPaxr",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "photo",
+        "url": "https://www.threads.net/@soren.iverson/post/C8PdJ59pMLr",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "mixed media",
+        "url": "https://www.threads.net/@snazzahguy/post/C8Q7UZDseWz",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "giphy gif",
+        "url": "https://www.threads.net/@zjy4fun/post/DDvzVsgpZNQ",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]