Merge branch 'wukko:current' into current

docs/examples/cookies.example.json

@@ -2,7 +2,13 @@
     "instagram": [
         "mid=<replace>; ig_did=<with>; csrftoken=<your>; ds_user_id=<own>; sessionid=<cookies>"
     ],
+    "instagram_bearer": [
+        "token=<token_with_no_bearer_in_front>", "token=IGT:2:<looks_like_this>"
+    ],
     "reddit": [
         "client_id=<replace_this>; client_secret=<replace_this>; refresh_token=<replace_this>"
+    ],
+    "twitter": [
+        "auth_token=<replace_this>; ct0=<replace_this>"
     ]
 }

src/front/cobalt.js

@@ -8,7 +8,7 @@ const isOldFirefox = ua.match("firefox/") && ua.split("firefox/")[1].split('.')[
 const switchers = {
     "theme": ["auto", "light", "dark"],
     "vCodec": ["h264", "av1", "vp9"],
-    "vQuality": ["720", "max", "2160", "1440", "1080", "480", "360"],
+    "vQuality": ["720", "max", "2160", "1440", "1080", "480", "360", "240", "144"],
     "aFormat": ["mp3", "best", "ogg", "wav", "opus"],
     "audioMode": ["false", "true"],
     "filenamePattern": ["classic", "pretty", "basic", "nerdy"]

src/modules/pageRender/page.js

@@ -349,6 +349,12 @@ export default function(obj) {
                         }, {
                             action: "360",
                             text: "360p"
+                        }, {
+                            action: "240",
+                            text: "240p"
+                        }, {
+                            action: "144",
+                            text: "144p"
                         }]
                     })
                 })

src/modules/processing/match.js

@@ -80,7 +80,8 @@ export default async function(host, patternMatch, url, lang, obj) {
             case "reddit":
                 r = await reddit({
                     sub: patternMatch.sub,
-                    id: patternMatch.id
+                    id: patternMatch.id,
+                    user: patternMatch.user
                 });
                 break;
             case "tiktok":

src/modules/processing/services/instagram.js

@@ -86,24 +86,26 @@ async function request(url, cookie, method = 'GET', requestData) {
     updateCookie(cookie, data.headers);
     return data.json();
 }
-
-async function requestMobileApi(id, cookie) {
+async function getMediaId(id, { cookie, token } = {}) {
     const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/');
     oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`);
 
     const oembed = await fetch(oembedURL, {
         headers: {
             ...mobileHeaders,
+            ...( token && { authorization: `Bearer ${token}` } ),
             cookie
         }
     }).then(r => r.json()).catch(() => {});
 
-    const mediaId = oembed?.media_id;
-    if (!mediaId) return false;
+    return oembed?.media_id;
+}
 
+async function requestMobileApi(mediaId, { cookie, token } = {}) {
     const mediaInfo = await fetch(`https://i.instagram.com/api/v1/media/${mediaId}/info/`, {
         headers: {
             ...mobileHeaders,
+            ...( token && { authorization: `Bearer ${token}` } ),
             cookie
         }
     }).then(r => r.json()).catch(() => {});
@@ -237,9 +239,20 @@ async function getPost(id) {
     try {
         const cookie = getCookie('instagram');
     
+        const bearer = getCookie('instagram_bearer');
+        const token = bearer?.values()?.token;
+
+        // get media_id for mobile api, three methods
+        let media_id = await getMediaId(id);
+        if (!media_id && token) media_id = await getMediaId(id, { token });
+        if (!media_id && cookie) media_id = await getMediaId(id, { cookie });
+
+        // mobile api (bearer)
+        if (media_id && token) data = await requestMobileApi(id, { token });
+
         // mobile api (no cookie, cookie)
-        data = await requestMobileApi(id);
-        if (!data && cookie) data = await requestMobileApi(id, cookie);
+        if (!data && media_id) data = await requestMobileApi(id);
+        if (!data && media_id && cookie) data = await requestMobileApi(id, { cookie });
 
         // html embed (no cookie, cookie)
         if (!data) data = await requestHTML(id);

src/modules/processing/services/reddit.js

@@ -48,43 +48,73 @@ async function getAccessToken() {
 }
 
 export default async function(obj) {
-    const url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`);
+    let url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`);
+
+    if (obj.user) {
+        url.pathname = `/user/${obj.user}/comments/${obj.id}.json`;
+    }
 
     const accessToken = await getAccessToken();
     if (accessToken) url.hostname = 'oauth.reddit.com';
 
     let data = await fetch(
-        url, { headers: accessToken && { authorization: `Bearer ${accessToken}` } }
-    ).then((r) => { return r.json() }).catch(() => { return false });
-    if (!data) return { error: 'ErrorCouldntFetch' };
+        url, {
+            headers: accessToken && { authorization: `Bearer ${accessToken}` }
+        }
+    ).then(r => r.json() ).catch(() => {});
 
-    data = data[0]["data"]["children"][0]["data"];
+    if (!data || !Array.isArray(data)) return { error: 'ErrorCouldntFetch' };
 
-    if (data.url.endsWith('.gif')) return { typeId: 1, urls: data.url };
+    data = data[0]?.data?.children[0]?.data;
 
-    if (!("reddit_video" in data["secure_media"])) return { error: 'ErrorEmptyDownload' };
-    if (data["secure_media"]["reddit_video"]["duration"] * 1000 > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+    if (data?.url?.endsWith('.gif')) return {
+        typeId: 1,
+        urls: data.url
+    }
+
+    if (!data.secure_media?.reddit_video)
+        return { error: 'ErrorEmptyDownload' };
+
+    if (data.secure_media?.reddit_video?.duration * 1000 > maxVideoDuration)
+        return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
 
     let audio = false,
-        video = data["secure_media"]["reddit_video"]["fallback_url"].split('?')[0],
-        audioFileLink = video.match('.mp4') ? `${video.split('_')[0]}_audio.mp4` : `${data["secure_media"]["reddit_video"]["fallback_url"].split('DASH')[0]}audio`;
+        video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0],
+        audioFileLink = `${data.secure_media?.reddit_video?.fallback_url?.split('DASH')[0]}audio`;
 
-    await fetch(audioFileLink, { method: "HEAD" }).then((r) => { if (Number(r.status) === 200) audio = true }).catch(() => { audio = false });
+    if (video.match('.mp4')) {
+        audioFileLink = `${video.split('_')[0]}_audio.mp4`
+    }
+
+    // test the existence of audio
+    await fetch(audioFileLink, { method: "HEAD" }).then((r) => {
+        if (Number(r.status) === 200) {
+            audio = true
+        }
+    }).catch(() => {})
 
     // fallback for videos with variable audio quality
     if (!audio) {
         audioFileLink = `${video.split('_')[0]}_AUDIO_128.mp4`
-        await fetch(audioFileLink, { method: "HEAD" }).then((r) => { if (Number(r.status) === 200) audio = true }).catch(() => { audio = false });
+        await fetch(audioFileLink, { method: "HEAD" }).then((r) => {
+            if (Number(r.status) === 200) {
+                audio = true
+            }
+        }).catch(() => {})
     }
 
     let id = video.split('/')[3];
 
-    if (!audio) return { typeId: 1, urls: video };
+    if (!audio) return {
+        typeId: 1,
+        urls: video
+    }
+
     return {
         typeId: 2,
         type: "render",
         urls: [video, audioFileLink],
         audioFilename: `reddit_${id}_audio`,
         filename: `reddit_${id}.mp4`
-    };
+    }
 }

src/modules/processing/services/twitter.js

@@ -1,5 +1,6 @@
 import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
+import { getCookie, updateCookie } from "../cookie/manager.js";
 
 const graphqlURL = 'https://twitter.com/i/api/graphql/5GOHgZe-8U2j5sVHQzEm9A/TweetResultByRestId';
 const tokenURL = 'https://api.twitter.com/1.1/guest/activate.json';
@@ -49,9 +50,26 @@ const getGuestToken = async (forceReload = false) => {
     }
 }
 
-const requestTweet = (tweetId, token) => {
+const requestTweet = async(tweetId, token, cookie) => {
     const graphqlTweetURL = new URL(graphqlURL);
 
+    let headers = {
+        ...commonHeaders,
+        'content-type': 'application/json',
+        'x-guest-token': token,
+        cookie: `guest_id=${encodeURIComponent(`v1:${token}`)}`
+    }
+
+    if (cookie) {
+        headers = {
+            ...commonHeaders,
+            'content-type': 'application/json',
+            'X-Twitter-Auth-Type': 'OAuth2Session',
+            'x-csrf-token': cookie.values().ct0,
+            cookie
+        }
+    }
+
     graphqlTweetURL.searchParams.set('variables',
         JSON.stringify({
             tweetId,
@@ -62,31 +80,39 @@ const requestTweet = (tweetId, token) => {
     );
     graphqlTweetURL.searchParams.set('features', tweetFeatures);
 
-    return fetch(graphqlTweetURL, {
-        headers: {
-            ...commonHeaders,
-            'content-type': 'application/json',
-            'x-guest-token': token,
-            cookie: `guest_id=${encodeURIComponent(`v1:${token}`)}`
-        }
-    })
+    let result = await fetch(graphqlTweetURL, { headers });
+    updateCookie(cookie, result.headers);
+
+    // we might have been missing the `ct0` cookie, retry
+    if (result.status === 403 && result.headers.get('set-cookie')) {
+        result = await fetch(graphqlTweetURL, {
+            headers: {
+                ...headers,
+                'x-csrf-token': cookie.values().ct0
+            }
+        });
+    }
+
+    return result
 }
 
 export default async function({ id, index, toGif }) {
+    const cookie = await getCookie('twitter');
+
     let guestToken = await getGuestToken();
     if (!guestToken) return { error: 'ErrorCouldntFetch' };
 
     let tweet = await requestTweet(id, guestToken);
 
-    if ([403, 429].includes(tweet.status)) { // get new token & retry
+    // get new token & retry if old one expired
+    if ([403, 429].includes(tweet.status)) {
         guestToken = await getGuestToken(true);
         tweet = await requestTweet(id, guestToken)
     }
 
     tweet = await tweet.json();
 
-    // {"data":{"tweetResult":{"result":{"__typename":"TweetUnavailable","reason":"Protected"}}}}
-    const tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+    let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
 
     if (tweetTypename === "TweetUnavailable") {
         const reason = tweet?.data?.tweetResult?.result?.reason;
@@ -94,21 +120,32 @@ export default async function({ id, index, toGif }) {
             case "Protected":
                 return { error: 'ErrorTweetProtected' }
             case "NsfwLoggedOut":
-                return { error: 'ErrorTweetNSFW' }
+                if (cookie) {
+                    tweet = await requestTweet(id, guestToken, cookie);
+                    tweet = await tweet.json();
+                    tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+                } else return { error: 'ErrorTweetNSFW' }
         }
     }
-    if (tweetTypename !== "Tweet") {
+
+    if (!["Tweet", "TweetWithVisibilityResults"].includes(tweetTypename)) {
         return { error: 'ErrorTweetUnavailable' }
     }
 
-    const baseTweet = tweet.data.tweetResult.result.legacy,
-          repostedTweet = baseTweet.retweeted_status_result?.result.legacy.extended_entities;
+    let tweetResult = tweet.data.tweetResult.result,
+        baseTweet = tweetResult.legacy,
+        repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
+
+    if (tweetTypename === "TweetWithVisibilityResults") {
+        baseTweet = tweetResult.tweet.legacy;
+        repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities;
+    }
 
     let media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
     media = media?.filter(m => m.video_info?.variants?.length);
 
     // check if there's a video at given index (/video/<index>)
-    if ([0, 1, 2, 3].includes(index) && index < media?.length) {
+    if (index >= 0 && index < media?.length) {
         media = [media[index]]
     }
 

src/modules/processing/servicesConfig.json

@@ -12,7 +12,7 @@
         },
         "reddit": {
             "alias": "reddit videos & gifs",
-            "patterns": ["r/:sub/comments/:id/:title"],
+            "patterns": ["r/:sub/comments/:id/:title", "user/:user/comments/:id/:title"],
             "subdomains": "*",
             "enabled": true
         },

src/modules/processing/servicesPatternTesters.js

@@ -16,7 +16,8 @@ export const testers = {
         patternMatch.id?.length <= 128 || patternMatch.shortLink?.length <= 32,
 
     "reddit": (patternMatch) =>
-        patternMatch.sub?.length <= 22 && patternMatch.id?.length <= 10,
+        (patternMatch.sub?.length <= 22 && patternMatch.id?.length <= 10)
+        || (patternMatch.user?.length <= 22 && patternMatch.id?.length <= 10),
 
     "rutube": (patternMatch) =>
         patternMatch.id?.length === 32 || patternMatch.yappyId?.length === 32,