tiktok: implement new webapp-based downloading method

src/modules/processing/match.js

@@ -111,6 +111,7 @@ export default async function(host, patternMatch, lang, obj) {
                 r = await tiktok({
                     postId: patternMatch.postId,
                     id: patternMatch.id,
+                    user: patternMatch.user,
                     fullAudio: obj.isTTFullAudio,
                     isAudioOnly: isAudioOnly,
                     h265: obj.tiktokH265

src/modules/processing/matchActionDecider.js

@@ -2,6 +2,7 @@ import { audioIgnore, services, supportedAudio } from "../config.js";
 import { createResponse } from "../processing/request.js";
 import loc from "../../localization/manager.js";
 import createFilename from "./createFilename.js";
+import { createStream } from "../stream/manage.js";
 
 export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) {
     let action,
@@ -76,8 +77,13 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                     params = {
                         type: pickerType,
                         picker: r.picker,
-                        u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
+                        u: createStream({
-                        copy: audioFormat === "best" ? true : false
+                            service: "tiktok",
+                            type: pickerType,
+                            u: r.urls,
+                            filename: r.audioFilename,
+                        }),
+                        copy: audioFormat === "best"
                     }
             }
             break;

src/modules/processing/services/tiktok.js

@@ -1,15 +1,16 @@
-import { genericUserAgent, env } from "../../config.js";
+import { genericUserAgent } from "../../config.js";
+import { updateCookie } from "../cookie/manager.js";
+import Cookie from "../cookie/cookie.js";
 
+const fullDomain = "https://tiktok.com/";
 const shortDomain = "https://vt.tiktok.com/";
-const apiPath = "https://api22-normal-c-alisg.tiktokv.com/aweme/v1/feed/?region=US&carrier_region=US";
+export const cookie = new Cookie({})
-const apiUserAgent = "TikTok/338014 CFNetwork/1410.1 Darwin/22.6.0";
 
 export default async function(obj) {
-    let postId = obj.postId ? obj.postId : false;
+    let postId = obj.postId || false
+    let username = obj.user || false
 
-    if (!env.tiktokDeviceInfo) return { error: 'ErrorCouldntFetch' };
+    if (!username || !postId) {
-
-    if (!postId) {
         let html = await fetch(`${shortDomain}${obj.id}`, {
             redirect: "manual",
             headers: {
@@ -20,52 +21,47 @@ export default async function(obj) {
         if (!html) return { error: 'ErrorCouldntFetch' };
 
         if (html.slice(0, 17) === '<a href="https://') {
-            postId = html.split('<a href="https://')[1].split('?')[0].split('/')[3]
+            const fullLink = html.split('<a href="https://')[1].split('?')[0].split('/')
-        } else if (html.slice(0, 32) === '<a href="https://m.tiktok.com/v/' && html.includes('/v/')) {
+            username = fullLink[1]
-            postId = html.split('/v/')[1].split('.html')[0].replace("/", '')
+            postId = fullLink[3]
         }
     }
-    if (!postId) return { error: 'ErrorCantGetID' };
+    if (!username || !postId) return { error: 'ErrorCantGetID' };
 
-    let deviceInfo = new URLSearchParams(env.tiktokDeviceInfo).toString();
+    // should always be /video/, even for photos
-
+    const res = await fetch(`${fullDomain}${username}/video/${postId}`, {
-    let apiURL = new URL(apiPath);
-        apiURL.searchParams.append("aweme_id", postId);
-
-    let detail = await fetch(`${apiURL.href}&${deviceInfo}`, {
         headers: {
-            "user-agent": apiUserAgent
+            "user-agent": genericUserAgent,
+            cookie,
         }
-    }).then(r => r.json()).catch(() => {});
+    })
+    updateCookie(cookie, res.headers)
 
-    detail = detail?.aweme_list?.find(v => v.aweme_id === postId);
+    const html = await res.text()
-    if (!detail) return { error: 'ErrorCouldntFetch' };
+    const json = html
+        .split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1]
+        .split('</script>')[0]
+    const data = JSON.parse(json)
+    const detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
 
     let video, videoFilename, audioFilename, audio, images,
         filenameBase = `tiktok_${detail.author.unique_id}_${postId}`,
         bestAudio = 'm4a';
 
-    images = detail.image_post_info?.images;
+    images = detail.imagePost?.images;
 
-    let playAddr = detail.video.play_addr_h264;
+    let playAddr = detail.video.playAddr;
     if (obj.h265) {
-        playAddr = detail.video.bit_rate[0].play_addr
+        const h265PlayAddr = detail.video.bitrateInfo.find(b => b.CodecType.contains("h265"))?.PlayAddr.UrlList[0]
-    }
+        playAddr = h265PlayAddr || playAddr
-    if (!playAddr && detail.video.play_addr) {
-        playAddr = detail.video.play_addr
     }
 
     if (!obj.isAudioOnly && !images) {
-        video = playAddr.url_list[0];
+        video = playAddr;
         videoFilename = `${filenameBase}.mp4`;
     } else {
-        let fallback = playAddr.url_list[0];
+        audio = detail.music.playUrl || playAddr;
-        audio = fallback;
         audioFilename = `${filenameBase}_audio`;
-        if (obj.fullAudio || fallback.includes("music")) {
-            audio = detail.music.play_url.url_list[0]
-            audioFilename = `${filenameBase}_audio_original`
-        }
         if (audio.slice(-4) === ".mp3") bestAudio = 'mp3';
     }
 
@@ -80,12 +76,9 @@ export default async function(obj) {
         bestAudio
     }
     if (images) {
-        let imageLinks = [];
+        let imageLinks = images
-        for (let i in images) {
+            .map(i => i.imageURL.urlList.find(p => p.includes(".jpeg?")))
-            let sel = images[i].display_image.url_list;
+            .map(url => ({ url }))
-            sel = sel.filter(p => p.includes(".jpeg?"))
-            imageLinks.push({url: sel[0]})
-        }
         return {
             picker: imageLinks,
             urls: audio,

src/modules/stream/shared.js

@@ -1,4 +1,5 @@
 import { genericUserAgent } from "../config.js";
+import { cookie as tiktokCookie } from "../processing/services/tiktok.js";
 
 const defaultHeaders = {
     'user-agent': genericUserAgent
@@ -13,9 +14,14 @@ const serviceHeaders = {
         origin: 'https://www.youtube.com',
         referer: 'https://www.youtube.com',
         DNT: '?1'
+    },
+    tiktok: {
+        cookie: tiktokCookie
     }
 }
 
 export function getHeaders(service) {
-    return { ...defaultHeaders, ...serviceHeaders[service] }
+    // Converting all header values to strings
+    return Object.entries({ ...defaultHeaders, ...serviceHeaders[service] })
+        .reduce((p, c) => ({ ...p, [c[0]]: String(c[1]) }), {})
 }