diff --git a/api/package.json b/api/package.json index fc5304ed..4f2b21dc 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.7.7", + "version": "10.7.10", "author": "imput", "exports": "./src/cobalt.js", "type": "module", diff --git a/api/src/config.js b/api/src/config.js index 191e8441..bff1eda3 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,3 +1,4 @@ +import { Constants } from "youtubei.js"; import { getVersion } from "@imput/version-info"; import { services } from "./processing/service-config.js"; import { supportsReusePort } from "./misc/cluster.js"; @@ -52,6 +53,7 @@ const env = { keyReloadInterval: 900, enabledServices, + customInnertubeClient: process.env.CUSTOM_INNERTUBE_CLIENT, } const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; @@ -74,6 +76,12 @@ if (env.instanceCount > 1 && !env.redisURL) { throw new Error('SO_REUSEPORT is not supported'); } +if (env.customInnertubeClient && !Constants.SUPPORTED_CLIENTS.includes(env.customInnertubeClient)) { + console.error("CUSTOM_INNERTUBE_CLIENT is invalid. Provided client is not supported."); + console.error(`Supported clients are: ${Constants.SUPPORTED_CLIENTS.join(', ')}\n`); + throw new Error("Invalid CUSTOM_INNERTUBE_CLIENT"); +} + export { env, genericUserAgent, diff --git a/api/src/processing/services/pinterest.js b/api/src/processing/services/pinterest.js index ea4275cb..15566cc4 100644 --- a/api/src/processing/services/pinterest.js +++ b/api/src/processing/services/pinterest.js @@ -23,7 +23,7 @@ export default async function(o) { const videoLink = [...html.matchAll(videoRegex)] .map(([, link]) => link) - .find(a => a.endsWith('.mp4') && a.includes('720p')); + .find(a => a.endsWith('.mp4')); if (videoLink) return { urls: videoLink, diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index b4a1d557..a4f4505e 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -24,6 +24,11 @@ const badContainerEnd = new Date(1702605600000); function needsFixing(media) { const representativeId = media.source_status_id_str ?? media.id_str; + + // syndication api doesn't have media ids in its response, + // so we just assume it's all good + if (!representativeId) return false; + const mediaTimestamp = new Date( Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH) ); @@ -53,6 +58,25 @@ const getGuestToken = async (dispatcher, forceReload = false) => { } } +const requestSyndication = async(dispatcher, tweetId) => { + // thank you + // https://github.com/yt-dlp/yt-dlp/blob/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9/yt_dlp/extractor/twitter.py#L1334 + const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, ''); + const syndicationUrl = new URL("https://cdn.syndication.twimg.com/tweet-result"); + + syndicationUrl.searchParams.set("id", tweetId); + syndicationUrl.searchParams.set("token", token(tweetId)); + + const result = await fetch(syndicationUrl, { + headers: { + "user-agent": genericUserAgent + }, + dispatcher + }); + + return result; +} + const requestTweet = async(dispatcher, tweetId, token, cookie) => { const graphqlTweetURL = new URL(graphqlURL); @@ -87,36 +111,24 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => { let result = await fetch(graphqlTweetURL, { headers, dispatcher }); updateCookie(cookie, result.headers); - // we might have been missing the `ct0` cookie, retry + // we might have been missing the ct0 cookie, retry if (result.status === 403 && result.headers.get('set-cookie')) { - result = await fetch(graphqlTweetURL, { - headers: { - ...headers, - 'x-csrf-token': cookie.values().ct0 - }, - dispatcher - }); + const cookieValues = cookie?.values(); + if (cookieValues?.ct0) { + result = await fetch(graphqlTweetURL, { + headers: { + ...headers, + 'x-csrf-token': cookieValues.ct0 + }, + dispatcher + }); + } } return result } -export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { - const cookie = await getCookie('twitter'); - - let guestToken = await getGuestToken(dispatcher); - if (!guestToken) return { error: "fetch.fail" }; - - let tweet = await requestTweet(dispatcher, id, guestToken); - - // get new token & retry if old one expired - if ([403, 429].includes(tweet.status)) { - guestToken = await getGuestToken(dispatcher, true); - tweet = await requestTweet(dispatcher, id, guestToken) - } - - tweet = await tweet.json(); - +const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => { let tweetTypename = tweet?.data?.tweetResult?.result?.__typename; if (!tweetTypename) { @@ -127,13 +139,13 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { const reason = tweet?.data?.tweetResult?.result?.reason; switch(reason) { case "Protected": - return { error: "content.post.private" } + return { error: "content.post.private" }; case "NsfwLoggedOut": if (cookie) { tweet = await requestTweet(dispatcher, id, guestToken, cookie); tweet = await tweet.json(); tweetTypename = tweet?.data?.tweetResult?.result?.__typename; - } else return { error: "content.post.age" } + } else return { error: "content.post.age" }; } } @@ -150,7 +162,69 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities; } - let media = (repostedTweet?.media || baseTweet?.extended_entities?.media); + return (repostedTweet?.media || baseTweet?.extended_entities?.media); +} + +const testResponse = (result) => { + const contentLength = result.headers.get("content-length"); + + if (!contentLength || contentLength === '0') { + return false; + } + + if (!result.headers.get("content-type").startsWith("application/json")) { + return false; + } + + return true; +} + +export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { + const cookie = await getCookie('twitter'); + + let syndication = false; + + let guestToken = await getGuestToken(dispatcher); + if (!guestToken) return { error: "fetch.fail" }; + + // for now we assume that graphql api will come back after some time, + // so we try it first + + let tweet = await requestTweet(dispatcher, id, guestToken); + + // get new token & retry if old one expired + if ([403, 429].includes(tweet.status)) { + guestToken = await getGuestToken(dispatcher, true); + if (cookie) { + tweet = await requestTweet(dispatcher, id, guestToken, cookie); + } else { + tweet = await requestTweet(dispatcher, id, guestToken); + } + } + + const testGraphql = testResponse(tweet); + + // if graphql requests fail, then resort to tweet embed api + if (!testGraphql) { + syndication = true; + tweet = await requestSyndication(dispatcher, id); + + const testSyndication = testResponse(tweet); + + // if even syndication request failed, then cry out loud + if (!testSyndication) { + return { error: "fetch.fail" }; + } + } + + tweet = await tweet.json(); + + let media = + syndication + ? tweet.mediaDetails + : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie); + + if (!media) return { error: "fetch.empty" }; // check if there's a video at given index (/video/) if (index >= 0 && index < media?.length) { @@ -163,7 +237,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { service: "twitter", type: "proxy", url, filename, - }) + }); switch (media?.length) { case undefined: diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index a4c54a82..fc712e24 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -41,6 +41,8 @@ const hlsCodecList = { } } +const clientsWithNoCipher = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDROID']; + const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320]; const transformSessionData = (cookie) => { @@ -149,7 +151,7 @@ export default async function (o) { useHLS = false; } - let innertubeClient = o.innertubeClient || "ANDROID"; + let innertubeClient = o.innertubeClient || env.customInnertubeClient || "ANDROID"; if (cookie) { useHLS = false; @@ -428,6 +430,10 @@ export default async function (o) { } } + if (video?.drm_families || audio?.drm_families) { + return { error: "youtube.drm" }; + } + const fileMetadata = { title: basicInfo.title.trim(), artist: basicInfo.author.replace("- Topic", "").trim() @@ -474,7 +480,7 @@ export default async function (o) { urls = audio.uri; } - if (innertubeClient === "WEB" && innertube) { + if (!clientsWithNoCipher.includes(innertubeClient) && innertube) { urls = audio.decipher(innertube.session.player); } @@ -509,7 +515,7 @@ export default async function (o) { filenameAttributes.resolution = `${video.width}x${video.height}`; filenameAttributes.extension = codecList[codec].container; - if (innertubeClient === "WEB" && innertube) { + if (!clientsWithNoCipher.includes(innertubeClient) && innertube) { video = video.decipher(innertube.session.player); audio = audio.decipher(innertube.session.player); } else { diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json index 4fc5900f..4139e39d 100644 --- a/api/src/util/tests/twitter.json +++ b/api/src/util/tests/twitter.json @@ -169,6 +169,15 @@ "status": "tunnel" } }, + { + "name": "gif", + "url": "https://x.com/thelastromances/status/1897839691212202479", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "inexistent post", "url": "https://twitter.com/test/status/9487653", diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index ea31cfc5..fe8316f5 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -80,6 +80,7 @@ sudo service nscd start | `API_REDIS_URL` | ➖ | `redis://localhost:6379` | when set, cobalt uses redis instead of internal memory for the tunnel cache. | | `API_INSTANCE_COUNT` | ➖ | `2` | supported only on Linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. | | `DISABLED_SERVICES` | ➖ | `bilibili,youtube` | comma-separated list which disables certain services from being used. | +| `CUSTOM_INNERTUBE_CLIENT` | ➖ | `IOS` | innertube client that will be used instead of the default one. | \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)). diff --git a/web/i18n/en/error.json b/web/i18n/en/error.json index 2788d9e4..fe276aa0 100644 --- a/web/i18n/en/error.json +++ b/web/i18n/en/error.json @@ -67,5 +67,6 @@ "api.youtube.token_expired": "couldn't get this video because the youtube token expired and i couldn't refresh it. try again in a few seconds, but if it still doesn't work, tell the instance owner about this error!", "api.youtube.no_hls_streams": "couldn't find any matching HLS streams for this video. try downloading it without HLS!", "api.youtube.api_error": "youtube updated something about its api and i couldn't get any info about this video. try again in a few seconds, but if this issue sticks, please report it!", - "api.youtube.temporary_disabled": "youtube downloading is temporarily disabled due to restrictions from youtube's side. we're already looking for ways to go around them.\n\nwe apologize for the inconvenience and are doing our best to restore this functionality. check cobalt's socials or github for timely updates!" + "api.youtube.temporary_disabled": "youtube downloading is temporarily disabled due to restrictions from youtube's side. we're already looking for ways to go around them.\n\nwe apologize for the inconvenience and are doing our best to restore this functionality. check cobalt's socials or github for timely updates!", + "api.youtube.drm": "this youtube video is protected by widevine DRM, so i can't download it. try a different link!" }