mirrors/cobalt
1
0
Fork 0
mirror of https://github.com/imputnet/cobalt.git synced 2025-06-28 17:38:31 +00:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

Merge branch 'imputnet:main' into main

Browse Source
This commit is contained in:
ZEREX 2025-02-10 08:48:49 +01:00 committed by GitHub
commit ce9f649a9f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 333 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@imput/cobalt-api",
"description": "save what you love",
"version": "10.6",
"version": "10.7.2",
"author": "imput",
"exports": "./src/cobalt.js",
"type": "module",

5
api/src/misc/run-test.js
View File

@ -25,6 +25,11 @@ export async function runTest(url, params, expect) {
error.push(`status mismatch: ${detail}`);
}
if (expect.errorCode && expect.errorCode !== result.body?.error?.code) {
const detail = `${expect.errorCode} (expected) != ${result.body.error.code} (actual)`
error.push(`error mismatch: ${detail}`);
}
if (expect.code !== result.status) {
const detail = `${expect.code} (expected) != ${result.status} (actual)`;
error.push(`status code mismatch: ${detail}`);

14
api/src/misc/utils.js
View File

@ -1,12 +1,14 @@
import { request } from 'undici';
const redirectStatuses = new Set([301, 302, 303, 307, 308]);
export async function getRedirectingURL(url, dispatcher) {
const location = await fetch(url, {
redirect: 'manual',
export async function getRedirectingURL(url, dispatcher, userAgent) {
const location = await request(url, {
dispatcher,
}).then((r) => {
if (redirectStatuses.has(r.status) && r.headers.has('location')) {
return r.headers.get('location');
method: 'HEAD',
headers: { 'user-agent': userAgent }
}).then(r => {
if (redirectStatuses.has(r.statusCode) && r.headers['location']) {
return r.headers['location'];
}
}).catch(() => null);

5
api/src/processing/match.js
View File

@ -120,9 +120,8 @@ export default async function({ host, patternMatch, params }) {
case "reddit":
r = await reddit({
sub: patternMatch.sub,
id: patternMatch.id,
user: patternMatch.user
...patternMatch,
dispatcher,
});
break;

37
api/src/processing/service-config.js
View File

@ -35,13 +35,25 @@ export const services = {
},
instagram: {
patterns: [
"reels/:postId",
":username/reel/:postId",
"reel/:postId",
"p/:postId",
":username/p/:postId",
"tv/:postId",
"stories/:username/:storyId"
"reel/:postId",
"reels/:postId",
"stories/:username/:storyId",
/*
share & username links use the same url pattern,
so we test the share pattern first, cuz id type is different.
however, if someone has the "share" username and the user
somehow gets a link of this ancient style, it's joever.
*/
"share/:shareId",
"share/p/:shareId",
"share/reel/:shareId",
":username/p/:postId",
":username/reel/:postId",
],
altDomains: ["ddinstagram.com"],
},
@ -64,8 +76,21 @@ export const services = {
},
reddit: {
patterns: [
"comments/:id",
"r/:sub/comments/:id",
"r/:sub/comments/:id/:title",
"user/:user/comments/:id/:title"
"r/:sub/comments/:id/comment/:commentId",
"user/:user/comments/:id",
"user/:user/comments/:id/:title",
"user/:user/comments/:id/comment/:commentId",
"r/u_:user/comments/:id",
"r/u_:user/comments/:id/:title",
"r/u_:user/comments/:id/comment/:commentId",
"r/:sub/s/:shareId"
],
subdomains: "*",
},

9
api/src/processing/service-patterns.js
View File

@ -6,7 +6,8 @@ export const testers = {
"dailymotion": pattern => pattern.id?.length <= 32,
"instagram": pattern =>
pattern.postId?.length <= 12
pattern.postId?.length <= 48
|| pattern.shareId?.length <= 16
|| (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
"loom": pattern =>
@ -19,8 +20,10 @@ export const testers = {
pattern.id?.length <= 128 || pattern.shortLink?.length <= 32,
"reddit": pattern =>
(pattern.sub?.length <= 22 && pattern.id?.length <= 10)
|| (pattern.user?.length <= 22 && pattern.id?.length <= 10),
pattern.id?.length <= 16 && !pattern.sub && !pattern.user
|| (pattern.sub?.length <= 22 && pattern.id?.length <= 16)
|| (pattern.user?.length <= 22 && pattern.id?.length <= 16)
|| (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16),
"rutube": pattern =>
(pattern.id?.length === 32 && pattern.key?.length <= 32) ||

16
api/src/processing/services/bilibili.js
View File

@ -1,19 +1,8 @@
import { genericUserAgent, env } from "../../config.js";
import { resolveRedirectingURL } from "../url.js";
// TO-DO: higher quality downloads (currently requires an account)
function com_resolveShortlink(shortId) {
return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' })
.then(r => r.status > 300 && r.status < 400 && r.headers.get('location'))
.then(url => {
if (!url) return;
const path = new URL(url).pathname;
if (path.startsWith('/video/'))
return path.split('/')[2];
})
.catch(() => {})
}
function getBest(content) {
return content?.filter(v => v.baseUrl || v.url)
.map(v => (v.baseUrl = v.baseUrl || v.url, v))
@ -99,7 +88,8 @@ async function tv_download(id) {
export default async function({ comId, tvId, comShortLink }) {
if (comShortLink) {
comId = await com_resolveShortlink(comShortLink);
const patternMatch = await resolveRedirectingURL(`https://b23.tv/${comShortLink}`);
comId = patternMatch?.comId;
}
if (comId) {

234
api/src/processing/services/instagram.js
View File

@ -1,3 +1,5 @@
import { randomBytes } from "node:crypto";
import { resolveRedirectingURL } from "../url.js";
import { genericUserAgent } from "../../config.js";
import { createStream } from "../../stream/manage.js";
import { getCookie, updateCookie } from "../cookie/manager.js";
@ -8,6 +10,7 @@ const commonHeaders = {
"sec-fetch-site": "same-origin",
"x-ig-app-id": "936619743392459"
}
const mobileHeaders = {
"x-ig-app-locale": "en_US",
"x-ig-device-locale": "en_US",
@ -19,6 +22,7 @@ const mobileHeaders = {
"x-fb-server-cluster": "True",
"content-length": "0",
}
const embedHeaders = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Accept-Language": "en-GB,en;q=0.9",
@ -33,7 +37,7 @@ const embedHeaders = {
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"User-Agent": genericUserAgent,
}
const cachedDtsg = {
@ -41,7 +45,17 @@ const cachedDtsg = {
expiry: 0
}
export default function(obj) {
const getNumberFromQuery = (name, data) => {
const s = data?.match(new RegExp(name + '=(\\d+)'))?.[1];
if (+s) return +s;
}
const getObjectFromEntries = (name, data) => {
const obj = data?.match(new RegExp('\\["' + name + '",.*?,({.*?}),\\d+\\]'))?.[1];
return obj && JSON.parse(obj);
}
export default function instagram(obj) {
const dispatcher = obj.dispatcher;
async function findDtsgId(cookie) {
@ -91,6 +105,7 @@ export default function(obj) {
updateCookie(cookie, data.headers);
return data.json();
}
async function getMediaId(id, { cookie, token } = {}) {
const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/');
oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`);
@ -119,6 +134,7 @@ export default function(obj) {
return mediaInfo?.items?.[0];
}
async function requestHTML(id, cookie) {
const data = await fetch(`https://www.instagram.com/p/${id}/embed/captioned/`, {
headers: {
@ -136,40 +152,167 @@ export default function(obj) {
return embedData;
}
async function requestGQL(id, cookie) {
let dtsgId;
if (cookie) {
dtsgId = await findDtsgId(cookie);
}
const url = new URL('https://www.instagram.com/api/graphql/');
async function getGQLParams(id, cookie) {
const req = await fetch(`https://www.instagram.com/p/${id}/`, {
headers: {
...embedHeaders,
cookie
},
dispatcher
});
const requestData = {
jazoest: '26406',
variables: JSON.stringify({
shortcode: id,
__relay_internal__pv__PolarisShareMenurelayprovider: false
}),
doc_id: '7153618348081770'
const html = await req.text();
const siteData = getObjectFromEntries('SiteData', html);
const polarisSiteData = getObjectFromEntries('PolarisSiteData', html);
const webConfig = getObjectFromEntries('DGWWebConfig', html);
const pushInfo = getObjectFromEntries('InstagramWebPushInfo', html);
const lsd = getObjectFromEntries('LSD', html)?.token || randomBytes(8).toString('base64url');
const csrf = getObjectFromEntries('InstagramSecurityConfig', html)?.csrf_token;
const anon_cookie = [
csrf && "csrftoken=" + csrf,
polarisSiteData?.device_id && "ig_did=" + polarisSiteData?.device_id,
"wd=1280x720",
"dpr=2",
polarisSiteData?.machine_id && "mid=" + polarisSiteData.machine_id,
"ig_nrcb=1"
].filter(a => a).join('; ');
return {
headers: {
'x-ig-app-id': webConfig?.appId || '936619743392459',
'X-FB-LSD': lsd,
'X-CSRFToken': csrf,
'X-Bloks-Version-Id': getObjectFromEntries('WebBloksVersioningID', html)?.versioningID,
'x-asbd-id': 129477,
cookie: anon_cookie
},
body: {
__d: 'www',
__a: '1',
__s: '::' + Math.random().toString(36).substring(2).replace(/\d/g, '').slice(0, 6),
__hs: siteData?.haste_session || '20126.HYP:instagram_web_pkg.2.1...0',
__req: 'b',
__ccg: 'EXCELLENT',
__rev: pushInfo?.rollout_hash || '1019933358',
__hsi: siteData?.hsi || '7436540909012459023',
__dyn: randomBytes(154).toString('base64url'),
__csr: randomBytes(154).toString('base64url'),
__user: '0',
__comet_req: getNumberFromQuery('__comet_req', html) || '7',
av: '0',
dpr: '2',
lsd,
jazoest: getNumberFromQuery('jazoest', html) || Math.floor(Math.random() * 10000),
__spin_r: siteData?.__spin_r || '1019933358',
__spin_b: siteData?.__spin_b || 'trunk',
__spin_t: siteData?.__spin_t || Math.floor(new Date().getTime() / 1000),
}
};
if (dtsgId) {
requestData.fb_dtsg = dtsgId;
}
async function requestGQL(id, cookie) {
const { headers, body } = await getGQLParams(id, cookie);
const req = await fetch('https://www.instagram.com/graphql/query', {
method: 'POST',
dispatcher,
headers: {
...embedHeaders,
...headers,
cookie,
'content-type': 'application/x-www-form-urlencoded',
'X-FB-Friendly-Name': 'PolarisPostActionLoadPostQueryQuery',
},
body: new URLSearchParams({
...body,
fb_api_caller_class: 'RelayModern',
fb_api_req_friendly_name: 'PolarisPostActionLoadPostQueryQuery',
variables: JSON.stringify({
shortcode: id,
fetch_tagged_user_count: null,
hoisted_comment_id: null,
hoisted_reply_id: null
}),
server_timestamps: true,
doc_id: '8845758582119845'
}).toString()
});
return {
gql_data: await req.json()
.then(r => r.data)
.catch(() => null)
};
}
async function getErrorContext(id) {
try {
const { headers, body } = await getGQLParams(id);
const req = await fetch('https://www.instagram.com/ajax/bulk-route-definitions/', {
method: 'POST',
dispatcher,
headers: {
...embedHeaders,
...headers,
'content-type': 'application/x-www-form-urlencoded',
'X-Ig-D': 'www',
},
body: new URLSearchParams({
'route_urls[0]': `/p/${id}/`,
routing_namespace: 'igx_www',
...body
}).toString()
});
const response = await req.text();
if (response.includes('"tracePolicy":"polaris.privatePostPage"'))
return { error: 'content.post.private' };
const [, mediaId, mediaOwnerId] = response.match(
/"media_id":\s*?"(\d+)","media_owner_id":\s*?"(\d+)"/
) || [];
if (mediaId && mediaOwnerId) {
const rulingURL = new URL('https://www.instagram.com/api/v1/web/get_ruling_for_media_content_logged_out');
rulingURL.searchParams.set('media_id', mediaId);
rulingURL.searchParams.set('owner_id', mediaOwnerId);
const rulingResponse = await fetch(rulingURL, {
headers: {
...headers,
...commonHeaders
},
dispatcher,
}).then(a => a.json()).catch(() => ({}));
if (rulingResponse?.title?.includes('Restricted'))
return { error: "content.post.age" };
}
} catch {
return { error: "fetch.fail" };
}
return (await request(url, cookie, 'POST', requestData))
.data
?.xdt_api__v1__media__shortcode__web_info
?.items
?.[0];
return { error: "fetch.empty" };
}
function extractOldPost(data, id, alwaysProxy) {
const sidecar = data?.gql_data?.shortcode_media?.edge_sidecar_to_children;
const shortcodeMedia = data?.gql_data?.shortcode_media || data?.gql_data?.xdt_shortcode_media;
const sidecar = shortcodeMedia?.edge_sidecar_to_children;
if (sidecar) {
const picker = sidecar.edges.filter(e => e.node?.display_url)
.map((e, i) => {
const type = e.node?.is_video ? "video" : "photo";
const url = type === "video" ? e.node?.video_url : e.node?.display_url;
let url;
if (type === 'video') {
url = e.node?.video_url;
} else if (type === 'photo') {
url = e.node?.display_url;
}
let itemExt = type === "video" ? "mp4" : "jpg";
@ -196,16 +339,21 @@ export default function(obj) {
});
if (picker.length) return { picker }
} else if (data?.gql_data?.shortcode_media?.video_url) {
}
if (shortcodeMedia?.video_url) {
return {
urls: data.gql_data.shortcode_media.video_url,
urls: shortcodeMedia.video_url,
filename: `instagram_${id}.mp4`,
audioFilename: `instagram_${id}_audio`
}
} else if (data?.gql_data?.shortcode_media?.display_url) {
}
if (shortcodeMedia?.display_url) {
return {
urls: data.gql_data?.shortcode_media.display_url,
isPhoto: true
urls: shortcodeMedia.display_url,
isPhoto: true,
filename: `instagram_${id}.jpg`,
}
}
}
@ -266,7 +414,9 @@ export default function(obj) {
}
async function getPost(id, alwaysProxy) {
const hasData = (data) => data && data.gql_data !== null;
const hasData = (data) => data
&& data.gql_data !== null
&& data?.gql_data?.xdt_shortcode_media !== null;
let data, result;
try {
const cookie = getCookie('instagram');
@ -295,7 +445,9 @@ export default function(obj) {
if (!hasData(data) && cookie) data = await requestGQL(id, cookie);
} catch {}
if (!data) return { error: "fetch.fail" };
if (!hasData(data)) {
return getErrorContext(id);
}
if (data?.gql_data) {
result = extractOldPost(data, id, alwaysProxy)
@ -358,14 +510,30 @@ export default function(obj) {
if (item.image_versions2?.candidates) {
return {
urls: item.image_versions2.candidates[0].url,
isPhoto: true
isPhoto: true,
filename: `instagram_${id}.jpg`,
}
}
return { error: "link.unsupported" };
}
const { postId, storyId, username, alwaysProxy } = obj;
const { postId, shareId, storyId, username, alwaysProxy } = obj;
if (shareId) {
return resolveRedirectingURL(
`https://www.instagram.com/share/${shareId}/`,
dispatcher,
// for some reason instagram decides to return HTML
// instead of a redirect when requesting with a normal
// browser user-agent
'curl/7.88.1'
).then(match => instagram({
...obj, ...match,
shareId: undefined
}));
}
if (postId) return getPost(postId, alwaysProxy);
if (username && storyId) return getStory(username, storyId);

13
api/src/processing/services/pinterest.js
View File

@ -1,4 +1,5 @@
import { genericUserAgent } from "../../config.js";
import { resolveRedirectingURL } from "../url.js";
const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
@ -7,10 +8,10 @@ export default async function(o) {
let id = o.id;
if (!o.id && o.shortLink) {
id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" })
.then(r => r.headers.get("location").split('pin/')[1].split('/')[0])
.catch(() => {});
const patternMatch = await resolveRedirectingURL(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`);
id = patternMatch?.id;
}
if (id.includes("--")) id = id.split("--")[1];
if (!id) return { error: "fetch.fail" };
@ -26,8 +27,8 @@ export default async function(o) {
if (videoLink) return {
urls: videoLink,
filename: `pinterest_${o.id}.mp4`,
audioFilename: `pinterest_${o.id}_audio`
filename: `pinterest_${id}.mp4`,
audioFilename: `pinterest_${id}_audio`
}
const imageLink = [...html.matchAll(imageRegex)]
@ -39,7 +40,7 @@ export default async function(o) {
if (imageLink) return {
urls: imageLink,
isPhoto: true,
filename: `pinterest_${o.id}.${imageType}`
filename: `pinterest_${id}.${imageType}`
}
return { error: "fetch.empty" };

31
api/src/processing/services/reddit.js
View File

@ -1,3 +1,4 @@
import { resolveRedirectingURL } from "../url.js";
import { genericUserAgent, env } from "../../config.js";
import { getCookie, updateCookieValues } from "../cookie/manager.js";
@ -48,12 +49,20 @@ async function getAccessToken() {
}
export default async function(obj) {
let url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`);
let params = obj;
if (obj.user) {
url.pathname = `/user/${obj.user}/comments/${obj.id}.json`;
if (!params.id && params.shareId) {
params = await resolveRedirectingURL(
`https://www.reddit.com/r/${params.sub}/s/${params.shareId}`,
obj.dispatcher,
genericUserAgent
);
}
if (!params?.id) return { error: "fetch.short_link" };
const url = new URL(`https://www.reddit.com/comments/${params.id}.json`);
const accessToken = await getAccessToken();
if (accessToken) url.hostname = 'oauth.reddit.com';
@ -73,12 +82,17 @@ export default async function(obj) {
data = data[0]?.data?.children[0]?.data;
const id = `${String(obj.sub).toLowerCase()}_${obj.id}`;
let sourceId;
if (params.sub || params.user) {
sourceId = `${String(params.sub || params.user).toLowerCase()}_${params.id}`;
} else {
sourceId = params.id;
}
if (data?.url?.endsWith('.gif')) return {
typeId: "redirect",
urls: data.url,
filename: `reddit_${id}.gif`,
filename: `reddit_${sourceId}.gif`,
}
if (!data.secure_media?.reddit_video)
@ -87,8 +101,9 @@ export default async function(obj) {
if (data.secure_media?.reddit_video?.duration > env.durationLimit)
return { error: "content.too_long" };
const video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0];
let audio = false,
video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0],
audioFileLink = `${data.secure_media?.reddit_video?.fallback_url?.split('DASH')[0]}audio`;
if (video.match('.mp4')) {
@ -121,7 +136,7 @@ export default async function(obj) {
typeId: "tunnel",
type: "merge",
urls: [video, audioFileLink],
audioFilename: `reddit_${id}_audio`,
filename: `reddit_${id}.mp4`
audioFilename: `reddit_${sourceId}_audio`,
filename: `reddit_${sourceId}.mp4`
}
}

22
api/src/processing/services/snapchat.js
View File

@ -1,7 +1,6 @@
import { extract, normalizeURL } from "../url.js";
import { resolveRedirectingURL } from "../url.js";
import { genericUserAgent } from "../../config.js";
import { createStream } from "../../stream/manage.js";
import { getRedirectingURL } from "../../misc/utils.js";
const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="([^"]+)" as="video"\/>/;
const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
@ -41,9 +40,9 @@ async function getStory(username, storyId, alwaysProxy) {
const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
if (nextDataString) {
const data = JSON.parse(nextDataString);
const storyIdParam = data.query.profileParams[1];
const storyIdParam = data?.query?.profileParams?.[1];
if (storyIdParam && data.props.pageProps.story) {
if (storyIdParam && data?.props?.pageProps?.story) {
const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
if (story) {
if (story.snapMediaType === 0) {
@ -62,7 +61,7 @@ async function getStory(username, storyId, alwaysProxy) {
}
}
const defaultStory = data.props.pageProps.curatedHighlights[0];
const defaultStory = data?.props?.pageProps?.curatedHighlights?.[0];
if (defaultStory) {
return {
picker: defaultStory.snapList.map(snap => {
@ -100,18 +99,7 @@ async function getStory(username, storyId, alwaysProxy) {
export default async function (obj) {
let params = obj;
if (obj.shortLink) {
const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
if (!link?.startsWith('https://www.snapchat.com/')) {
return { error: "fetch.short_link" };
}
const extractResult = extract(normalizeURL(link));
if (extractResult?.host !== 'snapchat') {
return { error: "fetch.short_link" };
}
params = extractResult.patternMatch;
params = await resolveRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
}
if (params.spotlightId) {

15
api/src/processing/services/xiaohongshu.js
View File

@ -1,7 +1,6 @@
import { extract, normalizeURL } from "../url.js";
import { resolveRedirectingURL } from "../url.js";
import { genericUserAgent } from "../../config.js";
import { createStream } from "../../stream/manage.js";
import { getRedirectingURL } from "../../misc/utils.js";
const https = (url) => {
return url.replace(/^http:/i, 'https:');
@ -12,19 +11,13 @@ export default async function ({ id, token, shareId, h265, isAudioOnly, dispatch
let xsecToken = token;
if (!noteId) {
const extractedURL = await getRedirectingURL(
const patternMatch = await resolveRedirectingURL(
`https://xhslink.com/a/${shareId}`,
dispatcher
);
if (extractedURL) {
const { patternMatch } = extract(normalizeURL(extractedURL));
if (patternMatch) {
noteId = patternMatch.id;
xsecToken = patternMatch.token;
}
}
noteId = patternMatch?.id;
xsecToken = patternMatch?.token;
}
if (!noteId || !xsecToken) return { error: "fetch.short_link" };

3
api/src/processing/services/youtube.js
View File

@ -165,7 +165,8 @@ export default async function (o) {
info = await yt.getBasicInfo(o.id, innertubeClient);
} catch (e) {
if (e?.info) {
const errorInfo = JSON.parse(e?.info);
let errorInfo;
try { errorInfo = JSON.parse(e?.info); } catch {}
if (errorInfo?.reason === "This video is private") {
return { error: "content.video.private" };

15
api/src/processing/url.js
View File

@ -3,6 +3,7 @@ import { strict as assert } from "node:assert";
import { env } from "../config.js";
import { services } from "./service-config.js";
import { getRedirectingURL } from "../misc/utils.js";
import { friendlyServiceName } from "./service-alias.js";
function aliasURL(url) {
@ -221,3 +222,17 @@ export function extract(url) {
return { host, patternMatch };
}
export async function resolveRedirectingURL(url, dispatcher, userAgent) {
const originalService = getHostIfValid(normalizeURL(url));
if (!originalService) return;
const canonicalURL = await getRedirectingURL(url, dispatcher, userAgent);
if (!canonicalURL) return;
const { host, patternMatch } = extract(normalizeURL(canonicalURL));
if (host === originalService) {
return patternMatch;
}
}

2
api/src/util/test.js
View File

@ -13,7 +13,7 @@ const getTests = (service) => loadJSON(getTestPath(service));
// services that are known to frequently fail due to external
// factors (e.g. rate limiting)
const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter']);
const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']);
const runTestsFor = async (service) => {
const tests = getTests(service);

4
api/src/util/tests/bilibili.json
View File

@ -41,7 +41,7 @@
},
{
"name": "b23.tv shortlink",
"url": "https://b23.tv/lbMyOI9",
"url": "https://b23.tv/av32430100",
"params": {},
"expected": {
"code": 200,
@ -57,4 +57,4 @@
"status": "tunnel"
}
}
]
]

22
api/src/util/tests/instagram.json
View File

@ -1,11 +1,11 @@
[
{
"name": "single photo post",
"url": "https://www.instagram.com/p/CwIgW8Yu5-I/",
"url": "https://www.instagram.com/p/DFx6KVduFWy/",
"params": {},
"expected": {
"code": 200,
"status": "redirect"
"status": "tunnel"
}
},
{
@ -19,7 +19,7 @@
},
{
"name": "reel",
"url": "https://www.instagram.com/reel/CoEBV3eM4QR/",
"url": "https://www.instagram.com/reel/DFQe23tOWKz/",
"params": {},
"expected": {
"code": 200,
@ -37,7 +37,7 @@
},
{
"name": "reel (isAudioOnly)",
"url": "https://www.instagram.com/reel/CoEBV3eM4QR/",
"url": "https://www.instagram.com/reel/DFQe23tOWKz/",
"params": {
"downloadMode": "audio"
},
@ -48,7 +48,7 @@
},
{
"name": "reel (isAudioMuted)",
"url": "https://www.instagram.com/reel/CoEBV3eM4QR/",
"url": "https://www.instagram.com/reel/DFQe23tOWKz/",
"params": {
"downloadMode": "mute"
},
@ -119,5 +119,15 @@
"code": 200,
"status": "redirect"
}
},
{
"name": "private instagram post",
"url": "https://www.instagram.com/p/C5_A1TQNPrYw4c2g9KAUTPUl8RVHqiAdAcOOSY0",
"params": {},
"expected": {
"code": 400,
"status": "error",
"errorCode": "error.api.content.post.private"
}
}
]
]

2
api/src/util/tests/snapchat.json
View File

@ -26,4 +26,4 @@
"status": "picker"
}
}
]
]