diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index 76d7a3eb..62bf6351 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -1,17 +1,26 @@ import { request } from 'undici'; const redirectStatuses = new Set([301, 302, 303, 307, 308]); -export async function getRedirectingURL(url, dispatcher, userAgent) { - const location = await request(url, { +export async function getRedirectingURL(url, dispatcher, headers) { + const params = { dispatcher, method: 'HEAD', - headers: { 'user-agent': userAgent } - }).then(r => { + headers, + redirect: 'manual' + }; + + let location = await request(url, params).then(r => { if (redirectStatuses.has(r.statusCode) && r.headers['location']) { return r.headers['location']; } }).catch(() => null); + location ??= await fetch(url, params).then(r => { + if (redirectStatuses.has(r.status) && r.headers.has('location')) { + return r.headers.get('location'); + } + }).catch(() => null); + return location; } diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 00fa4ebf..87a71c38 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -90,7 +90,9 @@ export const services = { "r/u_:user/comments/:id/:title", "r/u_:user/comments/:id/comment/:commentId", - "r/:sub/s/:shareId" + "r/:sub/s/:shareId", + + "video/:shortId", ], subdomains: "*", }, diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index 8735f123..2412fd46 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -23,7 +23,8 @@ export const testers = { pattern.id?.length <= 16 && !pattern.sub && !pattern.user || (pattern.sub?.length <= 22 && pattern.id?.length <= 16) || (pattern.user?.length <= 22 && pattern.id?.length <= 16) - || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16), + || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16) + || (pattern.shortId?.length <= 16), "rutube": pattern => (pattern.id?.length === 32 && pattern.key?.length <= 32) || diff --git a/api/src/processing/services/instagram.js b/api/src/processing/services/instagram.js index 9cc7dbdf..0fa25527 100644 --- a/api/src/processing/services/instagram.js +++ b/api/src/processing/services/instagram.js @@ -527,7 +527,7 @@ export default function instagram(obj) { // for some reason instagram decides to return HTML // instead of a redirect when requesting with a normal // browser user-agent - 'curl/7.88.1' + {'User-Agent': 'curl/7.88.1'} ).then(match => instagram({ ...obj, ...match, shareId: undefined diff --git a/api/src/processing/services/reddit.js b/api/src/processing/services/reddit.js index 50c78d35..0f506eea 100644 --- a/api/src/processing/services/reddit.js +++ b/api/src/processing/services/reddit.js @@ -50,12 +50,24 @@ async function getAccessToken() { export default async function(obj) { let params = obj; + const accessToken = await getAccessToken(); + const headers = { + 'user-agent': genericUserAgent, + authorization: accessToken && `Bearer ${accessToken}`, + accept: 'application/json' + }; + + if (params.shortId) { + params = await resolveRedirectingURL( + `https://www.reddit.com/video/${params.shortId}`, + obj.dispatcher, headers + ); + } if (!params.id && params.shareId) { params = await resolveRedirectingURL( `https://www.reddit.com/r/${params.sub}/s/${params.shareId}`, - obj.dispatcher, - genericUserAgent + obj.dispatcher, headers ); } @@ -63,17 +75,10 @@ export default async function(obj) { const url = new URL(`https://www.reddit.com/comments/${params.id}.json`); - const accessToken = await getAccessToken(); if (accessToken) url.hostname = 'oauth.reddit.com'; let data = await fetch( - url, { - headers: { - 'User-Agent': genericUserAgent, - accept: 'application/json', - authorization: accessToken && `Bearer ${accessToken}` - } - } + url, { headers } ).then(r => r.json()).catch(() => {}); if (!data || !Array.isArray(data)) { diff --git a/api/src/processing/url.js b/api/src/processing/url.js index 82299999..86c333f6 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -106,6 +106,14 @@ function aliasURL(url) { url.pathname = `/share/${idPart.slice(-32)}`; } break; + + case "redd": + /* reddit short video links can be treated by changing https://v.redd.it/ + to https://reddit.com/video/.*/ + if (url.hostname === "v.redd.it" && parts.length === 2) { + url = new URL(`https://www.reddit.com/video/${parts[1]}`); + } + break; } return url; @@ -231,11 +239,11 @@ export function extract(url) { return { host, patternMatch }; } -export async function resolveRedirectingURL(url, dispatcher, userAgent) { +export async function resolveRedirectingURL(url, dispatcher, headers) { const originalService = getHostIfValid(normalizeURL(url)); if (!originalService) return; - const canonicalURL = await getRedirectingURL(url, dispatcher, userAgent); + const canonicalURL = await getRedirectingURL(url, dispatcher, headers); if (!canonicalURL) return; const { host, patternMatch } = extract(normalizeURL(canonicalURL)); diff --git a/api/src/util/tests/instagram.json b/api/src/util/tests/instagram.json index 1df87b9a..4adcf6f8 100644 --- a/api/src/util/tests/instagram.json +++ b/api/src/util/tests/instagram.json @@ -123,6 +123,7 @@ { "name": "private instagram post", "url": "https://www.instagram.com/p/C5_A1TQNPrYw4c2g9KAUTPUl8RVHqiAdAcOOSY0", + "canFail": true, "params": {}, "expected": { "code": 400, diff --git a/api/src/util/tests/reddit.json b/api/src/util/tests/reddit.json index 3afc6126..1dd10ee5 100644 --- a/api/src/util/tests/reddit.json +++ b/api/src/util/tests/reddit.json @@ -56,5 +56,23 @@ "code": 200, "status": "tunnel" } + }, + { + "name": "shortened video link", + "url": "https://v.redd.it/ifg2emt5ck0e1", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "shortened video link (alternative)", + "url": "https://reddit.com/video/ifg2emt5ck0e1", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } } ] \ No newline at end of file