From 83af16bb12c19371b458947d7a49f634eb980b63 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 25 Jul 2024 11:57:02 +0600 Subject: [PATCH 1/4] facebook: do not request the source url & clean up fixed a vulnerability, removed useless variables, and cleaned up thankfully we haven't built the image yesterday --- src/modules/processing/match.js | 3 +- src/modules/processing/services/facebook.js | 46 ++++++++----------- src/modules/processing/servicesConfig.json | 2 +- .../processing/servicesPatternTesters.js | 3 +- 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index b7d46333..ec052e60 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -203,8 +203,7 @@ export default async function(host, patternMatch, lang, obj) { break; case "facebook": r = await facebook({ - ...patternMatch, - sourceUrl: url.href + ...patternMatch }); break; default: diff --git a/src/modules/processing/services/facebook.js b/src/modules/processing/services/facebook.js index 45d31b5f..17dedab4 100644 --- a/src/modules/processing/services/facebook.js +++ b/src/modules/processing/services/facebook.js @@ -4,59 +4,53 @@ const headers = { 'User-Agent': genericUserAgent, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate, br', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', } -function resolveUrl(url) { +const resolveUrl = (url) => { return fetch(url, { headers }) .then(r => { if (r.headers.get('location')) { - return decodeURIComponent(r.headers.get('location')) + return decodeURIComponent(r.headers.get('location')); } if (r.headers.get('link')) { - const linkMatch = r.headers.get('link').match(/<(.*?)\/>/) - return decodeURIComponent(linkMatch[1]) + const linkMatch = r.headers.get('link').match(/<(.*?)\/>/); + return decodeURIComponent(linkMatch[1]); } - return false + return false; }) - .catch(() => false) + .catch(() => false); } -export default async function({ sourceUrl, shortLink, username, id }) { - const isShortLink = !!shortLink?.length - const isSharedLink = !!sourceUrl.match(/\/share\/\w\//)?.length +export default async function({ id, shareType, shortLink }) { + let url = `https://web.facebook.com/i/videos/${id}`; - let url = isShortLink - ? `https://fb.watch/${shortLink}` - : `https://web.facebook.com/${username}/videos/${id}` - - if (isShortLink) url = await resolveUrl(url) - if (isSharedLink) url = sourceUrl + if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`; + if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`); const html = await fetch(url, { headers }) .then(r => r.text()) - .catch(() => false) + .catch(() => false); if (!html) return { error: 'ErrorCouldntFetch' }; - const urls = [] - const hd = html.match('"browser_native_hd_url":(".*?")') - const sd = html.match('"browser_native_sd_url":(".*?")') + const urls = []; + const hd = html.match('"browser_native_hd_url":(".*?")'); + const sd = html.match('"browser_native_sd_url":(".*?")'); - if (hd?.[1]) urls.push(JSON.parse(hd[1])) - if (sd?.[1]) urls.push(JSON.parse(sd[1])) + if (hd?.[1]) urls.push(JSON.parse(hd[1])); + if (sd?.[1]) urls.push(JSON.parse(sd[1])); if (!urls.length) { return { error: 'ErrorEmptyDownload' }; } - let filename = `facebook_${id || shortLink}.mp4` + const baseFilename = `facebook_${id || shortLink}`; return { urls: urls[0], - filename, - audioFilename: `${filename.slice(0, -4)}_audio`, + filename: `${baseFilename}.mp4`, + audioFilename: `${baseFilename}_audio`, }; -} \ No newline at end of file +} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 9161117a..6955a946 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -134,7 +134,7 @@ ":username/videos/:caption/:id", ":username/videos/:id", "reel/:id", - "share/:shortLink/:id" + "share/:shareType/:id" ], "enabled": true } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 9f4e0f91..48f953c1 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -68,5 +68,6 @@ export const testers = { patternMatch.shortLink?.length <= 11 || patternMatch.username?.length <= 30 || patternMatch.caption?.length <= 255 - || patternMatch.id?.length <= 20, + || patternMatch.id?.length <= 20 && !patternMatch.shareType + || patternMatch.id?.length <= 20 && patternMatch.shareType?.length === 1, } From a388839e15dd1002cb781dcec12bf2112ac4617e Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 25 Jul 2024 11:58:43 +0600 Subject: [PATCH 2/4] snapchat: clean up --- src/modules/processing/match.js | 2 +- src/modules/processing/services/snapchat.js | 26 ++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index ec052e60..6e600cf0 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -192,7 +192,7 @@ export default async function(host, patternMatch, lang, obj) { break; case "snapchat": r = await snapchat({ - url, + hostname: url.hostname, ...patternMatch }); break; diff --git a/src/modules/processing/services/snapchat.js b/src/modules/processing/services/snapchat.js index 44a2b84e..a93f0933 100644 --- a/src/modules/processing/services/snapchat.js +++ b/src/modules/processing/services/snapchat.js @@ -9,8 +9,8 @@ async function getSpotlight(id) { const html = await fetch(`https://www.snapchat.com/spotlight/${id}`, { headers: { 'User-Agent': genericUserAgent } }).then((r) => r.text()).catch(() => null); - if (!html) { - return { error: 'ErrorCouldntFetch' }; + if (!html) { + return { error: 'ErrorCouldntFetch' }; } const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1]; @@ -27,9 +27,9 @@ async function getStory(username, storyId) { const html = await fetch(`https://www.snapchat.com/add/${username}${storyId ? `/${storyId}` : ''}`, { headers: { 'User-Agent': genericUserAgent } }).then((r) => r.text()).catch(() => null); - if (!html) { - return { error: 'ErrorCouldntFetch' }; - } + if (!html) { + return { error: 'ErrorCouldntFetch' }; + } const nextDataString = html.match(NEXT_DATA_REGEX)?.[1]; if (nextDataString) { @@ -67,18 +67,18 @@ async function getStory(username, storyId) { } } -export default async function(obj) { +export default async function (obj) { let params = obj; - if (obj.url.hostname === 't.snapchat.com' && obj.shortLink) { + if (obj.hostname === 't.snapchat.com' && obj.shortLink) { const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`); - - if (!link?.startsWith('https://www.snapchat.com/')) { - return { error: 'ErrorCouldntFetch' }; - } + + if (!link?.startsWith('https://www.snapchat.com/')) { + return { error: 'ErrorCouldntFetch' }; + } const extractResult = extract(normalizeURL(link)); - if (extractResult?.host !== 'snapchat') { - return { error: 'ErrorCouldntFetch' }; + if (extractResult?.host !== 'snapchat') { + return { error: 'ErrorCouldntFetch' }; } params = extractResult.patternMatch; From 8ab0661f1aa7e4229fbc2bfc02b46ef633e7c589 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 25 Jul 2024 12:08:37 +0600 Subject: [PATCH 3/4] tests: update rutube yappy link --- src/util/tests.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/tests.json b/src/util/tests.json index ac65bf0b..6399d8ac 100644 --- a/src/util/tests.json +++ b/src/util/tests.json @@ -1064,7 +1064,7 @@ } }, { "name": "yappy", - "url": "https://rutube.ru/yappy/a06b1bf53bce403b9a069107f23c47eb/", + "url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/", "params": {}, "expected": { "code": 200, From cb2bde5b5175089bdd32c0d3d1552637393c0547 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 25 Jul 2024 12:14:25 +0600 Subject: [PATCH 4/4] tests: allow facebook reel test to fail --- src/util/tests.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/tests.json b/src/util/tests.json index 6399d8ac..1093d75c 100644 --- a/src/util/tests.json +++ b/src/util/tests.json @@ -1231,6 +1231,7 @@ }, { "name": "reel video", "url": "https://web.facebook.com/reel/730293269054758", + "canFail": true, "params": {}, "expected": { "code": 200,