diff --git a/src/modules/processing/services/bilibili.js b/src/modules/processing/services/bilibili.js deleted file mode 100644 index c562406a..00000000 --- a/src/modules/processing/services/bilibili.js +++ /dev/null @@ -1,105 +0,0 @@ -import { genericUserAgent, env } from "../../config.js"; - -// TO-DO: higher quality downloads (currently requires an account) - -function com_resolveShortlink(shortId) { - return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' }) - .then(r => r.status > 300 && r.status < 400 && r.headers.get('location')) - .then(url => { - if (!url) return; - const path = new URL(url).pathname; - if (path.startsWith('/video/')) - return path.split('/')[2]; - }) - .catch(() => {}) -} - -function getBest(content) { - return content?.filter(v => v.baseUrl || v.url) - .map(v => (v.baseUrl = v.baseUrl || v.url, v)) - .reduce((a, b) => a?.bandwidth > b?.bandwidth ? a : b); -} - -function extractBestQuality(dashData) { - const bestVideo = getBest(dashData.video), - bestAudio = getBest(dashData.audio); - - if (!bestVideo || !bestAudio) return []; - return [ bestVideo, bestAudio ]; -} - -async function com_download(id) { - let html = await fetch(`https://bilibili.com/video/${id}`, { - headers: { "user-agent": genericUserAgent } - }).then(r => r.text()).catch(() => {}); - if (!html) return { error: 'ErrorCouldntFetch' }; - - if (!(html.includes('')[0]); - if (streamData.data.timelength > env.durationLimit * 1000) { - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - } - - const [ video, audio ] = extractBestQuality(streamData.data.dash); - if (!video || !audio) { - return { error: 'ErrorEmptyDownload' }; - } - - return { - urls: [video.baseUrl, audio.baseUrl], - audioFilename: `bilibili_${id}_audio`, - filename: `bilibili_${id}_${video.width}x${video.height}.mp4` - }; -} - -async function tv_download(id) { - const url = new URL( - 'https://api.bilibili.tv/intl/gateway/web/playurl' - + '?s_locale=en_US&platform=web&qn=64&type=0&device=wap' - + '&tf=0&spm_id=bstar-web.ugc-video-detail.0.0&from_spm_id=' - ); - - url.searchParams.set('aid', id); - - const { data } = await fetch(url).then(a => a.json()); - if (!data?.playurl?.video) { - return { error: 'ErrorEmptyDownload' }; - } - - const [ video, audio ] = extractBestQuality({ - video: data.playurl.video.map(s => s.video_resource) - .filter(s => s.codecs.includes('avc1')), - audio: data.playurl.audio_resource - }); - - if (!video || !audio) { - return { error: 'ErrorEmptyDownload' }; - } - - if (video.duration > env.durationLimit * 1000) { - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - } - - return { - urls: [video.url, audio.url], - audioFilename: `bilibili_tv_${id}_audio`, - filename: `bilibili_tv_${id}.mp4` - }; -} - -export default async function({ comId, tvId, comShortLink }) { - if (comShortLink) { - comId = await com_resolveShortlink(comShortLink); - } - - if (comId) { - return com_download(comId); - } else if (tvId) { - return tv_download(tvId); - } - - return { error: 'ErrorCouldntFetch' }; -} diff --git a/src/modules/processing/services/dailymotion.js b/src/modules/processing/services/dailymotion.js deleted file mode 100644 index c1fca95d..00000000 --- a/src/modules/processing/services/dailymotion.js +++ /dev/null @@ -1,107 +0,0 @@ -import HLSParser from 'hls-parser'; -import { env } from '../../config.js'; - -let _token; - -function getExp(token) { - return JSON.parse( - Buffer.from(token.split('.')[1], 'base64') - ).exp * 1000; -} - -const getToken = async () => { - if (_token && getExp(_token) > new Date().getTime()) { - return _token; - } - - const req = await fetch('https://graphql.api.dailymotion.com/oauth/token', { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8', - 'User-Agent': 'dailymotion/240213162706 CFNetwork/1492.0.1 Darwin/23.3.0', - 'Authorization': 'Basic MGQyZDgyNjQwOWFmOWU3MmRiNWQ6ODcxNmJmYTVjYmEwMmUwMGJkYTVmYTg1NTliNDIwMzQ3NzIyYWMzYQ==' - }, - body: 'traffic_segment=&grant_type=client_credentials' - }).then(r => r.json()).catch(() => {}); - - if (req.access_token) { - return _token = req.access_token; - } -} - -export default async function({ id }) { - const token = await getToken(); - if (!token) return { error: 'ErrorSomethingWentWrong' }; - - const req = await fetch('https://graphql.api.dailymotion.com/', - { - method: 'POST', - headers: { - 'User-Agent': 'dailymotion/240213162706 CFNetwork/1492.0.1 Darwin/23.3.0', - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - 'X-DM-AppInfo-Version': '7.16.0_240213162706', - 'X-DM-AppInfo-Type': 'iosapp', - 'X-DM-AppInfo-Id': 'com.dailymotion.dailymotion' - }, - body: JSON.stringify({ - operationName: "Media", - query: ` - query Media($xid: String!, $password: String) { - media(xid: $xid, password: $password) { - __typename - ... on Video { - xid - hlsURL - duration - title - channel { - displayName - } - } - } - } - `, - variables: { xid: id } - }) - } - ).then(r => r.status === 200 && r.json()).catch(() => {}); - - const media = req?.data?.media; - - if (media?.__typename !== 'Video' || !media.hlsURL) { - return { error: 'ErrorEmptyDownload' } - } - - if (media.duration > env.durationLimit) { - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - } - - const manifest = await fetch(media.hlsURL).then(r => r.text()).catch(() => {}); - if (!manifest) return { error: 'ErrorSomethingWentWrong' }; - - const bestQuality = HLSParser.parse(manifest).variants - .filter(v => v.codecs.includes('avc1')) - .reduce((a, b) => a.bandwidth > b.bandwidth ? a : b); - if (!bestQuality) return { error: 'ErrorEmptyDownload' } - - const fileMetadata = { - title: media.title, - artist: media.channel.displayName - } - - return { - urls: bestQuality.uri, - isM3U8: true, - filenameAttributes: { - service: 'dailymotion', - id: media.xid, - title: fileMetadata.title, - author: fileMetadata.artist, - resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`, - qualityLabel: `${bestQuality.resolution.height}p`, - extension: 'mp4' - }, - fileMetadata - } -} \ No newline at end of file diff --git a/src/modules/processing/services/loom.js b/src/modules/processing/services/loom.js deleted file mode 100644 index ecb6c534..00000000 --- a/src/modules/processing/services/loom.js +++ /dev/null @@ -1,39 +0,0 @@ -import { genericUserAgent } from "../../config.js"; - -export default async function({ id }) { - const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/transcoded-url`, { - method: "POST", - headers: { - "user-agent": genericUserAgent, - origin: "https://www.loom.com", - referer: `https://www.loom.com/share/${id}`, - cookie: `loom_referral_video=${id};`, - - "apollographql-client-name": "web", - "apollographql-client-version": "14c0b42", - "x-loom-request-source": "loom_web_14c0b42", - }, - body: JSON.stringify({ - force_original: false, - password: null, - anonID: null, - deviceID: null - }) - }) - .then(r => r.status === 200 ? r.json() : false) - .catch(() => {}); - - if (!gql) return { error: 'ErrorEmptyDownload' }; - - const videoUrl = gql?.url; - - if (videoUrl?.includes('.mp4?')) { - return { - urls: videoUrl, - filename: `loom_${id}.mp4`, - audioFilename: `loom_${id}_audio` - } - } - - return { error: 'ErrorEmptyDownload' } -} diff --git a/src/modules/processing/services/ok.js b/src/modules/processing/services/ok.js deleted file mode 100644 index 295d5b81..00000000 --- a/src/modules/processing/services/ok.js +++ /dev/null @@ -1,64 +0,0 @@ -import { genericUserAgent, env } from "../../config.js"; -import { cleanString } from "../../sub/utils.js"; - -const resolutions = { - "ultra": "2160", - "quad": "1440", - "full": "1080", - "hd": "720", - "sd": "480", - "low": "360", - "lowest": "240", - "mobile": "144" -} - -export default async function(o) { - let quality = o.quality === "max" ? "2160" : o.quality; - - let html = await fetch(`https://ok.ru/video/${o.id}`, { - headers: { "user-agent": genericUserAgent } - }).then(r => r.text()).catch(() => {}); - - if (!html) return { error: 'ErrorCouldntFetch' }; - if (!html.includes(`
env.durationLimit) - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - - let videos = videoData.videos.filter(v => !v.disallowed); - let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1]; - - let fileMetadata = { - title: cleanString(videoData.movie.title.trim()), - author: cleanString(videoData.author.name.trim()), - } - - if (bestVideo) return { - urls: bestVideo.url, - filenameAttributes: { - service: "ok", - id: o.id, - title: fileMetadata.title, - author: fileMetadata.author, - resolution: `${resolutions[bestVideo.name]}p`, - qualityLabel: `${resolutions[bestVideo.name]}p`, - extension: "mp4" - } - } - - return { error: 'ErrorEmptyDownload' } -} diff --git a/src/modules/processing/services/pinterest.js b/src/modules/processing/services/pinterest.js deleted file mode 100644 index f823d6bb..00000000 --- a/src/modules/processing/services/pinterest.js +++ /dev/null @@ -1,43 +0,0 @@ -import { genericUserAgent } from "../../config.js"; - -const videoRegex = /"url":"(https:\/\/v1.pinimg.com\/videos\/.*?)"/g; -const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g; - -export default async function(o) { - let id = o.id; - - if (!o.id && o.shortLink) { - id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" }) - .then(r => r.headers.get("location").split('pin/')[1].split('/')[0]) - .catch(() => {}); - } - if (id.includes("--")) id = id.split("--")[1]; - if (!id) return { error: 'ErrorCouldntFetch' }; - - let html = await fetch(`https://www.pinterest.com/pin/${id}/`, { - headers: { "user-agent": genericUserAgent } - }).then(r => r.text()).catch(() => {}); - - if (!html) return { error: 'ErrorCouldntFetch' }; - - let videoLink = [...html.matchAll(videoRegex)] - .map(([, link]) => link) - .find(a => a.endsWith('.mp4') && a.includes('720p')); - - if (videoLink) return { - urls: videoLink, - filename: `pinterest_${o.id}.mp4`, - audioFilename: `pinterest_${o.id}_audio` - } - - let imageLink = [...html.matchAll(imageRegex)] - .map(([, link]) => link) - .find(a => a.endsWith('.jpg') || a.endsWith('.gif')); - - if (imageLink) return { - urls: imageLink, - isPhoto: true - } - - return { error: 'ErrorEmptyDownload' }; -} diff --git a/src/modules/processing/services/reddit.js b/src/modules/processing/services/reddit.js deleted file mode 100644 index 41f9efb4..00000000 --- a/src/modules/processing/services/reddit.js +++ /dev/null @@ -1,124 +0,0 @@ -import { genericUserAgent, env } from "../../config.js"; -import { getCookie, updateCookieValues } from "../cookie/manager.js"; - -async function getAccessToken() { - /* "cookie" in cookiefile needs to contain: - * client_id, client_secret, refresh_token - * e.g. client_id=bla; client_secret=bla; refresh_token=bla - * - * you can get these by making a reddit app and - * authenticating an account against reddit's oauth2 api - * see: https://github.com/reddit-archive/reddit/wiki/OAuth2 - * - * any additional cookie fields are managed by this code and you - * should not touch them unless you know what you're doing. **/ - const cookie = await getCookie('reddit'); - if (!cookie) return; - - const values = cookie.values(), - needRefresh = !values.access_token - || !values.expiry - || Number(values.expiry) < new Date().getTime(); - if (!needRefresh) return values.access_token; - - const data = await fetch('https://www.reddit.com/api/v1/access_token', { - method: 'POST', - headers: { - 'authorization': `Basic ${Buffer.from( - [values.client_id, values.client_secret].join(':') - ).toString('base64')}`, - 'content-type': 'application/x-www-form-urlencoded', - 'user-agent': genericUserAgent, - 'accept': 'application/json' - }, - body: `grant_type=refresh_token&refresh_token=${encodeURIComponent(values.refresh_token)}` - }).then(r => r.json()).catch(() => {}); - if (!data) return; - - const { access_token, refresh_token, expires_in } = data; - if (!access_token) return; - - updateCookieValues(cookie, { - ...cookie.values(), - access_token, refresh_token, - expiry: new Date().getTime() + (expires_in * 1000), - }); - - return access_token; -} - -export default async function(obj) { - let url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`); - - if (obj.user) { - url.pathname = `/user/${obj.user}/comments/${obj.id}.json`; - } - - const accessToken = await getAccessToken(); - if (accessToken) url.hostname = 'oauth.reddit.com'; - - let data = await fetch( - url, { - headers: { - 'User-Agent': genericUserAgent, - accept: 'application/json', - authorization: accessToken && `Bearer ${accessToken}` - } - } - ).then(r => r.json()).catch(() => {}); - - if (!data || !Array.isArray(data)) return { error: 'ErrorCouldntFetch' }; - - data = data[0]?.data?.children[0]?.data; - - if (data?.url?.endsWith('.gif')) return { - typeId: "redirect", - urls: data.url - } - - if (!data.secure_media?.reddit_video) - return { error: 'ErrorEmptyDownload' }; - - if (data.secure_media?.reddit_video?.duration > env.durationLimit) - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - - let audio = false, - video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0], - audioFileLink = `${data.secure_media?.reddit_video?.fallback_url?.split('DASH')[0]}audio`; - - if (video.match('.mp4')) { - audioFileLink = `${video.split('_')[0]}_audio.mp4` - } - - // test the existence of audio - await fetch(audioFileLink, { method: "HEAD" }).then(r => { - if (Number(r.status) === 200) { - audio = true - } - }).catch(() => {}) - - // fallback for videos with variable audio quality - if (!audio) { - audioFileLink = `${video.split('_')[0]}_AUDIO_128.mp4` - await fetch(audioFileLink, { method: "HEAD" }).then(r => { - if (Number(r.status) === 200) { - audio = true - } - }).catch(() => {}) - } - - let id = `${String(obj.sub).toLowerCase()}_${obj.id}`; - - if (!audio) return { - typeId: "redirect", - urls: video - } - - return { - typeId: "stream", - type: "render", - urls: [video, audioFileLink], - audioFilename: `reddit_${id}_audio`, - filename: `reddit_${id}.mp4` - } -} diff --git a/src/modules/processing/services/rutube.js b/src/modules/processing/services/rutube.js deleted file mode 100644 index a8d0abbe..00000000 --- a/src/modules/processing/services/rutube.js +++ /dev/null @@ -1,74 +0,0 @@ -import HLS from 'hls-parser'; - -import { env } from "../../config.js"; -import { cleanString } from '../../sub/utils.js'; - -async function requestJSON(url) { - try { - const r = await fetch(url); - return await r.json(); - } catch {} -} - -export default async function(obj) { - if (obj.yappyId) { - const yappy = await requestJSON( - `https://rutube.ru/pangolin/api/web/yappy/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=15` - ) - const yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link; - if (!yappyURL) return { error: 'ErrorEmptyDownload' }; - - return { - urls: yappyURL, - filename: `rutube_yappy_${obj.yappyId}.mp4`, - audioFilename: `rutube_yappy_${obj.yappyId}_audio` - } - } - - const quality = obj.quality === "max" ? "9000" : obj.quality; - - const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`); - if (obj.key) requestURL.searchParams.set('p', obj.key); - - const play = await requestJSON(requestURL); - if (!play) return { error: 'ErrorCouldntFetch' }; - - if (play.detail || !play.video_balancer) return { error: 'ErrorEmptyDownload' }; - if (play.live_streams?.hls) return { error: 'ErrorLiveVideo' }; - - if (play.duration > env.durationLimit * 1000) - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - - let m3u8 = await fetch(play.video_balancer.m3u8) - .then(r => r.text()) - .catch(() => {}); - - if (!m3u8) return { error: 'ErrorCouldntFetch' }; - - m3u8 = HLS.parse(m3u8).variants.sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); - - let bestQuality = m3u8[0]; - if (Number(quality) < bestQuality.resolution.height) { - bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height)); - } - - const fileMetadata = { - title: cleanString(play.title.trim()), - artist: cleanString(play.author.name.trim()), - } - - return { - urls: bestQuality.uri, - isM3U8: true, - filenameAttributes: { - service: "rutube", - id: obj.id, - title: fileMetadata.title, - author: fileMetadata.artist, - resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`, - qualityLabel: `${bestQuality.resolution.height}p`, - extension: "mp4" - }, - fileMetadata: fileMetadata - } -} diff --git a/src/modules/processing/services/soundcloud.js b/src/modules/processing/services/soundcloud.js deleted file mode 100644 index 389eaf6c..00000000 --- a/src/modules/processing/services/soundcloud.js +++ /dev/null @@ -1,105 +0,0 @@ -import { env } from "../../config.js"; -import { cleanString } from "../../sub/utils.js"; - -const cachedID = { - version: '', - id: '' -} - -async function findClientID() { - try { - let sc = await fetch('https://soundcloud.com/').then(r => r.text()).catch(() => {}); - let scVersion = String(sc.match(/')[0] - const data = JSON.parse(json) - detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"] - } catch { - return { error: 'ErrorCouldntFetch' }; - } - - let video, videoFilename, audioFilename, audio, images, - filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`, - bestAudio = 'm4a'; - - images = detail.imagePost?.images; - - let playAddr = detail.video.playAddr; - if (obj.h265) { - const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0] - playAddr = h265PlayAddr || playAddr - } - - if (!obj.isAudioOnly && !images) { - video = playAddr; - videoFilename = `${filenameBase}.mp4`; - } else { - audio = playAddr; - audioFilename = `${filenameBase}_audio`; - - if (obj.fullAudio || !audio) { - audio = detail.music.playUrl; - audioFilename += `_original` - } - if (audio.includes("mime_type=audio_mpeg")) bestAudio = 'mp3'; - } - - if (video) { - return { - urls: video, - filename: videoFilename, - headers: { cookie } - } - } - - if (images && obj.isAudioOnly) { - return { - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - bestAudio, - headers: { cookie } - } - } - - if (images) { - let imageLinks = images - .map(i => i.imageURL.urlList.find(p => p.includes(".jpeg?"))) - .map(url => ({ url })); - - return { - picker: imageLinks, - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - bestAudio, - headers: { cookie } - } - } - - if (audio) { - return { - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - bestAudio, - headers: { cookie } - } - } -} diff --git a/src/modules/processing/services/tumblr.js b/src/modules/processing/services/tumblr.js deleted file mode 100644 index b2866c8f..00000000 --- a/src/modules/processing/services/tumblr.js +++ /dev/null @@ -1,70 +0,0 @@ -import psl from "psl"; - -const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z'; -const API_BASE = 'https://api-http2.tumblr.com'; - -function request(domain, id) { - const url = new URL(`/v2/blog/${domain}/posts/${id}/permalink`, API_BASE); - url.searchParams.set('api_key', API_KEY); - url.searchParams.set('fields[blogs]', 'uuid,name,avatar,?description,?can_message,?can_be_followed,?is_adult,?reply_conditions,' - + '?theme,?title,?url,?is_blocked_from_primary,?placement_id,?primary,?updated,?followed,' - + '?ask,?can_subscribe,?paywall_access,?subscription_plan,?is_blogless_advertiser,?tumblrmart_accessories'); - - return fetch(url, { - headers: { - 'User-Agent': 'Tumblr/iPhone/33.3/333010/17.3.1/tumblr', - 'X-Version': 'iPhone/33.3/333010/17.3.1/tumblr' - } - }).then(a => a.json()).catch(() => {}); -} - -export default async function(input) { - let { subdomain } = psl.parse(input.url.hostname); - - if (subdomain?.includes('.')) { - return { error: ['ErrorBrokenLink', 'tumblr'] } - } else if (subdomain === 'www' || subdomain === 'at') { - subdomain = undefined - } - - const domain = `${subdomain ?? input.user}.tumblr.com`; - const data = await request(domain, input.id); - - const element = data?.response?.timeline?.elements?.[0]; - if (!element) return { error: 'ErrorEmptyDownload' }; - - const contents = [ - ...element.content, - ...element?.trail?.map(t => t.content).flat() - ] - - const audio = contents.find(c => c.type === 'audio'); - if (audio && audio.provider === 'tumblr') { - const fileMetadata = { - title: audio?.title, - artist: audio?.artist - }; - - return { - urls: audio.media.url, - filenameAttributes: { - service: 'tumblr', - id: input.id, - title: fileMetadata.title, - author: fileMetadata.artist - }, - isAudioOnly: true - } - } - - const video = contents.find(c => c.type === 'video'); - if (video && video.provider === 'tumblr') { - return { - urls: video.media.url, - filename: `tumblr_${input.id}.mp4`, - audioFilename: `tumblr_${input.id}_audio` - } - } - - return { error: 'ErrorEmptyDownload' } -} diff --git a/src/modules/processing/services/twitch.js b/src/modules/processing/services/twitch.js deleted file mode 100644 index 13cee19f..00000000 --- a/src/modules/processing/services/twitch.js +++ /dev/null @@ -1,87 +0,0 @@ -import { env } from "../../config.js"; -import { cleanString } from '../../sub/utils.js'; - -const gqlURL = "https://gql.twitch.tv/gql"; -const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; - -export default async function (obj) { - let req_metadata = await fetch(gqlURL, { - method: "POST", - headers: clientIdHead, - body: JSON.stringify({ - query: `{ - clip(slug: "${obj.clipId}") { - broadcaster { - login - } - createdAt - curator { - login - } - durationSeconds - id - medium: thumbnailURL(width: 480, height: 272) - title - videoQualities { - quality - sourceURL - } - } - }` - }) - }).then(r => r.status === 200 ? r.json() : false).catch(() => {}); - if (!req_metadata) return { error: 'ErrorCouldntFetch' }; - - let clipMetadata = req_metadata.data.clip; - - if (clipMetadata.durationSeconds > env.durationLimit) - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) - return { error: 'ErrorEmptyDownload' }; - - let req_token = await fetch(gqlURL, { - method: "POST", - headers: clientIdHead, - body: JSON.stringify([ - { - "operationName": "VideoAccessToken_Clip", - "variables": { - "slug": obj.clipId - }, - "extensions": { - "persistedQuery": { - "version": 1, - "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11" - } - } - } - ]) - }).then(r => r.status === 200 ? r.json() : false).catch(() => {}); - - if (!req_token) return { error: 'ErrorCouldntFetch' }; - - let formats = clipMetadata.videoQualities; - let format = formats.find(f => f.quality === obj.quality) || formats[0]; - - return { - type: "bridge", - urls: `${format.sourceURL}?${new URLSearchParams({ - sig: req_token[0].data.clip.playbackAccessToken.signature, - token: req_token[0].data.clip.playbackAccessToken.value - })}`, - fileMetadata: { - title: cleanString(clipMetadata.title.trim()), - artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, - }, - filenameAttributes: { - service: "twitch", - id: clipMetadata.id, - title: cleanString(clipMetadata.title.trim()), - author: `${clipMetadata.broadcaster.login}, clipped by ${clipMetadata.curator.login}`, - qualityLabel: `${format.quality}p`, - extension: 'mp4' - }, - filename: `twitchclip_${clipMetadata.id}_${format.quality}p.mp4`, - audioFilename: `twitchclip_${clipMetadata.id}_audio` - } -} diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js deleted file mode 100644 index 36a8669b..00000000 --- a/src/modules/processing/services/twitter.js +++ /dev/null @@ -1,191 +0,0 @@ -import { genericUserAgent } from "../../config.js"; -import { createStream } from "../../stream/manage.js"; -import { getCookie, updateCookie } from "../cookie/manager.js"; - -const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId'; -const tokenURL = 'https://api.x.com/1.1/guest/activate.json'; - -const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}); - -const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false}); - -const commonHeaders = { - "user-agent": genericUserAgent, - "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", - "x-twitter-client-language": "en", - "x-twitter-active-user": "yes", - "accept-language": "en" -} - -// fix all videos affected by the container bug in twitter muxer (took them over two weeks to fix it????) -const TWITTER_EPOCH = 1288834974657n; -const badContainerStart = new Date(1701446400000); -const badContainerEnd = new Date(1702605600000); - -function needsFixing(media) { - const representativeId = media.source_status_id_str ?? media.id_str; - const mediaTimestamp = new Date( - Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH) - ); - return mediaTimestamp > badContainerStart && mediaTimestamp < badContainerEnd -} - -function bestQuality(arr) { - return arr.filter(v => v.content_type === "video/mp4") - .reduce((a, b) => Number(a?.bitrate) > Number(b?.bitrate) ? a : b) - .url -} - -let _cachedToken; -const getGuestToken = async (dispatcher, forceReload = false) => { - if (_cachedToken && !forceReload) { - return _cachedToken; - } - - const tokenResponse = await fetch(tokenURL, { - method: 'POST', - headers: commonHeaders, - dispatcher - }).then(r => r.status === 200 && r.json()).catch(() => {}) - - if (tokenResponse?.guest_token) { - return _cachedToken = tokenResponse.guest_token - } -} - -const requestTweet = async(dispatcher, tweetId, token, cookie) => { - const graphqlTweetURL = new URL(graphqlURL); - - let headers = { - ...commonHeaders, - 'content-type': 'application/json', - 'x-guest-token': token, - cookie: `guest_id=${encodeURIComponent(`v1:${token}`)}` - } - - if (cookie) { - headers = { - ...commonHeaders, - 'content-type': 'application/json', - 'X-Twitter-Auth-Type': 'OAuth2Session', - 'x-csrf-token': cookie.values().ct0, - cookie - } - } - - graphqlTweetURL.searchParams.set('variables', - JSON.stringify({ - tweetId, - withCommunity: false, - includePromotedContent: false, - withVoice: false - }) - ); - graphqlTweetURL.searchParams.set('features', tweetFeatures); - graphqlTweetURL.searchParams.set('fieldToggles', tweetFieldToggles); - - let result = await fetch(graphqlTweetURL, { headers, dispatcher }); - updateCookie(cookie, result.headers); - - // we might have been missing the `ct0` cookie, retry - if (result.status === 403 && result.headers.get('set-cookie')) { - result = await fetch(graphqlTweetURL, { - headers: { - ...headers, - 'x-csrf-token': cookie.values().ct0 - }, - dispatcher - }); - } - - return result -} - -export default async function({ id, index, toGif, dispatcher }) { - const cookie = await getCookie('twitter'); - - let guestToken = await getGuestToken(dispatcher); - if (!guestToken) return { error: 'ErrorCouldntFetch' }; - - let tweet = await requestTweet(dispatcher, id, guestToken); - - // get new token & retry if old one expired - if ([403, 429].includes(tweet.status)) { - guestToken = await getGuestToken(dispatcher, true); - tweet = await requestTweet(dispatcher, id, guestToken) - } - - tweet = await tweet.json(); - - let tweetTypename = tweet?.data?.tweetResult?.result?.__typename; - - if (tweetTypename === "TweetUnavailable") { - const reason = tweet?.data?.tweetResult?.result?.reason; - switch(reason) { - case "Protected": - return { error: 'ErrorTweetProtected' } - case "NsfwLoggedOut": - if (cookie) { - tweet = await requestTweet(dispatcher, id, guestToken, cookie); - tweet = await tweet.json(); - tweetTypename = tweet?.data?.tweetResult?.result?.__typename; - } else return { error: 'ErrorTweetNSFW' } - } - } - - if (!["Tweet", "TweetWithVisibilityResults"].includes(tweetTypename)) { - return { error: 'ErrorTweetUnavailable' } - } - - let tweetResult = tweet.data.tweetResult.result, - baseTweet = tweetResult.legacy, - repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities; - - if (tweetTypename === "TweetWithVisibilityResults") { - baseTweet = tweetResult.tweet.legacy; - repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities; - } - - let media = (repostedTweet?.media || baseTweet?.extended_entities?.media); - media = media?.filter(m => m.video_info?.variants?.length); - - // check if there's a video at given index (/video/) - if (index >= 0 && index < media?.length) { - media = [media[index]] - } - - switch (media?.length) { - case undefined: - case 0: - return { error: 'ErrorNoVideosInTweet' }; - case 1: - return { - type: needsFixing(media[0]) ? "remux" : "normal", - urls: bestQuality(media[0].video_info.variants), - filename: `twitter_${id}.mp4`, - audioFilename: `twitter_${id}_audio`, - isGif: media[0].type === "animated_gif" - }; - default: - const picker = media.map((content, i) => { - let url = bestQuality(content.video_info.variants); - const shouldRenderGif = content.type === 'animated_gif' && toGif; - - if (needsFixing(content) || shouldRenderGif) { - url = createStream({ - service: 'twitter', - type: shouldRenderGif ? 'gif' : 'remux', - u: url, - filename: `twitter_${id}_${i + 1}.mp4` - }) - } - - return { - type: 'video', - url, - thumb: content.media_url_https, - } - }); - return { picker }; - } -} diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js deleted file mode 100644 index 69a36eca..00000000 --- a/src/modules/processing/services/vimeo.js +++ /dev/null @@ -1,109 +0,0 @@ -import { env } from "../../config.js"; -import { cleanString } from '../../sub/utils.js'; - -const resolutionMatch = { - "3840": "2160", - "2732": "1440", - "2560": "1440", - "2048": "1080", - "1920": "1080", - "1366": "720", - "1280": "720", - "960": "480", - "640": "360", - "426": "240" -} - -const qualityMatch = { - "2160": "4K", - "1440": "2K", - "480": "540", - - "4K": "2160", - "2K": "1440", - "540": "480" -} - -export default async function(obj) { - let quality = obj.quality === "max" ? "9000" : obj.quality; - if (!quality || obj.isAudioOnly) quality = "9000"; - - const url = new URL(`https://player.vimeo.com/video/${obj.id}/config`); - if (obj.password) { - url.searchParams.set('h', obj.password); - } - - let api = await fetch(url) - .then(r => r.json()) - .catch(() => {}); - if (!api) return { error: 'ErrorCouldntFetch' }; - - let downloadType = "dash"; - - if (!obj.isAudioOnly && JSON.stringify(api).includes('"progressive":[{')) - downloadType = "progressive"; - - let fileMetadata = { - title: cleanString(api.video.title.trim()), - artist: cleanString(api.video.owner.name.trim()), - } - - if (downloadType !== "dash") { - if (qualityMatch[quality]) quality = qualityMatch[quality]; - let all = api.request.files.progressive.sort((a, b) => Number(b.width) - Number(a.width)); - let best = all[0]; - - let bestQuality = all[0].quality.split('p')[0]; - if (qualityMatch[bestQuality]) { - bestQuality = qualityMatch[bestQuality] - } - - if (Number(quality) < Number(bestQuality)) { - best = all.find(i => i.quality.split('p')[0] === quality); - } - - if (!best) return { error: 'ErrorEmptyDownload' }; - - return { - urls: best.url, - audioFilename: `vimeo_${obj.id}_audio`, - filename: `vimeo_${obj.id}_${best.width}x${best.height}.mp4` - } - } - - if (api.video.duration > env.durationLimit) - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - - let masterJSONURL = api.request.files.dash.cdns.akfire_interconnect_quic.url; - let masterJSON = await fetch(masterJSONURL).then(r => r.json()).catch(() => {}); - - if (!masterJSON) return { error: 'ErrorCouldntFetch' }; - if (!masterJSON.video) return { error: 'ErrorEmptyDownload' }; - - let masterJSON_Video = masterJSON.video - .sort((a, b) => Number(b.width) - Number(a.width)) - .filter(a => ["dash", "mp42"].includes(a.format)); - - let bestVideo = masterJSON_Video[0]; - if (Number(quality) < Number(resolutionMatch[bestVideo.width])) { - bestVideo = masterJSON_Video.find(i => resolutionMatch[i.width] === quality) - } - - let masterM3U8 = `${masterJSONURL.split("/sep/")[0]}/sep/video/${bestVideo.id}/master.m3u8`; - const fallbackResolution = bestVideo.height > bestVideo.width ? bestVideo.width : bestVideo.height; - - return { - urls: masterM3U8, - isM3U8: true, - fileMetadata: fileMetadata, - filenameAttributes: { - service: "vimeo", - id: obj.id, - title: fileMetadata.title, - author: fileMetadata.artist, - resolution: `${bestVideo.width}x${bestVideo.height}`, - qualityLabel: `${resolutionMatch[bestVideo.width] || fallbackResolution}p`, - extension: "mp4" - } - } -} diff --git a/src/modules/processing/services/vine.js b/src/modules/processing/services/vine.js deleted file mode 100644 index 25163227..00000000 --- a/src/modules/processing/services/vine.js +++ /dev/null @@ -1,15 +0,0 @@ -export default async function(obj) { - let post = await fetch(`https://archive.vine.co/posts/${obj.id}.json`) - .then(r => r.json()) - .catch(() => {}); - - if (!post) return { error: 'ErrorEmptyDownload' }; - - if (post.videoUrl) return { - urls: post.videoUrl.replace("http://", "https://"), - filename: `vine_${obj.id}.mp4`, - audioFilename: `vine_${obj.id}_audio` - } - - return { error: 'ErrorEmptyDownload' } -} diff --git a/src/modules/processing/services/vk.js b/src/modules/processing/services/vk.js deleted file mode 100644 index e95f12be..00000000 --- a/src/modules/processing/services/vk.js +++ /dev/null @@ -1,55 +0,0 @@ -import { genericUserAgent, env } from "../../config.js"; -import { cleanString } from "../../sub/utils.js"; - -const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"]; - -export default async function(o) { - let html, url, quality = o.quality === "max" ? 2160 : o.quality; - - html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, { - headers: { "user-agent": genericUserAgent } - }).then(r => r.arrayBuffer()).catch(() => {}); - - if (!html) return { error: 'ErrorCouldntFetch' }; - - // decode cyrillic from windows-1251 because vk still uses apis from prehistoric times - let decoder = new TextDecoder('windows-1251'); - html = decoder.decode(html); - - if (!html.includes(`{"lang":`)) return { error: 'ErrorEmptyDownload' }; - - let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]); - - if (Number(js.mvData.is_active_live) !== 0) return { error: 'ErrorLiveVideo' }; - if (js.mvData.duration > env.durationLimit) - return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - - for (let i in resolutions) { - if (js.player.params[0][`url${resolutions[i]}`]) { - quality = resolutions[i]; - break - } - } - if (Number(quality) > Number(o.quality)) quality = o.quality; - - url = js.player.params[0][`url${quality}`]; - - let fileMetadata = { - title: cleanString(js.player.params[0].md_title.trim()), - author: cleanString(js.player.params[0].md_author.trim()), - } - - if (url) return { - urls: url, - filenameAttributes: { - service: "vk", - id: `${o.userId}_${o.videoId}`, - title: fileMetadata.title, - author: fileMetadata.author, - resolution: `${quality}p`, - qualityLabel: `${quality}p`, - extension: "mp4" - } - } - return { error: 'ErrorEmptyDownload' } -} diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js deleted file mode 100644 index 1c86b4ed..00000000 --- a/src/modules/processing/services/youtube.js +++ /dev/null @@ -1,235 +0,0 @@ -import { Innertube, Session } from 'youtubei.js'; -import { env } from '../../config.js'; -import { cleanString } from '../../sub/utils.js'; -import { fetch } from 'undici' -import { getCookie, updateCookieValues } from '../cookie/manager.js' - -const ytBase = Innertube.create().catch(e => e); - -const codecMatch = { - h264: { - codec: "avc1", - aCodec: "mp4a", - container: "mp4" - }, - av1: { - codec: "av01", - aCodec: "mp4a", - container: "mp4" - }, - vp9: { - codec: "vp9", - aCodec: "opus", - container: "webm" - } -} - -const transformSessionData = (cookie) => { - if (!cookie) - return; - - const values = cookie.values(); - const REQUIRED_VALUES = [ - 'access_token', 'refresh_token', - 'client_id', 'client_secret', - 'expires' - ]; - - if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) { - return; - } - return { - ...values, - expires: new Date(values.expires), - }; -} - -const cloneInnertube = async (customFetch) => { - const innertube = await ytBase; - if (innertube instanceof Error) { - throw innertube; - } - - const session = new Session( - innertube.session.context, - innertube.session.key, - innertube.session.api_version, - innertube.session.account_index, - innertube.session.player, - undefined, - customFetch ?? innertube.session.http.fetch, - innertube.session.cache - ); - - const cookie = getCookie('youtube_oauth'); - const oauthData = transformSessionData(cookie); - - if (!session.logged_in && oauthData) { - await session.oauth.init(oauthData); - session.logged_in = true; - } - - if (session.logged_in) { - await session.oauth.refreshIfRequired(); - const oldExpiry = new Date(cookie.values().expires); - const newExpiry = session.oauth.credentials.expires; - - if (oldExpiry.getTime() !== newExpiry.getTime()) { - updateCookieValues(cookie, { - ...session.oauth.credentials, - expires: session.oauth.credentials.expires.toISOString() - }); - } - } - - const yt = new Innertube(session); - return yt; -} - -export default async function(o) { - const yt = await cloneInnertube( - (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher }) - ); - - let info, isDubbed, format = o.format || "h264"; - let quality = o.quality === "max" ? "9000" : o.quality; // 9000(p) - max quality - - function qual(i) { - if (!i.quality_label) { - return; - } - - return i.quality_label.split('p')[0].split('s')[0] - } - - try { - info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS'); - } catch(e) { - if (e?.message === 'This video is unavailable') { - return { error: 'ErrorCouldntFetch' }; - } else { - return { error: 'ErrorCantConnectToServiceAPI' }; - } - } - - if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; - - const playability = info.playability_status; - - if (playability.status === 'LOGIN_REQUIRED') { - if (playability.reason.endsWith('bot')) { - return { error: 'ErrorYTLogin' } - } - if (playability.reason.endsWith('age')) { - return { error: 'ErrorYTAgeRestrict' } - } - } - if (playability.status === "UNPLAYABLE" && playability.reason.endsWith('request limit.')) { - return { error: 'ErrorYTRateLimit' } - } - - if (playability.status !== 'OK') return { error: 'ErrorYTUnavailable' }; - if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' }; - - // return a critical error if returned video is "Video Not Available" - // or a similar stub by youtube - if (info.basic_info.id !== o.id) { - return { - error: 'ErrorCantConnectToServiceAPI', - critical: true - } - } - - let bestQuality, hasAudio; - - const filterByCodec = (formats) => formats.filter(e => - e.mime_type.includes(codecMatch[format].codec) || e.mime_type.includes(codecMatch[format].aCodec) - ).sort((a, b) => Number(b.bitrate) - Number(a.bitrate)); - - let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats); - if (adaptive_formats.length === 0 && format === "vp9") { - format = "h264" - adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats) - } - - bestQuality = adaptive_formats.find(i => i.has_video && i.content_length); - hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length); - - if (bestQuality) bestQuality = qual(bestQuality); - if (!bestQuality && !o.isAudioOnly || !hasAudio) return { error: 'ErrorYTTryOtherCodec' }; - if (info.basic_info.duration > env.durationLimit) return { error: ['ErrorLengthLimit', env.durationLimit / 60] }; - - let checkBestAudio = (i) => (i.has_audio && !i.has_video), - audio = adaptive_formats.find(i => checkBestAudio(i) && !i.is_dubbed); - - if (o.dubLang) { - let dubbedAudio = adaptive_formats.find(i => - checkBestAudio(i) && i.language === o.dubLang && i.audio_track && !i.audio_track.audio_is_default - ); - if (dubbedAudio) { - audio = dubbedAudio; - isDubbed = true - } - } - let fileMetadata = { - title: cleanString(info.basic_info.title.trim()), - artist: cleanString(info.basic_info.author.replace("- Topic", "").trim()), - } - if (info.basic_info.short_description && info.basic_info.short_description.startsWith("Provided to YouTube by")) { - let descItems = info.basic_info.short_description.split("\n\n"); - fileMetadata.album = descItems[2]; - fileMetadata.copyright = descItems[3]; - if (descItems[4].startsWith("Released on:")) { - fileMetadata.date = descItems[4].replace("Released on: ", '').trim() - } - } - - let filenameAttributes = { - service: "youtube", - id: o.id, - title: fileMetadata.title, - author: fileMetadata.artist, - youtubeDubName: isDubbed ? o.dubLang : false - } - - if (hasAudio && o.isAudioOnly) return { - type: "render", - isAudioOnly: true, - urls: audio.decipher(yt.session.player), - filenameAttributes: filenameAttributes, - fileMetadata: fileMetadata, - bestAudio: format === "h264" ? 'm4a' : 'opus' - } - const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality, - checkSingle = i => qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].codec), - checkRender = i => qual(i) === matchingQuality && i.has_video && !i.has_audio; - - let match, type, urls; - if (!o.isAudioOnly && !o.isAudioMuted && format === 'h264') { - match = info.streaming_data.formats.find(checkSingle); - type = "bridge"; - urls = match?.decipher(yt.session.player); - } - - const video = adaptive_formats.find(checkRender); - if (!match && video) { - match = video; - type = "render"; - urls = [video.decipher(yt.session.player), audio.decipher(yt.session.player)]; - } - - if (match) { - filenameAttributes.qualityLabel = match.quality_label; - filenameAttributes.resolution = `${match.width}x${match.height}`; - filenameAttributes.extension = codecMatch[format].container; - filenameAttributes.youtubeFormat = format; - return { - type, - urls, - filenameAttributes, - fileMetadata - } - } - - return { error: 'ErrorYTTryOtherCodec' } -}