diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 89951fe0..00000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -custom: https://boosty.to/wukko/donate diff --git a/.github/test.sh b/.github/test.sh index 85683b91..80de1fcd 100755 --- a/.github/test.sh +++ b/.github/test.sh @@ -18,7 +18,7 @@ test_api() { -X POST \ -H "Accept: application/json" \ -H "Content-Type: application/json" \ - -d '{"url":"https://www.youtube.com/watch?v=jNQXAC9IVRw"}') + -d '{"url":"https://vine.co/v/huwVJIEJW50", "isAudioOnly": true}') echo "$API_RESPONSE" STATUS=$(echo "$API_RESPONSE" | jq -r .status) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 193ddd01..4ac2daf3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,3 +36,27 @@ jobs: uses: actions/checkout@v4 - name: Run test script run: .github/test.sh api + + check-services: + name: test service functionality + runs-on: ubuntu-latest + outputs: + services: ${{ steps.checkServices.outputs.service_list }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - id: checkServices + run: npm ci && echo "service_list=$(node src/util/test-ci get-services)" >> "$GITHUB_OUTPUT" + + test-services: + needs: check-services + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service: ${{ fromJson(needs.check-services.outputs.services) }} + name: "test service: ${{ matrix.service }}" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - run: npm ci && node src/util/test-ci run-tests-for ${{ matrix.service }} \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd2140eb..2668242b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ the scope is not strictly defined, you can write whatever you find most fitting if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it. -if your contribution has uninformative commit messages, you may be asked to interactively rebase your branch and amend each commit to include a meaningful message. +if your contribution has uninformative commit titles, you may be asked to interactively rebase your branch and amend each commit to include a meaningful title. ### clean commit history if your branch is out of date and/or has some merge conflicts with the `current` branch, you should **rebase** it instead of merging. this prevents meaningless merge commits from being included in your branch, which would then end up in the cobalt git history. diff --git a/README.md b/README.md index d2bb064c..d5fe8164 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,13 @@ this list is not final and keeps expanding over time. if support for a service y | bilibili.com & bilibili.tv | ✅ | ✅ | ✅ | ➖ | ➖ | | dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ | | instagram posts & reels | ✅ | ✅ | ✅ | ➖ | ➖ | +| facebook videos | ✅ | ❌ | ❌ | ➖ | ➖ | | loom | ✅ | ❌ | ✅ | ✅ | ➖ | | ok video | ✅ | ❌ | ✅ | ✅ | ✅ | | pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | | reddit | ✅ | ✅ | ✅ | ❌ | ❌ | | rutube | ✅ | ✅ | ✅ | ✅ | ✅ | +| snapchat stories & spotlights | ✅ | ✅ | ✅ | ➖ | ➖ | | soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ | | streamable | ✅ | ✅ | ✅ | ➖ | ➖ | | tiktok | ✅ | ✅ | ✅ | ❌ | ❌ | @@ -45,8 +47,10 @@ this list is not final and keeps expanding over time. if support for a service y | service | notes or features | | :-------- | :----- | | instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. | +| facebook | supports public accessible videos content only. | | pinterest | supports photos, gifs, videos and stories. | | reddit | supports gifs and videos. | +| snapchat | supports spotlights and stories. lets you pick what to save from stories. | | rutube | supports yappy & private links. | | soundcloud | supports private links. | | tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. | diff --git a/docs/api.md b/docs/api.md index b63c9216..6cd66bba 100644 --- a/docs/api.md +++ b/docs/api.md @@ -49,9 +49,9 @@ item type: `object` | key | type | variables | description | |:--------|:---------|:--------------------------------------------------------|:---------------------------------------| -| `type` | `string` | `video` | used only if `pickerType`is `various`. | +| `type` | `string` | `video / photo / gif` | used only if `pickerType` is `various` | | `url` | `string` | direct link to a file or a link to cobalt's live render | | -| `thumb` | `string` | item thumbnail that's displayed in the picker | used only for `video` type. | +| `thumb` | `string` | item thumbnail that's displayed in the picker | used for `video` and `gif` types | ## GET: `/api/stream` cobalt's live render (or stream) endpoint. usually, you will receive a url to this endpoint diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index a440d11d..d31a67f5 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -56,8 +56,9 @@ sudo service nscd start | `API_LISTEN_ADDRESS` | `0.0.0.0` | `127.0.0.1` | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** | | `API_URL` | ➖ | `https://api.cobalt.tools/` | changes url from which api server is accessible.
***REQUIRED TO RUN THE API***. | | `API_NAME` | `unknown` | `ams-1` | api server name that is shown in `/api/serverInfo`. | +| `API_EXTERNAL_PROXY` | ➖ | `http://user:password@127.0.0.1:8080`| url of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only. | | `CORS_WILDCARD` | `1` | `0` | toggles cross-origin resource sharing.
`0`: disabled. `1`: enabled. | -| `CORS_URL` | not used | `https://cobalt.tools/` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. | +| `CORS_URL` | not used | `https://cobalt.tools` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. | | `COOKIE_PATH` | not used | `/cookies.json` | path for cookie file relative to main folder. | | `PROCESSING_PRIORITY` | not used | `10` | changes `nice` value* for ffmpeg subprocess. available only on unix systems. | | `FREEBIND_CIDR` | ➖ | `2001:db8::/32` | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. see below for more info. | diff --git a/package-lock.json b/package-lock.json index 9c5434d9..87e11661 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cobalt", - "version": "7.14.5", + "version": "7.15", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cobalt", - "version": "7.14.5", + "version": "7.15", "license": "AGPL-3.0", "dependencies": { "content-disposition-header": "0.6.0", @@ -24,7 +24,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^9.3.0" + "youtubei.js": "^10.2.0" }, "engines": { "node": ">=18" @@ -73,9 +73,9 @@ } }, "node_modules/acorn": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", - "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "version": "8.12.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", + "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", "bin": { "acorn": "bin/acorn" }, @@ -683,9 +683,9 @@ } }, "node_modules/jintr": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/jintr/-/jintr-1.1.0.tgz", - "integrity": "sha512-Tu9wk3BpN2v+kb8yT6YBtue+/nbjeLFv4vvVC4PJ7oCidHKbifWhvORrAbQfxVIQZG+67am/mDagpiGSVtvrZg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jintr/-/jintr-2.0.0.tgz", + "integrity": "sha512-RiVlevxttZ4eHEYB2dXKXDXluzHfRuw0DJQGsYuKCc5IvZj5/GbOakeqVX+Bar/G9kTty9xDJREcxukurkmYLA==", "funding": [ "https://github.com/sponsors/LuanRT" ], @@ -1123,14 +1123,15 @@ } }, "node_modules/youtubei.js": { - "version": "9.4.0", - "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-9.4.0.tgz", - "integrity": "sha512-8plCOZD2WabqWSEgZU3RjzigIIeR7sF028EERJENYrC9xO/6awpLMZfeoE1gNrNEbKcA+bzbMvonqlvBdxGdKg==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-10.2.0.tgz", + "integrity": "sha512-JLKW9AHQ1qrTwBbre1aDkH8UJFmNcc4+kOSaVou5jSY7AzfFPFJK0yvX6afnLst0UVC9wfXHrLiNx93sutVErA==", "funding": [ "https://github.com/sponsors/LuanRT" ], + "license": "MIT", "dependencies": { - "jintr": "^1.1.0", + "jintr": "^2.0.0", "tslib": "^2.5.0", "undici": "^5.19.1" } diff --git a/package.json b/package.json index b2224a50..58fdec83 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.14.5", + "version": "7.15", "author": "imput", "exports": "./src/cobalt.js", "type": "module", @@ -40,7 +40,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^9.3.0" + "youtubei.js": "^10.2.0" }, "optionalDependencies": { "freebind": "^0.2.2" diff --git a/src/core/api.js b/src/core/api.js index 8eb4cb40..5f4ee804 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -1,5 +1,6 @@ import cors from "cors"; import rateLimit from "express-rate-limit"; +import { setGlobalDispatcher, ProxyAgent } from "undici"; import { env, version } from "../modules/config.js"; @@ -26,7 +27,7 @@ const corsConfig = env.corsWildcard ? {} : { export function runAPI(express, app, gitCommit, gitBranch, __dirname) { const startTime = new Date(); const startTimestamp = startTime.getTime(); - + const serverInfo = { version: version, commit: gitCommit, @@ -81,38 +82,23 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { app.use((req, res, next) => { try { decodeURIComponent(req.path) - } catch { + } catch { return res.redirect('/') } next(); }) - app.use('/api/json', express.json({ - verify: (req, res, buf) => { - if (String(req.header('Accept')) === "application/json") { - if (buf.length > 720) throw new Error(); - JSON.parse(buf); - } else { - throw new Error(); - } - } - })) - - // handle express.json errors properly (https://github.com/expressjs/express/issues/4065) - app.use('/api/json', (err, req, res, next) => { - let errorText = "invalid json body"; - const acceptHeader = String(req.header('Accept')) !== "application/json"; - - if (err || acceptHeader) { - if (acceptHeader) errorText = "invalid accept header"; + app.use('/api/json', express.json({ limit: 1024 })); + app.use('/api/json', (err, _, res, next) => { + if (err) { return res.status(400).json({ status: "error", - text: errorText + text: "invalid json body" }); - } else { - next(); } - }) + + next(); + }); app.post('/api/json', async (req, res) => { const request = req.body; @@ -123,6 +109,10 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { res.status(status).json(body); } + if (!acceptRegex.test(req.header('Accept'))) { + return fail('ErrorInvalidAcceptHeader'); + } + if (!acceptRegex.test(req.header('Content-Type'))) { return fail('ErrorInvalidContentType'); } @@ -219,6 +209,14 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { randomizeCiphers(); setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes + if (env.externalProxy) { + if (env.freebindCIDR) { + throw new Error('Freebind is not available when external proxy is enabled') + } + + setGlobalDispatcher(new ProxyAgent(env.externalProxy)) + } + app.listen(env.apiPort, env.listenAddress, () => { console.log(`\n` + `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index eecd9ac1..2b10f41d 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -159,6 +159,7 @@ "UpdateOneMillion": "1 million users and blazing speed", "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!", "ErrorYTLogin": "couldn't get this youtube video because it requires an account to view.\n\nthis limitation is done by google to seemingly stop scraping, affecting all 3rd party tools and even their own clients.\n\ntry again, but if issue persists, {ContactLink}.", - "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}." + "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}.", + "ErrorInvalidAcceptHeader": "invalid accept header" } } diff --git a/src/modules/config.js b/src/modules/config.js index 530c5f0b..662d8b05 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -48,7 +48,9 @@ const processingPriority: process.platform !== 'win32' && process.env.PROCESSING_PRIORITY - && parseInt(process.env.PROCESSING_PRIORITY) + && parseInt(process.env.PROCESSING_PRIORITY), + + externalProxy: process.env.API_EXTERNAL_PROXY, } export const diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 66ebd88e..a176587b 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -24,8 +24,10 @@ import streamable from "./services/streamable.js"; import twitch from "./services/twitch.js"; import rutube from "./services/rutube.js"; import dailymotion from "./services/dailymotion.js"; +import snapchat from "./services/snapchat.js"; import loom from "./services/loom.js"; import videoclip from "./services/videoclip.js"; +import facebook from "./services/facebook.js"; let freebind; @@ -194,11 +196,22 @@ export default async function(host, patternMatch, lang, obj) { id: patternMatch.id }); break; + case "snapchat": + r = await snapchat({ + hostname: url.hostname, + ...patternMatch + }); + break; case "loom": r = await loom({ id: patternMatch.id }); break; + case "facebook": + r = await facebook({ + ...patternMatch + }); + break; default: return createResponse("error", { t: loc(lang, 'ErrorUnsupported') diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index fb469386..27ceedfe 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -73,6 +73,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di switch (host) { case "instagram": case "twitter": + case "snapchat": params = { picker: r.picker }; break; case "tiktok": @@ -137,11 +138,13 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di params = { type: "bridge" }; break; + case "facebook": case "vine": case "instagram": case "tumblr": case "pinterest": case "streamable": + case "snapchat": case "loom": responseType = "redirect"; break; diff --git a/src/modules/processing/services/facebook.js b/src/modules/processing/services/facebook.js new file mode 100644 index 00000000..17dedab4 --- /dev/null +++ b/src/modules/processing/services/facebook.js @@ -0,0 +1,56 @@ +import { genericUserAgent } from "../../config.js"; + +const headers = { + 'User-Agent': genericUserAgent, + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', +} + +const resolveUrl = (url) => { + return fetch(url, { headers }) + .then(r => { + if (r.headers.get('location')) { + return decodeURIComponent(r.headers.get('location')); + } + if (r.headers.get('link')) { + const linkMatch = r.headers.get('link').match(/<(.*?)\/>/); + return decodeURIComponent(linkMatch[1]); + } + return false; + }) + .catch(() => false); +} + +export default async function({ id, shareType, shortLink }) { + let url = `https://web.facebook.com/i/videos/${id}`; + + if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`; + if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`); + + const html = await fetch(url, { headers }) + .then(r => r.text()) + .catch(() => false); + + if (!html) return { error: 'ErrorCouldntFetch' }; + + const urls = []; + const hd = html.match('"browser_native_hd_url":(".*?")'); + const sd = html.match('"browser_native_sd_url":(".*?")'); + + if (hd?.[1]) urls.push(JSON.parse(hd[1])); + if (sd?.[1]) urls.push(JSON.parse(sd[1])); + + if (!urls.length) { + return { error: 'ErrorEmptyDownload' }; + } + + const baseFilename = `facebook_${id || shortLink}`; + + return { + urls: urls[0], + filename: `${baseFilename}.mp4`, + audioFilename: `${baseFilename}_audio`, + }; +} diff --git a/src/modules/processing/services/ok.js b/src/modules/processing/services/ok.js index 97bbcf82..33847cd8 100644 --- a/src/modules/processing/services/ok.js +++ b/src/modules/processing/services/ok.js @@ -45,7 +45,7 @@ export default async function(o) { let fileMetadata = { title: cleanString(videoData.movie.title.trim()), - author: cleanString(videoData.author.name.trim()), + author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()), } if (bestVideo) return { diff --git a/src/modules/processing/services/rutube.js b/src/modules/processing/services/rutube.js index a8d0abbe..325e9ccc 100644 --- a/src/modules/processing/services/rutube.js +++ b/src/modules/processing/services/rutube.js @@ -10,6 +10,8 @@ async function requestJSON(url) { } catch {} } +const delta = (a, b) => Math.abs(a - b); + export default async function(obj) { if (obj.yappyId) { const yappy = await requestJSON( @@ -25,7 +27,7 @@ export default async function(obj) { } } - const quality = obj.quality === "max" ? "9000" : obj.quality; + const quality = Number(obj.quality) || 9000; const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`); if (obj.key) requestURL.searchParams.set('p', obj.key); @@ -45,12 +47,16 @@ export default async function(obj) { if (!m3u8) return { error: 'ErrorCouldntFetch' }; - m3u8 = HLS.parse(m3u8).variants.sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); + m3u8 = HLS.parse(m3u8).variants; - let bestQuality = m3u8[0]; - if (Number(quality) < bestQuality.resolution.height) { - bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height)); - } + const matchingQuality = m3u8.reduce((prev, next) => { + const diff = { + prev: delta(quality, prev.resolution.height), + next: delta(quality, next.resolution.height) + }; + + return diff.prev < diff.next ? prev : next; + }); const fileMetadata = { title: cleanString(play.title.trim()), @@ -58,15 +64,15 @@ export default async function(obj) { } return { - urls: bestQuality.uri, + urls: matchingQuality.uri, isM3U8: true, filenameAttributes: { service: "rutube", id: obj.id, title: fileMetadata.title, author: fileMetadata.artist, - resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`, - qualityLabel: `${bestQuality.resolution.height}p`, + resolution: `${matchingQuality.resolution.width}x${matchingQuality.resolution.height}`, + qualityLabel: `${matchingQuality.resolution.height}p`, extension: "mp4" }, fileMetadata: fileMetadata diff --git a/src/modules/processing/services/snapchat.js b/src/modules/processing/services/snapchat.js new file mode 100644 index 00000000..a93f0933 --- /dev/null +++ b/src/modules/processing/services/snapchat.js @@ -0,0 +1,96 @@ +import { genericUserAgent } from "../../config.js"; +import { getRedirectingURL } from "../../sub/utils.js"; +import { extract, normalizeURL } from "../url.js"; + +const SPOTLIGHT_VIDEO_REGEX = //; +const NEXT_DATA_REGEX = /