diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4b71a621c..e2411ecfa 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -256,7 +256,7 @@ jobs:
with:
path: |
~/yt-dlp-build-venv
- key: cache-reqs-${{ github.job }}
+ key: cache-reqs-${{ github.job }}-${{ github.ref }}
- name: Install Requirements
run: |
@@ -331,19 +331,16 @@ jobs:
if: steps.restore-cache.outputs.cache-hit == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- cache_key: cache-reqs-${{ github.job }}
- repository: ${{ github.repository }}
- branch: ${{ github.ref }}
+ cache_key: cache-reqs-${{ github.job }}-${{ github.ref }}
run: |
- gh extension install actions/gh-actions-cache
- gh actions-cache delete "${cache_key}" -R "${repository}" -B "${branch}" --confirm
+ gh cache delete "${cache_key}"
- name: Cache requirements
uses: actions/cache/save@v4
with:
path: |
~/yt-dlp-build-venv
- key: cache-reqs-${{ github.job }}
+ key: cache-reqs-${{ github.job }}-${{ github.ref }}
macos_legacy:
needs: process
diff --git a/.gitignore b/.gitignore
index 8fcd0de64..40bb34d2a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -105,6 +105,8 @@ README.txt
*.zsh
*.spec
test/testdata/sigs/player-*.js
+test/testdata/thumbnails/empty.webp
+test/testdata/thumbnails/foo\ %d\ bar/foo_%d.*
# Binary
/youtube-dl
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 5710f9a9e..6aa52c595 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -770,3 +770,8 @@ NeonMan
pj47x
troex
WouterGordts
+baierjan
+GeoffreyFrogeye
+Pawka
+v3DJG6GL
+yozel
diff --git a/Changelog.md b/Changelog.md
index 513724bf4..80b72da05 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,52 @@ # Changelog
# To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
-->
+### 2025.05.22
+
+#### Core changes
+- **cookies**: [Fix Linux desktop environment detection](https://github.com/yt-dlp/yt-dlp/commit/e491fd4d090db3af52a82863fb0553dd5e17fb85) ([#13197](https://github.com/yt-dlp/yt-dlp/issues/13197)) by [mbway](https://github.com/mbway)
+- **jsinterp**: [Fix increment/decrement evaluation](https://github.com/yt-dlp/yt-dlp/commit/167d7a9f0ffd1b4fe600193441bdb7358db2740b) ([#13238](https://github.com/yt-dlp/yt-dlp/issues/13238)) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev)
+
+#### Extractor changes
+- **1tv**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/41c0a1fb89628696f8bb88e2b9f3a68f355b8c26) ([#13168](https://github.com/yt-dlp/yt-dlp/issues/13168)) by [bashonly](https://github.com/bashonly)
+- **amcnetworks**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/464c84fedf78eef822a431361155f108b5df96d7) ([#13147](https://github.com/yt-dlp/yt-dlp/issues/13147)) by [bashonly](https://github.com/bashonly)
+- **bitchute**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/1d0f6539c47e5d5c68c3c47cdb7075339e2885ac) ([#13081](https://github.com/yt-dlp/yt-dlp/issues/13081)) by [bashonly](https://github.com/bashonly)
+- **cartoonnetwork**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/7dbb47f84f0ee1266a3a01f58c9bc4c76d76794a) ([#13148](https://github.com/yt-dlp/yt-dlp/issues/13148)) by [bashonly](https://github.com/bashonly)
+- **iprima**: [Fix login support](https://github.com/yt-dlp/yt-dlp/commit/a7d9a5eb79ceeecb851389f3f2c88597871ca3f2) ([#12937](https://github.com/yt-dlp/yt-dlp/issues/12937)) by [baierjan](https://github.com/baierjan)
+- **jiosaavn**
+ - artist: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/586b557b124f954d3f625360ebe970989022ad97) ([#12803](https://github.com/yt-dlp/yt-dlp/issues/12803)) by [subrat-lima](https://github.com/subrat-lima)
+ - playlist, show: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/317f4b8006c2c0f0f64f095b1485163ad97c9053) ([#12803](https://github.com/yt-dlp/yt-dlp/issues/12803)) by [subrat-lima](https://github.com/subrat-lima)
+ - show: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/6839276496d8814cf16f58b637e45663467928e6) ([#12803](https://github.com/yt-dlp/yt-dlp/issues/12803)) by [subrat-lima](https://github.com/subrat-lima)
+- **lrtradio**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/abf58dcd6a09e14eec4ea82ae12f79a0337cb383) ([#13200](https://github.com/yt-dlp/yt-dlp/issues/13200)) by [Pawka](https://github.com/Pawka)
+- **nebula**: [Support `--mark-watched`](https://github.com/yt-dlp/yt-dlp/commit/20f288bdc2173c7cc58d709d25ca193c1f6001e7) ([#13120](https://github.com/yt-dlp/yt-dlp/issues/13120)) by [GeoffreyFrogeye](https://github.com/GeoffreyFrogeye)
+- **niconico**
+ - [Fix error handling](https://github.com/yt-dlp/yt-dlp/commit/f569be4602c2a857087e495d5d7ed6060cd97abe) ([#13236](https://github.com/yt-dlp/yt-dlp/issues/13236)) by [bashonly](https://github.com/bashonly)
+ - live: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/7a7b85c9014d96421e18aa7ea5f4c1bee5ceece0) ([#13045](https://github.com/yt-dlp/yt-dlp/issues/13045)) by [doe1080](https://github.com/doe1080)
+- **nytimesarticle**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/b26bc32579c00ef579d75a835807ccc87d20ee0a) ([#13104](https://github.com/yt-dlp/yt-dlp/issues/13104)) by [bashonly](https://github.com/bashonly)
+- **once**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/f475e8b529d18efdad603ffda02a56e707fe0e2c) ([#13164](https://github.com/yt-dlp/yt-dlp/issues/13164)) by [bashonly](https://github.com/bashonly)
+- **picarto**: vod: [Support `/profile/` video URLs](https://github.com/yt-dlp/yt-dlp/commit/31e090cb787f3504ec25485adff9a2a51d056734) ([#13227](https://github.com/yt-dlp/yt-dlp/issues/13227)) by [subrat-lima](https://github.com/subrat-lima)
+- **playsuisse**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/d880e060803ae8ed5a047e578cca01e1f0e630ce) ([#12466](https://github.com/yt-dlp/yt-dlp/issues/12466)) by [v3DJG6GL](https://github.com/v3DJG6GL)
+- **sprout**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/cbcfe6378dde33a650e3852ab17ad4503b8e008d) ([#13149](https://github.com/yt-dlp/yt-dlp/issues/13149)) by [bashonly](https://github.com/bashonly)
+- **svtpage**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/ea8498ed534642dd7e925961b97b934987142fd3) ([#12957](https://github.com/yt-dlp/yt-dlp/issues/12957)) by [diman8](https://github.com/diman8)
+- **twitch**: [Support `--live-from-start`](https://github.com/yt-dlp/yt-dlp/commit/00b1bec55249cf2ad6271d36492c51b34b6459d1) ([#13202](https://github.com/yt-dlp/yt-dlp/issues/13202)) by [bashonly](https://github.com/bashonly)
+- **vimeo**: event: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/545c1a5b6f2fe88722b41aef0e7485bf3be3f3f9) ([#13216](https://github.com/yt-dlp/yt-dlp/issues/13216)) by [bashonly](https://github.com/bashonly)
+- **wat.tv**: [Improve error handling](https://github.com/yt-dlp/yt-dlp/commit/f123cc83b3aea45053f5fa1d9141048b01fc2774) ([#13111](https://github.com/yt-dlp/yt-dlp/issues/13111)) by [bashonly](https://github.com/bashonly)
+- **weverse**: [Fix live extraction](https://github.com/yt-dlp/yt-dlp/commit/5328eda8820cc5f21dcf917684d23fbdca41831d) ([#13084](https://github.com/yt-dlp/yt-dlp/issues/13084)) by [bashonly](https://github.com/bashonly)
+- **xinpianchang**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/83fabf352489d52843f67e6e9cc752db86d27e6e) ([#13245](https://github.com/yt-dlp/yt-dlp/issues/13245)) by [garret1317](https://github.com/garret1317)
+- **youtube**
+ - [Add PO token support for subtitles](https://github.com/yt-dlp/yt-dlp/commit/32ed5f107c6c641958d1cd2752e130de4db55a13) ([#13234](https://github.com/yt-dlp/yt-dlp/issues/13234)) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz)
+ - [Add `web_embedded` client for age-restricted videos](https://github.com/yt-dlp/yt-dlp/commit/0feec6dc131f488428bf881519e7c69766fbb9ae) ([#13089](https://github.com/yt-dlp/yt-dlp/issues/13089)) by [bashonly](https://github.com/bashonly)
+ - [Add a PO Token Provider Framework](https://github.com/yt-dlp/yt-dlp/commit/2685654a37141cca63eda3a92da0e2706e23ccfd) ([#12840](https://github.com/yt-dlp/yt-dlp/issues/12840)) by [coletdjnz](https://github.com/coletdjnz)
+ - [Extract `media_type` for all videos](https://github.com/yt-dlp/yt-dlp/commit/ded11ebc9afba6ba33923375103e9be2d7c804e7) ([#13136](https://github.com/yt-dlp/yt-dlp/issues/13136)) by [bashonly](https://github.com/bashonly)
+ - [Fix `--live-from-start` support for premieres](https://github.com/yt-dlp/yt-dlp/commit/8f303afb43395be360cafd7ad4ce2b6e2eedfb8a) ([#13079](https://github.com/yt-dlp/yt-dlp/issues/13079)) by [arabcoders](https://github.com/arabcoders)
+ - [Fix geo-restriction error handling](https://github.com/yt-dlp/yt-dlp/commit/c7e575e31608c19c5b26c10a4229db89db5fc9a8) ([#13217](https://github.com/yt-dlp/yt-dlp/issues/13217)) by [yozel](https://github.com/yozel)
+
+#### Misc. changes
+- **build**
+ - [Bump PyInstaller to v6.13.0](https://github.com/yt-dlp/yt-dlp/commit/17cf9088d0d535e4a7feffbf02bd49cd9dae5ab9) ([#13082](https://github.com/yt-dlp/yt-dlp/issues/13082)) by [bashonly](https://github.com/bashonly)
+ - [Bump run-on-arch-action to v3](https://github.com/yt-dlp/yt-dlp/commit/9064d2482d1fe722bbb4a49731fe0711c410d1c8) ([#13088](https://github.com/yt-dlp/yt-dlp/issues/13088)) by [bashonly](https://github.com/bashonly)
+- **cleanup**: Miscellaneous: [7977b32](https://github.com/yt-dlp/yt-dlp/commit/7977b329ed97b216e37bd402f4935f28c00eac9e) by [bashonly](https://github.com/bashonly)
+
### 2025.04.30
#### Important changes
diff --git a/Makefile b/Makefile
index 6c72ead1e..273cb3cc0 100644
--- a/Makefile
+++ b/Makefile
@@ -18,10 +18,11 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \
tar pypi-files lazy-extractors install uninstall
clean-test:
- rm -rf test/testdata/sigs/player-*.js tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
+ rm -rf tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
*.frag.aria2 *.frag.urls *.info.json *.live_chat.json *.meta *.part* *.tmp *.temp *.unknown_video *.ytdl \
*.3gp *.ape *.ass *.avi *.desktop *.f4v *.flac *.flv *.gif *.jpeg *.jpg *.lrc *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 *.mp4 \
- *.mpg *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.ssa *.swf *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
+ *.mpg *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.ssa *.swf *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp \
+ test/testdata/sigs/player-*.js test/testdata/thumbnails/empty.webp "test/testdata/thumbnails/foo %d bar/foo_%d."*
clean-dist:
rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS
diff --git a/README.md b/README.md
index a87b52832..6e2dc6243 100644
--- a/README.md
+++ b/README.md
@@ -44,6 +44,7 @@
* [Post-processing Options](#post-processing-options)
* [SponsorBlock Options](#sponsorblock-options)
* [Extractor Options](#extractor-options)
+ * [Preset Aliases](#preset-aliases)
* [CONFIGURATION](#configuration)
* [Configuration file encoding](#configuration-file-encoding)
* [Authentication with netrc](#authentication-with-netrc)
@@ -348,8 +349,8 @@ ## General Options:
--no-flat-playlist Fully extract the videos of a playlist
(default)
--live-from-start Download livestreams from the start.
- Currently only supported for YouTube
- (Experimental)
+ Currently experimental and only supported
+ for YouTube and Twitch
--no-live-from-start Download livestreams from the current time
(default)
--wait-for-video MIN[-MAX] Wait for scheduled streams to become
@@ -375,12 +376,12 @@ ## General Options:
an alias starts with a dash "-", it is
prefixed with "--". Arguments are parsed
according to the Python string formatting
- mini-language. E.g. --alias get-audio,-X
- "-S=aext:{0},abr -x --audio-format {0}"
- creates options "--get-audio" and "-X" that
- takes an argument (ARG0) and expands to
- "-S=aext:ARG0,abr -x --audio-format ARG0".
- All defined aliases are listed in the --help
+ mini-language. E.g. --alias get-audio,-X "-S
+ aext:{0},abr -x --audio-format {0}" creates
+ options "--get-audio" and "-X" that takes an
+ argument (ARG0) and expands to "-S
+ aext:ARG0,abr -x --audio-format ARG0". All
+ defined aliases are listed in the --help
output. Alias options can trigger more
aliases; so be careful to avoid defining
recursive options. As a safety measure, each
@@ -1105,6 +1106,10 @@ ## Extractor Options:
arguments for different extractors
## Preset Aliases:
+Predefined aliases for convenience and ease of use. Note that future
+ versions of yt-dlp may add or adjust presets, but the existing preset
+ names will not be changed or removed
+
-t mp3 -f 'ba[acodec^=mp3]/ba/b' -x --audio-format
mp3
@@ -1795,6 +1800,7 @@ #### youtube
* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `android`, `android_vr`, `tv` and `tv_embedded`. By default, `tv,ios,web` is used, or `tv,web` is used when authenticating with cookies. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only works if the video is embeddable. The `tv_embedded` and `web_creator` clients are added for age-restricted videos if account age-verification is required. Some clients, such as `web` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-ios`
* `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player), `initial_data` (skip initial data/next ep request). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause issues such as missing formats or metadata. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) and [#12826](https://github.com/yt-dlp/yt-dlp/issues/12826) for more details
* `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp.
+* `player_js_variant`: The player javascript variant to use for signature and nsig deciphering. The known variants are: `main`, `tce`, `tv`, `tv_es6`, `phone`, `tablet`. Only `main` is recommended as a possible workaround; the others are for debugging purposes. The default is to use what is prescribed by the site, and can be selected with `actual`
* `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
* `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all`
* E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total
@@ -1804,8 +1810,12 @@ #### youtube
* `raise_incomplete_data`: `Incomplete Data Received` raises an error instead of reporting a warning
* `data_sync_id`: Overrides the account Data Sync ID used in Innertube API requests. This may be needed if you are using an account with `youtube:player_skip=webpage,configs` or `youtubetab:skip=webpage`
* `visitor_data`: Overrides the Visitor Data used in Innertube API requests. This should be used with `player_skip=webpage,configs` and without cookies. Note: this may have adverse effects if used improperly. If a session from a browser is wanted, you should pass cookies instead (which contain the Visitor ID)
-* `po_token`: Proof of Origin (PO) Token(s) to use. Comma seperated list of PO Tokens in the format `CLIENT.CONTEXT+PO_TOKEN`, e.g. `youtube:po_token=web.gvs+XXX,web.player=XXX,web_safari.gvs+YYY`. Context can be either `gvs` (Google Video Server URLs) or `player` (Innertube player request)
-* `player_js_variant`: The player javascript variant to use for signature and nsig deciphering. The known variants are: `main`, `tce`, `tv`, `tv_es6`, `phone`, `tablet`. Only `main` is recommended as a possible workaround; the others are for debugging purposes. The default is to use what is prescribed by the site, and can be selected with `actual`
+* `po_token`: Proof of Origin (PO) Token(s) to use. Comma seperated list of PO Tokens in the format `CLIENT.CONTEXT+PO_TOKEN`, e.g. `youtube:po_token=web.gvs+XXX,web.player=XXX,web_safari.gvs+YYY`. Context can be any of `gvs` (Google Video Server URLs), `player` (Innertube player request) or `subs` (Subtitles)
+* `pot_trace`: Enable debug logging for PO Token fetching. Either `true` or `false` (default)
+* `fetch_pot`: Policy to use for fetching a PO Token from providers. One of `always` (always try fetch a PO Token regardless if the client requires one for the given context), `never` (never fetch a PO Token), or `auto` (default; only fetch a PO Token if the client requires one for the given context)
+
+#### youtubepot-webpo
+* `bind_to_visitor_id`: Whether to use the Visitor ID instead of Visitor Data for caching WebPO tokens. Either `true` (default) or `false`
#### youtubetab (YouTube playlists, channels, feeds, etc.)
* `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details)
diff --git a/bundle/docker/static/entrypoint.sh b/bundle/docker/static/entrypoint.sh
index 220275974..8049e6820 100755
--- a/bundle/docker/static/entrypoint.sh
+++ b/bundle/docker/static/entrypoint.sh
@@ -2,6 +2,7 @@
set -e
source ~/.local/share/pipx/venvs/pyinstaller/bin/activate
+python -m devscripts.install_deps -o --include build
python -m devscripts.install_deps --include secretstorage --include curl-cffi
python -m devscripts.make_lazy_extractors
python devscripts/update-version.py -c "${channel}" -r "${origin}" "${version}"
diff --git a/bundle/pyinstaller.py b/bundle/pyinstaller.py
index 4184c4bc9..c2f651121 100755
--- a/bundle/pyinstaller.py
+++ b/bundle/pyinstaller.py
@@ -36,6 +36,9 @@ def main():
f'--name={name}',
'--icon=devscripts/logo.ico',
'--upx-exclude=vcruntime140.dll',
+ # Ref: https://github.com/yt-dlp/yt-dlp/issues/13311
+ # https://github.com/pyinstaller/pyinstaller/issues/9149
+ '--exclude-module=pkg_resources',
'--noconfirm',
'--additional-hooks-dir=yt_dlp/__pyinstaller',
*opts,
diff --git a/pyproject.toml b/pyproject.toml
index 7accaeeb9..3775251e1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -65,7 +65,7 @@ build = [
"build",
"hatchling",
"pip",
- "setuptools>=71.0.2", # 71.0.0 broke pyinstaller
+ "setuptools>=71.0.2,<81", # See https://github.com/pyinstaller/pyinstaller/issues/9149
"wheel",
]
dev = [
diff --git a/supportedsites.md b/supportedsites.md
index 03bd8a7c3..c2d7b4555 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -246,7 +246,6 @@ # Supported sites
- **Canalplus**: mycanal.fr and piwiplus.fr
- **Canalsurmas**
- **CaracolTvPlay**: [*caracoltv-play*](## "netrc machine")
- - **CartoonNetwork**
- **cbc.ca**
- **cbc.ca:player**
- **cbc.ca:player:playlist**
@@ -649,7 +648,10 @@ # Supported sites
- **jiocinema**: [*jiocinema*](## "netrc machine")
- **jiocinema:series**: [*jiocinema*](## "netrc machine")
- **jiosaavn:album**
+ - **jiosaavn:artist**
- **jiosaavn:playlist**
+ - **jiosaavn:show**
+ - **jiosaavn:show:playlist**
- **jiosaavn:song**
- **Joj**
- **JoqrAg**: 超!A&G+ 文化放送 (f.k.a. AGQR) Nippon Cultural Broadcasting, Inc. (JOQR)
@@ -1081,8 +1083,8 @@ # Supported sites
- **Photobucket**
- **PiaLive**
- **Piapro**: [*piapro*](## "netrc machine")
- - **Picarto**
- - **PicartoVod**
+ - **picarto**
+ - **picarto:vod**
- **Piksel**
- **Pinkbike**
- **Pinterest**
@@ -1390,7 +1392,6 @@ # Supported sites
- **Spreaker**
- **SpreakerShow**
- **SpringboardPlatform**
- - **Sprout**
- **SproutVideo**
- **sr:mediathek**: Saarländischer Rundfunk (**Currently broken**)
- **SRGSSR**
@@ -1656,6 +1657,7 @@ # Supported sites
- **vimeo**: [*vimeo*](## "netrc machine")
- **vimeo:album**: [*vimeo*](## "netrc machine")
- **vimeo:channel**: [*vimeo*](## "netrc machine")
+ - **vimeo:event**: [*vimeo*](## "netrc machine")
- **vimeo:group**: [*vimeo*](## "netrc machine")
- **vimeo:likes**: [*vimeo*](## "netrc machine") Vimeo user likes
- **vimeo:ondemand**: [*vimeo*](## "netrc machine")
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 708a04f92..91312e4e5 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -1435,6 +1435,27 @@ def test_load_plugins_compat(self):
FakeYDL().close()
assert all_plugins_loaded.value
+ def test_close_hooks(self):
+ # Should call all registered close hooks on close
+ close_hook_called = False
+ close_hook_two_called = False
+
+ def close_hook():
+ nonlocal close_hook_called
+ close_hook_called = True
+
+ def close_hook_two():
+ nonlocal close_hook_two_called
+ close_hook_two_called = True
+
+ ydl = FakeYDL()
+ ydl.add_close_hook(close_hook)
+ ydl.add_close_hook(close_hook_two)
+
+ ydl.close()
+ self.assertTrue(close_hook_called, 'Close hook was not called')
+ self.assertTrue(close_hook_two_called, 'Close hook two was not called')
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/test_cookies.py b/test/test_cookies.py
index 4b9b9b5a9..f956ab187 100644
--- a/test/test_cookies.py
+++ b/test/test_cookies.py
@@ -58,6 +58,14 @@ def test_get_desktop_environment(self):
({'DESKTOP_SESSION': 'kde'}, _LinuxDesktopEnvironment.KDE3),
({'DESKTOP_SESSION': 'xfce'}, _LinuxDesktopEnvironment.XFCE),
+ ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'gnome'}, _LinuxDesktopEnvironment.GNOME),
+ ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'mate'}, _LinuxDesktopEnvironment.GNOME),
+ ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE4),
+ ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'kde'}, _LinuxDesktopEnvironment.KDE3),
+ ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'xfce'}, _LinuxDesktopEnvironment.XFCE),
+
+ ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'my_custom_de', 'GNOME_DESKTOP_SESSION_ID': 1}, _LinuxDesktopEnvironment.GNOME),
+
({'GNOME_DESKTOP_SESSION_ID': 1}, _LinuxDesktopEnvironment.GNOME),
({'KDE_FULL_SESSION': 1}, _LinuxDesktopEnvironment.KDE3),
({'KDE_FULL_SESSION': 1, 'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE4),
diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index b14069ccc..2e3cdc2a5 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -478,6 +478,14 @@ def test_extract_function_with_global_stack(self):
func = jsi.extract_function('c', {'e': 10}, {'f': 100, 'g': 1000})
self.assertEqual(func([1]), 1111)
+ def test_increment_decrement(self):
+ self._test('function f() { var x = 1; return ++x; }', 2)
+ self._test('function f() { var x = 1; return x++; }', 1)
+ self._test('function f() { var x = 1; x--; return x }', 0)
+ self._test('function f() { var y; var x = 1; x++, --x, x--, x--, y="z", "abc", x++; return --x }', -1)
+ self._test('function f() { var a = "test--"; return a; }', 'test--')
+ self._test('function f() { var b = 1; var a = "b--"; return a; }', 'b--')
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/test_networking_utils.py b/test/test_networking_utils.py
index 204fe87bd..a2feacba7 100644
--- a/test/test_networking_utils.py
+++ b/test/test_networking_utils.py
@@ -20,7 +20,6 @@
add_accept_encoding_header,
get_redirect_method,
make_socks_proxy_opts,
- select_proxy,
ssl_load_certs,
)
from yt_dlp.networking.exceptions import (
@@ -28,7 +27,7 @@
IncompleteRead,
)
from yt_dlp.socks import ProxyType
-from yt_dlp.utils.networking import HTTPHeaderDict
+from yt_dlp.utils.networking import HTTPHeaderDict, select_proxy
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py
index 603f85c65..ecc73e39e 100644
--- a/test/test_postprocessors.py
+++ b/test/test_postprocessors.py
@@ -8,6 +8,8 @@
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import subprocess
+
from yt_dlp import YoutubeDL
from yt_dlp.utils import shell_quote
from yt_dlp.postprocessor import (
@@ -47,7 +49,18 @@ def test_escaping(self):
print('Skipping: ffmpeg not found')
return
- file = 'test/testdata/thumbnails/foo %d bar/foo_%d.{}'
+ test_data_dir = 'test/testdata/thumbnails'
+ generated_file = f'{test_data_dir}/empty.webp'
+
+ subprocess.check_call([
+ pp.executable, '-y', '-f', 'lavfi', '-i', 'color=c=black:s=320x320',
+ '-c:v', 'libwebp', '-pix_fmt', 'yuv420p', '-vframes', '1', generated_file,
+ ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+ file = test_data_dir + '/foo %d bar/foo_%d.{}'
+ initial_file = file.format('webp')
+ os.replace(generated_file, initial_file)
+
tests = (('webp', 'png'), ('png', 'jpg'))
for inp, out in tests:
@@ -55,11 +68,13 @@ def test_escaping(self):
if os.path.exists(out_file):
os.remove(out_file)
pp.convert_thumbnail(file.format(inp), out)
- assert os.path.exists(out_file)
+ self.assertTrue(os.path.exists(out_file))
for _, out in tests:
os.remove(file.format(out))
+ os.remove(initial_file)
+
class TestExec(unittest.TestCase):
def test_parse_cmd(self):
@@ -610,3 +625,7 @@ def test_quote_for_concat_QuotesAtEnd(self):
self.assertEqual(
r"'special '\'' characters '\'' galore'\'\'\'",
self._pp._quote_for_ffmpeg("special ' characters ' galore'''"))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/test_pot/conftest.py b/test/test_pot/conftest.py
new file mode 100644
index 000000000..ff0667e92
--- /dev/null
+++ b/test/test_pot/conftest.py
@@ -0,0 +1,71 @@
+import collections
+
+import pytest
+
+from yt_dlp import YoutubeDL
+from yt_dlp.cookies import YoutubeDLCookieJar
+from yt_dlp.extractor.common import InfoExtractor
+from yt_dlp.extractor.youtube.pot._provider import IEContentProviderLogger
+from yt_dlp.extractor.youtube.pot.provider import PoTokenRequest, PoTokenContext
+from yt_dlp.utils.networking import HTTPHeaderDict
+
+
+class MockLogger(IEContentProviderLogger):
+
+ log_level = IEContentProviderLogger.LogLevel.TRACE
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.messages = collections.defaultdict(list)
+
+ def trace(self, message: str):
+ self.messages['trace'].append(message)
+
+ def debug(self, message: str):
+ self.messages['debug'].append(message)
+
+ def info(self, message: str):
+ self.messages['info'].append(message)
+
+ def warning(self, message: str, *, once=False):
+ self.messages['warning'].append(message)
+
+ def error(self, message: str):
+ self.messages['error'].append(message)
+
+
+@pytest.fixture
+def ie() -> InfoExtractor:
+ ydl = YoutubeDL()
+ return ydl.get_info_extractor('Youtube')
+
+
+@pytest.fixture
+def logger() -> MockLogger:
+ return MockLogger()
+
+
+@pytest.fixture()
+def pot_request() -> PoTokenRequest:
+ return PoTokenRequest(
+ context=PoTokenContext.GVS,
+ innertube_context={'client': {'clientName': 'WEB'}},
+ innertube_host='youtube.com',
+ session_index=None,
+ player_url=None,
+ is_authenticated=False,
+ video_webpage=None,
+
+ visitor_data='example-visitor-data',
+ data_sync_id='example-data-sync-id',
+ video_id='example-video-id',
+
+ request_cookiejar=YoutubeDLCookieJar(),
+ request_proxy=None,
+ request_headers=HTTPHeaderDict(),
+ request_timeout=None,
+ request_source_address=None,
+ request_verify_tls=True,
+
+ bypass_cache=False,
+ )
diff --git a/test/test_pot/test_pot_builtin_memorycache.py b/test/test_pot/test_pot_builtin_memorycache.py
new file mode 100644
index 000000000..ea19fbe29
--- /dev/null
+++ b/test/test_pot/test_pot_builtin_memorycache.py
@@ -0,0 +1,117 @@
+import threading
+import time
+from collections import OrderedDict
+import pytest
+from yt_dlp.extractor.youtube.pot._provider import IEContentProvider, BuiltinIEContentProvider
+from yt_dlp.utils import bug_reports_message
+from yt_dlp.extractor.youtube.pot._builtin.memory_cache import MemoryLRUPCP, memorylru_preference, initialize_global_cache
+from yt_dlp.version import __version__
+from yt_dlp.extractor.youtube.pot._registry import _pot_cache_providers, _pot_memory_cache
+
+
+class TestMemoryLRUPCS:
+
+ def test_base_type(self):
+ assert issubclass(MemoryLRUPCP, IEContentProvider)
+ assert issubclass(MemoryLRUPCP, BuiltinIEContentProvider)
+
+ @pytest.fixture
+ def pcp(self, ie, logger) -> MemoryLRUPCP:
+ return MemoryLRUPCP(ie, logger, {}, initialize_cache=lambda max_size: (OrderedDict(), threading.Lock(), max_size))
+
+ def test_is_registered(self):
+ assert _pot_cache_providers.value.get('MemoryLRU') == MemoryLRUPCP
+
+ def test_initialization(self, pcp):
+ assert pcp.PROVIDER_NAME == 'memory'
+ assert pcp.PROVIDER_VERSION == __version__
+ assert pcp.BUG_REPORT_MESSAGE == bug_reports_message(before='')
+ assert pcp.is_available()
+
+ def test_store_and_get(self, pcp):
+ pcp.store('key1', 'value1', int(time.time()) + 60)
+ assert pcp.get('key1') == 'value1'
+ assert len(pcp.cache) == 1
+
+ def test_store_ignore_expired(self, pcp):
+ pcp.store('key1', 'value1', int(time.time()) - 1)
+ assert len(pcp.cache) == 0
+ assert pcp.get('key1') is None
+ assert len(pcp.cache) == 0
+
+ def test_store_override_existing_key(self, ie, logger):
+ MAX_SIZE = 2
+ pcp = MemoryLRUPCP(ie, logger, {}, initialize_cache=lambda max_size: (OrderedDict(), threading.Lock(), MAX_SIZE))
+ pcp.store('key1', 'value1', int(time.time()) + 60)
+ pcp.store('key2', 'value2', int(time.time()) + 60)
+ assert len(pcp.cache) == 2
+ pcp.store('key1', 'value2', int(time.time()) + 60)
+ # Ensure that the override key gets added to the end of the cache instead of in the same position
+ pcp.store('key3', 'value3', int(time.time()) + 60)
+ assert pcp.get('key1') == 'value2'
+
+ def test_store_ignore_expired_existing_key(self, pcp):
+ pcp.store('key1', 'value2', int(time.time()) + 60)
+ pcp.store('key1', 'value1', int(time.time()) - 1)
+ assert len(pcp.cache) == 1
+ assert pcp.get('key1') == 'value2'
+ assert len(pcp.cache) == 1
+
+ def test_get_key_expired(self, pcp):
+ pcp.store('key1', 'value1', int(time.time()) + 60)
+ assert pcp.get('key1') == 'value1'
+ assert len(pcp.cache) == 1
+ pcp.cache['key1'] = ('value1', int(time.time()) - 1)
+ assert pcp.get('key1') is None
+ assert len(pcp.cache) == 0
+
+ def test_lru_eviction(self, ie, logger):
+ MAX_SIZE = 2
+ provider = MemoryLRUPCP(ie, logger, {}, initialize_cache=lambda max_size: (OrderedDict(), threading.Lock(), MAX_SIZE))
+ provider.store('key1', 'value1', int(time.time()) + 5)
+ provider.store('key2', 'value2', int(time.time()) + 5)
+ assert len(provider.cache) == 2
+
+ assert provider.get('key1') == 'value1'
+
+ provider.store('key3', 'value3', int(time.time()) + 5)
+ assert len(provider.cache) == 2
+
+ assert provider.get('key2') is None
+
+ provider.store('key4', 'value4', int(time.time()) + 5)
+ assert len(provider.cache) == 2
+
+ assert provider.get('key1') is None
+ assert provider.get('key3') == 'value3'
+ assert provider.get('key4') == 'value4'
+
+ def test_delete(self, pcp):
+ pcp.store('key1', 'value1', int(time.time()) + 5)
+ assert len(pcp.cache) == 1
+ assert pcp.get('key1') == 'value1'
+ pcp.delete('key1')
+ assert len(pcp.cache) == 0
+ assert pcp.get('key1') is None
+
+ def test_use_global_cache_default(self, ie, logger):
+ pcp = MemoryLRUPCP(ie, logger, {})
+ assert pcp.max_size == _pot_memory_cache.value['max_size'] == 25
+ assert pcp.cache is _pot_memory_cache.value['cache']
+ assert pcp.lock is _pot_memory_cache.value['lock']
+
+ pcp2 = MemoryLRUPCP(ie, logger, {})
+ assert pcp.max_size == pcp2.max_size == _pot_memory_cache.value['max_size'] == 25
+ assert pcp.cache is pcp2.cache is _pot_memory_cache.value['cache']
+ assert pcp.lock is pcp2.lock is _pot_memory_cache.value['lock']
+
+ def test_fail_max_size_change_global(self, ie, logger):
+ pcp = MemoryLRUPCP(ie, logger, {})
+ assert pcp.max_size == _pot_memory_cache.value['max_size'] == 25
+ with pytest.raises(ValueError, match='Cannot change max_size of initialized global memory cache'):
+ initialize_global_cache(50)
+
+ assert pcp.max_size == _pot_memory_cache.value['max_size'] == 25
+
+ def test_memory_lru_preference(self, pcp, ie, pot_request):
+ assert memorylru_preference(pcp, pot_request) == 10000
diff --git a/test/test_pot/test_pot_builtin_utils.py b/test/test_pot/test_pot_builtin_utils.py
new file mode 100644
index 000000000..a95fc4e15
--- /dev/null
+++ b/test/test_pot/test_pot_builtin_utils.py
@@ -0,0 +1,47 @@
+import pytest
+from yt_dlp.extractor.youtube.pot.provider import (
+ PoTokenContext,
+
+)
+
+from yt_dlp.extractor.youtube.pot.utils import get_webpo_content_binding, ContentBindingType
+
+
+class TestGetWebPoContentBinding:
+
+ @pytest.mark.parametrize('client_name, context, is_authenticated, expected', [
+ *[(client, context, is_authenticated, expected) for client in [
+ 'WEB', 'MWEB', 'TVHTML5', 'WEB_EMBEDDED_PLAYER', 'WEB_CREATOR', 'TVHTML5_SIMPLY_EMBEDDED_PLAYER']
+ for context, is_authenticated, expected in [
+ (PoTokenContext.GVS, False, ('example-visitor-data', ContentBindingType.VISITOR_DATA)),
+ (PoTokenContext.PLAYER, False, ('example-video-id', ContentBindingType.VIDEO_ID)),
+ (PoTokenContext.SUBS, False, ('example-video-id', ContentBindingType.VIDEO_ID)),
+ (PoTokenContext.GVS, True, ('example-data-sync-id', ContentBindingType.DATASYNC_ID)),
+ ]],
+ ('WEB_REMIX', PoTokenContext.GVS, False, ('example-visitor-data', ContentBindingType.VISITOR_DATA)),
+ ('WEB_REMIX', PoTokenContext.PLAYER, False, ('example-visitor-data', ContentBindingType.VISITOR_DATA)),
+ ('ANDROID', PoTokenContext.GVS, False, (None, None)),
+ ('IOS', PoTokenContext.GVS, False, (None, None)),
+ ])
+ def test_get_webpo_content_binding(self, pot_request, client_name, context, is_authenticated, expected):
+ pot_request.innertube_context['client']['clientName'] = client_name
+ pot_request.context = context
+ pot_request.is_authenticated = is_authenticated
+ assert get_webpo_content_binding(pot_request) == expected
+
+ def test_extract_visitor_id(self, pot_request):
+ pot_request.visitor_data = 'CgsxMjNhYmNYWVpfLSiA4s%2DqBg%3D%3D'
+ assert get_webpo_content_binding(pot_request, bind_to_visitor_id=True) == ('123abcXYZ_-', ContentBindingType.VISITOR_ID)
+
+ def test_invalid_visitor_id(self, pot_request):
+ # visitor id not alphanumeric (i.e. protobuf extraction failed)
+ pot_request.visitor_data = 'CggxMjM0NTY3OCiA4s-qBg%3D%3D'
+ assert get_webpo_content_binding(pot_request, bind_to_visitor_id=True) == (pot_request.visitor_data, ContentBindingType.VISITOR_DATA)
+
+ def test_no_visitor_id(self, pot_request):
+ pot_request.visitor_data = 'KIDiz6oG'
+ assert get_webpo_content_binding(pot_request, bind_to_visitor_id=True) == (pot_request.visitor_data, ContentBindingType.VISITOR_DATA)
+
+ def test_invalid_base64(self, pot_request):
+ pot_request.visitor_data = 'invalid-base64'
+ assert get_webpo_content_binding(pot_request, bind_to_visitor_id=True) == (pot_request.visitor_data, ContentBindingType.VISITOR_DATA)
diff --git a/test/test_pot/test_pot_builtin_webpospec.py b/test/test_pot/test_pot_builtin_webpospec.py
new file mode 100644
index 000000000..c5fb6f382
--- /dev/null
+++ b/test/test_pot/test_pot_builtin_webpospec.py
@@ -0,0 +1,92 @@
+import pytest
+
+from yt_dlp.extractor.youtube.pot._provider import IEContentProvider, BuiltinIEContentProvider
+from yt_dlp.extractor.youtube.pot.cache import CacheProviderWritePolicy
+from yt_dlp.utils import bug_reports_message
+from yt_dlp.extractor.youtube.pot.provider import (
+ PoTokenRequest,
+ PoTokenContext,
+
+)
+from yt_dlp.version import __version__
+
+from yt_dlp.extractor.youtube.pot._builtin.webpo_cachespec import WebPoPCSP
+from yt_dlp.extractor.youtube.pot._registry import _pot_pcs_providers
+
+
+@pytest.fixture()
+def pot_request(pot_request) -> PoTokenRequest:
+ pot_request.visitor_data = 'CgsxMjNhYmNYWVpfLSiA4s%2DqBg%3D%3D' # visitor_id=123abcXYZ_-
+ return pot_request
+
+
+class TestWebPoPCSP:
+ def test_base_type(self):
+ assert issubclass(WebPoPCSP, IEContentProvider)
+ assert issubclass(WebPoPCSP, BuiltinIEContentProvider)
+
+ def test_init(self, ie, logger):
+ pcs = WebPoPCSP(ie=ie, logger=logger, settings={})
+ assert pcs.PROVIDER_NAME == 'webpo'
+ assert pcs.PROVIDER_VERSION == __version__
+ assert pcs.BUG_REPORT_MESSAGE == bug_reports_message(before='')
+ assert pcs.is_available()
+
+ def test_is_registered(self):
+ assert _pot_pcs_providers.value.get('WebPo') == WebPoPCSP
+
+ @pytest.mark.parametrize('client_name, context, is_authenticated', [
+ ('ANDROID', PoTokenContext.GVS, False),
+ ('IOS', PoTokenContext.GVS, False),
+ ('IOS', PoTokenContext.PLAYER, False),
+ ])
+ def test_not_supports(self, ie, logger, pot_request, client_name, context, is_authenticated):
+ pcs = WebPoPCSP(ie=ie, logger=logger, settings={})
+ pot_request.innertube_context['client']['clientName'] = client_name
+ pot_request.context = context
+ pot_request.is_authenticated = is_authenticated
+ assert pcs.generate_cache_spec(pot_request) is None
+
+ @pytest.mark.parametrize('client_name, context, is_authenticated, remote_host, source_address, request_proxy, expected', [
+ *[(client, context, is_authenticated, remote_host, source_address, request_proxy, expected) for client in [
+ 'WEB', 'MWEB', 'TVHTML5', 'WEB_EMBEDDED_PLAYER', 'WEB_CREATOR', 'TVHTML5_SIMPLY_EMBEDDED_PLAYER']
+ for context, is_authenticated, remote_host, source_address, request_proxy, expected in [
+ (PoTokenContext.GVS, False, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': '123abcXYZ_-', 'cbt': 'visitor_id'}),
+ (PoTokenContext.PLAYER, False, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': '123abcXYZ_-', 'cbt': 'video_id'}),
+ (PoTokenContext.GVS, True, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': 'example-data-sync-id', 'cbt': 'datasync_id'}),
+ ]],
+ ('WEB_REMIX', PoTokenContext.PLAYER, False, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': '123abcXYZ_-', 'cbt': 'visitor_id'}),
+ ('WEB', PoTokenContext.GVS, False, None, None, None, {'t': 'webpo', 'cb': '123abcXYZ_-', 'cbt': 'visitor_id', 'ip': None, 'sa': None, 'px': None}),
+ ('TVHTML5', PoTokenContext.PLAYER, False, None, None, 'http://example.com', {'t': 'webpo', 'cb': '123abcXYZ_-', 'cbt': 'video_id', 'ip': None, 'sa': None, 'px': 'http://example.com'}),
+
+ ])
+ def test_generate_key_bindings(self, ie, logger, pot_request, client_name, context, is_authenticated, remote_host, source_address, request_proxy, expected):
+ pcs = WebPoPCSP(ie=ie, logger=logger, settings={})
+ pot_request.innertube_context['client']['clientName'] = client_name
+ pot_request.context = context
+ pot_request.is_authenticated = is_authenticated
+ pot_request.innertube_context['client']['remoteHost'] = remote_host
+ pot_request.request_source_address = source_address
+ pot_request.request_proxy = request_proxy
+ pot_request.video_id = '123abcXYZ_-' # same as visitor id to test type
+
+ assert pcs.generate_cache_spec(pot_request).key_bindings == expected
+
+ def test_no_bind_visitor_id(self, ie, logger, pot_request):
+ # Should not bind to visitor id if setting is set to False
+ pcs = WebPoPCSP(ie=ie, logger=logger, settings={'bind_to_visitor_id': ['false']})
+ pot_request.innertube_context['client']['clientName'] = 'WEB'
+ pot_request.context = PoTokenContext.GVS
+ pot_request.is_authenticated = False
+ assert pcs.generate_cache_spec(pot_request).key_bindings == {'t': 'webpo', 'ip': None, 'sa': None, 'px': None, 'cb': 'CgsxMjNhYmNYWVpfLSiA4s%2DqBg%3D%3D', 'cbt': 'visitor_data'}
+
+ def test_default_ttl(self, ie, logger, pot_request):
+ pcs = WebPoPCSP(ie=ie, logger=logger, settings={})
+ assert pcs.generate_cache_spec(pot_request).default_ttl == 6 * 60 * 60 # should default to 6 hours
+
+ def test_write_policy(self, ie, logger, pot_request):
+ pcs = WebPoPCSP(ie=ie, logger=logger, settings={})
+ pot_request.context = PoTokenContext.GVS
+ assert pcs.generate_cache_spec(pot_request).write_policy == CacheProviderWritePolicy.WRITE_ALL
+ pot_request.context = PoTokenContext.PLAYER
+ assert pcs.generate_cache_spec(pot_request).write_policy == CacheProviderWritePolicy.WRITE_FIRST
diff --git a/test/test_pot/test_pot_director.py b/test/test_pot/test_pot_director.py
new file mode 100644
index 000000000..bbfdd0e98
--- /dev/null
+++ b/test/test_pot/test_pot_director.py
@@ -0,0 +1,1529 @@
+from __future__ import annotations
+import abc
+import base64
+import dataclasses
+import hashlib
+import json
+import time
+import pytest
+
+from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider, IEContentProvider
+
+from yt_dlp.extractor.youtube.pot.provider import (
+ PoTokenRequest,
+ PoTokenContext,
+ PoTokenProviderError,
+ PoTokenProviderRejectedRequest,
+)
+from yt_dlp.extractor.youtube.pot._director import (
+ PoTokenCache,
+ validate_cache_spec,
+ clean_pot,
+ validate_response,
+ PoTokenRequestDirector,
+ provider_display_list,
+)
+
+from yt_dlp.extractor.youtube.pot.cache import (
+ PoTokenCacheSpec,
+ PoTokenCacheSpecProvider,
+ PoTokenCacheProvider,
+ CacheProviderWritePolicy,
+ PoTokenCacheProviderError,
+)
+
+
+from yt_dlp.extractor.youtube.pot.provider import (
+ PoTokenResponse,
+ PoTokenProvider,
+)
+
+
+class BaseMockPoTokenProvider(PoTokenProvider, abc.ABC):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.available_called_times = 0
+ self.request_called_times = 0
+ self.close_called = False
+
+ def is_available(self) -> bool:
+ self.available_called_times += 1
+ return True
+
+ def request_pot(self, *args, **kwargs):
+ self.request_called_times += 1
+ return super().request_pot(*args, **kwargs)
+
+ def close(self):
+ self.close_called = True
+ super().close()
+
+
+class ExamplePTP(BaseMockPoTokenProvider):
+ PROVIDER_NAME = 'example'
+ PROVIDER_VERSION = '0.0.1'
+ BUG_REPORT_LOCATION = 'https://example.com/issues'
+
+ _SUPPORTED_CLIENTS = ('WEB',)
+ _SUPPORTED_CONTEXTS = (PoTokenContext.GVS, )
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ if request.data_sync_id == 'example':
+ return PoTokenResponse(request.video_id)
+ return PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+
+def success_ptp(response: PoTokenResponse | None = None, key: str | None = None):
+ class SuccessPTP(BaseMockPoTokenProvider):
+ PROVIDER_NAME = 'success'
+ PROVIDER_VERSION = '0.0.1'
+ BUG_REPORT_LOCATION = 'https://success.example.com/issues'
+
+ _SUPPORTED_CLIENTS = ('WEB',)
+ _SUPPORTED_CONTEXTS = (PoTokenContext.GVS,)
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ return response or PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ if key:
+ SuccessPTP.PROVIDER_KEY = key
+ return SuccessPTP
+
+
+@pytest.fixture
+def pot_provider(ie, logger):
+ return success_ptp()(ie=ie, logger=logger, settings={})
+
+
+class UnavailablePTP(BaseMockPoTokenProvider):
+ PROVIDER_NAME = 'unavailable'
+ BUG_REPORT_LOCATION = 'https://unavailable.example.com/issues'
+ _SUPPORTED_CLIENTS = None
+ _SUPPORTED_CONTEXTS = None
+
+ def is_available(self) -> bool:
+ super().is_available()
+ return False
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ raise PoTokenProviderError('something went wrong')
+
+
+class UnsupportedPTP(BaseMockPoTokenProvider):
+ PROVIDER_NAME = 'unsupported'
+ BUG_REPORT_LOCATION = 'https://unsupported.example.com/issues'
+ _SUPPORTED_CLIENTS = None
+ _SUPPORTED_CONTEXTS = None
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ raise PoTokenProviderRejectedRequest('unsupported request')
+
+
+class ErrorPTP(BaseMockPoTokenProvider):
+ PROVIDER_NAME = 'error'
+ BUG_REPORT_LOCATION = 'https://error.example.com/issues'
+ _SUPPORTED_CLIENTS = None
+ _SUPPORTED_CONTEXTS = None
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ expected = request.video_id == 'expected'
+ raise PoTokenProviderError('an error occurred', expected=expected)
+
+
+class UnexpectedErrorPTP(BaseMockPoTokenProvider):
+ PROVIDER_NAME = 'unexpected_error'
+ BUG_REPORT_LOCATION = 'https://unexpected.example.com/issues'
+ _SUPPORTED_CLIENTS = None
+ _SUPPORTED_CONTEXTS = None
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ raise ValueError('an unexpected error occurred')
+
+
+class InvalidPTP(BaseMockPoTokenProvider):
+ PROVIDER_NAME = 'invalid'
+ BUG_REPORT_LOCATION = 'https://invalid.example.com/issues'
+ _SUPPORTED_CLIENTS = None
+ _SUPPORTED_CONTEXTS = None
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ if request.video_id == 'invalid_type':
+ return 'invalid-response'
+ else:
+ return PoTokenResponse('example-token?', expires_at='123')
+
+
+class BaseMockCacheSpecProvider(PoTokenCacheSpecProvider, abc.ABC):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.generate_called_times = 0
+ self.is_available_called_times = 0
+ self.close_called = False
+
+ def is_available(self) -> bool:
+ self.is_available_called_times += 1
+ return super().is_available()
+
+ def generate_cache_spec(self, request: PoTokenRequest):
+ self.generate_called_times += 1
+
+ def close(self):
+ self.close_called = True
+ super().close()
+
+
+class ExampleCacheSpecProviderPCSP(BaseMockCacheSpecProvider):
+
+ PROVIDER_NAME = 'example'
+ PROVIDER_VERSION = '0.0.1'
+ BUG_REPORT_LOCATION = 'https://example.com/issues'
+
+ def generate_cache_spec(self, request: PoTokenRequest):
+ super().generate_cache_spec(request)
+ return PoTokenCacheSpec(
+ key_bindings={'v': request.video_id, 'e': None},
+ default_ttl=60,
+ )
+
+
+class UnavailableCacheSpecProviderPCSP(BaseMockCacheSpecProvider):
+
+ PROVIDER_NAME = 'unavailable'
+ PROVIDER_VERSION = '0.0.1'
+
+ def is_available(self) -> bool:
+ super().is_available()
+ return False
+
+ def generate_cache_spec(self, request: PoTokenRequest):
+ super().generate_cache_spec(request)
+ return None
+
+
+class UnsupportedCacheSpecProviderPCSP(BaseMockCacheSpecProvider):
+
+ PROVIDER_NAME = 'unsupported'
+ PROVIDER_VERSION = '0.0.1'
+
+ def generate_cache_spec(self, request: PoTokenRequest):
+ super().generate_cache_spec(request)
+ return None
+
+
+class InvalidSpecCacheSpecProviderPCSP(BaseMockCacheSpecProvider):
+
+ PROVIDER_NAME = 'invalid'
+ PROVIDER_VERSION = '0.0.1'
+
+ def generate_cache_spec(self, request: PoTokenRequest):
+ super().generate_cache_spec(request)
+ return 'invalid-spec'
+
+
+class ErrorSpecCacheSpecProviderPCSP(BaseMockCacheSpecProvider):
+
+ PROVIDER_NAME = 'invalid'
+ PROVIDER_VERSION = '0.0.1'
+
+ def generate_cache_spec(self, request: PoTokenRequest):
+ super().generate_cache_spec(request)
+ raise ValueError('something went wrong')
+
+
+class BaseMockCacheProvider(PoTokenCacheProvider, abc.ABC):
+ BUG_REPORT_MESSAGE = 'example bug report message'
+
+ def __init__(self, *args, available=True, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.store_calls = 0
+ self.delete_calls = 0
+ self.get_calls = 0
+ self.available_called_times = 0
+ self.available = available
+
+ def is_available(self) -> bool:
+ self.available_called_times += 1
+ return self.available
+
+ def store(self, *args, **kwargs):
+ self.store_calls += 1
+
+ def delete(self, *args, **kwargs):
+ self.delete_calls += 1
+
+ def get(self, *args, **kwargs):
+ self.get_calls += 1
+
+ def close(self):
+ self.close_called = True
+ super().close()
+
+
+class ErrorPCP(BaseMockCacheProvider):
+ PROVIDER_NAME = 'error'
+
+ def store(self, *args, **kwargs):
+ super().store(*args, **kwargs)
+ raise PoTokenCacheProviderError('something went wrong')
+
+ def get(self, *args, **kwargs):
+ super().get(*args, **kwargs)
+ raise PoTokenCacheProviderError('something went wrong')
+
+
+class UnexpectedErrorPCP(BaseMockCacheProvider):
+ PROVIDER_NAME = 'unexpected_error'
+
+ def store(self, *args, **kwargs):
+ super().store(*args, **kwargs)
+ raise ValueError('something went wrong')
+
+ def get(self, *args, **kwargs):
+ super().get(*args, **kwargs)
+ raise ValueError('something went wrong')
+
+
+class MockMemoryPCP(BaseMockCacheProvider):
+ PROVIDER_NAME = 'memory'
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.cache = {}
+
+ def store(self, key, value, expires_at):
+ super().store(key, value, expires_at)
+ self.cache[key] = (value, expires_at)
+
+ def delete(self, key):
+ super().delete(key)
+ self.cache.pop(key, None)
+
+ def get(self, key):
+ super().get(key)
+ return self.cache.get(key, [None])[0]
+
+
+def create_memory_pcp(ie, logger, provider_key='memory', provider_name='memory', available=True):
+ cache = MockMemoryPCP(ie, logger, {}, available=available)
+ cache.PROVIDER_KEY = provider_key
+ cache.PROVIDER_NAME = provider_name
+ return cache
+
+
+@pytest.fixture
+def memorypcp(ie, logger) -> MockMemoryPCP:
+ return create_memory_pcp(ie, logger)
+
+
+@pytest.fixture
+def pot_cache(ie, logger):
+ class MockPoTokenCache(PoTokenCache):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.get_calls = 0
+ self.store_calls = 0
+ self.close_called = False
+
+ def get(self, *args, **kwargs):
+ self.get_calls += 1
+ return super().get(*args, **kwargs)
+
+ def store(self, *args, **kwargs):
+ self.store_calls += 1
+ return super().store(*args, **kwargs)
+
+ def close(self):
+ self.close_called = True
+ super().close()
+
+ return MockPoTokenCache(
+ cache_providers=[MockMemoryPCP(ie, logger, {})],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie, logger, settings={})],
+ logger=logger,
+ )
+
+
+EXAMPLE_PO_TOKEN = base64.urlsafe_b64encode(b'example-token').decode()
+
+
+class TestPoTokenCache:
+
+ def test_cache_success(self, memorypcp, pot_request, ie, logger):
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ assert cache.get(pot_request) is None
+ cache.store(pot_request, response)
+
+ cached_response = cache.get(pot_request)
+ assert cached_response is not None
+ assert cached_response.po_token == EXAMPLE_PO_TOKEN
+ assert cached_response.expires_at is not None
+
+ assert cache.get(dataclasses.replace(pot_request, video_id='another-video-id')) is None
+
+ def test_unsupported_cache_spec_no_fallback(self, memorypcp, pot_request, ie, logger):
+ unsupported_provider = UnsupportedCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[unsupported_provider],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ assert cache.get(pot_request) is None
+ assert unsupported_provider.generate_called_times == 1
+ cache.store(pot_request, response)
+ assert len(memorypcp.cache) == 0
+ assert unsupported_provider.generate_called_times == 2
+ assert cache.get(pot_request) is None
+ assert unsupported_provider.generate_called_times == 3
+ assert len(logger.messages.get('error', [])) == 0
+
+ def test_unsupported_cache_spec_fallback(self, memorypcp, pot_request, ie, logger):
+ unsupported_provider = UnsupportedCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ example_provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[unsupported_provider, example_provider],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ assert cache.get(pot_request) is None
+ assert unsupported_provider.generate_called_times == 1
+ assert example_provider.generate_called_times == 1
+
+ cache.store(pot_request, response)
+ assert unsupported_provider.generate_called_times == 2
+ assert example_provider.generate_called_times == 2
+
+ cached_response = cache.get(pot_request)
+ assert unsupported_provider.generate_called_times == 3
+ assert example_provider.generate_called_times == 3
+ assert cached_response is not None
+ assert cached_response.po_token == EXAMPLE_PO_TOKEN
+ assert cached_response.expires_at is not None
+
+ assert len(logger.messages.get('error', [])) == 0
+
+ def test_invalid_cache_spec_no_fallback(self, memorypcp, pot_request, ie, logger):
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[InvalidSpecCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ assert cache.get(pot_request) is None
+ cache.store(pot_request, response)
+
+ assert cache.get(pot_request) is None
+
+ assert 'PoTokenCacheSpecProvider "InvalidSpecCacheSpecProvider" generate_cache_spec() returned invalid spec invalid-spec; please report this issue to the provider developer at (developer has not provided a bug report location) .' in logger.messages['error']
+
+ def test_invalid_cache_spec_fallback(self, memorypcp, pot_request, ie, logger):
+
+ invalid_provider = InvalidSpecCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ example_provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[invalid_provider, example_provider],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ assert cache.get(pot_request) is None
+ assert invalid_provider.generate_called_times == example_provider.generate_called_times == 1
+
+ cache.store(pot_request, response)
+ assert invalid_provider.generate_called_times == example_provider.generate_called_times == 2
+
+ cached_response = cache.get(pot_request)
+ assert invalid_provider.generate_called_times == example_provider.generate_called_times == 3
+ assert cached_response is not None
+ assert cached_response.po_token == EXAMPLE_PO_TOKEN
+ assert cached_response.expires_at is not None
+
+ assert 'PoTokenCacheSpecProvider "InvalidSpecCacheSpecProvider" generate_cache_spec() returned invalid spec invalid-spec; please report this issue to the provider developer at (developer has not provided a bug report location) .' in logger.messages['error']
+
+ def test_unavailable_cache_spec_no_fallback(self, memorypcp, pot_request, ie, logger):
+ unavailable_provider = UnavailableCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[unavailable_provider],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ assert cache.get(pot_request) is None
+ cache.store(pot_request, response)
+ assert cache.get(pot_request) is None
+ assert unavailable_provider.generate_called_times == 0
+
+ def test_unavailable_cache_spec_fallback(self, memorypcp, pot_request, ie, logger):
+ unavailable_provider = UnavailableCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ example_provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[unavailable_provider, example_provider],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ assert cache.get(pot_request) is None
+ assert unavailable_provider.generate_called_times == 0
+ assert unavailable_provider.is_available_called_times == 1
+ assert example_provider.generate_called_times == 1
+
+ cache.store(pot_request, response)
+ assert unavailable_provider.generate_called_times == 0
+ assert unavailable_provider.is_available_called_times == 2
+ assert example_provider.generate_called_times == 2
+
+ cached_response = cache.get(pot_request)
+ assert unavailable_provider.generate_called_times == 0
+ assert unavailable_provider.is_available_called_times == 3
+ assert example_provider.generate_called_times == 3
+ assert example_provider.is_available_called_times == 3
+ assert cached_response is not None
+ assert cached_response.po_token == EXAMPLE_PO_TOKEN
+ assert cached_response.expires_at is not None
+
+ def test_unexpected_error_cache_spec(self, memorypcp, pot_request, ie, logger):
+ error_provider = ErrorSpecCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[error_provider],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ assert cache.get(pot_request) is None
+ cache.store(pot_request, response)
+ assert cache.get(pot_request) is None
+ assert error_provider.generate_called_times == 3
+ assert error_provider.is_available_called_times == 3
+
+ assert 'Error occurred with "invalid" PO Token cache spec provider: ValueError(\'something went wrong\'); please report this issue to the provider developer at (developer has not provided a bug report location) .' in logger.messages['error']
+
+ def test_unexpected_error_cache_spec_fallback(self, memorypcp, pot_request, ie, logger):
+ error_provider = ErrorSpecCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ example_provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[error_provider, example_provider],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ assert cache.get(pot_request) is None
+ assert error_provider.generate_called_times == 1
+ assert error_provider.is_available_called_times == 1
+ assert example_provider.generate_called_times == 1
+
+ cache.store(pot_request, response)
+ assert error_provider.generate_called_times == 2
+ assert error_provider.is_available_called_times == 2
+ assert example_provider.generate_called_times == 2
+
+ cached_response = cache.get(pot_request)
+ assert error_provider.generate_called_times == 3
+ assert error_provider.is_available_called_times == 3
+ assert example_provider.generate_called_times == 3
+ assert example_provider.is_available_called_times == 3
+ assert cached_response is not None
+ assert cached_response.po_token == EXAMPLE_PO_TOKEN
+ assert cached_response.expires_at is not None
+
+ assert 'Error occurred with "invalid" PO Token cache spec provider: ValueError(\'something went wrong\'); please report this issue to the provider developer at (developer has not provided a bug report location) .' in logger.messages['error']
+
+ def test_key_bindings_spec_provider(self, memorypcp, pot_request, ie, logger):
+
+ class ExampleProviderPCSP(PoTokenCacheSpecProvider):
+ PROVIDER_NAME = 'example'
+
+ def generate_cache_spec(self, request: PoTokenRequest):
+ return PoTokenCacheSpec(
+ key_bindings={'v': request.video_id},
+ default_ttl=60,
+ )
+
+ class ExampleProviderTwoPCSP(ExampleProviderPCSP):
+ pass
+
+ example_provider = ExampleProviderPCSP(ie=ie, logger=logger, settings={})
+ example_provider_two = ExampleProviderTwoPCSP(ie=ie, logger=logger, settings={})
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[example_provider],
+ logger=logger,
+ )
+
+ assert cache.get(pot_request) is None
+ cache.store(pot_request, response)
+ assert len(memorypcp.cache) == 1
+ assert hashlib.sha256(
+ f"{{'_dlp_cache': 'v1', '_p': 'ExampleProvider', 'v': '{pot_request.video_id}'}}".encode()).hexdigest() in memorypcp.cache
+
+ # The second spec provider returns the exact same key bindings as the first one,
+ # however the PoTokenCache should use the provider key to differentiate between them
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[example_provider_two],
+ logger=logger,
+ )
+
+ assert cache.get(pot_request) is None
+ cache.store(pot_request, response)
+ assert len(memorypcp.cache) == 2
+ assert hashlib.sha256(
+ f"{{'_dlp_cache': 'v1', '_p': 'ExampleProviderTwo', 'v': '{pot_request.video_id}'}}".encode()).hexdigest() in memorypcp.cache
+
+ def test_cache_provider_preferences(self, pot_request, ie, logger):
+ pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one')
+ pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two')
+
+ cache = PoTokenCache(
+ cache_providers=[pcp_one, pcp_two],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN), write_policy=CacheProviderWritePolicy.WRITE_FIRST)
+ assert len(pcp_one.cache) == 1
+ assert len(pcp_two.cache) == 0
+
+ assert cache.get(pot_request)
+ assert pcp_one.get_calls == 1
+ assert pcp_two.get_calls == 0
+
+ standard_preference_called = False
+ pcp_one_preference_claled = False
+
+ def standard_preference(provider, request, *_, **__):
+ nonlocal standard_preference_called
+ standard_preference_called = True
+ assert isinstance(provider, PoTokenCacheProvider)
+ assert isinstance(request, PoTokenRequest)
+ return 1
+
+ def pcp_one_preference(provider, request, *_, **__):
+ nonlocal pcp_one_preference_claled
+ pcp_one_preference_claled = True
+ assert isinstance(provider, PoTokenCacheProvider)
+ assert isinstance(request, PoTokenRequest)
+ if provider.PROVIDER_KEY == pcp_one.PROVIDER_KEY:
+ return -100
+ return 0
+
+ # test that it can hanldle multiple preferences
+ cache.cache_provider_preferences.append(standard_preference)
+ cache.cache_provider_preferences.append(pcp_one_preference)
+
+ cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN), write_policy=CacheProviderWritePolicy.WRITE_FIRST)
+ assert cache.get(pot_request)
+ assert len(pcp_one.cache) == len(pcp_two.cache) == 1
+ assert pcp_two.get_calls == pcp_one.get_calls == 1
+ assert pcp_one.store_calls == pcp_two.store_calls == 1
+ assert standard_preference_called
+ assert pcp_one_preference_claled
+
+ def test_secondary_cache_provider_hit(self, pot_request, ie, logger):
+ pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one')
+ pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two')
+
+ cache = PoTokenCache(
+ cache_providers=[pcp_two],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ # Given the lower priority provider has the cache hit, store the response in the higher priority provider
+ cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN))
+ assert cache.get(pot_request)
+
+ cache.cache_providers[pcp_one.PROVIDER_KEY] = pcp_one
+
+ def pcp_one_pref(provider, *_, **__):
+ if provider.PROVIDER_KEY == pcp_one.PROVIDER_KEY:
+ return 1
+ return -1
+
+ cache.cache_provider_preferences.append(pcp_one_pref)
+
+ assert cache.get(pot_request)
+ assert pcp_one.get_calls == 1
+ assert pcp_two.get_calls == 2
+ # Should write back to pcp_one (now the highest priority cache provider)
+ assert pcp_one.store_calls == pcp_two.store_calls == 1
+ assert 'Writing PO Token response to highest priority cache provider' in logger.messages['trace']
+
+ def test_cache_provider_no_hits(self, pot_request, ie, logger):
+ pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one')
+ pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two')
+
+ cache = PoTokenCache(
+ cache_providers=[pcp_one, pcp_two],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ assert cache.get(pot_request) is None
+ assert pcp_one.get_calls == pcp_two.get_calls == 1
+
+ def test_get_invalid_po_token_response(self, pot_request, ie, logger):
+ # Test various scenarios where the po token response stored in the cache provider is invalid
+ pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one')
+ pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two')
+
+ cache = PoTokenCache(
+ cache_providers=[pcp_one, pcp_two],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ valid_response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, valid_response)
+ assert len(pcp_one.cache) == len(pcp_two.cache) == 1
+ # Overwrite the valid response with an invalid one in the cache
+ pcp_one.store(next(iter(pcp_one.cache.keys())), json.dumps(dataclasses.asdict(PoTokenResponse(None))), int(time.time() + 1000))
+ assert cache.get(pot_request).po_token == valid_response.po_token
+ assert pcp_one.get_calls == pcp_two.get_calls == 1
+ assert pcp_one.delete_calls == 1 # Invalid response should be deleted from cache
+ assert pcp_one.store_calls == 3 # Since response was fetched from second cache provider, it should be stored in the first one
+ assert len(pcp_one.cache) == 1
+ assert 'Invalid PO Token response retrieved from cache provider "memory": {"po_token": null, "expires_at": null}; example bug report message' in logger.messages['error']
+
+ # Overwrite the valid response with an invalid json in the cache
+ pcp_one.store(next(iter(pcp_one.cache.keys())), 'invalid-json', int(time.time() + 1000))
+ assert cache.get(pot_request).po_token == valid_response.po_token
+ assert pcp_one.get_calls == pcp_two.get_calls == 2
+ assert pcp_one.delete_calls == 2
+ assert pcp_one.store_calls == 5 # 3 + 1 store we made in the test + 1 store from lower priority cache provider
+ assert len(pcp_one.cache) == 1
+
+ assert 'Invalid PO Token response retrieved from cache provider "memory": invalid-json; example bug report message' in logger.messages['error']
+
+ # Valid json, but missing required fields
+ pcp_one.store(next(iter(pcp_one.cache.keys())), '{"unknown_param": 0}', int(time.time() + 1000))
+ assert cache.get(pot_request).po_token == valid_response.po_token
+ assert pcp_one.get_calls == pcp_two.get_calls == 3
+ assert pcp_one.delete_calls == 3
+ assert pcp_one.store_calls == 7 # 5 + 1 store from test + 1 store from lower priority cache provider
+ assert len(pcp_one.cache) == 1
+
+ assert 'Invalid PO Token response retrieved from cache provider "memory": {"unknown_param": 0}; example bug report message' in logger.messages['error']
+
+ def test_store_invalid_po_token_response(self, pot_request, ie, logger):
+ # Should not store an invalid po token response
+ pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one')
+
+ cache = PoTokenCache(
+ cache_providers=[pcp_one],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ cache.store(pot_request, PoTokenResponse(po_token=EXAMPLE_PO_TOKEN, expires_at=80))
+ assert cache.get(pot_request) is None
+ assert pcp_one.store_calls == 0
+ assert 'Invalid PO Token response provided to PoTokenCache.store()' in logger.messages['error'][0]
+
+ def test_store_write_policy(self, pot_request, ie, logger):
+ pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one')
+ pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two')
+
+ cache = PoTokenCache(
+ cache_providers=[pcp_one, pcp_two],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN), write_policy=CacheProviderWritePolicy.WRITE_FIRST)
+ assert pcp_one.store_calls == 1
+ assert pcp_two.store_calls == 0
+
+ cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN), write_policy=CacheProviderWritePolicy.WRITE_ALL)
+ assert pcp_one.store_calls == 2
+ assert pcp_two.store_calls == 1
+
+ def test_store_write_first_policy_cache_spec(self, pot_request, ie, logger):
+ pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one')
+ pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two')
+
+ class WriteFirstPCSP(BaseMockCacheSpecProvider):
+ def generate_cache_spec(self, request: PoTokenRequest):
+ super().generate_cache_spec(request)
+ return PoTokenCacheSpec(
+ key_bindings={'v': request.video_id, 'e': None},
+ default_ttl=60,
+ write_policy=CacheProviderWritePolicy.WRITE_FIRST,
+ )
+
+ cache = PoTokenCache(
+ cache_providers=[pcp_one, pcp_two],
+ cache_spec_providers=[WriteFirstPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN))
+ assert pcp_one.store_calls == 1
+ assert pcp_two.store_calls == 0
+
+ def test_store_write_all_policy_cache_spec(self, pot_request, ie, logger):
+ pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one')
+ pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two')
+
+ class WriteAllPCSP(BaseMockCacheSpecProvider):
+ def generate_cache_spec(self, request: PoTokenRequest):
+ super().generate_cache_spec(request)
+ return PoTokenCacheSpec(
+ key_bindings={'v': request.video_id, 'e': None},
+ default_ttl=60,
+ write_policy=CacheProviderWritePolicy.WRITE_ALL,
+ )
+
+ cache = PoTokenCache(
+ cache_providers=[pcp_one, pcp_two],
+ cache_spec_providers=[WriteAllPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN))
+ assert pcp_one.store_calls == 1
+ assert pcp_two.store_calls == 1
+
+ def test_expires_at_pot_response(self, pot_request, memorypcp, ie, logger):
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=10000000000)
+ cache.store(pot_request, response)
+ assert next(iter(memorypcp.cache.values()))[1] == 10000000000
+
+ def test_expires_at_default_spec(self, pot_request, memorypcp, ie, logger):
+
+ class TtlPCSP(BaseMockCacheSpecProvider):
+ def generate_cache_spec(self, request: PoTokenRequest):
+ super().generate_cache_spec(request)
+ return PoTokenCacheSpec(
+ key_bindings={'v': request.video_id, 'e': None},
+ default_ttl=10000000000,
+ )
+
+ cache = PoTokenCache(
+ cache_providers=[memorypcp],
+ cache_spec_providers=[TtlPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, response)
+ assert next(iter(memorypcp.cache.values()))[1] >= 10000000000
+
+ def test_cache_provider_error_no_fallback(self, pot_request, ie, logger):
+ error_pcp = ErrorPCP(ie, logger, {})
+ cache = PoTokenCache(
+ cache_providers=[error_pcp],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, response)
+ assert cache.get(pot_request) is None
+ assert error_pcp.get_calls == 1
+ assert error_pcp.store_calls == 1
+
+ assert logger.messages['warning'].count("Error from \"error\" PO Token cache provider: PoTokenCacheProviderError('something went wrong'); example bug report message") == 2
+
+ def test_cache_provider_error_fallback(self, pot_request, ie, logger):
+ error_pcp = ErrorPCP(ie, logger, {})
+ memory_pcp = create_memory_pcp(ie, logger, provider_key='memory')
+
+ cache = PoTokenCache(
+ cache_providers=[error_pcp, memory_pcp],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, response)
+
+ # 1. Store fails for error_pcp, stored in memory_pcp
+ # 2. Get fails for error_pcp, fetched from memory_pcp
+ # 3. Since fetched from lower priority, it should be stored in the highest priority cache provider
+ # 4. Store fails in error_pcp. Since write policy is WRITE_FIRST, it should not try to store in memory_pcp regardless of if the store in error_pcp fails
+
+ assert cache.get(pot_request)
+ assert error_pcp.get_calls == 1
+ assert error_pcp.store_calls == 2 # since highest priority, when fetched from lower priority, it should be stored in the highest priority cache provider
+ assert memory_pcp.get_calls == 1
+ assert memory_pcp.store_calls == 1
+
+ assert logger.messages['warning'].count("Error from \"error\" PO Token cache provider: PoTokenCacheProviderError('something went wrong'); example bug report message") == 3
+
+ def test_cache_provider_unexpected_error_no_fallback(self, pot_request, ie, logger):
+ error_pcp = UnexpectedErrorPCP(ie, logger, {})
+ cache = PoTokenCache(
+ cache_providers=[error_pcp],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, response)
+ assert cache.get(pot_request) is None
+ assert error_pcp.get_calls == 1
+ assert error_pcp.store_calls == 1
+
+ assert logger.messages['error'].count("Error occurred with \"unexpected_error\" PO Token cache provider: ValueError('something went wrong'); example bug report message") == 2
+
+ def test_cache_provider_unexpected_error_fallback(self, pot_request, ie, logger):
+ error_pcp = UnexpectedErrorPCP(ie, logger, {})
+ memory_pcp = create_memory_pcp(ie, logger, provider_key='memory')
+
+ cache = PoTokenCache(
+ cache_providers=[error_pcp, memory_pcp],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, response)
+
+ # 1. Store fails for error_pcp, stored in memory_pcp
+ # 2. Get fails for error_pcp, fetched from memory_pcp
+ # 3. Since fetched from lower priority, it should be stored in the highest priority cache provider
+ # 4. Store fails in error_pcp. Since write policy is WRITE_FIRST, it should not try to store in memory_pcp regardless of if the store in error_pcp fails
+
+ assert cache.get(pot_request)
+ assert error_pcp.get_calls == 1
+ assert error_pcp.store_calls == 2 # since highest priority, when fetched from lower priority, it should be stored in the highest priority cache provider
+ assert memory_pcp.get_calls == 1
+ assert memory_pcp.store_calls == 1
+
+ assert logger.messages['error'].count("Error occurred with \"unexpected_error\" PO Token cache provider: ValueError('something went wrong'); example bug report message") == 3
+
+ def test_cache_provider_unavailable_no_fallback(self, pot_request, ie, logger):
+ provider = create_memory_pcp(ie, logger, available=False)
+
+ cache = PoTokenCache(
+ cache_providers=[provider],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, response)
+ assert cache.get(pot_request) is None
+ assert provider.get_calls == 0
+ assert provider.store_calls == 0
+ assert provider.available_called_times
+
+ def test_cache_provider_unavailable_fallback(self, pot_request, ie, logger):
+ provider_unavailable = create_memory_pcp(ie, logger, provider_key='unavailable', provider_name='unavailable', available=False)
+ provider_available = create_memory_pcp(ie, logger, provider_key='available', provider_name='available')
+
+ cache = PoTokenCache(
+ cache_providers=[provider_unavailable, provider_available],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, response)
+ assert cache.get(pot_request) is not None
+ assert provider_unavailable.get_calls == 0
+ assert provider_unavailable.store_calls == 0
+ assert provider_available.get_calls == 1
+ assert provider_available.store_calls == 1
+ assert provider_unavailable.available_called_times
+ assert provider_available.available_called_times
+
+ # should not even try to use the provider for the request
+ assert 'Attempting to fetch a PO Token response from "unavailable" provider' not in logger.messages['trace']
+ assert 'Attempting to fetch a PO Token response from "available" provider' not in logger.messages['trace']
+
+ def test_available_not_called(self, ie, pot_request, logger):
+ # Test that the available method is not called when provider higher in the list is available
+ provider_unavailable = create_memory_pcp(
+ ie, logger, provider_key='unavailable', provider_name='unavailable', available=False)
+ provider_available = create_memory_pcp(ie, logger, provider_key='available', provider_name='available')
+
+ logger.log_level = logger.LogLevel.INFO
+
+ cache = PoTokenCache(
+ cache_providers=[provider_available, provider_unavailable],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, response, write_policy=CacheProviderWritePolicy.WRITE_FIRST)
+ assert cache.get(pot_request) is not None
+ assert provider_unavailable.get_calls == 0
+ assert provider_unavailable.store_calls == 0
+ assert provider_available.get_calls == 1
+ assert provider_available.store_calls == 1
+ assert provider_unavailable.available_called_times == 0
+ assert provider_available.available_called_times
+ assert 'PO Token Cache Providers: available-0.0.0 (external), unavailable-0.0.0 (external, unavailable)' not in logger.messages.get('trace', [])
+
+ def test_available_called_trace(self, ie, pot_request, logger):
+ # But if logging level is trace should call available (as part of debug logging)
+ provider_unavailable = create_memory_pcp(
+ ie, logger, provider_key='unavailable', provider_name='unavailable', available=False)
+ provider_available = create_memory_pcp(ie, logger, provider_key='available', provider_name='available')
+
+ logger.log_level = logger.LogLevel.TRACE
+
+ cache = PoTokenCache(
+ cache_providers=[provider_available, provider_unavailable],
+ cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})],
+ logger=logger,
+ )
+
+ response = PoTokenResponse(EXAMPLE_PO_TOKEN)
+ cache.store(pot_request, response, write_policy=CacheProviderWritePolicy.WRITE_FIRST)
+ assert cache.get(pot_request) is not None
+ assert provider_unavailable.get_calls == 0
+ assert provider_unavailable.store_calls == 0
+ assert provider_available.get_calls == 1
+ assert provider_available.store_calls == 1
+ assert provider_unavailable.available_called_times
+ assert provider_available.available_called_times
+ assert 'PO Token Cache Providers: available-0.0.0 (external), unavailable-0.0.0 (external, unavailable)' in logger.messages.get('trace', [])
+
+ def test_close(self, ie, pot_request, logger):
+ # Should call close on the cache providers and cache specs
+ memory_pcp = create_memory_pcp(ie, logger, provider_key='memory')
+ memory2_pcp = create_memory_pcp(ie, logger, provider_key='memory2')
+
+ spec1 = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ spec2 = UnavailableCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+
+ cache = PoTokenCache(
+ cache_providers=[memory2_pcp, memory_pcp],
+ cache_spec_providers=[spec1, spec2],
+ logger=logger,
+ )
+
+ cache.close()
+ assert memory_pcp.close_called
+ assert memory2_pcp.close_called
+ assert spec1.close_called
+ assert spec2.close_called
+
+
+class TestPoTokenRequestDirector:
+
+ def test_request_pot_success(self, ie, pot_request, pot_cache, pot_provider, logger):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ director.register_provider(pot_provider)
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+
+ def test_request_and_cache(self, ie, pot_request, pot_cache, pot_provider, logger):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ director.register_provider(pot_provider)
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert pot_provider.request_called_times == 1
+ assert pot_cache.get_calls == 1
+ assert pot_cache.store_calls == 1
+
+ # Second request, should be cached
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert pot_cache.get_calls == 2
+ assert pot_cache.store_calls == 1
+ assert pot_provider.request_called_times == 1
+
+ def test_bypass_cache(self, ie, pot_request, pot_cache, logger, pot_provider):
+ pot_request.bypass_cache = True
+
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ director.register_provider(pot_provider)
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert pot_provider.request_called_times == 1
+ assert pot_cache.get_calls == 0
+ assert pot_cache.store_calls == 1
+
+ # Second request, should not get from cache
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert pot_provider.request_called_times == 2
+ assert pot_cache.get_calls == 0
+ assert pot_cache.store_calls == 2
+
+ # POT is still cached, should get from cache
+ pot_request.bypass_cache = False
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert pot_provider.request_called_times == 2
+ assert pot_cache.get_calls == 1
+ assert pot_cache.store_calls == 2
+
+ def test_clean_pot_generate(self, ie, pot_request, pot_cache, logger):
+ # Token should be cleaned before returning
+ base_token = base64.urlsafe_b64encode(b'token').decode()
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = success_ptp(PoTokenResponse(base_token + '?extra=params'))(ie, logger, settings={})
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == base_token
+ assert provider.request_called_times == 1
+
+ # Confirm the cleaned version was stored in the cache
+ cached_token = pot_cache.get(pot_request)
+ assert cached_token.po_token == base_token
+
+ def test_clean_pot_cache(self, ie, pot_request, pot_cache, logger, pot_provider):
+ # Token retrieved from cache should be cleaned before returning
+ base_token = base64.urlsafe_b64encode(b'token').decode()
+ pot_cache.store(pot_request, PoTokenResponse(base_token + '?extra=params'))
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ director.register_provider(pot_provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == base_token
+ assert pot_cache.get_calls == 1
+ assert pot_provider.request_called_times == 0
+
+ def test_cache_expires_at_none(self, ie, pot_request, pot_cache, logger, pot_provider):
+ # Should cache if expires_at=None in the response
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = success_ptp(PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=None))(ie, logger, settings={})
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert pot_cache.store_calls == 1
+ assert pot_cache.get(pot_request).po_token == EXAMPLE_PO_TOKEN
+
+ def test_cache_expires_at_positive(self, ie, pot_request, pot_cache, logger, pot_provider):
+ # Should cache if expires_at is a positive number in the response
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = success_ptp(PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=99999999999))(ie, logger, settings={})
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert pot_cache.store_calls == 1
+ assert pot_cache.get(pot_request).po_token == EXAMPLE_PO_TOKEN
+
+ @pytest.mark.parametrize('expires_at', [0, -1])
+ def test_not_cache_expires_at(self, ie, pot_request, pot_cache, logger, pot_provider, expires_at):
+ # Should not cache if expires_at <= 0 in the response
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = success_ptp(PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=expires_at))(ie, logger, settings={})
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert pot_cache.store_calls == 0
+ assert pot_cache.get(pot_request) is None
+
+ def test_no_providers(self, ie, pot_request, pot_cache, logger):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ response = director.get_po_token(pot_request)
+ assert response is None
+
+ def test_try_cache_no_providers(self, ie, pot_request, pot_cache, logger):
+ # Should still try the cache even if no providers are configured
+ pot_cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN))
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+
+ def test_close(self, ie, pot_request, pot_cache, pot_provider, logger):
+ # Should call close on the pot cache and any providers
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+
+ provider2 = UnavailablePTP(ie, logger, {})
+ director.register_provider(pot_provider)
+ director.register_provider(provider2)
+
+ director.close()
+ assert pot_provider.close_called
+ assert provider2.close_called
+ assert pot_cache.close_called
+
+ def test_pot_provider_preferences(self, pot_request, pot_cache, ie, logger):
+ pot_request.bypass_cache = True
+ provider_two_pot = base64.urlsafe_b64encode(b'token2').decode()
+
+ example_provider = success_ptp(response=PoTokenResponse(EXAMPLE_PO_TOKEN), key='exampleone')(ie, logger, settings={})
+ example_provider_two = success_ptp(response=PoTokenResponse(provider_two_pot), key='exampletwo')(ie, logger, settings={})
+
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ director.register_provider(example_provider)
+ director.register_provider(example_provider_two)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert example_provider.request_called_times == 1
+ assert example_provider_two.request_called_times == 0
+
+ standard_preference_called = False
+ example_preference_called = False
+
+ # Test that the provider preferences are respected
+ def standard_preference(provider, request, *_, **__):
+ nonlocal standard_preference_called
+ standard_preference_called = True
+ assert isinstance(provider, PoTokenProvider)
+ assert isinstance(request, PoTokenRequest)
+ return 1
+
+ def example_preference(provider, request, *_, **__):
+ nonlocal example_preference_called
+ example_preference_called = True
+ assert isinstance(provider, PoTokenProvider)
+ assert isinstance(request, PoTokenRequest)
+ if provider.PROVIDER_KEY == example_provider.PROVIDER_KEY:
+ return -100
+ return 0
+
+ # test that it can handle multiple preferences
+ director.register_preference(example_preference)
+ director.register_preference(standard_preference)
+
+ response = director.get_po_token(pot_request)
+ assert response == provider_two_pot
+ assert example_provider.request_called_times == 1
+ assert example_provider_two.request_called_times == 1
+ assert standard_preference_called
+ assert example_preference_called
+
+ def test_unsupported_request_no_fallback(self, ie, logger, pot_cache, pot_request):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = UnsupportedPTP(ie, logger, {})
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response is None
+ assert provider.request_called_times == 1
+
+ def test_unsupported_request_fallback(self, ie, logger, pot_cache, pot_request, pot_provider):
+ # Should fallback to the next provider if the first one does not support the request
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = UnsupportedPTP(ie, logger, {})
+ director.register_provider(provider)
+ director.register_provider(pot_provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert provider.request_called_times == 1
+ assert pot_provider.request_called_times == 1
+ assert 'PO Token Provider "unsupported" rejected this request, trying next available provider. Reason: unsupported request' in logger.messages['trace']
+
+ def test_unavailable_request_no_fallback(self, ie, logger, pot_cache, pot_request):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = UnavailablePTP(ie, logger, {})
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response is None
+ assert provider.request_called_times == 0
+ assert provider.available_called_times
+
+ def test_unavailable_request_fallback(self, ie, logger, pot_cache, pot_request, pot_provider):
+ # Should fallback to the next provider if the first one is unavailable
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = UnavailablePTP(ie, logger, {})
+ director.register_provider(provider)
+ director.register_provider(pot_provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert provider.request_called_times == 0
+ assert provider.available_called_times
+ assert pot_provider.request_called_times == 1
+ assert pot_provider.available_called_times
+ # should not even try use the provider for the request
+ assert 'Attempting to fetch a PO Token from "unavailable" provider' not in logger.messages['trace']
+ assert 'Attempting to fetch a PO Token from "success" provider' in logger.messages['trace']
+
+ def test_available_not_called(self, ie, logger, pot_cache, pot_request, pot_provider):
+ # Test that the available method is not called when provider higher in the list is available
+ logger.log_level = logger.LogLevel.INFO
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = UnavailablePTP(ie, logger, {})
+ director.register_provider(pot_provider)
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert provider.request_called_times == 0
+ assert provider.available_called_times == 0
+ assert pot_provider.request_called_times == 1
+ assert pot_provider.available_called_times == 2
+ assert 'PO Token Providers: success-0.0.1 (external), unavailable-0.0.0 (external, unavailable)' not in logger.messages.get('trace', [])
+
+ def test_available_called_trace(self, ie, logger, pot_cache, pot_request, pot_provider):
+ # But if logging level is trace should call available (as part of debug logging)
+ logger.log_level = logger.LogLevel.TRACE
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = UnavailablePTP(ie, logger, {})
+ director.register_provider(pot_provider)
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert provider.request_called_times == 0
+ assert provider.available_called_times == 1
+ assert pot_provider.request_called_times == 1
+ assert pot_provider.available_called_times == 3
+ assert 'PO Token Providers: success-0.0.1 (external), unavailable-0.0.0 (external, unavailable)' in logger.messages['trace']
+
+ def test_provider_error_no_fallback_unexpected(self, ie, logger, pot_cache, pot_request):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = ErrorPTP(ie, logger, {})
+ director.register_provider(provider)
+ pot_request.video_id = 'unexpected'
+ response = director.get_po_token(pot_request)
+ assert response is None
+ assert provider.request_called_times == 1
+ assert "Error fetching PO Token from \"error\" provider: PoTokenProviderError('an error occurred'); please report this issue to the provider developer at https://error.example.com/issues ." in logger.messages['warning']
+
+ def test_provider_error_no_fallback_expected(self, ie, logger, pot_cache, pot_request):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = ErrorPTP(ie, logger, {})
+ director.register_provider(provider)
+ pot_request.video_id = 'expected'
+ response = director.get_po_token(pot_request)
+ assert response is None
+ assert provider.request_called_times == 1
+ assert "Error fetching PO Token from \"error\" provider: PoTokenProviderError('an error occurred')" in logger.messages['warning']
+
+ def test_provider_error_fallback(self, ie, logger, pot_cache, pot_request, pot_provider):
+ # Should fallback to the next provider if the first one raises an error
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = ErrorPTP(ie, logger, {})
+ director.register_provider(provider)
+ director.register_provider(pot_provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert provider.request_called_times == 1
+ assert pot_provider.request_called_times == 1
+ assert "Error fetching PO Token from \"error\" provider: PoTokenProviderError('an error occurred'); please report this issue to the provider developer at https://error.example.com/issues ." in logger.messages['warning']
+
+ def test_provider_unexpected_error_no_fallback(self, ie, logger, pot_cache, pot_request):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = UnexpectedErrorPTP(ie, logger, {})
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response is None
+ assert provider.request_called_times == 1
+ assert "Unexpected error when fetching PO Token from \"unexpected_error\" provider: ValueError('an unexpected error occurred'); please report this issue to the provider developer at https://unexpected.example.com/issues ." in logger.messages['error']
+
+ def test_provider_unexpected_error_fallback(self, ie, logger, pot_cache, pot_request, pot_provider):
+ # Should fallback to the next provider if the first one raises an unexpected error
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = UnexpectedErrorPTP(ie, logger, {})
+ director.register_provider(provider)
+ director.register_provider(pot_provider)
+
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert provider.request_called_times == 1
+ assert pot_provider.request_called_times == 1
+ assert "Unexpected error when fetching PO Token from \"unexpected_error\" provider: ValueError('an unexpected error occurred'); please report this issue to the provider developer at https://unexpected.example.com/issues ." in logger.messages['error']
+
+ def test_invalid_po_token_response_type(self, ie, logger, pot_cache, pot_request, pot_provider):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = InvalidPTP(ie, logger, {})
+ director.register_provider(provider)
+
+ pot_request.video_id = 'invalid_type'
+
+ response = director.get_po_token(pot_request)
+ assert response is None
+ assert provider.request_called_times == 1
+ assert 'Invalid PO Token response received from "invalid" provider: invalid-response; please report this issue to the provider developer at https://invalid.example.com/issues .' in logger.messages['error']
+
+ # Should fallback to next available provider
+ director.register_provider(pot_provider)
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert provider.request_called_times == 2
+ assert pot_provider.request_called_times == 1
+
+ def test_invalid_po_token_response(self, ie, logger, pot_cache, pot_request, pot_provider):
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+ provider = InvalidPTP(ie, logger, {})
+ director.register_provider(provider)
+
+ response = director.get_po_token(pot_request)
+ assert response is None
+ assert provider.request_called_times == 1
+ assert "Invalid PO Token response received from \"invalid\" provider: PoTokenResponse(po_token='example-token?', expires_at='123'); please report this issue to the provider developer at https://invalid.example.com/issues ." in logger.messages['error']
+
+ # Should fallback to next available provider
+ director.register_provider(pot_provider)
+ response = director.get_po_token(pot_request)
+ assert response == EXAMPLE_PO_TOKEN
+ assert provider.request_called_times == 2
+ assert pot_provider.request_called_times == 1
+
+ def test_copy_request_provider(self, ie, logger, pot_cache, pot_request):
+
+ class BadProviderPTP(BaseMockPoTokenProvider):
+ _SUPPORTED_CONTEXTS = None
+ _SUPPORTED_CLIENTS = None
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ # Providers should not modify the request object, but we should guard against it
+ request.video_id = 'bad'
+ raise PoTokenProviderRejectedRequest('bad request')
+
+ class GoodProviderPTP(BaseMockPoTokenProvider):
+ _SUPPORTED_CONTEXTS = None
+ _SUPPORTED_CLIENTS = None
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ return PoTokenResponse(base64.urlsafe_b64encode(request.video_id.encode()).decode())
+
+ director = PoTokenRequestDirector(logger=logger, cache=pot_cache)
+
+ bad_provider = BadProviderPTP(ie, logger, {})
+ good_provider = GoodProviderPTP(ie, logger, {})
+
+ director.register_provider(bad_provider)
+ director.register_provider(good_provider)
+
+ pot_request.video_id = 'good'
+ response = director.get_po_token(pot_request)
+ assert response == base64.urlsafe_b64encode(b'good').decode()
+ assert bad_provider.request_called_times == 1
+ assert good_provider.request_called_times == 1
+ assert pot_request.video_id == 'good'
+
+
+@pytest.mark.parametrize('spec, expected', [
+ (None, False),
+ (PoTokenCacheSpec(key_bindings={'v': 'video-id'}, default_ttl=60, write_policy=None), False), # type: ignore
+ (PoTokenCacheSpec(key_bindings={'v': 'video-id'}, default_ttl='invalid'), False), # type: ignore
+ (PoTokenCacheSpec(key_bindings='invalid', default_ttl=60), False), # type: ignore
+ (PoTokenCacheSpec(key_bindings={2: 'video-id'}, default_ttl=60), False), # type: ignore
+ (PoTokenCacheSpec(key_bindings={'v': 2}, default_ttl=60), False), # type: ignore
+ (PoTokenCacheSpec(key_bindings={'v': None}, default_ttl=60), False), # type: ignore
+
+ (PoTokenCacheSpec(key_bindings={'v': 'video_id', 'e': None}, default_ttl=60), True),
+ (PoTokenCacheSpec(key_bindings={'v': 'video_id'}, default_ttl=60, write_policy=CacheProviderWritePolicy.WRITE_FIRST), True),
+])
+def test_validate_cache_spec(spec, expected):
+ assert validate_cache_spec(spec) == expected
+
+
+@pytest.mark.parametrize('po_token', [
+ 'invalid-token?',
+ '123',
+])
+def test_clean_pot_fail(po_token):
+ with pytest.raises(ValueError, match='Invalid PO Token'):
+ clean_pot(po_token)
+
+
+@pytest.mark.parametrize('po_token,expected', [
+ ('TwAA/+8=', 'TwAA_-8='),
+ ('TwAA%5F%2D9VA6Q92v%5FvEQ4==?extra-param=2', 'TwAA_-9VA6Q92v_vEQ4='),
+])
+def test_clean_pot(po_token, expected):
+ assert clean_pot(po_token) == expected
+
+
+@pytest.mark.parametrize(
+ 'response, expected',
+ [
+ (None, False),
+ (PoTokenResponse(None), False),
+ (PoTokenResponse(1), False),
+ (PoTokenResponse('invalid-token?'), False),
+ (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at='abc'), False), # type: ignore
+ (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=100), False),
+ (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=time.time() + 10000.0), False), # type: ignore
+ (PoTokenResponse(EXAMPLE_PO_TOKEN), True),
+ (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=-1), True),
+ (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=0), True),
+ (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=int(time.time()) + 10000), True),
+ ],
+)
+def test_validate_pot_response(response, expected):
+ assert validate_response(response) == expected
+
+
+def test_built_in_provider(ie, logger):
+ class BuiltinProviderDefaultT(BuiltinIEContentProvider, suffix='T'):
+ def is_available(self):
+ return True
+
+ class BuiltinProviderCustomNameT(BuiltinIEContentProvider, suffix='T'):
+ PROVIDER_NAME = 'CustomName'
+
+ def is_available(self):
+ return True
+
+ class ExternalProviderDefaultT(IEContentProvider, suffix='T'):
+ def is_available(self):
+ return True
+
+ class ExternalProviderCustomT(IEContentProvider, suffix='T'):
+ PROVIDER_NAME = 'custom'
+ PROVIDER_VERSION = '5.4b2'
+
+ def is_available(self):
+ return True
+
+ class ExternalProviderUnavailableT(IEContentProvider, suffix='T'):
+ def is_available(self) -> bool:
+ return False
+
+ class BuiltinProviderUnavailableT(IEContentProvider, suffix='T'):
+ def is_available(self) -> bool:
+ return False
+
+ built_in_default = BuiltinProviderDefaultT(ie=ie, logger=logger, settings={})
+ built_in_custom_name = BuiltinProviderCustomNameT(ie=ie, logger=logger, settings={})
+ built_in_unavailable = BuiltinProviderUnavailableT(ie=ie, logger=logger, settings={})
+ external_default = ExternalProviderDefaultT(ie=ie, logger=logger, settings={})
+ external_custom = ExternalProviderCustomT(ie=ie, logger=logger, settings={})
+ external_unavailable = ExternalProviderUnavailableT(ie=ie, logger=logger, settings={})
+
+ assert provider_display_list([]) == 'none'
+ assert provider_display_list([built_in_default]) == 'BuiltinProviderDefault'
+ assert provider_display_list([external_unavailable]) == 'ExternalProviderUnavailable-0.0.0 (external, unavailable)'
+ assert provider_display_list([
+ built_in_default,
+ built_in_custom_name,
+ external_default,
+ external_custom,
+ external_unavailable,
+ built_in_unavailable],
+ ) == 'BuiltinProviderDefault, CustomName, ExternalProviderDefault-0.0.0 (external), custom-5.4b2 (external), ExternalProviderUnavailable-0.0.0 (external, unavailable), BuiltinProviderUnavailable-0.0.0 (external, unavailable)'
diff --git a/test/test_pot/test_pot_framework.py b/test/test_pot/test_pot_framework.py
new file mode 100644
index 000000000..bc94653f4
--- /dev/null
+++ b/test/test_pot/test_pot_framework.py
@@ -0,0 +1,629 @@
+import pytest
+
+from yt_dlp.extractor.youtube.pot._provider import IEContentProvider
+from yt_dlp.cookies import YoutubeDLCookieJar
+from yt_dlp.utils.networking import HTTPHeaderDict
+from yt_dlp.extractor.youtube.pot.provider import (
+ PoTokenRequest,
+ PoTokenContext,
+ ExternalRequestFeature,
+
+)
+
+from yt_dlp.extractor.youtube.pot.cache import (
+ PoTokenCacheProvider,
+ PoTokenCacheSpec,
+ PoTokenCacheSpecProvider,
+ CacheProviderWritePolicy,
+)
+
+import yt_dlp.extractor.youtube.pot.cache as cache
+
+from yt_dlp.networking import Request
+from yt_dlp.extractor.youtube.pot.provider import (
+ PoTokenResponse,
+ PoTokenProvider,
+ PoTokenProviderRejectedRequest,
+ provider_bug_report_message,
+ register_provider,
+ register_preference,
+)
+
+from yt_dlp.extractor.youtube.pot._registry import _pot_providers, _ptp_preferences, _pot_pcs_providers, _pot_cache_providers, _pot_cache_provider_preferences
+
+
+class ExamplePTP(PoTokenProvider):
+ PROVIDER_NAME = 'example'
+ PROVIDER_VERSION = '0.0.1'
+ BUG_REPORT_LOCATION = 'https://example.com/issues'
+
+ _SUPPORTED_CLIENTS = ('WEB',)
+ _SUPPORTED_CONTEXTS = (PoTokenContext.GVS, )
+
+ _SUPPORTED_EXTERNAL_REQUEST_FEATURES = (
+ ExternalRequestFeature.PROXY_SCHEME_HTTP,
+ ExternalRequestFeature.PROXY_SCHEME_SOCKS5H,
+ )
+
+ def is_available(self) -> bool:
+ return True
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ return PoTokenResponse('example-token', expires_at=123)
+
+
+class ExampleCacheProviderPCP(PoTokenCacheProvider):
+
+ PROVIDER_NAME = 'example'
+ PROVIDER_VERSION = '0.0.1'
+ BUG_REPORT_LOCATION = 'https://example.com/issues'
+
+ def is_available(self) -> bool:
+ return True
+
+ def get(self, key: str):
+ return 'example-cache'
+
+ def store(self, key: str, value: str, expires_at: int):
+ pass
+
+ def delete(self, key: str):
+ pass
+
+
+class ExampleCacheSpecProviderPCSP(PoTokenCacheSpecProvider):
+
+ PROVIDER_NAME = 'example'
+ PROVIDER_VERSION = '0.0.1'
+ BUG_REPORT_LOCATION = 'https://example.com/issues'
+
+ def generate_cache_spec(self, request: PoTokenRequest):
+ return PoTokenCacheSpec(
+ key_bindings={'field': 'example-key'},
+ default_ttl=60,
+ write_policy=CacheProviderWritePolicy.WRITE_FIRST,
+ )
+
+
+class TestPoTokenProvider:
+
+ def test_base_type(self):
+ assert issubclass(PoTokenProvider, IEContentProvider)
+
+ def test_create_provider_missing_fetch_method(self, ie, logger):
+ class MissingMethodsPTP(PoTokenProvider):
+ def is_available(self) -> bool:
+ return True
+
+ with pytest.raises(TypeError):
+ MissingMethodsPTP(ie=ie, logger=logger, settings={})
+
+ def test_create_provider_missing_available_method(self, ie, logger):
+ class MissingMethodsPTP(PoTokenProvider):
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ raise PoTokenProviderRejectedRequest('Not implemented')
+
+ with pytest.raises(TypeError):
+ MissingMethodsPTP(ie=ie, logger=logger, settings={})
+
+ def test_barebones_provider(self, ie, logger):
+ class BarebonesProviderPTP(PoTokenProvider):
+ def is_available(self) -> bool:
+ return True
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ raise PoTokenProviderRejectedRequest('Not implemented')
+
+ provider = BarebonesProviderPTP(ie=ie, logger=logger, settings={})
+ assert provider.PROVIDER_NAME == 'BarebonesProvider'
+ assert provider.PROVIDER_KEY == 'BarebonesProvider'
+ assert provider.PROVIDER_VERSION == '0.0.0'
+ assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at (developer has not provided a bug report location) .'
+
+ def test_example_provider_success(self, ie, logger, pot_request):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={})
+ assert provider.PROVIDER_NAME == 'example'
+ assert provider.PROVIDER_KEY == 'Example'
+ assert provider.PROVIDER_VERSION == '0.0.1'
+ assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at https://example.com/issues .'
+ assert provider.is_available()
+
+ response = provider.request_pot(pot_request)
+
+ assert response.po_token == 'example-token'
+ assert response.expires_at == 123
+
+ def test_provider_unsupported_context(self, ie, logger, pot_request):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={})
+ pot_request.context = PoTokenContext.PLAYER
+
+ with pytest.raises(PoTokenProviderRejectedRequest):
+ provider.request_pot(pot_request)
+
+ def test_provider_unsupported_client(self, ie, logger, pot_request):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={})
+ pot_request.innertube_context['client']['clientName'] = 'ANDROID'
+
+ with pytest.raises(PoTokenProviderRejectedRequest):
+ provider.request_pot(pot_request)
+
+ def test_provider_unsupported_proxy_scheme(self, ie, logger, pot_request):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={})
+ pot_request.request_proxy = 'socks4://example.com'
+
+ with pytest.raises(
+ PoTokenProviderRejectedRequest,
+ match='External requests by "example" provider do not support proxy scheme "socks4". Supported proxy '
+ 'schemes: http, socks5h',
+ ):
+ provider.request_pot(pot_request)
+
+ pot_request.request_proxy = 'http://example.com'
+
+ assert provider.request_pot(pot_request)
+
+ def test_provider_ignore_external_request_features(self, ie, logger, pot_request):
+ class InternalPTP(ExamplePTP):
+ _SUPPORTED_EXTERNAL_REQUEST_FEATURES = None
+
+ provider = InternalPTP(ie=ie, logger=logger, settings={})
+
+ pot_request.request_proxy = 'socks5://example.com'
+ assert provider.request_pot(pot_request)
+ pot_request.request_source_address = '0.0.0.0'
+ assert provider.request_pot(pot_request)
+
+ def test_provider_unsupported_external_request_source_address(self, ie, logger, pot_request):
+ class InternalPTP(ExamplePTP):
+ _SUPPORTED_EXTERNAL_REQUEST_FEATURES = tuple()
+
+ provider = InternalPTP(ie=ie, logger=logger, settings={})
+
+ pot_request.request_source_address = None
+ assert provider.request_pot(pot_request)
+
+ pot_request.request_source_address = '0.0.0.0'
+ with pytest.raises(
+ PoTokenProviderRejectedRequest,
+ match='External requests by "example" provider do not support setting source address',
+ ):
+ provider.request_pot(pot_request)
+
+ def test_provider_supported_external_request_source_address(self, ie, logger, pot_request):
+ class InternalPTP(ExamplePTP):
+ _SUPPORTED_EXTERNAL_REQUEST_FEATURES = (
+ ExternalRequestFeature.SOURCE_ADDRESS,
+ )
+
+ provider = InternalPTP(ie=ie, logger=logger, settings={})
+
+ pot_request.request_source_address = None
+ assert provider.request_pot(pot_request)
+
+ pot_request.request_source_address = '0.0.0.0'
+ assert provider.request_pot(pot_request)
+
+ def test_provider_unsupported_external_request_tls_verification(self, ie, logger, pot_request):
+ class InternalPTP(ExamplePTP):
+ _SUPPORTED_EXTERNAL_REQUEST_FEATURES = tuple()
+
+ provider = InternalPTP(ie=ie, logger=logger, settings={})
+
+ pot_request.request_verify_tls = True
+ assert provider.request_pot(pot_request)
+
+ pot_request.request_verify_tls = False
+ with pytest.raises(
+ PoTokenProviderRejectedRequest,
+ match='External requests by "example" provider do not support ignoring TLS certificate failures',
+ ):
+ provider.request_pot(pot_request)
+
+ def test_provider_supported_external_request_tls_verification(self, ie, logger, pot_request):
+ class InternalPTP(ExamplePTP):
+ _SUPPORTED_EXTERNAL_REQUEST_FEATURES = (
+ ExternalRequestFeature.DISABLE_TLS_VERIFICATION,
+ )
+
+ provider = InternalPTP(ie=ie, logger=logger, settings={})
+
+ pot_request.request_verify_tls = True
+ assert provider.request_pot(pot_request)
+
+ pot_request.request_verify_tls = False
+ assert provider.request_pot(pot_request)
+
+ def test_provider_request_webpage(self, ie, logger, pot_request):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={})
+
+ cookiejar = YoutubeDLCookieJar()
+ pot_request.request_headers = HTTPHeaderDict({'User-Agent': 'example-user-agent'})
+ pot_request.request_proxy = 'socks5://example-proxy.com'
+ pot_request.request_cookiejar = cookiejar
+
+ def mock_urlopen(request):
+ return request
+
+ ie._downloader.urlopen = mock_urlopen
+
+ sent_request = provider._request_webpage(Request(
+ 'https://example.com',
+ ), pot_request=pot_request)
+
+ assert sent_request.url == 'https://example.com'
+ assert sent_request.headers['User-Agent'] == 'example-user-agent'
+ assert sent_request.proxies == {'all': 'socks5://example-proxy.com'}
+ assert sent_request.extensions['cookiejar'] is cookiejar
+ assert 'Requesting webpage' in logger.messages['info']
+
+ def test_provider_request_webpage_override(self, ie, logger, pot_request):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={})
+
+ cookiejar_request = YoutubeDLCookieJar()
+ pot_request.request_headers = HTTPHeaderDict({'User-Agent': 'example-user-agent'})
+ pot_request.request_proxy = 'socks5://example-proxy.com'
+ pot_request.request_cookiejar = cookiejar_request
+
+ def mock_urlopen(request):
+ return request
+
+ ie._downloader.urlopen = mock_urlopen
+
+ sent_request = provider._request_webpage(Request(
+ 'https://example.com',
+ headers={'User-Agent': 'override-user-agent-override'},
+ proxies={'http': 'http://example-proxy-override.com'},
+ extensions={'cookiejar': YoutubeDLCookieJar()},
+ ), pot_request=pot_request, note='Custom requesting webpage')
+
+ assert sent_request.url == 'https://example.com'
+ assert sent_request.headers['User-Agent'] == 'override-user-agent-override'
+ assert sent_request.proxies == {'http': 'http://example-proxy-override.com'}
+ assert sent_request.extensions['cookiejar'] is not cookiejar_request
+ assert 'Custom requesting webpage' in logger.messages['info']
+
+ def test_provider_request_webpage_no_log(self, ie, logger, pot_request):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={})
+
+ def mock_urlopen(request):
+ return request
+
+ ie._downloader.urlopen = mock_urlopen
+
+ sent_request = provider._request_webpage(Request(
+ 'https://example.com',
+ ), note=False)
+
+ assert sent_request.url == 'https://example.com'
+ assert 'info' not in logger.messages
+
+ def test_provider_request_webpage_no_pot_request(self, ie, logger):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={})
+
+ def mock_urlopen(request):
+ return request
+
+ ie._downloader.urlopen = mock_urlopen
+
+ sent_request = provider._request_webpage(Request(
+ 'https://example.com',
+ ), pot_request=None)
+
+ assert sent_request.url == 'https://example.com'
+
+ def test_get_config_arg(self, ie, logger):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={'abc': ['123D'], 'xyz': ['456a', '789B']})
+
+ assert provider._configuration_arg('abc') == ['123d']
+ assert provider._configuration_arg('abc', default=['default']) == ['123d']
+ assert provider._configuration_arg('ABC', default=['default']) == ['default']
+ assert provider._configuration_arg('abc', casesense=True) == ['123D']
+ assert provider._configuration_arg('xyz', casesense=False) == ['456a', '789b']
+
+ def test_require_class_end_with_suffix(self, ie, logger):
+ class InvalidSuffix(PoTokenProvider):
+ PROVIDER_NAME = 'invalid-suffix'
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ raise PoTokenProviderRejectedRequest('Not implemented')
+
+ def is_available(self) -> bool:
+ return True
+
+ provider = InvalidSuffix(ie=ie, logger=logger, settings={})
+
+ with pytest.raises(AssertionError):
+ provider.PROVIDER_KEY # noqa: B018
+
+
+class TestPoTokenCacheProvider:
+
+ def test_base_type(self):
+ assert issubclass(PoTokenCacheProvider, IEContentProvider)
+
+ def test_create_provider_missing_get_method(self, ie, logger):
+ class MissingMethodsPCP(PoTokenCacheProvider):
+ def store(self, key: str, value: str, expires_at: int):
+ pass
+
+ def delete(self, key: str):
+ pass
+
+ def is_available(self) -> bool:
+ return True
+
+ with pytest.raises(TypeError):
+ MissingMethodsPCP(ie=ie, logger=logger, settings={})
+
+ def test_create_provider_missing_store_method(self, ie, logger):
+ class MissingMethodsPCP(PoTokenCacheProvider):
+ def get(self, key: str):
+ pass
+
+ def delete(self, key: str):
+ pass
+
+ def is_available(self) -> bool:
+ return True
+
+ with pytest.raises(TypeError):
+ MissingMethodsPCP(ie=ie, logger=logger, settings={})
+
+ def test_create_provider_missing_delete_method(self, ie, logger):
+ class MissingMethodsPCP(PoTokenCacheProvider):
+ def get(self, key: str):
+ pass
+
+ def store(self, key: str, value: str, expires_at: int):
+ pass
+
+ def is_available(self) -> bool:
+ return True
+
+ with pytest.raises(TypeError):
+ MissingMethodsPCP(ie=ie, logger=logger, settings={})
+
+ def test_create_provider_missing_is_available_method(self, ie, logger):
+ class MissingMethodsPCP(PoTokenCacheProvider):
+ def get(self, key: str):
+ pass
+
+ def store(self, key: str, value: str, expires_at: int):
+ pass
+
+ def delete(self, key: str):
+ pass
+
+ with pytest.raises(TypeError):
+ MissingMethodsPCP(ie=ie, logger=logger, settings={})
+
+ def test_barebones_provider(self, ie, logger):
+ class BarebonesProviderPCP(PoTokenCacheProvider):
+
+ def is_available(self) -> bool:
+ return True
+
+ def get(self, key: str):
+ return 'example-cache'
+
+ def store(self, key: str, value: str, expires_at: int):
+ pass
+
+ def delete(self, key: str):
+ pass
+
+ provider = BarebonesProviderPCP(ie=ie, logger=logger, settings={})
+ assert provider.PROVIDER_NAME == 'BarebonesProvider'
+ assert provider.PROVIDER_KEY == 'BarebonesProvider'
+ assert provider.PROVIDER_VERSION == '0.0.0'
+ assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at (developer has not provided a bug report location) .'
+
+ def test_create_provider_example(self, ie, logger):
+ provider = ExampleCacheProviderPCP(ie=ie, logger=logger, settings={})
+ assert provider.PROVIDER_NAME == 'example'
+ assert provider.PROVIDER_KEY == 'ExampleCacheProvider'
+ assert provider.PROVIDER_VERSION == '0.0.1'
+ assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at https://example.com/issues .'
+ assert provider.is_available()
+
+ def test_get_config_arg(self, ie, logger):
+ provider = ExampleCacheProviderPCP(ie=ie, logger=logger, settings={'abc': ['123D'], 'xyz': ['456a', '789B']})
+ assert provider._configuration_arg('abc') == ['123d']
+ assert provider._configuration_arg('abc', default=['default']) == ['123d']
+ assert provider._configuration_arg('ABC', default=['default']) == ['default']
+ assert provider._configuration_arg('abc', casesense=True) == ['123D']
+ assert provider._configuration_arg('xyz', casesense=False) == ['456a', '789b']
+
+ def test_require_class_end_with_suffix(self, ie, logger):
+ class InvalidSuffix(PoTokenCacheProvider):
+ def get(self, key: str):
+ return 'example-cache'
+
+ def store(self, key: str, value: str, expires_at: int):
+ pass
+
+ def delete(self, key: str):
+ pass
+
+ def is_available(self) -> bool:
+ return True
+
+ provider = InvalidSuffix(ie=ie, logger=logger, settings={})
+
+ with pytest.raises(AssertionError):
+ provider.PROVIDER_KEY # noqa: B018
+
+
+class TestPoTokenCacheSpecProvider:
+
+ def test_base_type(self):
+ assert issubclass(PoTokenCacheSpecProvider, IEContentProvider)
+
+ def test_create_provider_missing_supports_method(self, ie, logger):
+ class MissingMethodsPCS(PoTokenCacheSpecProvider):
+ pass
+
+ with pytest.raises(TypeError):
+ MissingMethodsPCS(ie=ie, logger=logger, settings={})
+
+ def test_create_provider_barebones(self, ie, pot_request, logger):
+ class BarebonesProviderPCSP(PoTokenCacheSpecProvider):
+ def generate_cache_spec(self, request: PoTokenRequest):
+ return PoTokenCacheSpec(
+ default_ttl=100,
+ key_bindings={},
+ )
+
+ provider = BarebonesProviderPCSP(ie=ie, logger=logger, settings={})
+ assert provider.PROVIDER_NAME == 'BarebonesProvider'
+ assert provider.PROVIDER_KEY == 'BarebonesProvider'
+ assert provider.PROVIDER_VERSION == '0.0.0'
+ assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at (developer has not provided a bug report location) .'
+ assert provider.is_available()
+ assert provider.generate_cache_spec(request=pot_request).default_ttl == 100
+ assert provider.generate_cache_spec(request=pot_request).key_bindings == {}
+ assert provider.generate_cache_spec(request=pot_request).write_policy == CacheProviderWritePolicy.WRITE_ALL
+
+ def test_create_provider_example(self, ie, pot_request, logger):
+ provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})
+ assert provider.PROVIDER_NAME == 'example'
+ assert provider.PROVIDER_KEY == 'ExampleCacheSpecProvider'
+ assert provider.PROVIDER_VERSION == '0.0.1'
+ assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at https://example.com/issues .'
+ assert provider.is_available()
+ assert provider.generate_cache_spec(pot_request)
+ assert provider.generate_cache_spec(pot_request).key_bindings == {'field': 'example-key'}
+ assert provider.generate_cache_spec(pot_request).default_ttl == 60
+ assert provider.generate_cache_spec(pot_request).write_policy == CacheProviderWritePolicy.WRITE_FIRST
+
+ def test_get_config_arg(self, ie, logger):
+ provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={'abc': ['123D'], 'xyz': ['456a', '789B']})
+
+ assert provider._configuration_arg('abc') == ['123d']
+ assert provider._configuration_arg('abc', default=['default']) == ['123d']
+ assert provider._configuration_arg('ABC', default=['default']) == ['default']
+ assert provider._configuration_arg('abc', casesense=True) == ['123D']
+ assert provider._configuration_arg('xyz', casesense=False) == ['456a', '789b']
+
+ def test_require_class_end_with_suffix(self, ie, logger):
+ class InvalidSuffix(PoTokenCacheSpecProvider):
+ def generate_cache_spec(self, request: PoTokenRequest):
+ return None
+
+ provider = InvalidSuffix(ie=ie, logger=logger, settings={})
+
+ with pytest.raises(AssertionError):
+ provider.PROVIDER_KEY # noqa: B018
+
+
+class TestPoTokenRequest:
+ def test_copy_request(self, pot_request):
+ copied_request = pot_request.copy()
+
+ assert copied_request is not pot_request
+ assert copied_request.context == pot_request.context
+ assert copied_request.innertube_context == pot_request.innertube_context
+ assert copied_request.innertube_context is not pot_request.innertube_context
+ copied_request.innertube_context['client']['clientName'] = 'ANDROID'
+ assert pot_request.innertube_context['client']['clientName'] != 'ANDROID'
+ assert copied_request.innertube_host == pot_request.innertube_host
+ assert copied_request.session_index == pot_request.session_index
+ assert copied_request.player_url == pot_request.player_url
+ assert copied_request.is_authenticated == pot_request.is_authenticated
+ assert copied_request.visitor_data == pot_request.visitor_data
+ assert copied_request.data_sync_id == pot_request.data_sync_id
+ assert copied_request.video_id == pot_request.video_id
+ assert copied_request.request_cookiejar is pot_request.request_cookiejar
+ assert copied_request.request_proxy == pot_request.request_proxy
+ assert copied_request.request_headers == pot_request.request_headers
+ assert copied_request.request_headers is not pot_request.request_headers
+ assert copied_request.request_timeout == pot_request.request_timeout
+ assert copied_request.request_source_address == pot_request.request_source_address
+ assert copied_request.request_verify_tls == pot_request.request_verify_tls
+ assert copied_request.bypass_cache == pot_request.bypass_cache
+
+
+def test_provider_bug_report_message(ie, logger):
+ provider = ExamplePTP(ie=ie, logger=logger, settings={})
+ assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at https://example.com/issues .'
+
+ message = provider_bug_report_message(provider)
+ assert message == '; please report this issue to the provider developer at https://example.com/issues .'
+
+ message_before = provider_bug_report_message(provider, before='custom message!')
+ assert message_before == 'custom message! Please report this issue to the provider developer at https://example.com/issues .'
+
+
+def test_register_provider(ie):
+
+ @register_provider
+ class UnavailableProviderPTP(PoTokenProvider):
+ def is_available(self) -> bool:
+ return False
+
+ def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
+ raise PoTokenProviderRejectedRequest('Not implemented')
+
+ assert _pot_providers.value.get('UnavailableProvider') == UnavailableProviderPTP
+ _pot_providers.value.pop('UnavailableProvider')
+
+
+def test_register_pot_preference(ie):
+ before = len(_ptp_preferences.value)
+
+ @register_preference(ExamplePTP)
+ def unavailable_preference(provider: PoTokenProvider, request: PoTokenRequest):
+ return 1
+
+ assert len(_ptp_preferences.value) == before + 1
+
+
+def test_register_cache_provider(ie):
+
+ @cache.register_provider
+ class UnavailableCacheProviderPCP(PoTokenCacheProvider):
+ def is_available(self) -> bool:
+ return False
+
+ def get(self, key: str):
+ return 'example-cache'
+
+ def store(self, key: str, value: str, expires_at: int):
+ pass
+
+ def delete(self, key: str):
+ pass
+
+ assert _pot_cache_providers.value.get('UnavailableCacheProvider') == UnavailableCacheProviderPCP
+ _pot_cache_providers.value.pop('UnavailableCacheProvider')
+
+
+def test_register_cache_provider_spec(ie):
+
+ @cache.register_spec
+ class UnavailableCacheProviderPCSP(PoTokenCacheSpecProvider):
+ def is_available(self) -> bool:
+ return False
+
+ def generate_cache_spec(self, request: PoTokenRequest):
+ return None
+
+ assert _pot_pcs_providers.value.get('UnavailableCacheProvider') == UnavailableCacheProviderPCSP
+ _pot_pcs_providers.value.pop('UnavailableCacheProvider')
+
+
+def test_register_cache_provider_preference(ie):
+ before = len(_pot_cache_provider_preferences.value)
+
+ @cache.register_preference(ExampleCacheProviderPCP)
+ def unavailable_preference(provider: PoTokenCacheProvider, request: PoTokenRequest):
+ return 1
+
+ assert len(_pot_cache_provider_preferences.value) == before + 1
+
+
+def test_logger_log_level(logger):
+ assert logger.LogLevel('INFO') == logger.LogLevel.INFO
+ assert logger.LogLevel('debuG') == logger.LogLevel.DEBUG
+ assert logger.LogLevel(10) == logger.LogLevel.DEBUG
+ assert logger.LogLevel('UNKNOWN') == logger.LogLevel.INFO
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 0f0885366..3f777aed7 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -316,6 +316,10 @@
'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js',
'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE',
),
+ (
+ 'https://www.youtube.com/s/player/59b252b9/player_ias.vflset/en_US/base.js',
+ 'D3XWVpYgwhLLKNK4AGX', 'aZrQ1qWJ5yv5h',
+ ),
]
diff --git a/test/testdata/thumbnails/foo %d bar/foo_%d.webp b/test/testdata/thumbnails/foo %d bar/foo_%d.webp
deleted file mode 100644
index d64d0839f..000000000
Binary files a/test/testdata/thumbnails/foo %d bar/foo_%d.webp and /dev/null differ
diff --git a/test/testdata/thumbnails/foo %d bar/placeholder b/test/testdata/thumbnails/foo %d bar/placeholder
new file mode 100644
index 000000000..e69de29bb
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index e219068cd..c2cbc154f 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -645,6 +645,7 @@ def __init__(self, params=None, auto_init=True):
self._printed_messages = set()
self._first_webpage_request = True
self._post_hooks = []
+ self._close_hooks = []
self._progress_hooks = []
self._postprocessor_hooks = []
self._download_retcode = 0
@@ -913,6 +914,11 @@ def add_post_hook(self, ph):
"""Add the post hook"""
self._post_hooks.append(ph)
+ def add_close_hook(self, ch):
+ """Add a close hook, called when YoutubeDL.close() is called"""
+ assert callable(ch), 'Close hook must be callable'
+ self._close_hooks.append(ch)
+
def add_progress_hook(self, ph):
"""Add the download progress hook"""
self._progress_hooks.append(ph)
@@ -1021,6 +1027,9 @@ def close(self):
self._request_director.close()
del self._request_director
+ for close_hook in self._close_hooks:
+ close_hook()
+
def trouble(self, message=None, tb=None, is_error=True):
"""Determine action to take when a download problem appears.
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index fad323c90..5675445ac 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -764,11 +764,11 @@ def _get_linux_desktop_environment(env, logger):
GetDesktopEnvironment
"""
xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None)
- desktop_session = env.get('DESKTOP_SESSION', None)
+ desktop_session = env.get('DESKTOP_SESSION', '')
if xdg_current_desktop is not None:
for part in map(str.strip, xdg_current_desktop.split(':')):
if part == 'Unity':
- if desktop_session is not None and 'gnome-fallback' in desktop_session:
+ if 'gnome-fallback' in desktop_session:
return _LinuxDesktopEnvironment.GNOME
else:
return _LinuxDesktopEnvironment.UNITY
@@ -797,35 +797,34 @@ def _get_linux_desktop_environment(env, logger):
return _LinuxDesktopEnvironment.UKUI
elif part == 'LXQt':
return _LinuxDesktopEnvironment.LXQT
- logger.info(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"')
+ logger.debug(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"')
- elif desktop_session is not None:
- if desktop_session == 'deepin':
- return _LinuxDesktopEnvironment.DEEPIN
- elif desktop_session in ('mate', 'gnome'):
- return _LinuxDesktopEnvironment.GNOME
- elif desktop_session in ('kde4', 'kde-plasma'):
+ if desktop_session == 'deepin':
+ return _LinuxDesktopEnvironment.DEEPIN
+ elif desktop_session in ('mate', 'gnome'):
+ return _LinuxDesktopEnvironment.GNOME
+ elif desktop_session in ('kde4', 'kde-plasma'):
+ return _LinuxDesktopEnvironment.KDE4
+ elif desktop_session == 'kde':
+ if 'KDE_SESSION_VERSION' in env:
return _LinuxDesktopEnvironment.KDE4
- elif desktop_session == 'kde':
- if 'KDE_SESSION_VERSION' in env:
- return _LinuxDesktopEnvironment.KDE4
- else:
- return _LinuxDesktopEnvironment.KDE3
- elif 'xfce' in desktop_session or desktop_session == 'xubuntu':
- return _LinuxDesktopEnvironment.XFCE
- elif desktop_session == 'ukui':
- return _LinuxDesktopEnvironment.UKUI
else:
- logger.info(f'DESKTOP_SESSION is set to an unknown value: "{desktop_session}"')
-
+ return _LinuxDesktopEnvironment.KDE3
+ elif 'xfce' in desktop_session or desktop_session == 'xubuntu':
+ return _LinuxDesktopEnvironment.XFCE
+ elif desktop_session == 'ukui':
+ return _LinuxDesktopEnvironment.UKUI
else:
- if 'GNOME_DESKTOP_SESSION_ID' in env:
- return _LinuxDesktopEnvironment.GNOME
- elif 'KDE_FULL_SESSION' in env:
- if 'KDE_SESSION_VERSION' in env:
- return _LinuxDesktopEnvironment.KDE4
- else:
- return _LinuxDesktopEnvironment.KDE3
+ logger.debug(f'DESKTOP_SESSION is set to an unknown value: "{desktop_session}"')
+
+ if 'GNOME_DESKTOP_SESSION_ID' in env:
+ return _LinuxDesktopEnvironment.GNOME
+ elif 'KDE_FULL_SESSION' in env:
+ if 'KDE_SESSION_VERSION' in env:
+ return _LinuxDesktopEnvironment.KDE4
+ else:
+ return _LinuxDesktopEnvironment.KDE3
+
return _LinuxDesktopEnvironment.OTHER
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index bb1c3db16..b0c52e0fc 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -300,7 +300,6 @@
BrainPOPIlIE,
BrainPOPJrIE,
)
-from .bravotv import BravoTVIE
from .breitbart import BreitBartIE
from .brightcove import (
BrightcoveLegacyIE,
@@ -338,7 +337,6 @@
from .canalplus import CanalplusIE
from .canalsurmas import CanalsurmasIE
from .caracoltv import CaracolTvPlayIE
-from .cartoonnetwork import CartoonNetworkIE
from .cbc import (
CBCIE,
CBCGemIE,
@@ -929,7 +927,10 @@
)
from .jiosaavn import (
JioSaavnAlbumIE,
+ JioSaavnArtistIE,
JioSaavnPlaylistIE,
+ JioSaavnShowIE,
+ JioSaavnShowPlaylistIE,
JioSaavnSongIE,
)
from .joj import JojIE
@@ -1260,6 +1261,7 @@
)
from .nbc import (
NBCIE,
+ BravoTVIE,
NBCNewsIE,
NBCOlympicsIE,
NBCOlympicsStreamIE,
@@ -1267,6 +1269,7 @@
NBCSportsStreamIE,
NBCSportsVPlayerIE,
NBCStationsIE,
+ SyfyIE,
)
from .ndr import (
NDRIE,
@@ -1964,7 +1967,6 @@
SpreakerShowIE,
)
from .springboardplatform import SpringboardPlatformIE
-from .sprout import SproutIE
from .sproutvideo import (
SproutVideoIE,
VidsIoIE,
@@ -2021,7 +2023,6 @@
SVTSeriesIE,
)
from .swearnet import SwearnetEpisodeIE
-from .syfy import SyfyIE
from .syvdk import SYVDKIE
from .sztvhu import SztvHuIE
from .tagesschau import TagesschauIE
@@ -2146,6 +2147,7 @@
from .toggo import ToggoIE
from .tonline import TOnlineIE
from .toongoggles import ToonGogglesIE
+from .toutiao import ToutiaoIE
from .toutv import TouTvIE
from .toypics import (
ToypicsIE,
@@ -2368,6 +2370,7 @@
VHXEmbedIE,
VimeoAlbumIE,
VimeoChannelIE,
+ VimeoEventIE,
VimeoGroupsIE,
VimeoIE,
VimeoLikesIE,
diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py
index f1b877927..91c40b32e 100644
--- a/yt_dlp/extractor/adobepass.py
+++ b/yt_dlp/extractor/adobepass.py
@@ -3,6 +3,7 @@
import re
import time
import urllib.parse
+import uuid
import xml.etree.ElementTree as etree
from .common import InfoExtractor
@@ -10,6 +11,7 @@
from ..utils import (
NO_DEFAULT,
ExtractorError,
+ parse_qs,
unescapeHTML,
unified_timestamp,
urlencode_postdata,
@@ -45,6 +47,8 @@
'name': 'Comcast XFINITY',
'username_field': 'user',
'password_field': 'passwd',
+ 'login_hostname': 'login.xfinity.com',
+ 'needs_newer_ua': True,
},
'TWC': {
'name': 'Time Warner Cable | Spectrum',
@@ -74,6 +78,12 @@
'name': 'Verizon FiOS',
'username_field': 'IDToken1',
'password_field': 'IDToken2',
+ 'login_hostname': 'ssoauth.verizon.com',
+ },
+ 'Fubo': {
+ 'name': 'Fubo',
+ 'username_field': 'username',
+ 'password_field': 'password',
},
'Cablevision': {
'name': 'Optimum/Cablevision',
@@ -1338,6 +1348,7 @@
'name': 'Sling TV',
'username_field': 'username',
'password_field': 'password',
+ 'login_hostname': 'identity.sling.com',
},
'Suddenlink': {
'name': 'Suddenlink',
@@ -1355,7 +1366,6 @@
class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should end with BaseIE/InfoExtractor
_SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
_USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
- _MODERN_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; rv:131.0) Gecko/20100101 Firefox/131.0'
_MVPD_CACHE = 'ap-mvpd'
_DOWNLOADING_LOGIN_PAGE = 'Downloading Provider Login Page'
@@ -1367,6 +1377,14 @@ def _download_webpage_handle(self, *args, **kwargs):
return super()._download_webpage_handle(
*args, **kwargs)
+ @staticmethod
+ def _get_mso_headers(mso_info):
+ # yt-dlp's default user-agent is usually too old for some MSO's like Comcast_SSO
+ # See: https://github.com/yt-dlp/yt-dlp/issues/10848
+ return {
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:131.0) Gecko/20100101 Firefox/131.0',
+ } if mso_info.get('needs_newer_ua') else {}
+
@staticmethod
def _get_mvpd_resource(provider_id, title, guid, rating):
channel = etree.Element('channel')
@@ -1382,7 +1400,13 @@ def _get_mvpd_resource(provider_id, title, guid, rating):
resource_rating.text = rating
return '' + etree.tostring(channel).decode() + ''
- def _extract_mvpd_auth(self, url, video_id, requestor_id, resource):
+ def _extract_mvpd_auth(self, url, video_id, requestor_id, resource, software_statement):
+ mso_id = self.get_param('ap_mso')
+ if mso_id:
+ mso_info = MSO_INFO[mso_id]
+ else:
+ mso_info = {}
+
def xml_text(xml_str, tag):
return self._search_regex(
f'<{tag}>(.+?)', xml_str, tag)
@@ -1391,15 +1415,27 @@ def is_expired(token, date_ele):
token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(token, date_ele)))
return token_expires and token_expires <= int(time.time())
- def post_form(form_page_res, note, data={}):
+ def post_form(form_page_res, note, data={}, validate_url=False):
form_page, urlh = form_page_res
post_url = self._html_search_regex(r']+action=(["\'])(?P.+?)\1', form_page, 'post url', group='url')
if not re.match(r'https?://', post_url):
post_url = urllib.parse.urljoin(urlh.url, post_url)
+ if validate_url:
+ # This request is submitting credentials so we should validate it when possible
+ url_parsed = urllib.parse.urlparse(post_url)
+ expected_hostname = mso_info.get('login_hostname')
+ if expected_hostname and expected_hostname != url_parsed.hostname:
+ raise ExtractorError(
+ f'Unexpected login URL hostname; expected "{expected_hostname}" but got '
+ f'"{url_parsed.hostname}". Aborting before submitting credentials')
+ if url_parsed.scheme != 'https':
+ self.write_debug('Upgrading login URL scheme to https')
+ post_url = urllib.parse.urlunparse(url_parsed._replace(scheme='https'))
form_data = self._hidden_inputs(form_page)
form_data.update(data)
return self._download_webpage_handle(
post_url, video_id, note, data=urlencode_postdata(form_data), headers={
+ **self._get_mso_headers(mso_info),
'Content-Type': 'application/x-www-form-urlencoded',
})
@@ -1432,40 +1468,72 @@ def extract_redirect_url(html, url=None, fatal=False):
}
guid = xml_text(resource, 'guid') if '<' in resource else resource
- count = 0
- while count < 2:
+ for _ in range(2):
requestor_info = self.cache.load(self._MVPD_CACHE, requestor_id) or {}
authn_token = requestor_info.get('authn_token')
if authn_token and is_expired(authn_token, 'simpleTokenExpires'):
authn_token = None
if not authn_token:
- mso_id = self.get_param('ap_mso')
- if mso_id:
- username, password = self._get_login_info('ap_username', 'ap_password', mso_id)
- if not username or not password:
- raise_mvpd_required()
- mso_info = MSO_INFO[mso_id]
-
- provider_redirect_page_res = self._download_webpage_handle(
- self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
- 'Downloading Provider Redirect Page', query={
- 'noflash': 'true',
- 'mso_id': mso_id,
- 'requestor_id': requestor_id,
- 'no_iframe': 'false',
- 'domain_name': 'adobe.com',
- 'redirect_url': url,
- }, headers={
- # yt-dlp's default user-agent is usually too old for Comcast_SSO
- # See: https://github.com/yt-dlp/yt-dlp/issues/10848
- 'User-Agent': self._MODERN_USER_AGENT,
- } if mso_id == 'Comcast_SSO' else None)
- elif not self._cookies_passed:
+ if not mso_id:
+ raise_mvpd_required()
+ username, password = self._get_login_info('ap_username', 'ap_password', mso_id)
+ if not username or not password:
raise_mvpd_required()
- if not mso_id:
- pass
- elif mso_id == 'Comcast_SSO':
+ device_info, urlh = self._download_json_handle(
+ 'https://sp.auth.adobe.com/indiv/devices',
+ video_id, 'Registering device with Adobe',
+ data=json.dumps({'fingerprint': uuid.uuid4().hex}).encode(),
+ headers={'Content-Type': 'application/json; charset=UTF-8'})
+
+ device_id = device_info['deviceId']
+ mvpd_headers['pass_sfp'] = urlh.get_header('pass_sfp')
+ mvpd_headers['Ap_21'] = device_id
+
+ registration = self._download_json(
+ 'https://sp.auth.adobe.com/o/client/register',
+ video_id, 'Registering client with Adobe',
+ data=json.dumps({'software_statement': software_statement}).encode(),
+ headers={'Content-Type': 'application/json; charset=UTF-8'})
+
+ access_token = self._download_json(
+ 'https://sp.auth.adobe.com/o/client/token', video_id,
+ 'Obtaining access token', data=urlencode_postdata({
+ 'grant_type': 'client_credentials',
+ 'client_id': registration['client_id'],
+ 'client_secret': registration['client_secret'],
+ }),
+ headers={
+ 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+ })['access_token']
+ mvpd_headers['Authorization'] = f'Bearer {access_token}'
+
+ reg_code = self._download_json(
+ f'https://sp.auth.adobe.com/reggie/v1/{requestor_id}/regcode',
+ video_id, 'Obtaining registration code',
+ data=urlencode_postdata({
+ 'requestor': requestor_id,
+ 'deviceId': device_id,
+ 'format': 'json',
+ }),
+ headers={
+ 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+ 'Authorization': f'Bearer {access_token}',
+ })['code']
+
+ provider_redirect_page_res = self._download_webpage_handle(
+ self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
+ 'Downloading Provider Redirect Page', query={
+ 'noflash': 'true',
+ 'mso_id': mso_id,
+ 'requestor_id': requestor_id,
+ 'no_iframe': 'false',
+ 'domain_name': 'adobe.com',
+ 'redirect_url': url,
+ 'reg_code': reg_code,
+ }, headers=self._get_mso_headers(mso_info))
+
+ if mso_id == 'Comcast_SSO':
# Comcast page flow varies by video site and whether you
# are on Comcast's network.
provider_redirect_page, urlh = provider_redirect_page_res
@@ -1489,8 +1557,8 @@ def extract_redirect_url(html, url=None, fatal=False):
oauth_redirect_url = extract_redirect_url(
provider_redirect_page, fatal=True)
provider_login_page_res = self._download_webpage_handle(
- oauth_redirect_url, video_id,
- self._DOWNLOADING_LOGIN_PAGE)
+ oauth_redirect_url, video_id, self._DOWNLOADING_LOGIN_PAGE,
+ headers=self._get_mso_headers(mso_info))
else:
provider_login_page_res = post_form(
provider_redirect_page_res,
@@ -1500,7 +1568,7 @@ def extract_redirect_url(html, url=None, fatal=False):
provider_login_page_res, 'Logging in', {
mso_info['username_field']: username,
mso_info['password_field']: password,
- })
+ }, validate_url=True)
mvpd_confirm_page, urlh = mvpd_confirm_page_res
if '' in mvpd_confirm_page:
post_form(mvpd_confirm_page_res, 'Confirming Login')
@@ -1539,7 +1607,7 @@ def extract_redirect_url(html, url=None, fatal=False):
provider_redirect_page_res, 'Logging in', {
mso_info['username_field']: username,
mso_info['password_field']: password,
- })
+ }, validate_url=True)
saml_login_page, urlh = saml_login_page_res
if 'Please try again.' in saml_login_page:
raise ExtractorError(
@@ -1560,7 +1628,7 @@ def extract_redirect_url(html, url=None, fatal=False):
[saml_login_page, saml_redirect_url], 'Logging in', {
mso_info['username_field']: username,
mso_info['password_field']: password,
- })
+ }, validate_url=True)
if 'Please try again.' in saml_login_page:
raise ExtractorError(
'Failed to login, incorrect User ID or Password.')
@@ -1631,7 +1699,7 @@ def extract_redirect_url(html, url=None, fatal=False):
provider_login_page_res, 'Logging in', {
mso_info['username_field']: username,
mso_info['password_field']: password,
- })
+ }, validate_url=True)
provider_refresh_redirect_url = extract_redirect_url(
provider_association_redirect, url=urlh.url)
@@ -1682,7 +1750,7 @@ def extract_redirect_url(html, url=None, fatal=False):
provider_login_page_res, 'Logging in', {
mso_info['username_field']: username,
mso_info['password_field']: password,
- })
+ }, validate_url=True)
provider_refresh_redirect_url = extract_redirect_url(
provider_association_redirect, url=urlh.url)
@@ -1699,6 +1767,27 @@ def extract_redirect_url(html, url=None, fatal=False):
query=hidden_data)
post_form(mvpd_confirm_page_res, 'Confirming Login')
+ elif mso_id == 'Fubo':
+ _, urlh = provider_redirect_page_res
+
+ fubo_response = self._download_json(
+ 'https://api.fubo.tv/partners/tve/connect', video_id,
+ 'Authenticating with Fubo', 'Unable to authenticate with Fubo',
+ query=parse_qs(urlh.url), data=json.dumps({
+ 'username': username,
+ 'password': password,
+ }).encode(), headers={
+ 'Accept': 'application/json',
+ 'Content-Type': 'application/json',
+ })
+
+ self._request_webpage(
+ 'https://sp.auth.adobe.com/adobe-services/oauth2', video_id,
+ 'Authenticating with Adobe', 'Failed to authenticate with Adobe',
+ query={
+ 'code': fubo_response['code'],
+ 'state': fubo_response['state'],
+ })
else:
# Some providers (e.g. DIRECTV NOW) have another meta refresh
# based redirect that should be followed.
@@ -1717,7 +1806,8 @@ def extract_redirect_url(html, url=None, fatal=False):
}
if mso_id in ('Cablevision', 'AlticeOne'):
form_data['_eventId_proceed'] = ''
- mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', form_data)
+ mvpd_confirm_page_res = post_form(
+ provider_login_page_res, 'Logging in', form_data, validate_url=True)
if mso_id != 'Rogers':
post_form(mvpd_confirm_page_res, 'Confirming Login')
@@ -1727,6 +1817,7 @@ def extract_redirect_url(html, url=None, fatal=False):
'Retrieving Session', data=urlencode_postdata({
'_method': 'GET',
'requestor_id': requestor_id,
+ 'reg_code': reg_code,
}), headers=mvpd_headers)
except ExtractorError as e:
if not mso_id and isinstance(e.cause, HTTPError) and e.cause.status == 401:
@@ -1734,7 +1825,6 @@ def extract_redirect_url(html, url=None, fatal=False):
raise
if 'amc|bbcamerica|ifc|(?:we|sundance)tv)\.com/(?P(?:movies|shows(?:/[^/]+)+)/[^/?#&]+)'
+class AMCNetworksIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?(?:amc|bbcamerica|ifc|(?:we|sundance)tv)\.com/(?P(?:movies|shows(?:/[^/?#]+)+)/[^/?#&]+)'
_TESTS = [{
- 'url': 'https://www.bbcamerica.com/shows/the-graham-norton-show/videos/tina-feys-adorable-airline-themed-family-dinner--51631',
+ 'url': 'https://www.amc.com/shows/dark-winds/videos/dark-winds-a-look-at-season-3--1072027',
'info_dict': {
- 'id': '4Lq1dzOnZGt0',
+ 'id': '6369261343112',
'ext': 'mp4',
- 'title': "The Graham Norton Show - Season 28 - Tina Fey's Adorable Airline-Themed Family Dinner",
- 'description': "It turns out child stewardesses are very generous with the wine! All-new episodes of 'The Graham Norton Show' premiere Fridays at 11/10c on BBC America.",
- 'upload_date': '20201120',
- 'timestamp': 1605904350,
- 'uploader': 'AMCN',
+ 'title': 'Dark Winds: A Look at Season 3',
+ 'uploader_id': '6240731308001',
+ 'duration': 176.427,
+ 'thumbnail': r're:https://[^/]+\.boltdns\.net/.+/image\.jpg',
+ 'tags': [],
+ 'timestamp': 1740414792,
+ 'upload_date': '20250224',
},
- 'params': {
- # m3u8 download
- 'skip_download': True,
- },
- 'skip': '404 Not Found',
+ 'params': {'skip_download': 'm3u8'},
}, {
'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge',
'only_matching': True,
@@ -52,96 +44,18 @@ class AMCNetworksIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'url': 'https://www.sundancetv.com/shows/riviera/full-episodes/season-1/episode-01-episode-1',
'only_matching': True,
}]
- _REQUESTOR_ID_MAP = {
- 'amc': 'AMC',
- 'bbcamerica': 'BBCA',
- 'ifc': 'IFC',
- 'sundancetv': 'SUNDANCE',
- 'wetv': 'WETV',
- }
def _real_extract(self, url):
- site, display_id = self._match_valid_url(url).groups()
- requestor_id = self._REQUESTOR_ID_MAP[site]
- page_data = self._download_json(
- f'https://content-delivery-gw.svc.ds.amcn.com/api/v2/content/amcn/{requestor_id.lower()}/url/{display_id}',
- display_id)['data']
- properties = page_data.get('properties') or {}
- query = {
- 'mbr': 'true',
- 'manifest': 'm3u',
- }
+ display_id = self._match_id(url)
+ webpage = self._download_webpage(url, display_id)
+ initial_data = self._search_json(
+ r'window\.initialData\s*=\s*JSON\.parse\(String\.raw`', webpage, 'initial data', display_id)
+ video_id = traverse_obj(initial_data, ('initialData', 'properties', 'videoId', {str}))
+ if not video_id: # All locked videos are now DRM-protected
+ self.report_drm(display_id)
+ account_id = initial_data['config']['brightcove']['accountId']
+ player_id = initial_data['config']['brightcove']['playerId']
- video_player_count = 0
- try:
- for v in page_data['children']:
- if v.get('type') == 'video-player':
- release_pid = v['properties']['currentVideo']['meta']['releasePid']
- tp_path = 'M_UwQC/' + release_pid
- media_url = 'https://link.theplatform.com/s/' + tp_path
- video_player_count += 1
- except KeyError:
- pass
- if video_player_count > 1:
- self.report_warning(
- f'The JSON data has {video_player_count} video players. Only one will be extracted')
-
- # Fall back to videoPid if releasePid not found.
- # TODO: Fall back to videoPid if releasePid manifest uses DRM.
- if not video_player_count:
- tp_path = 'M_UwQC/media/' + properties['videoPid']
- media_url = 'https://link.theplatform.com/s/' + tp_path
-
- theplatform_metadata = self._download_theplatform_metadata(tp_path, display_id)
- info = self._parse_theplatform_metadata(theplatform_metadata)
- video_id = theplatform_metadata['pid']
- title = theplatform_metadata['title']
- rating = try_get(
- theplatform_metadata, lambda x: x['ratings'][0]['rating'])
- video_category = properties.get('videoCategory')
- if video_category and video_category.endswith('-Auth'):
- resource = self._get_mvpd_resource(
- requestor_id, title, video_id, rating)
- query['auth'] = self._extract_mvpd_auth(
- url, video_id, requestor_id, resource)
- media_url = update_url_query(media_url, query)
- formats, subtitles = self._extract_theplatform_smil(
- media_url, video_id)
-
- thumbnails = []
- thumbnail_urls = [properties.get('imageDesktop')]
- if 'thumbnail' in info:
- thumbnail_urls.append(info.pop('thumbnail'))
- for thumbnail_url in thumbnail_urls:
- if not thumbnail_url:
- continue
- mobj = re.search(r'(\d+)x(\d+)', thumbnail_url)
- thumbnails.append({
- 'url': thumbnail_url,
- 'width': int(mobj.group(1)) if mobj else None,
- 'height': int(mobj.group(2)) if mobj else None,
- })
-
- info.update({
- 'age_limit': parse_age_limit(rating),
- 'formats': formats,
- 'id': video_id,
- 'subtitles': subtitles,
- 'thumbnails': thumbnails,
- })
- ns_keys = theplatform_metadata.get('$xmlns', {}).keys()
- if ns_keys:
- ns = next(iter(ns_keys))
- episode = theplatform_metadata.get(ns + '$episodeTitle') or None
- episode_number = int_or_none(
- theplatform_metadata.get(ns + '$episode'))
- season_number = int_or_none(
- theplatform_metadata.get(ns + '$season'))
- series = theplatform_metadata.get(ns + '$show') or None
- info.update({
- 'episode': episode,
- 'episode_number': episode_number,
- 'season_number': season_number,
- 'series': series,
- })
- return info
+ return self.url_result(
+ f'https://players.brightcove.net/{account_id}/{player_id}_default/index.html?videoId={video_id}',
+ BrightcoveNewIE, video_id)
diff --git a/yt_dlp/extractor/bravotv.py b/yt_dlp/extractor/bravotv.py
deleted file mode 100644
index 0b2c44798..000000000
--- a/yt_dlp/extractor/bravotv.py
+++ /dev/null
@@ -1,188 +0,0 @@
-from .adobepass import AdobePassIE
-from ..networking import HEADRequest
-from ..utils import (
- extract_attributes,
- float_or_none,
- get_element_html_by_class,
- int_or_none,
- merge_dicts,
- parse_age_limit,
- remove_end,
- str_or_none,
- traverse_obj,
- unescapeHTML,
- unified_timestamp,
- update_url_query,
- url_or_none,
-)
-
-
-class BravoTVIE(AdobePassIE):
- _VALID_URL = r'https?://(?:www\.)?(?Pbravotv|oxygen)\.com/(?:[^/]+/)+(?P[^/?#]+)'
- _TESTS = [{
- 'url': 'https://www.bravotv.com/top-chef/season-16/episode-15/videos/the-top-chef-season-16-winner-is',
- 'info_dict': {
- 'id': '3923059',
- 'ext': 'mp4',
- 'title': 'The Top Chef Season 16 Winner Is...',
- 'description': 'Find out who takes the title of Top Chef!',
- 'upload_date': '20190314',
- 'timestamp': 1552591860,
- 'season_number': 16,
- 'episode_number': 15,
- 'series': 'Top Chef',
- 'episode': 'The Top Chef Season 16 Winner Is...',
- 'duration': 190.357,
- 'season': 'Season 16',
- 'thumbnail': r're:^https://.+\.jpg',
- },
- 'params': {'skip_download': 'm3u8'},
- }, {
- 'url': 'https://www.bravotv.com/top-chef/season-20/episode-1/london-calling',
- 'info_dict': {
- 'id': '9000234570',
- 'ext': 'mp4',
- 'title': 'London Calling',
- 'description': 'md5:5af95a8cbac1856bd10e7562f86bb759',
- 'upload_date': '20230310',
- 'timestamp': 1678410000,
- 'season_number': 20,
- 'episode_number': 1,
- 'series': 'Top Chef',
- 'episode': 'London Calling',
- 'duration': 3266.03,
- 'season': 'Season 20',
- 'chapters': 'count:7',
- 'thumbnail': r're:^https://.+\.jpg',
- 'age_limit': 14,
- },
- 'params': {'skip_download': 'm3u8'},
- 'skip': 'This video requires AdobePass MSO credentials',
- }, {
- 'url': 'https://www.oxygen.com/in-ice-cold-blood/season-1/closing-night',
- 'info_dict': {
- 'id': '3692045',
- 'ext': 'mp4',
- 'title': 'Closing Night',
- 'description': 'md5:3170065c5c2f19548d72a4cbc254af63',
- 'upload_date': '20180401',
- 'timestamp': 1522623600,
- 'season_number': 1,
- 'episode_number': 1,
- 'series': 'In Ice Cold Blood',
- 'episode': 'Closing Night',
- 'duration': 2629.051,
- 'season': 'Season 1',
- 'chapters': 'count:6',
- 'thumbnail': r're:^https://.+\.jpg',
- 'age_limit': 14,
- },
- 'params': {'skip_download': 'm3u8'},
- 'skip': 'This video requires AdobePass MSO credentials',
- }, {
- 'url': 'https://www.oxygen.com/in-ice-cold-blood/season-2/episode-16/videos/handling-the-horwitz-house-after-the-murder-season-2',
- 'info_dict': {
- 'id': '3974019',
- 'ext': 'mp4',
- 'title': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
- 'description': 'md5:f9d638dd6946a1c1c0533a9c6100eae5',
- 'upload_date': '20190617',
- 'timestamp': 1560790800,
- 'season_number': 2,
- 'episode_number': 16,
- 'series': 'In Ice Cold Blood',
- 'episode': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
- 'duration': 68.235,
- 'season': 'Season 2',
- 'thumbnail': r're:^https://.+\.jpg',
- 'age_limit': 14,
- },
- 'params': {'skip_download': 'm3u8'},
- }, {
- 'url': 'https://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- site, display_id = self._match_valid_url(url).group('site', 'id')
- webpage = self._download_webpage(url, display_id)
- settings = self._search_json(
- r']+data-drupal-selector="drupal-settings-json"[^>]*>', webpage, 'settings', display_id)
- tve = extract_attributes(get_element_html_by_class('tve-video-deck-app', webpage) or '')
- query = {
- 'manifest': 'm3u',
- 'formats': 'm3u,mpeg4',
- }
-
- if tve:
- account_pid = tve.get('data-mpx-media-account-pid') or 'HNK2IC'
- account_id = tve['data-mpx-media-account-id']
- metadata = self._parse_json(
- tve.get('data-normalized-video', ''), display_id, fatal=False, transform_source=unescapeHTML)
- video_id = tve.get('data-guid') or metadata['guid']
- if tve.get('data-entitlement') == 'auth':
- auth = traverse_obj(settings, ('tve_adobe_auth', {dict})) or {}
- site = remove_end(site, 'tv')
- release_pid = tve['data-release-pid']
- resource = self._get_mvpd_resource(
- tve.get('data-adobe-pass-resource-id') or auth.get('adobePassResourceId') or site,
- tve['data-title'], release_pid, tve.get('data-rating'))
- query.update({
- 'switch': 'HLSServiceSecure',
- 'auth': self._extract_mvpd_auth(
- url, release_pid, auth.get('adobePassRequestorId') or site, resource),
- })
-
- else:
- ls_playlist = traverse_obj(settings, ('ls_playlist', ..., {dict}), get_all=False) or {}
- account_pid = ls_playlist.get('mpxMediaAccountPid') or 'PHSl-B'
- account_id = ls_playlist['mpxMediaAccountId']
- video_id = ls_playlist['defaultGuid']
- metadata = traverse_obj(
- ls_playlist, ('videos', lambda _, v: v['guid'] == video_id, {dict}), get_all=False)
-
- tp_url = f'https://link.theplatform.com/s/{account_pid}/media/guid/{account_id}/{video_id}'
- tp_metadata = self._download_json(
- update_url_query(tp_url, {'format': 'preview'}), video_id, fatal=False)
-
- chapters = traverse_obj(tp_metadata, ('chapters', ..., {
- 'start_time': ('startTime', {float_or_none(scale=1000)}),
- 'end_time': ('endTime', {float_or_none(scale=1000)}),
- }))
- # prune pointless single chapters that span the entire duration from short videos
- if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')):
- chapters = None
-
- m3u8_url = self._request_webpage(HEADRequest(
- update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').url
- if 'mpeg_cenc' in m3u8_url:
- self.report_drm(video_id)
- formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
-
- return {
- 'id': video_id,
- 'formats': formats,
- 'subtitles': subtitles,
- 'chapters': chapters,
- **merge_dicts(traverse_obj(tp_metadata, {
- 'title': 'title',
- 'description': 'description',
- 'duration': ('duration', {float_or_none(scale=1000)}),
- 'timestamp': ('pubDate', {float_or_none(scale=1000)}),
- 'season_number': (('pl1$seasonNumber', 'nbcu$seasonNumber'), {int_or_none}),
- 'episode_number': (('pl1$episodeNumber', 'nbcu$episodeNumber'), {int_or_none}),
- 'series': (('pl1$show', 'nbcu$show'), (None, ...), {str}),
- 'episode': (('title', 'pl1$episodeNumber', 'nbcu$episodeNumber'), {str_or_none}),
- 'age_limit': ('ratings', ..., 'rating', {parse_age_limit}),
- }, get_all=False), traverse_obj(metadata, {
- 'title': 'title',
- 'description': 'description',
- 'duration': ('durationInSeconds', {int_or_none}),
- 'timestamp': ('airDate', {unified_timestamp}),
- 'thumbnail': ('thumbnailUrl', {url_or_none}),
- 'season_number': ('seasonNumber', {int_or_none}),
- 'episode_number': ('episodeNumber', {int_or_none}),
- 'episode': 'episodeTitle',
- 'series': 'show',
- })),
- }
diff --git a/yt_dlp/extractor/brightcove.py b/yt_dlp/extractor/brightcove.py
index 3ada1fd5d..d4ac7a0c2 100644
--- a/yt_dlp/extractor/brightcove.py
+++ b/yt_dlp/extractor/brightcove.py
@@ -923,10 +923,18 @@ def extract_policy_key():
errors = json_data.get('errors')
if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
custom_fields = json_data['custom_fields']
+ missing_fields = ', '.join(
+ key for key in ('source_url', 'software_statement') if not smuggled_data.get(key))
+ if missing_fields:
+ raise ExtractorError(
+ f'Missing fields in smuggled data: {missing_fields}. '
+ f'This video can be only extracted from the webpage where it is embedded. '
+ f'Pass the URL of the embedding webpage instead of the Brightcove URL', expected=True)
tve_token = self._extract_mvpd_auth(
smuggled_data['source_url'], video_id,
custom_fields['bcadobepassrequestorid'],
- custom_fields['bcadobepassresourceid'])
+ custom_fields['bcadobepassresourceid'],
+ smuggled_data['software_statement'])
json_data = self._download_json(
api_url, video_id, headers={
'Accept': f'application/json;pk={policy_key}',
diff --git a/yt_dlp/extractor/cartoonnetwork.py b/yt_dlp/extractor/cartoonnetwork.py
deleted file mode 100644
index 1749a008a..000000000
--- a/yt_dlp/extractor/cartoonnetwork.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from .turner import TurnerBaseIE
-from ..utils import int_or_none
-
-
-class CartoonNetworkIE(TurnerBaseIE):
- _VALID_URL = r'https?://(?:www\.)?cartoonnetwork\.com/video/(?:[^/]+/)+(?P[^/?#]+)-(?:clip|episode)\.html'
- _TEST = {
- 'url': 'https://www.cartoonnetwork.com/video/ben-10/how-to-draw-upgrade-episode.html',
- 'info_dict': {
- 'id': '6e3375097f63874ebccec7ef677c1c3845fa850e',
- 'ext': 'mp4',
- 'title': 'How to Draw Upgrade',
- 'description': 'md5:2061d83776db7e8be4879684eefe8c0f',
- },
- 'params': {
- # m3u8 download
- 'skip_download': True,
- },
- }
-
- def _real_extract(self, url):
- display_id = self._match_id(url)
- webpage = self._download_webpage(url, display_id)
-
- def find_field(global_re, name, content_re=None, value_re='[^"]+', fatal=False):
- metadata_re = ''
- if content_re:
- metadata_re = r'|video_metadata\.content_' + content_re
- return self._search_regex(
- rf'(?:_cnglobal\.currentVideo\.{global_re}{metadata_re})\s*=\s*"({value_re})";',
- webpage, name, fatal=fatal)
-
- media_id = find_field('mediaId', 'media id', 'id', '[0-9a-f]{40}', True)
- title = find_field('episodeTitle', 'title', '(?:episodeName|name)', fatal=True)
-
- info = self._extract_ngtv_info(
- media_id, {'networkId': 'cartoonnetwork'}, {
- 'url': url,
- 'site_name': 'CartoonNetwork',
- 'auth_required': find_field('authType', 'auth type') != 'unauth',
- })
-
- series = find_field(
- 'propertyName', 'series', 'showName') or self._html_search_meta('partOfSeries', webpage)
- info.update({
- 'id': media_id,
- 'display_id': display_id,
- 'title': title,
- 'description': self._html_search_meta('description', webpage),
- 'series': series,
- 'episode': title,
- })
-
- for field in ('season', 'episode'):
- field_name = field + 'Number'
- info[field + '_number'] = int_or_none(find_field(
- field_name, field + ' number', value_re=r'\d+') or self._html_search_meta(field_name, webpage))
-
- return info
diff --git a/yt_dlp/extractor/espn.py b/yt_dlp/extractor/espn.py
index 552f9af12..ceba024bc 100644
--- a/yt_dlp/extractor/espn.py
+++ b/yt_dlp/extractor/espn.py
@@ -5,7 +5,6 @@
from .adobepass import AdobePassIE
from .common import InfoExtractor
-from .once import OnceIE
from ..utils import (
determine_ext,
dict_get,
@@ -16,7 +15,7 @@
)
-class ESPNIE(OnceIE):
+class ESPNIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://
(?:
@@ -131,9 +130,7 @@ def extract_source(source_url, source_id=None):
return
format_urls.add(source_url)
ext = determine_ext(source_url)
- if OnceIE.suitable(source_url):
- formats.extend(self._extract_once_formats(source_url))
- elif ext == 'smil':
+ if ext == 'smil':
formats.extend(self._extract_smil_formats(
source_url, video_id, fatal=False))
elif ext == 'f4m':
@@ -332,6 +329,7 @@ class WatchESPNIE(AdobePassIE):
}]
_API_KEY = 'ZXNwbiZicm93c2VyJjEuMC4w.ptUt7QxsteaRruuPmGZFaJByOoqKvDP2a5YkInHrc7c'
+ _SOFTWARE_STATEMENT = 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIyZGJmZWM4My03OWE1LTQyNzEtYTVmZC04NTZjYTMxMjRjNjMiLCJuYmYiOjE1NDAyMTI3NjEsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTQwMjEyNzYxfQ.yaK3r4AI2uLVvsyN1GLzqzgzRlxMPtasSaiYYBV0wIstqih5tvjTmeoLmi8Xy9Kp_U7Md-bOffwiyK3srHkpUkhhwXLH2x6RPjmS1tPmhaG7-3LBcHTf2ySPvXhVf7cN4ngldawK4tdtLtsw6rF_JoZE2yaC6XbS2F51nXSFEDDnOQWIHEQRG3aYAj-38P2CLGf7g-Yfhbp5cKXeksHHQ90u3eOO4WH0EAjc9oO47h33U8KMEXxJbvjV5J8Va2G2fQSgLDZ013NBI3kQnE313qgqQh2feQILkyCENpB7g-TVBreAjOaH1fU471htSoGGYepcAXv-UDtpgitDiLy7CQ'
def _call_bamgrid_api(self, path, video_id, payload=None, headers={}):
if 'Authorization' not in headers:
@@ -408,8 +406,8 @@ def _real_extract(self, url):
# TV Provider required
else:
- resource = self._get_mvpd_resource('ESPN', video_data['name'], video_id, None)
- auth = self._extract_mvpd_auth(url, video_id, 'ESPN', resource).encode()
+ resource = self._get_mvpd_resource('espn1', video_data['name'], video_id, None)
+ auth = self._extract_mvpd_auth(url, video_id, 'ESPN', resource, self._SOFTWARE_STATEMENT).encode()
asset = self._download_json(
f'https://watch.auth.api.espn.com/video/auth/media/{video_id}/asset?apikey=uiqlbgzdwuru14v627vdusswb',
diff --git a/yt_dlp/extractor/firsttv.py b/yt_dlp/extractor/firsttv.py
index ac7697bb3..878732c49 100644
--- a/yt_dlp/extractor/firsttv.py
+++ b/yt_dlp/extractor/firsttv.py
@@ -2,11 +2,15 @@
from .common import InfoExtractor
from ..utils import (
+ determine_ext,
int_or_none,
- qualities,
+ join_nonempty,
+ mimetype2ext,
+ parse_qs,
unified_strdate,
url_or_none,
)
+from ..utils.traversal import traverse_obj
class FirstTVIE(InfoExtractor):
@@ -15,40 +19,51 @@ class FirstTVIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?(?:sport)?1tv\.ru/(?:[^/?#]+/)+(?P[^/?#]+)'
_TESTS = [{
- # single format
- 'url': 'http://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015',
- 'md5': 'a1b6b60d530ebcf8daacf4565762bbaf',
+ # single format; has item.id
+ 'url': 'https://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015',
+ 'md5': '8011ae8e88ff4150107ab9c5a8f5b659',
'info_dict': {
'id': '40049',
'ext': 'mp4',
'title': 'Гость Людмила Сенчина. Наедине со всеми. Выпуск от 12.02.2015',
- 'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
'upload_date': '20150212',
'duration': 2694,
},
+ 'params': {'skip_download': 'm3u8'},
}, {
- # multiple formats
- 'url': 'http://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016',
+ # multiple formats; has item.id
+ 'url': 'https://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016',
'info_dict': {
'id': '364746',
'ext': 'mp4',
'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016',
- 'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
'upload_date': '20160407',
'duration': 179,
'formats': 'mincount:3',
},
- 'params': {
- 'skip_download': True,
- },
+ 'params': {'skip_download': 'm3u8'},
}, {
- 'url': 'http://www.1tv.ru/news/issue/2016-12-01/14:00',
+ 'url': 'https://www.1tv.ru/news/issue/2016-12-01/14:00',
'info_dict': {
'id': '14:00',
- 'title': 'Выпуск новостей в 14:00 1 декабря 2016 года. Новости. Первый канал',
- 'description': 'md5:2e921b948f8c1ff93901da78ebdb1dfd',
+ 'title': 'Выпуск программы «Время» в 20:00 1 декабря 2016 года. Новости. Первый канал',
+ 'thumbnail': 'https://static.1tv.ru/uploads/photo/image/8/big/338448_big_8fc7eb236f.jpg',
},
'playlist_count': 13,
+ }, {
+ # has timestamp; has item.uid but not item.id
+ 'url': 'https://www.1tv.ru/shows/segodnya-vecherom/vypuski/avtory-odnogo-hita-segodnya-vecherom-vypusk-ot-03-05-2025',
+ 'info_dict': {
+ 'id': '270411',
+ 'ext': 'mp4',
+ 'title': 'Авторы одного хита. Сегодня вечером. Выпуск от 03.05.2025',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'timestamp': 1746286020,
+ 'upload_date': '20250503',
+ },
+ 'params': {'skip_download': 'm3u8'},
}, {
'url': 'http://www.1tv.ru/shows/tochvtoch-supersezon/vystupleniya/evgeniy-dyatlov-vladimir-vysockiy-koni-priveredlivye-toch-v-toch-supersezon-fragment-vypuska-ot-06-11-2016',
'only_matching': True,
@@ -57,96 +72,60 @@ class FirstTVIE(InfoExtractor):
'only_matching': True,
}]
+ def _entries(self, items):
+ for item in items:
+ video_id = str(item.get('id') or item['uid'])
+
+ formats, subtitles = [], {}
+ for f in traverse_obj(item, ('sources', lambda _, v: url_or_none(v['src']))):
+ src = f['src']
+ ext = mimetype2ext(f.get('type'), default=determine_ext(src))
+ if ext == 'm3u8':
+ fmts, subs = self._extract_m3u8_formats_and_subtitles(
+ src, video_id, 'mp4', m3u8_id='hls', fatal=False)
+ elif ext == 'mpd':
+ fmts, subs = self._extract_mpd_formats_and_subtitles(
+ src, video_id, mpd_id='dash', fatal=False)
+ else:
+ tbr = self._search_regex(fr'_(\d{{3,}})\.{ext}', src, 'tbr', default=None)
+ formats.append({
+ 'url': src,
+ 'ext': ext,
+ 'format_id': join_nonempty('http', ext, tbr),
+ 'tbr': int_or_none(tbr),
+ # quality metadata of http formats may be incorrect
+ 'quality': -10,
+ })
+ continue
+ formats.extend(fmts)
+ self._merge_subtitles(subs, target=subtitles)
+
+ yield {
+ **traverse_obj(item, {
+ 'title': ('title', {str}),
+ 'thumbnail': ('poster', {url_or_none}),
+ 'timestamp': ('dvr_begin_at', {int_or_none}),
+ 'upload_date': ('date_air', {unified_strdate}),
+ 'duration': ('duration', {int_or_none}),
+ }),
+ 'id': video_id,
+ 'formats': formats,
+ 'subtitles': subtitles,
+ }
+
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
- playlist_url = urllib.parse.urljoin(url, self._search_regex(
+ playlist_url = urllib.parse.urljoin(url, self._html_search_regex(
r'data-playlist-url=(["\'])(?P(?:(?!\1).)+)\1',
webpage, 'playlist url', group='url'))
- parsed_url = urllib.parse.urlparse(playlist_url)
- qs = urllib.parse.parse_qs(parsed_url.query)
- item_ids = qs.get('videos_ids[]') or qs.get('news_ids[]')
+ item_ids = traverse_obj(parse_qs(playlist_url), 'video_id', 'videos_ids[]', 'news_ids[]')
+ items = traverse_obj(
+ self._download_json(playlist_url, display_id),
+ lambda _, v: v['uid'] and (str(v['uid']) in item_ids if item_ids else True))
- items = self._download_json(playlist_url, display_id)
-
- if item_ids:
- items = [
- item for item in items
- if item.get('uid') and str(item['uid']) in item_ids]
- else:
- items = [items[0]]
-
- entries = []
- QUALITIES = ('ld', 'sd', 'hd')
-
- for item in items:
- title = item['title']
- quality = qualities(QUALITIES)
- formats = []
- path = None
- for f in item.get('mbr', []):
- src = url_or_none(f.get('src'))
- if not src:
- continue
- tbr = int_or_none(self._search_regex(
- r'_(\d{3,})\.mp4', src, 'tbr', default=None))
- if not path:
- path = self._search_regex(
- r'//[^/]+/(.+?)_\d+\.mp4', src,
- 'm3u8 path', default=None)
- formats.append({
- 'url': src,
- 'format_id': f.get('name'),
- 'tbr': tbr,
- 'source_preference': quality(f.get('name')),
- # quality metadata of http formats may be incorrect
- 'preference': -10,
- })
- # m3u8 URL format is reverse engineered from [1] (search for
- # master.m3u8). dashEdges (that is currently balancer-vod.1tv.ru)
- # is taken from [2].
- # 1. http://static.1tv.ru/player/eump1tv-current/eump-1tv.all.min.js?rnd=9097422834:formatted
- # 2. http://static.1tv.ru/player/eump1tv-config/config-main.js?rnd=9097422834
- if not path and len(formats) == 1:
- path = self._search_regex(
- r'//[^/]+/(.+?$)', formats[0]['url'],
- 'm3u8 path', default=None)
- if path:
- if len(formats) == 1:
- m3u8_path = ','
- else:
- tbrs = [str(t) for t in sorted(f['tbr'] for f in formats)]
- m3u8_path = '_,{},{}'.format(','.join(tbrs), '.mp4')
- formats.extend(self._extract_m3u8_formats(
- f'http://balancer-vod.1tv.ru/{path}{m3u8_path}.urlset/master.m3u8',
- display_id, 'mp4',
- entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
-
- thumbnail = item.get('poster') or self._og_search_thumbnail(webpage)
- duration = int_or_none(item.get('duration') or self._html_search_meta(
- 'video:duration', webpage, 'video duration', fatal=False))
- upload_date = unified_strdate(self._html_search_meta(
- 'ya:ovs:upload_date', webpage, 'upload date', default=None))
-
- entries.append({
- 'id': str(item.get('id') or item['uid']),
- 'thumbnail': thumbnail,
- 'title': title,
- 'upload_date': upload_date,
- 'duration': int_or_none(duration),
- 'formats': formats,
- })
-
- title = self._html_search_regex(
- (r'\s*
([^<]*)',
- r"'title'\s*:\s*'([^']+)'"),
- webpage, 'title', default=None) or self._og_search_title(
- webpage, default=None)
- description = self._html_search_regex(
- r'',
- webpage, 'description', default=None) or self._html_search_meta(
- 'description', webpage, 'description', default=None)
-
- return self.playlist_result(entries, display_id, title, description)
+ return self.playlist_result(
+ self._entries(items), display_id, self._og_search_title(webpage, default=None),
+ thumbnail=self._og_search_thumbnail(webpage, default=None))
diff --git a/yt_dlp/extractor/gamespot.py b/yt_dlp/extractor/gamespot.py
index cd3f9655d..2799a27ba 100644
--- a/yt_dlp/extractor/gamespot.py
+++ b/yt_dlp/extractor/gamespot.py
@@ -1,9 +1,9 @@
import urllib.parse
-from .once import OnceIE
+from .common import InfoExtractor
-class GameSpotIE(OnceIE):
+class GameSpotIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article|review)s/(?:[^/]+/\d+-|embed/)(?P\d+)'
_TESTS = [{
'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
diff --git a/yt_dlp/extractor/go.py b/yt_dlp/extractor/go.py
index 83c1979db..4e138a828 100644
--- a/yt_dlp/extractor/go.py
+++ b/yt_dlp/extractor/go.py
@@ -7,161 +7,157 @@
int_or_none,
join_nonempty,
parse_age_limit,
- remove_end,
- remove_start,
- traverse_obj,
- try_get,
unified_timestamp,
urlencode_postdata,
)
+from ..utils.traversal import traverse_obj
class GoIE(AdobePassIE):
_SITE_INFO = {
'abc': {
'brand': '001',
- 'requestor_id': 'ABC',
+ 'requestor_id': 'dtci',
+ 'provider_id': 'ABC',
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI4OTcwMjlkYS0yYjM1LTQyOWUtYWQ0NS02ZjZiZjVkZTdhOTUiLCJuYmYiOjE2MjAxNzM5NjksImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNjIwMTczOTY5fQ.SC69DVJWSL8sIe-vVUrP6xS_kzHKqwz9PdKYexs_y-f7Vin6mM-7S-W1TE_-K55O0pyf-TL4xYgvm6LIye8CckG-nZfVwNPV4huduov0jmIcxCQFeUwkHULG2IaA44wfBVUBdaHgkhPweZ2amjycO_IXtez-gBXOLbE3B7Gx9j_5ISCFtyVUblThKfoGyQv6KT6t8Vpmc4ZSKCCQp74KWFFypydb9ucego1taW_nQD06Cdf4yByLd6NaTBceMcIKbug9b9gxFm3XBgJ5q3z7KGo1Kr6XalAV5j4m-fQ91wczlTilX8FM4AljMupyRM9mA_aEADILQ4hS79q4SM0w6w',
},
'freeform': {
'brand': '002',
'requestor_id': 'ABCFamily',
- },
- 'watchdisneychannel': {
- 'brand': '004',
- 'resource_id': 'Disney',
- },
- 'watchdisneyjunior': {
- 'brand': '008',
- 'resource_id': 'DisneyJunior',
- },
- 'watchdisneyxd': {
- 'brand': '009',
- 'resource_id': 'DisneyXD',
+ 'provider_id': 'ABCFamily',
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZWM2MGYyNC0xYzRjLTQ1NzQtYjc0Zi03ZmM4N2E5YWMzMzgiLCJuYmYiOjE1ODc2NjU5MjMsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTg3NjY1OTIzfQ.flCn3dhvmvPnWmV0JV8Fm0YFyj07yPez9-n1GFEwVIm_S2wQVWbWyJhqsAyLZVFrhOMZYTqmPS3OHxGwTwXkEYn6PD7o_vIVG3oqi-Xn1m5jRt_Gazw5qEtpat6VE7bvKGSD3ZhcidOrsCk8NcYyq75u61NHDvSl81pcedJjVRVUpsqrEwmo0aVbA0C8PX3ri0mEbGvkMKvHn8E60xp-PSE-VK8SDT0plwPu_TwUszkZ6-_I8_2xcv_WBqcXFkAVg7Q-iNJXgQvmNsrpcrYuLvi6hEH4ZLtoDcXU6MhwTQAJTiHSo8x9aHX1_qFP09CzlNOFQbC2ZEJdP9SvA53SLQ',
},
'disneynow': {
- 'brand': '011',
+ 'brand': '011', # also: '004', '008', '009'
+ 'requestor_id': 'DisneyChannels',
+ 'provider_id': 'DisneyChannels',
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI1MzAzNTRiOS04NDNiLTRkNjAtYTQ3ZS0yNzk1MzlkOTIyNTciLCJuYmYiOjE1NTg5ODc0NDksImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTU4OTg3NDQ5fQ.Jud6YS6-J2h0h6po0oMheDym0qRTJQGj4kzacrz4DFuEwhcBkkykW6pF5pKuAUJy9HCZ40oDAHe2KcTlDJjCZF5tDaUEfdihakZ9cC_rG7MU-QoRne8qaB_dPDKwGuk-ZyWD8eV3zwTJmbGo8hDxYTEU81YNCxwhyc_BPDr5TYiubbmpP3_pTnXmSpuL58isJ2peSKWlX9BacuXtBY25c_QnPFKk-_EETm7IHkTpDazde1QfHWGu4s4yJpKGk8RVVujVG6h6ELlL-ZeYLilBm7iS7h1TYG1u7fJhyZRL7isaom6NvAzsvN3ngss1fLwt8decP8wzdFHrbYTdTjW8qw',
'resource_id': 'Disney',
},
- 'fxnow.fxnetworks': {
- 'brand': '025',
+ 'fxnetworks': {
+ 'brand': '025', # also: '020'
'requestor_id': 'dtci',
+ 'provider_id': 'fx', # also 'fxx', 'fxm'
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIzYWRhYWZiNC02OTAxLTRlYzktOTdmNy1lYWZkZTJkODJkN2EiLCJuYmYiOjE1NjIwMjQwNzYsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTYyMDI0MDc2fQ.dhKMpZK50AObbZYrMiYPSfWtzXHUaeMP3jrIY4Cgfvh0GaEgk0Mns_zp78jypFeZgRtPVleQMQDNq2YEloRLcAGqP1aa6WVDglnK77ZWUm4IKai14Rwf3A6YBhSRoO2_lMmUGkuTf6gZY-kMIPqBYKqzTQiQl4HbniPFodIzFRiuI9QJVrkoyTGrJL4oqiX08PoFI3Z-TOti1Heu3EbFC-GveQHhlinYrzU7rbiAqLEz7FImtfBDsnXX1Y3uJDLYM3Bq4Oh0nrzTv1Fd62wNsCNErHHIbELidh1zZF0ujvt7ReuZUwAitm0UhEJ7OxNOUbEQWtae6pVNscvdvTFMpg',
+ },
+ 'nationalgeographic': {
+ 'brand': '026', # also '023'
+ 'requestor_id': 'dtci',
+ 'provider_id': 'ngc', # also 'ngw'
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxMzE4YTM1Ni05Mjc4LTQ4NjEtYTFmNi1jMTIzMzg1ZWMzYzMiLCJuYmYiOjE1NjIwMjM4MjgsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTYyMDIzODI4fQ.Le-2OzF9-jrhJ7ZfWtLWk5iSHGVZoxeU1w0_fO--Heli0OwRZsRq2slSmx-oZTzxuWmAgDEiBkWSDcDK6sM25DrCLsdsJa3MBuZ-slBRtH8aq3HpNoqqLkU-vg6gRUEKMtwBUtwCu_9aKUCayYtndWv4b1DjVQeSrteOW5NNudWVYleAe0kxeNJQHo5If9SCzDudKVJktFUjhNks4QPOC_uONPkRRlL9D0fNvtOY-LRFckfcHhf5z9l1iZjeukV0YhdKnuw1wyiaWrQXBUDiBfbkCRd2DM-KnelqPxfiXCaTjGKDURRBO3pz33ebge3IFXSiU5vl4qHQ8xvunzGpFw',
},
}
- _VALID_URL = r'''(?x)
- https?://
- (?P
- (?:{}\.)?go|fxnow\.fxnetworks|
- (?:www\.)?(?:abc|freeform|disneynow)
- )\.com/
- (?:
- (?:[^/]+/)*(?P[Vv][Dd][Kk][Aa]\w+)|
- (?:[^/]+/)*(?P[^/?\#]+)
- )
- '''.format(r'\.|'.join(list(_SITE_INFO.keys())))
+ _URL_PATH_RE = r'(?:video|episode|movies-and-specials)/(?P[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})'
+ _VALID_URL = [
+ fr'https?://(?:www\.)?(?Pabc)\.com/{_URL_PATH_RE}',
+ fr'https?://(?:www\.)?(?Pfreeform)\.com/{_URL_PATH_RE}',
+ fr'https?://(?:www\.)?(?Pdisneynow)\.com/{_URL_PATH_RE}',
+ fr'https?://fxnow\.(?Pfxnetworks)\.com/{_URL_PATH_RE}',
+ fr'https?://(?:www\.)?(?Pnationalgeographic)\.com/tv/{_URL_PATH_RE}',
+ ]
_TESTS = [{
- 'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
+ 'url': 'https://abc.com/episode/4192c0e6-26e5-47a8-817b-ce8272b9e440/playlist/PL551127435',
'info_dict': {
- 'id': 'VDKA3807643',
+ 'id': 'VDKA10805898',
'ext': 'mp4',
- 'title': 'The Traitor in the White House',
- 'description': 'md5:05b009d2d145a1e85d25111bd37222e8',
- },
- 'params': {
- # m3u8 download
- 'skip_download': True,
- },
- 'skip': 'This content is no longer available.',
- }, {
- 'url': 'https://disneynow.com/shows/big-hero-6-the-series',
- 'info_dict': {
- 'title': 'Doraemon',
- 'id': 'SH55574025',
- },
- 'playlist_mincount': 51,
- }, {
- 'url': 'http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood',
- 'info_dict': {
- 'id': 'VDKA3609139',
- 'title': 'This Guilty Blood',
- 'description': 'md5:f18e79ad1c613798d95fdabfe96cd292',
+ 'title': 'Switch the Flip',
+ 'description': 'To help get Brian’s life in order, Stewie and Brian swap bodies using a machine that Stewie invents.',
'age_limit': 14,
+ 'duration': 1297,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'Family Guy',
+ 'season': 'Season 16',
+ 'season_number': 16,
+ 'episode': 'Episode 17',
+ 'episode_number': 17,
+ 'timestamp': 1746082800.0,
+ 'upload_date': '20250501',
+ },
+ 'params': {'skip_download': 'm3u8'},
+ 'skip': 'This video requires AdobePass MSO credentials',
+ }, {
+ 'url': 'https://disneynow.com/episode/21029660-ba06-4406-adb0-a9a78f6e265e/playlist/PL553044961',
+ 'info_dict': {
+ 'id': 'VDKA39546942',
+ 'ext': 'mp4',
+ 'title': 'Zero Friends Again',
+ 'description': 'Relationships fray under the pressures of a difficult journey.',
+ 'age_limit': 0,
+ 'duration': 1721,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'Star Wars: Skeleton Crew',
+ 'season': 'Season 1',
+ 'season_number': 1,
+ 'episode': 'Episode 6',
+ 'episode_number': 6,
+ 'timestamp': 1746946800.0,
+ 'upload_date': '20250511',
+ },
+ 'params': {'skip_download': 'm3u8'},
+ 'skip': 'This video requires AdobePass MSO credentials',
+ }, {
+ 'url': 'https://fxnow.fxnetworks.com/episode/09f4fa6f-c293-469e-aebe-32c9ca5842a7/playlist/PL554408064',
+ 'info_dict': {
+ 'id': 'VDKA38112033',
+ 'ext': 'mp4',
+ 'title': 'The Return of Jerry',
+ 'description': 'The vampires’ long-lost fifth roommate returns. Written by Paul Simms; directed by Kyle Newacheck.',
+ 'age_limit': 17,
+ 'duration': 1493,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'What We Do in the Shadows',
+ 'season': 'Season 6',
+ 'season_number': 6,
'episode': 'Episode 1',
- 'upload_date': '20170102',
- 'season': 'Season 2',
- 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abcf/Shadowhunters/video/201/ae5f75608d86bf88aa4f9f4aa76ab1b7/579x325-Q100_ae5f75608d86bf88aa4f9f4aa76ab1b7.jpg',
- 'duration': 2544,
- 'season_number': 2,
- 'series': 'Shadowhunters',
'episode_number': 1,
- 'timestamp': 1483387200,
- 'ext': 'mp4',
- },
- 'params': {
- 'geo_bypass_ip_block': '3.244.239.0/24',
- # m3u8 download
- 'skip_download': True,
+ 'timestamp': 1729573200.0,
+ 'upload_date': '20241022',
},
+ 'params': {'skip_download': 'm3u8'},
+ 'skip': 'This video requires AdobePass MSO credentials',
}, {
- 'url': 'https://abc.com/shows/the-rookie/episode-guide/season-04/12-the-knock',
+ 'url': 'https://www.freeform.com/episode/bda0eaf7-761a-4838-aa44-96f794000844/playlist/PL553044961',
'info_dict': {
- 'id': 'VDKA26050359',
- 'title': 'The Knock',
- 'description': 'md5:0c2947e3ada4c31f28296db7db14aa64',
- 'age_limit': 14,
+ 'id': 'VDKA39007340',
'ext': 'mp4',
- 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abc/TheRookie/video/412/daf830d06e83b11eaf5c0a299d993ae3/1556x876-Q75_daf830d06e83b11eaf5c0a299d993ae3.jpg',
- 'episode': 'Episode 12',
- 'season_number': 4,
- 'season': 'Season 4',
- 'timestamp': 1642975200,
- 'episode_number': 12,
- 'upload_date': '20220123',
- 'series': 'The Rookie',
- 'duration': 2572,
- },
- 'params': {
- 'geo_bypass_ip_block': '3.244.239.0/24',
- # m3u8 download
- 'skip_download': True,
+ 'title': 'Angel\'s Landing',
+ 'description': 'md5:91bf084e785c968fab16734df7313446',
+ 'age_limit': 14,
+ 'duration': 2523,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'How I Escaped My Cult',
+ 'season': 'Season 1',
+ 'season_number': 1,
+ 'episode': 'Episode 2',
+ 'episode_number': 2,
+ 'timestamp': 1740038400.0,
+ 'upload_date': '20250220',
},
+ 'params': {'skip_download': 'm3u8'},
}, {
- 'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841',
+ 'url': 'https://www.nationalgeographic.com/tv/episode/ca694661-1186-41ae-8089-82f64d69b16d/playlist/PL554408064',
'info_dict': {
- 'id': 'VDKA12782841',
- 'title': 'First Look: Better Things - Season 2',
- 'description': 'md5:fa73584a95761c605d9d54904e35b407',
+ 'id': 'VDKA39492078',
'ext': 'mp4',
- 'age_limit': 14,
- 'upload_date': '20170825',
- 'duration': 161,
- 'series': 'Better Things',
- 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/fx/BetterThings/video/12782841/b6b05e58264121cc2c98811318e6d507/1556x876-Q75_b6b05e58264121cc2c98811318e6d507.jpg',
- 'timestamp': 1503661074,
- },
- 'params': {
- 'geo_bypass_ip_block': '3.244.239.0/24',
- # m3u8 download
- 'skip_download': True,
+ 'title': 'Heart of the Emperors',
+ 'description': 'md5:4fc50a2878f030bb3a7eac9124dca677',
+ 'age_limit': 0,
+ 'duration': 2775,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'Secrets of the Penguins',
+ 'season': 'Season 1',
+ 'season_number': 1,
+ 'episode': 'Episode 1',
+ 'episode_number': 1,
+ 'timestamp': 1745204400.0,
+ 'upload_date': '20250421',
},
+ 'params': {'skip_download': 'm3u8'},
}, {
- 'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
+ 'url': 'https://www.freeform.com/movies-and-specials/c38281fc-9f8f-47c7-8220-22394f9df2e1',
'only_matching': True,
}, {
- 'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland',
- 'only_matching': True,
- }, {
- # brand 004
- 'url': 'http://disneynow.go.com/shows/big-hero-6-the-series/season-01/episode-10-mr-sparkles-loses-his-sparkle/vdka4637915',
- 'only_matching': True,
- }, {
- # brand 008
- 'url': 'http://disneynow.go.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
- 'only_matching': True,
- }, {
- 'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
- 'only_matching': True,
- }, {
- 'url': 'https://www.freeform.com/shows/cruel-summer/episode-guide/season-01/01-happy-birthday-jeanette-turner',
+ 'url': 'https://abc.com/video/219a454a-172c-41bf-878a-d169e6bc0bdc/playlist/PL5523098420',
'only_matching': True,
}]
@@ -171,58 +167,29 @@ def _extract_videos(self, brand, video_id='-1', show_id='-1'):
f'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/{brand}/001/-1/{show_id}/-1/{video_id}/-1/-1.json',
display_id)['video']
+ def _extract_global_var(self, name, webpage, video_id):
+ return self._search_json(
+ fr'window\[["\']{re.escape(name)}["\']\]\s*=',
+ webpage, f'{name.strip("_")} JSON', video_id)
+
def _real_extract(self, url):
- mobj = self._match_valid_url(url)
- sub_domain = remove_start(remove_end(mobj.group('sub_domain') or '', '.go'), 'www.')
- video_id, display_id = mobj.group('id', 'display_id')
- site_info = self._SITE_INFO.get(sub_domain, {})
- brand = site_info.get('brand')
- if not video_id or not site_info:
- webpage = self._download_webpage(url, display_id or video_id)
- data = self._parse_json(
- self._search_regex(
- r'["\']__abc_com__["\']\s*\]\s*=\s*({.+?})\s*;', webpage,
- 'data', default='{}'),
- display_id or video_id, fatal=False)
- # https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot
- layout = try_get(data, lambda x: x['page']['content']['video']['layout'], dict)
- video_id = None
- if layout:
- video_id = try_get(
- layout,
- (lambda x: x['videoid'], lambda x: x['video']['id']),
- str)
- if not video_id:
- video_id = self._search_regex(
- (
- # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
- # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
- r'data-video-id=["\']*(VDKA\w+)',
- # page.analytics.videoIdCode
- r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\w+)',
- # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet
- r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)',
- ), webpage, 'video id', default=video_id)
- if not site_info:
- brand = self._search_regex(
- (r'data-brand=\s*["\']\s*(\d+)',
- r'data-page-brand=\s*["\']\s*(\d+)'), webpage, 'brand',
- default='004')
- site_info = next(
- si for _, si in self._SITE_INFO.items()
- if si.get('brand') == brand)
- if not video_id:
- # show extraction works for Disney, DisneyJunior and DisneyXD
- # ABC and Freeform has different layout
- show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id')
- videos = self._extract_videos(brand, show_id=show_id)
- show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False)
- entries = []
- for video in videos:
- entries.append(self.url_result(
- video['url'], 'Go', video.get('id'), video.get('title')))
- entries.reverse()
- return self.playlist_result(entries, show_id, show_title)
+ site, display_id = self._match_valid_url(url).group('site', 'id')
+ webpage = self._download_webpage(url, display_id)
+ config = self._extract_global_var('__CONFIG__', webpage, display_id)
+ data = self._extract_global_var(config['globalVar'], webpage, display_id)
+ video_id = traverse_obj(data, (
+ 'page', 'content', 'video', 'layout', (('video', 'id'), 'videoid'), {str}, any))
+ if not video_id:
+ video_id = self._search_regex([
+ # data-track-video_id="VDKA39492078"
+ # data-track-video_id_code="vdka39492078"
+ # data-video-id="'VDKA3609139'"
+ r'data-(?:track-)?video[_-]id(?:_code)?=["\']*((?:vdka|VDKA)\d+)',
+ # page.analytics.videoIdCode
+ r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\d+)'], webpage, 'video ID')
+
+ site_info = self._SITE_INFO[site]
+ brand = site_info['brand']
video_data = self._extract_videos(brand, video_id)[0]
video_id = video_data['id']
title = video_data['title']
@@ -238,26 +205,31 @@ def _real_extract(self, url):
if ext == 'm3u8':
video_type = video_data.get('type')
data = {
- 'video_id': video_data['id'],
+ 'video_id': video_id,
'video_type': video_type,
'brand': brand,
'device': '001',
+ 'app_name': 'webplayer-abc',
}
if video_data.get('accesslevel') == '1':
- requestor_id = site_info.get('requestor_id', 'DisneyChannels')
+ provider_id = site_info['provider_id']
+ software_statement = traverse_obj(data, ('app', 'config', (
+ ('features', 'auth', 'softwareStatement'),
+ ('tvAuth', 'SOFTWARE_STATEMENTS', 'PRODUCTION'),
+ ), {str}, any)) or site_info['software_statement']
resource = site_info.get('resource_id') or self._get_mvpd_resource(
- requestor_id, title, video_id, None)
+ provider_id, title, video_id, None)
auth = self._extract_mvpd_auth(
- url, video_id, requestor_id, resource)
+ url, video_id, site_info['requestor_id'], resource, software_statement)
data.update({
'token': auth,
'token_type': 'ap',
- 'adobe_requestor_id': requestor_id,
+ 'adobe_requestor_id': provider_id,
})
else:
self._initialize_geo_bypass({'countries': ['US']})
entitlement = self._download_json(
- 'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
+ 'https://prod.gatekeeper.us-abc.symphony.edgedatg.go.com/vp2/ws-secure/entitlement/2020/playmanifest_secure.json',
video_id, data=urlencode_postdata(data))
errors = entitlement.get('errors', {}).get('errors', [])
if errors:
@@ -267,7 +239,7 @@ def _real_extract(self, url):
error['message'], countries=['US'])
error_message = ', '.join([error['message'] for error in errors])
raise ExtractorError(f'{self.IE_NAME} said: {error_message}', expected=True)
- asset_url += '?' + entitlement['uplynkData']['sessionKey']
+ asset_url += '?' + entitlement['entitlement']['uplynkData']['sessionKey']
fmts, subs = self._extract_m3u8_formats_and_subtitles(
asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False)
formats.extend(fmts)
diff --git a/yt_dlp/extractor/iprima.py b/yt_dlp/extractor/iprima.py
index 9b91a454b..bf2344f72 100644
--- a/yt_dlp/extractor/iprima.py
+++ b/yt_dlp/extractor/iprima.py
@@ -1,3 +1,4 @@
+import json
import re
import time
@@ -6,9 +7,7 @@
ExtractorError,
determine_ext,
js_to_json,
- parse_qs,
traverse_obj,
- urlencode_postdata,
)
@@ -16,7 +15,6 @@ class IPrimaIE(InfoExtractor):
_VALID_URL = r'https?://(?!cnn)(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P[^/?#&]+)'
_GEO_BYPASS = False
_NETRC_MACHINE = 'iprima'
- _AUTH_ROOT = 'https://auth.iprima.cz'
access_token = None
_TESTS = [{
@@ -86,48 +84,18 @@ def _perform_login(self, username, password):
if self.access_token:
return
- login_page = self._download_webpage(
- f'{self._AUTH_ROOT}/oauth2/login', None, note='Downloading login page',
- errnote='Downloading login page failed')
-
- login_form = self._hidden_inputs(login_page)
-
- login_form.update({
- '_email': username,
- '_password': password})
-
- profile_select_html, login_handle = self._download_webpage_handle(
- f'{self._AUTH_ROOT}/oauth2/login', None, data=urlencode_postdata(login_form),
- note='Logging in')
-
- # a profile may need to be selected first, even when there is only a single one
- if '/profile-select' in login_handle.url:
- profile_id = self._search_regex(
- r'data-identifier\s*=\s*["\']?(\w+)', profile_select_html, 'profile id')
-
- login_handle = self._request_webpage(
- f'{self._AUTH_ROOT}/user/profile-select-perform/{profile_id}', None,
- query={'continueUrl': '/user/login?redirect_uri=/user/'}, note='Selecting profile')
-
- code = traverse_obj(login_handle.url, ({parse_qs}, 'code', 0))
- if not code:
- raise ExtractorError('Login failed', expected=True)
-
- token_request_data = {
- 'scope': 'openid+email+profile+phone+address+offline_access',
- 'client_id': 'prima_sso',
- 'grant_type': 'authorization_code',
- 'code': code,
- 'redirect_uri': f'{self._AUTH_ROOT}/sso/auth-check'}
-
token_data = self._download_json(
- f'{self._AUTH_ROOT}/oauth2/token', None,
- note='Downloading token', errnote='Downloading token failed',
- data=urlencode_postdata(token_request_data))
+ 'https://ucet.iprima.cz/api/session/create', None,
+ note='Logging in', errnote='Failed to log in',
+ data=json.dumps({
+ 'email': username,
+ 'password': password,
+ 'deviceName': 'Windows Chrome',
+ }).encode(), headers={'content-type': 'application/json'})
- self.access_token = token_data.get('access_token')
- if self.access_token is None:
- raise ExtractorError('Getting token failed', expected=True)
+ self.access_token = token_data['accessToken']['value']
+ if not self.access_token:
+ raise ExtractorError('Failed to fetch access token')
def _real_initialize(self):
if not self.access_token:
diff --git a/yt_dlp/extractor/jiosaavn.py b/yt_dlp/extractor/jiosaavn.py
index 030fe686b..3760e9abe 100644
--- a/yt_dlp/extractor/jiosaavn.py
+++ b/yt_dlp/extractor/jiosaavn.py
@@ -1,23 +1,33 @@
import functools
+import itertools
import math
import re
from .common import InfoExtractor
from ..utils import (
InAdvancePagedList,
+ ISO639Utils,
+ OnDemandPagedList,
clean_html,
int_or_none,
+ js_to_json,
make_archive_id,
+ orderedSet,
smuggle_url,
+ unified_strdate,
+ unified_timestamp,
unsmuggle_url,
url_basename,
url_or_none,
urlencode_postdata,
+ urljoin,
+ variadic,
)
from ..utils.traversal import traverse_obj
class JioSaavnBaseIE(InfoExtractor):
+ _URL_BASE_RE = r'https?://(?:www\.)?(?:jio)?saavn\.com'
_API_URL = 'https://www.jiosaavn.com/api.php'
_VALID_BITRATES = {'16', '32', '64', '128', '320'}
@@ -30,16 +40,20 @@ def requested_bitrates(self):
f'Valid bitrates are: {", ".join(sorted(self._VALID_BITRATES, key=int))}')
return requested_bitrates
- def _extract_formats(self, song_data):
+ def _extract_formats(self, item_data):
+ # Show/episode JSON data has a slightly different structure than song JSON data
+ if media_url := traverse_obj(item_data, ('more_info', 'encrypted_media_url', {str})):
+ item_data.setdefault('encrypted_media_url', media_url)
+
for bitrate in self.requested_bitrates:
media_data = self._download_json(
- self._API_URL, song_data['id'],
+ self._API_URL, item_data['id'],
f'Downloading format info for {bitrate}',
fatal=False, data=urlencode_postdata({
'__call': 'song.generateAuthToken',
'_format': 'json',
'bitrate': bitrate,
- 'url': song_data['encrypted_media_url'],
+ 'url': item_data['encrypted_media_url'],
}))
if not traverse_obj(media_data, ('auth_url', {url_or_none})):
self.report_warning(f'Unable to extract format info for {bitrate}')
@@ -53,24 +67,6 @@ def _extract_formats(self, song_data):
'vcodec': 'none',
}
- def _extract_song(self, song_data, url=None):
- info = traverse_obj(song_data, {
- 'id': ('id', {str}),
- 'title': ('song', {clean_html}),
- 'album': ('album', {clean_html}),
- 'thumbnail': ('image', {url_or_none}, {lambda x: re.sub(r'-\d+x\d+\.', '-500x500.', x)}),
- 'duration': ('duration', {int_or_none}),
- 'view_count': ('play_count', {int_or_none}),
- 'release_year': ('year', {int_or_none}),
- 'artists': ('primary_artists', {lambda x: x.split(', ') if x else None}),
- 'webpage_url': ('perma_url', {url_or_none}),
- })
- if webpage_url := info.get('webpage_url') or url:
- info['display_id'] = url_basename(webpage_url)
- info['_old_archive_ids'] = [make_archive_id(JioSaavnSongIE, info['display_id'])]
-
- return info
-
def _call_api(self, type_, token, note='API', params={}):
return self._download_json(
self._API_URL, token, f'Downloading {note} JSON', f'Unable to download {note} JSON',
@@ -84,19 +80,89 @@ def _call_api(self, type_, token, note='API', params={}):
**params,
})
- def _yield_songs(self, playlist_data):
- for song_data in traverse_obj(playlist_data, ('songs', lambda _, v: v['id'] and v['perma_url'])):
- song_info = self._extract_song(song_data)
- url = smuggle_url(song_info['webpage_url'], {
- 'id': song_data['id'],
- 'encrypted_media_url': song_data['encrypted_media_url'],
- })
- yield self.url_result(url, JioSaavnSongIE, url_transparent=True, **song_info)
+ @staticmethod
+ def _extract_song(song_data, url=None):
+ info = traverse_obj(song_data, {
+ 'id': ('id', {str}),
+ 'title': (('song', 'title'), {clean_html}, any),
+ 'album': ((None, 'more_info'), 'album', {clean_html}, any),
+ 'duration': ((None, 'more_info'), 'duration', {int_or_none}, any),
+ 'channel': ((None, 'more_info'), 'label', {str}, any),
+ 'channel_id': ((None, 'more_info'), 'label_id', {str}, any),
+ 'channel_url': ((None, 'more_info'), 'label_url', {urljoin('https://www.jiosaavn.com/')}, any),
+ 'release_date': ((None, 'more_info'), 'release_date', {unified_strdate}, any),
+ 'release_year': ('year', {int_or_none}),
+ 'thumbnail': ('image', {url_or_none}, {lambda x: re.sub(r'-\d+x\d+\.', '-500x500.', x)}),
+ 'view_count': ('play_count', {int_or_none}),
+ 'language': ('language', {lambda x: ISO639Utils.short2long(x.casefold()) or 'und'}),
+ 'webpage_url': ('perma_url', {url_or_none}),
+ 'artists': ('more_info', 'artistMap', 'primary_artists', ..., 'name', {str}, filter, all),
+ })
+ if webpage_url := info.get('webpage_url') or url:
+ info['display_id'] = url_basename(webpage_url)
+ info['_old_archive_ids'] = [make_archive_id(JioSaavnSongIE, info['display_id'])]
+
+ if primary_artists := traverse_obj(song_data, ('primary_artists', {lambda x: x.split(', ') if x else None})):
+ info['artists'].extend(primary_artists)
+ if featured_artists := traverse_obj(song_data, ('featured_artists', {str}, filter)):
+ info['artists'].extend(featured_artists.split(', '))
+ info['artists'] = orderedSet(info['artists']) or None
+
+ return info
+
+ @staticmethod
+ def _extract_episode(episode_data, url=None):
+ info = JioSaavnBaseIE._extract_song(episode_data, url)
+ info.pop('_old_archive_ids', None)
+ info.update(traverse_obj(episode_data, {
+ 'description': ('more_info', 'description', {str}),
+ 'timestamp': ('more_info', 'release_time', {unified_timestamp}),
+ 'series': ('more_info', 'show_title', {str}),
+ 'series_id': ('more_info', 'show_id', {str}),
+ 'season': ('more_info', 'season_title', {str}),
+ 'season_number': ('more_info', 'season_no', {int_or_none}),
+ 'season_id': ('more_info', 'season_id', {str}),
+ 'episode_number': ('more_info', 'episode_number', {int_or_none}),
+ 'cast': ('starring', {lambda x: x.split(', ') if x else None}),
+ }))
+ return info
+
+ def _extract_jiosaavn_result(self, url, endpoint, response_key, parse_func):
+ url, smuggled_data = unsmuggle_url(url)
+ data = traverse_obj(smuggled_data, ({
+ 'id': ('id', {str}),
+ 'encrypted_media_url': ('encrypted_media_url', {str}),
+ }))
+
+ if 'id' in data and 'encrypted_media_url' in data:
+ result = {'id': data['id']}
+ else:
+ # only extract metadata if this is not a url_transparent result
+ data = self._call_api(endpoint, self._match_id(url))[response_key][0]
+ result = parse_func(data, url)
+
+ result['formats'] = list(self._extract_formats(data))
+ return result
+
+ def _yield_items(self, playlist_data, keys=None, parse_func=None):
+ """Subclasses using this method must set _ENTRY_IE"""
+ if parse_func is None:
+ parse_func = self._extract_song
+
+ for item_data in traverse_obj(playlist_data, (
+ *variadic(keys, (str, bytes, dict, set)), lambda _, v: v['id'] and v['perma_url'],
+ )):
+ info = parse_func(item_data)
+ url = smuggle_url(info['webpage_url'], traverse_obj(item_data, {
+ 'id': ('id', {str}),
+ 'encrypted_media_url': ((None, 'more_info'), 'encrypted_media_url', {str}, any),
+ }))
+ yield self.url_result(url, self._ENTRY_IE, url_transparent=True, **info)
class JioSaavnSongIE(JioSaavnBaseIE):
IE_NAME = 'jiosaavn:song'
- _VALID_URL = r'https?://(?:www\.)?(?:jiosaavn\.com/song/[^/?#]+/|saavn\.com/s/song/(?:[^/?#]+/){3})(?P[^/?#]+)'
+ _VALID_URL = JioSaavnBaseIE._URL_BASE_RE + r'(?:/song/[^/?#]+/|/s/song/(?:[^/?#]+/){3})(?P[^/?#]+)'
_TESTS = [{
'url': 'https://www.jiosaavn.com/song/leja-re/OQsEfQFVUXk',
'md5': '3b84396d15ed9e083c3106f1fa589c04',
@@ -106,12 +172,38 @@ class JioSaavnSongIE(JioSaavnBaseIE):
'ext': 'm4a',
'title': 'Leja Re',
'album': 'Leja Re',
- 'thumbnail': r're:https?://c.saavncdn.com/258/Leja-Re-Hindi-2018-20181124024539-500x500.jpg',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
'duration': 205,
'view_count': int,
'release_year': 2018,
'artists': ['Sandesh Shandilya', 'Dhvani Bhanushali', 'Tanishk Bagchi'],
'_old_archive_ids': ['jiosaavnsong OQsEfQFVUXk'],
+ 'channel': 'T-Series',
+ 'language': 'hin',
+ 'channel_id': '34297',
+ 'channel_url': 'https://www.jiosaavn.com/label/t-series-albums/6DLuXO3VoTo_',
+ 'release_date': '20181124',
+ },
+ }, {
+ 'url': 'https://www.jiosaavn.com/song/chuttamalle/P1FfWjZkQ0Q',
+ 'md5': '96296c58d6ce488a417ef0728fd2d680',
+ 'info_dict': {
+ 'id': 'O94kBTtw',
+ 'display_id': 'P1FfWjZkQ0Q',
+ 'ext': 'm4a',
+ 'title': 'Chuttamalle',
+ 'album': 'Devara Part 1 - Telugu',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'duration': 222,
+ 'view_count': int,
+ 'release_year': 2024,
+ 'artists': 'count:3',
+ '_old_archive_ids': ['jiosaavnsong P1FfWjZkQ0Q'],
+ 'channel': 'T-Series',
+ 'language': 'tel',
+ 'channel_id': '34297',
+ 'channel_url': 'https://www.jiosaavn.com/label/t-series-albums/6DLuXO3VoTo_',
+ 'release_date': '20240926',
},
}, {
'url': 'https://www.saavn.com/s/song/hindi/Saathiya/O-Humdum-Suniyo-Re/KAMiazoCblU',
@@ -119,26 +211,51 @@ class JioSaavnSongIE(JioSaavnBaseIE):
}]
def _real_extract(self, url):
- url, smuggled_data = unsmuggle_url(url)
- song_data = traverse_obj(smuggled_data, ({
- 'id': ('id', {str}),
- 'encrypted_media_url': ('encrypted_media_url', {str}),
- }))
+ return self._extract_jiosaavn_result(url, 'song', 'songs', self._extract_song)
- if 'id' in song_data and 'encrypted_media_url' in song_data:
- result = {'id': song_data['id']}
- else:
- # only extract metadata if this is not a url_transparent result
- song_data = self._call_api('song', self._match_id(url))['songs'][0]
- result = self._extract_song(song_data, url)
- result['formats'] = list(self._extract_formats(song_data))
- return result
+class JioSaavnShowIE(JioSaavnBaseIE):
+ IE_NAME = 'jiosaavn:show'
+ _VALID_URL = JioSaavnBaseIE._URL_BASE_RE + r'/shows/[^/?#]+/(?P[^/?#]{11,})/?(?:$|[?#])'
+ _TESTS = [{
+ 'url': 'https://www.jiosaavn.com/shows/non-food-ways-to-boost-your-energy/XFMcKICOCgc_',
+ 'md5': '0733cd254cfe74ef88bea1eaedcf1f4f',
+ 'info_dict': {
+ 'id': 'qqzh3RKZ',
+ 'display_id': 'XFMcKICOCgc_',
+ 'ext': 'mp3',
+ 'title': 'Non-Food Ways To Boost Your Energy',
+ 'description': 'md5:26e7129644b5c6aada32b8851c3997c8',
+ 'episode': 'Episode 1',
+ 'timestamp': 1640563200,
+ 'series': 'Holistic Lifestyle With Neha Ranglani',
+ 'series_id': '52397',
+ 'season': 'Holistic Lifestyle With Neha Ranglani',
+ 'season_number': 1,
+ 'season_id': '61273',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'duration': 311,
+ 'view_count': int,
+ 'release_year': 2021,
+ 'language': 'eng',
+ 'channel': 'Saavn OG',
+ 'channel_id': '1953876',
+ 'episode_number': 1,
+ 'upload_date': '20211227',
+ 'release_date': '20211227',
+ },
+ }, {
+ 'url': 'https://www.jiosaavn.com/shows/himesh-reshammiya/Kr8fmfSN4vo_',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ return self._extract_jiosaavn_result(url, 'episode', 'episodes', self._extract_episode)
class JioSaavnAlbumIE(JioSaavnBaseIE):
IE_NAME = 'jiosaavn:album'
- _VALID_URL = r'https?://(?:www\.)?(?:jio)?saavn\.com/album/[^/?#]+/(?P[^/?#]+)'
+ _VALID_URL = JioSaavnBaseIE._URL_BASE_RE + r'/album/[^/?#]+/(?P[^/?#]+)'
_TESTS = [{
'url': 'https://www.jiosaavn.com/album/96/buIOjYZDrNA_',
'info_dict': {
@@ -147,18 +264,19 @@ class JioSaavnAlbumIE(JioSaavnBaseIE):
},
'playlist_count': 10,
}]
+ _ENTRY_IE = JioSaavnSongIE
def _real_extract(self, url):
display_id = self._match_id(url)
album_data = self._call_api('album', display_id)
return self.playlist_result(
- self._yield_songs(album_data), display_id, traverse_obj(album_data, ('title', {str})))
+ self._yield_items(album_data, 'songs'), display_id, traverse_obj(album_data, ('title', {str})))
class JioSaavnPlaylistIE(JioSaavnBaseIE):
IE_NAME = 'jiosaavn:playlist'
- _VALID_URL = r'https?://(?:www\.)?(?:jio)?saavn\.com/(?:s/playlist/(?:[^/?#]+/){2}|featured/[^/?#]+/)(?P[^/?#]+)'
+ _VALID_URL = JioSaavnBaseIE._URL_BASE_RE + r'/(?:s/playlist/(?:[^/?#]+/){2}|featured/[^/?#]+/)(?P[^/?#]+)'
_TESTS = [{
'url': 'https://www.jiosaavn.com/s/playlist/2279fbe391defa793ad7076929a2f5c9/mood-english/LlJ8ZWT1ibN5084vKHRj2Q__',
'info_dict': {
@@ -172,15 +290,16 @@ class JioSaavnPlaylistIE(JioSaavnBaseIE):
'id': 'DVR,pFUOwyXqIp77B1JF,A__',
'title': 'Mood Hindi',
},
- 'playlist_mincount': 801,
+ 'playlist_mincount': 750,
}, {
'url': 'https://www.jiosaavn.com/featured/taaza-tunes/Me5RridRfDk_',
'info_dict': {
'id': 'Me5RridRfDk_',
'title': 'Taaza Tunes',
},
- 'playlist_mincount': 301,
+ 'playlist_mincount': 50,
}]
+ _ENTRY_IE = JioSaavnSongIE
_PAGE_SIZE = 50
def _fetch_page(self, token, page):
@@ -189,7 +308,7 @@ def _fetch_page(self, token, page):
def _entries(self, token, first_page_data, page):
page_data = first_page_data if not page else self._fetch_page(token, page + 1)
- yield from self._yield_songs(page_data)
+ yield from self._yield_items(page_data, 'songs')
def _real_extract(self, url):
display_id = self._match_id(url)
@@ -199,3 +318,95 @@ def _real_extract(self, url):
return self.playlist_result(InAdvancePagedList(
functools.partial(self._entries, display_id, playlist_data),
total_pages, self._PAGE_SIZE), display_id, traverse_obj(playlist_data, ('listname', {str})))
+
+
+class JioSaavnShowPlaylistIE(JioSaavnBaseIE):
+ IE_NAME = 'jiosaavn:show:playlist'
+ _VALID_URL = JioSaavnBaseIE._URL_BASE_RE + r'/shows/(?P[^#/?]+)/(?P\d+)/[^/?#]+'
+ _TESTS = [{
+ 'url': 'https://www.jiosaavn.com/shows/talking-music/1/PjReFP-Sguk_',
+ 'info_dict': {
+ 'id': 'talking-music-1',
+ 'title': 'Talking Music',
+ },
+ 'playlist_mincount': 11,
+ }]
+ _ENTRY_IE = JioSaavnShowIE
+ _PAGE_SIZE = 10
+
+ def _fetch_page(self, show_id, season_id, page):
+ return self._call_api('show', show_id, f'show page {page}', {
+ 'p': page,
+ '__call': 'show.getAllEpisodes',
+ 'show_id': show_id,
+ 'season_number': season_id,
+ 'api_version': '4',
+ 'sort_order': 'desc',
+ })
+
+ def _entries(self, show_id, season_id, page):
+ page_data = self._fetch_page(show_id, season_id, page + 1)
+ yield from self._yield_items(page_data, keys=None, parse_func=self._extract_episode)
+
+ def _real_extract(self, url):
+ show_slug, season_id = self._match_valid_url(url).group('show', 'season')
+ playlist_id = f'{show_slug}-{season_id}'
+ webpage = self._download_webpage(url, playlist_id)
+
+ show_info = self._search_json(
+ r'window\.__INITIAL_DATA__\s*=', webpage, 'initial data',
+ playlist_id, transform_source=js_to_json)['showView']
+ show_id = show_info['current_id']
+
+ entries = OnDemandPagedList(functools.partial(self._entries, show_id, season_id), self._PAGE_SIZE)
+ return self.playlist_result(
+ entries, playlist_id, traverse_obj(show_info, ('show', 'title', 'text', {str})))
+
+
+class JioSaavnArtistIE(JioSaavnBaseIE):
+ IE_NAME = 'jiosaavn:artist'
+ _VALID_URL = JioSaavnBaseIE._URL_BASE_RE + r'/artist/[^/?#]+/(?P[^/?#]+)'
+ _TESTS = [{
+ 'url': 'https://www.jiosaavn.com/artist/krsna-songs/rYLBEve2z3U_',
+ 'info_dict': {
+ 'id': 'rYLBEve2z3U_',
+ 'title': 'KR$NA',
+ },
+ 'playlist_mincount': 38,
+ }, {
+ 'url': 'https://www.jiosaavn.com/artist/sanam-puri-songs/SkNEv3qRhDE_',
+ 'info_dict': {
+ 'id': 'SkNEv3qRhDE_',
+ 'title': 'Sanam Puri',
+ },
+ 'playlist_mincount': 51,
+ }]
+ _ENTRY_IE = JioSaavnSongIE
+ _PAGE_SIZE = 50
+
+ def _fetch_page(self, artist_id, page):
+ return self._call_api('artist', artist_id, f'artist page {page + 1}', {
+ 'p': page,
+ 'n_song': self._PAGE_SIZE,
+ 'n_album': self._PAGE_SIZE,
+ 'sub_type': '',
+ 'includeMetaTags': '',
+ 'api_version': '4',
+ 'category': 'alphabetical',
+ 'sort_order': 'asc',
+ })
+
+ def _entries(self, artist_id, first_page):
+ for page in itertools.count():
+ playlist_data = first_page if not page else self._fetch_page(artist_id, page)
+ if not traverse_obj(playlist_data, ('topSongs', ..., {dict})):
+ break
+ yield from self._yield_items(playlist_data, 'topSongs')
+
+ def _real_extract(self, url):
+ artist_id = self._match_id(url)
+ first_page = self._fetch_page(artist_id, 0)
+
+ return self.playlist_result(
+ self._entries(artist_id, first_page), artist_id,
+ traverse_obj(first_page, ('name', {str})))
diff --git a/yt_dlp/extractor/lrt.py b/yt_dlp/extractor/lrt.py
index e50194f88..caff9125e 100644
--- a/yt_dlp/extractor/lrt.py
+++ b/yt_dlp/extractor/lrt.py
@@ -2,7 +2,6 @@
from ..utils import (
clean_html,
merge_dicts,
- str_or_none,
traverse_obj,
unified_timestamp,
url_or_none,
@@ -138,13 +137,15 @@ def _real_extract(self, url):
'https://www.lrt.lt/radioteka/api/media', video_id,
query={'url': f'/mediateka/irasas/{video_id}/{path}'})
- return traverse_obj(media, {
- 'id': ('id', {int}, {str_or_none}),
- 'title': ('title', {str}),
- 'tags': ('tags', ..., 'name', {str}),
- 'categories': ('playlist_item', 'category', {str}, filter, all, filter),
- 'description': ('content', {clean_html}, {str}),
- 'timestamp': ('date', {lambda x: x.replace('.', '/')}, {unified_timestamp}),
- 'thumbnail': ('playlist_item', 'image', {urljoin('https://www.lrt.lt')}),
- 'formats': ('playlist_item', 'file', {lambda x: self._extract_m3u8_formats(x, video_id)}),
- })
+ return {
+ 'id': video_id,
+ 'formats': self._extract_m3u8_formats(media['playlist_item']['file'], video_id),
+ **traverse_obj(media, {
+ 'title': ('title', {str}),
+ 'tags': ('tags', ..., 'name', {str}),
+ 'categories': ('playlist_item', 'category', {str}, filter, all, filter),
+ 'description': ('content', {clean_html}, {str}),
+ 'timestamp': ('date', {lambda x: x.replace('.', '/')}, {unified_timestamp}),
+ 'thumbnail': ('playlist_item', 'image', {urljoin('https://www.lrt.lt')}),
+ }),
+ }
diff --git a/yt_dlp/extractor/nba.py b/yt_dlp/extractor/nba.py
index 91ae1d14c..c87046d8e 100644
--- a/yt_dlp/extractor/nba.py
+++ b/yt_dlp/extractor/nba.py
@@ -19,7 +19,8 @@
class NBACVPBaseIE(TurnerBaseIE):
def _extract_nba_cvp_info(self, path, video_id, fatal=False):
return self._extract_cvp_info(
- f'http://secure.nba.com/{path}', video_id, {
+ # XXX: The 3rd argument (None) needs to be the AdobePass software_statement
+ f'http://secure.nba.com/{path}', video_id, None, {
'default': {
'media_src': 'http://nba.cdn.turner.com/nba/big',
},
@@ -94,6 +95,7 @@ def _extract_video(self, filter_key, filter_value):
class NBAWatchEmbedIE(NBAWatchBaseIE):
+ _WORKING = False
IE_NAME = 'nba:watch:embed'
_VALID_URL = NBAWatchBaseIE._VALID_URL_BASE + r'embed\?.*?\bid=(?P\d+)'
_TESTS = [{
@@ -115,6 +117,7 @@ def _real_extract(self, url):
class NBAWatchIE(NBAWatchBaseIE):
+ _WORKING = False
IE_NAME = 'nba:watch'
_VALID_URL = NBAWatchBaseIE._VALID_URL_BASE + r'(?:nba/)?video/(?P.+?(?=/index\.html)|(?:[^/]+/)*[^/?#&]+)'
_TESTS = [{
@@ -167,6 +170,7 @@ def _real_extract(self, url):
class NBAWatchCollectionIE(NBAWatchBaseIE):
+ _WORKING = False
IE_NAME = 'nba:watch:collection'
_VALID_URL = NBAWatchBaseIE._VALID_URL_BASE + r'list/collection/(?P[^/?#&]+)'
_TESTS = [{
@@ -336,6 +340,7 @@ def _real_extract(self, url):
class NBAEmbedIE(NBABaseIE):
+ _WORKING = False
IE_NAME = 'nba:embed'
_VALID_URL = r'https?://secure\.nba\.com/assets/amp/include/video/(?:topI|i)frame\.html\?.*?\bcontentId=(?P[^?#&]+)'
_TESTS = [{
@@ -358,6 +363,7 @@ def _real_extract(self, url):
class NBAIE(NBABaseIE):
+ _WORKING = False
IE_NAME = 'nba'
_VALID_URL = NBABaseIE._VALID_URL_BASE + f'(?!{NBABaseIE._CHANNEL_PATH_REGEX})video/(?P(?:[^/]+/)*[^/?#&]+)'
_TESTS = [{
@@ -385,6 +391,7 @@ def _extract_url_results(self, team, content_id):
class NBAChannelIE(NBABaseIE):
+ _WORKING = False
IE_NAME = 'nba:channel'
_VALID_URL = NBABaseIE._VALID_URL_BASE + f'(?:{NBABaseIE._CHANNEL_PATH_REGEX})/(?P[^/?#&]+)'
_TESTS = [{
diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index d9aded09e..bd4862bde 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -6,7 +6,7 @@
from .adobepass import AdobePassIE
from .common import InfoExtractor
-from .theplatform import ThePlatformIE, default_ns
+from .theplatform import ThePlatformBaseIE, ThePlatformIE, default_ns
from ..networking import HEADRequest
from ..utils import (
ExtractorError,
@@ -14,26 +14,130 @@
UserNotLive,
clean_html,
determine_ext,
+ extract_attributes,
float_or_none,
+ get_element_html_by_class,
int_or_none,
join_nonempty,
+ make_archive_id,
mimetype2ext,
parse_age_limit,
parse_duration,
+ parse_iso8601,
remove_end,
- smuggle_url,
- traverse_obj,
try_get,
unescapeHTML,
unified_timestamp,
update_url_query,
url_basename,
+ url_or_none,
)
+from ..utils.traversal import require, traverse_obj
-class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
- _VALID_URL = r'https?(?P://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/]+/video/[^/]+/(?P(?:NBCE|n)?\d+))'
+class NBCUniversalBaseIE(ThePlatformBaseIE):
+ _GEO_COUNTRIES = ['US']
+ _GEO_BYPASS = False
+ _M3U8_RE = r'https?://[^/?#]+/prod/[\w-]+/(?P[^?#]+/)cmaf/mpeg_(?:cbcs|cenc)\w*/master_cmaf\w*\.m3u8'
+ def _download_nbcu_smil_and_extract_m3u8_url(self, tp_path, video_id, query):
+ smil = self._download_xml(
+ f'https://link.theplatform.com/s/{tp_path}', video_id,
+ 'Downloading SMIL manifest', 'Failed to download SMIL manifest', query={
+ **query,
+ 'format': 'SMIL', # XXX: Do not confuse "format" with "formats"
+ 'manifest': 'm3u',
+ 'switch': 'HLSServiceSecure', # Or else we get broken mp4 http URLs instead of HLS
+ }, headers=self.geo_verification_headers())
+
+ ns = f'//{{{default_ns}}}'
+ if url := traverse_obj(smil, (f'{ns}video/@src', lambda _, v: determine_ext(v) == 'm3u8', any)):
+ return url
+
+ exc = traverse_obj(smil, (f'{ns}param', lambda _, v: v.get('name') == 'exception', '@value', any))
+ if exc == 'GeoLocationBlocked':
+ self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+ raise ExtractorError(traverse_obj(smil, (f'{ns}ref/@abstract', ..., any)), expected=exc == 'Expired')
+
+ def _extract_nbcu_formats_and_subtitles(self, tp_path, video_id, query):
+ # formats='mpeg4' will return either a working m3u8 URL or an m3u8 template for non-DRM HLS
+ # formats='m3u+none,mpeg4' may return DRM HLS but w/the "folders" needed for non-DRM template
+ query['formats'] = 'm3u+none,mpeg4'
+ m3u8_url = self._download_nbcu_smil_and_extract_m3u8_url(tp_path, video_id, query)
+
+ if mobj := re.fullmatch(self._M3U8_RE, m3u8_url):
+ query['formats'] = 'mpeg4'
+ m3u8_tmpl = self._download_nbcu_smil_and_extract_m3u8_url(tp_path, video_id, query)
+ # Example: https://vod-lf-oneapp-prd.akamaized.net/prod/video/{folders}master_hls.m3u8
+ if '{folders}' in m3u8_tmpl:
+ self.write_debug('Found m3u8 URL template, formatting URL path')
+ m3u8_url = m3u8_tmpl.format(folders=mobj.group('folders'))
+
+ if '/mpeg_cenc' in m3u8_url or '/mpeg_cbcs' in m3u8_url:
+ self.report_drm(video_id)
+
+ return self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
+
+ def _extract_nbcu_video(self, url, display_id, old_ie_key=None):
+ webpage = self._download_webpage(url, display_id)
+ settings = self._search_json(
+ r']+data-drupal-selector="drupal-settings-json"[^>]*>',
+ webpage, 'settings', display_id)
+
+ query = {}
+ tve = extract_attributes(get_element_html_by_class('tve-video-deck-app', webpage) or '')
+ if tve:
+ account_pid = tve.get('data-mpx-media-account-pid') or tve['data-mpx-account-pid']
+ account_id = tve['data-mpx-media-account-id']
+ metadata = self._parse_json(
+ tve.get('data-normalized-video') or '', display_id, fatal=False, transform_source=unescapeHTML)
+ video_id = tve.get('data-guid') or metadata['guid']
+ if tve.get('data-entitlement') == 'auth':
+ auth = settings['tve_adobe_auth']
+ release_pid = tve['data-release-pid']
+ resource = self._get_mvpd_resource(
+ tve.get('data-adobe-pass-resource-id') or auth['adobePassResourceId'],
+ tve['data-title'], release_pid, tve.get('data-rating'))
+ query['auth'] = self._extract_mvpd_auth(
+ url, release_pid, auth['adobePassRequestorId'],
+ resource, auth['adobePassSoftwareStatement'])
+ else:
+ ls_playlist = traverse_obj(settings, (
+ 'ls_playlist', lambda _, v: v['defaultGuid'], any, {require('LS playlist')}))
+ video_id = ls_playlist['defaultGuid']
+ account_pid = ls_playlist.get('mpxMediaAccountPid') or ls_playlist['mpxAccountPid']
+ account_id = ls_playlist['mpxMediaAccountId']
+ metadata = traverse_obj(ls_playlist, ('videos', lambda _, v: v['guid'] == video_id, any)) or {}
+
+ tp_path = f'{account_pid}/media/guid/{account_id}/{video_id}'
+ formats, subtitles = self._extract_nbcu_formats_and_subtitles(tp_path, video_id, query)
+ tp_metadata = self._download_theplatform_metadata(tp_path, video_id, fatal=False)
+ parsed_info = self._parse_theplatform_metadata(tp_metadata)
+ self._merge_subtitles(parsed_info['subtitles'], target=subtitles)
+
+ return {
+ **parsed_info,
+ **traverse_obj(metadata, {
+ 'title': ('title', {str}),
+ 'description': ('description', {str}),
+ 'duration': ('durationInSeconds', {int_or_none}),
+ 'timestamp': ('airDate', {parse_iso8601}),
+ 'thumbnail': ('thumbnailUrl', {url_or_none}),
+ 'season_number': ('seasonNumber', {int_or_none}),
+ 'episode_number': ('episodeNumber', {int_or_none}),
+ 'episode': ('episodeTitle', {str}),
+ 'series': ('show', {str}),
+ }),
+ 'id': video_id,
+ 'display_id': display_id,
+ 'formats': formats,
+ 'subtitles': subtitles,
+ '_old_archive_ids': [make_archive_id(old_ie_key, video_id)] if old_ie_key else None,
+ }
+
+
+class NBCIE(NBCUniversalBaseIE):
+ _VALID_URL = r'https?(?P://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/?#]+/video/[^/?#]+/(?P\w+))'
_TESTS = [
{
'url': 'http://www.nbc.com/the-tonight-show/video/jimmy-fallon-surprises-fans-at-ben-jerrys/2848237',
@@ -49,47 +153,20 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'episode_number': 86,
'season': 'Season 2',
'season_number': 2,
- 'series': 'Tonight Show: Jimmy Fallon',
- 'duration': 237.0,
- 'chapters': 'count:1',
- 'tags': 'count:4',
+ 'series': 'Tonight',
+ 'duration': 236.504,
+ 'tags': 'count:2',
'thumbnail': r're:https?://.+\.jpg',
'categories': ['Series/The Tonight Show Starring Jimmy Fallon'],
'media_type': 'Full Episode',
+ 'age_limit': 14,
+ '_old_archive_ids': ['theplatform 2848237'],
},
'params': {
'skip_download': 'm3u8',
},
},
{
- 'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
- 'info_dict': {
- 'id': '2832821',
- 'ext': 'mp4',
- 'title': 'Star Wars Teaser',
- 'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
- 'timestamp': 1417852800,
- 'upload_date': '20141206',
- 'uploader': 'NBCU-COM',
- },
- 'skip': 'page not found',
- },
- {
- # HLS streams requires the 'hdnea3' cookie
- 'url': 'http://www.nbc.com/Kings/video/goliath/n1806',
- 'info_dict': {
- 'id': '101528f5a9e8127b107e98c5e6ce4638',
- 'ext': 'mp4',
- 'title': 'Goliath',
- 'description': 'When an unknown soldier saves the life of the King\'s son in battle, he\'s thrust into the limelight and politics of the kingdom.',
- 'timestamp': 1237100400,
- 'upload_date': '20090315',
- 'uploader': 'NBCU-COM',
- },
- 'skip': 'page not found',
- },
- {
- # manifest url does not have extension
'url': 'https://www.nbc.com/the-golden-globe-awards/video/oprah-winfrey-receives-cecil-b-de-mille-award-at-the-2018-golden-globes/3646439',
'info_dict': {
'id': '3646439',
@@ -99,48 +176,47 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'episode_number': 1,
'season': 'Season 75',
'season_number': 75,
- 'series': 'The Golden Globe Awards',
+ 'series': 'Golden Globes',
'description': 'Oprah Winfrey receives the Cecil B. de Mille Award at the 75th Annual Golden Globe Awards.',
'uploader': 'NBCU-COM',
'upload_date': '20180107',
'timestamp': 1515312000,
- 'duration': 570.0,
+ 'duration': 569.703,
'tags': 'count:8',
'thumbnail': r're:https?://.+\.jpg',
- 'chapters': 'count:1',
+ 'media_type': 'Highlight',
+ 'age_limit': 0,
+ 'categories': ['Series/The Golden Globe Awards'],
+ '_old_archive_ids': ['theplatform 3646439'],
},
'params': {
'skip_download': 'm3u8',
},
},
{
- # new video_id format
- 'url': 'https://www.nbc.com/quantum-leap/video/bens-first-leap-nbcs-quantum-leap/NBCE125189978',
+ # Needs to be extracted from webpage instead of GraphQL
+ 'url': 'https://www.nbc.com/paris2024/video/ali-truwit-found-purpose-pool-after-her-life-changed/para24_sww_alitruwittodayshow_240823',
'info_dict': {
- 'id': 'NBCE125189978',
+ 'id': 'para24_sww_alitruwittodayshow_240823',
'ext': 'mp4',
- 'title': 'Ben\'s First Leap | NBC\'s Quantum Leap',
- 'description': 'md5:a82762449b7ec4bb83291a7b355ebf8e',
- 'uploader': 'NBCU-COM',
- 'series': 'Quantum Leap',
- 'season': 'Season 1',
- 'season_number': 1,
- 'episode': 'Ben\'s First Leap | NBC\'s Quantum Leap',
- 'episode_number': 1,
- 'duration': 170.171,
- 'chapters': [],
- 'timestamp': 1663956155,
- 'upload_date': '20220923',
- 'tags': 'count:10',
- 'age_limit': 0,
+ 'title': 'Ali Truwit found purpose in the pool after her life changed',
+ 'description': 'md5:c16d7489e1516593de1cc5d3f39b9bdb',
+ 'uploader': 'NBCU-SPORTS',
+ 'duration': 311.077,
'thumbnail': r're:https?://.+\.jpg',
- 'categories': ['Series/Quantum Leap 2022'],
- 'media_type': 'Highlight',
+ 'episode': 'Ali Truwit found purpose in the pool after her life changed',
+ 'timestamp': 1724435902.0,
+ 'upload_date': '20240823',
+ '_old_archive_ids': ['theplatform para24_sww_alitruwittodayshow_240823'],
},
'params': {
'skip_download': 'm3u8',
},
},
+ {
+ 'url': 'https://www.nbc.com/quantum-leap/video/bens-first-leap-nbcs-quantum-leap/NBCE125189978',
+ 'only_matching': True,
+ },
{
'url': 'https://www.nbc.com/classic-tv/charles-in-charge/video/charles-in-charge-pilot/n3310',
'only_matching': True,
@@ -151,6 +227,7 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'only_matching': True,
},
]
+ _SOFTWARE_STATEMENT = 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI1Yzg2YjdkYy04NDI3LTRjNDUtOGQwZi1iNDkzYmE3MmQwYjQiLCJuYmYiOjE1Nzg3MDM2MzEsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTc4NzAzNjMxfQ.QQKIsBhAjGQTMdAqRTqhcz2Cddr4Y2hEjnSiOeKKki4nLrkDOsjQMmqeTR0hSRarraxH54wBgLvsxI7LHwKMvr7G8QpynNAxylHlQD3yhN9tFhxt4KR5wW3as02B-W2TznK9bhNWPKIyHND95Uo2Mi6rEQoq8tM9O09WPWaanE5BX_-r6Llr6dPq5F0Lpx2QOn2xYRb1T4nFxdFTNoss8GBds8OvChTiKpXMLHegLTc1OS4H_1a8tO_37jDwSdJuZ8iTyRLV4kZ2cpL6OL5JPMObD4-HQiec_dfcYgMKPiIfP9ZqdXpec2SVaCLsWEk86ZYvD97hLIQrK5rrKd1y-A'
def _real_extract(self, url):
permalink, video_id = self._match_valid_url(url).groups()
@@ -196,62 +273,50 @@ def _real_extract(self, url):
'userId': '0',
}),
})['data']['bonanzaPage']['metadata']
- query = {
- 'mbr': 'true',
- 'manifest': 'm3u',
- 'switch': 'HLSServiceSecure',
- }
+
+ if not video_data:
+ # Some videos are not available via GraphQL API
+ webpage = self._download_webpage(url, video_id)
+ video_data = self._search_json(
+ r'