diff --git a/.github/workflows/signature-tests.yml b/.github/workflows/signature-tests.yml
new file mode 100644
index 000000000..203172e0b
--- /dev/null
+++ b/.github/workflows/signature-tests.yml
@@ -0,0 +1,41 @@
+name: Signature Tests
+on:
+ push:
+ paths:
+ - .github/workflows/signature-tests.yml
+ - test/test_youtube_signature.py
+ - yt_dlp/jsinterp.py
+ pull_request:
+ paths:
+ - .github/workflows/signature-tests.yml
+ - test/test_youtube_signature.py
+ - yt_dlp/jsinterp.py
+permissions:
+ contents: read
+
+concurrency:
+ group: signature-tests-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ tests:
+ name: Signature Tests
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest]
+ python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', pypy-3.10, pypy-3.11]
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install test requirements
+ run: python3 ./devscripts/install_deps.py --only-optional --include test
+ - name: Run tests
+ timeout-minutes: 15
+ run: |
+ python3 -m yt_dlp -v || true # Print debug head
+ python3 ./devscripts/run_tests.py test/test_youtube_signature.py
diff --git a/.gitignore b/.gitignore
index 8fcd0de64..40bb34d2a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -105,6 +105,8 @@ README.txt
*.zsh
*.spec
test/testdata/sigs/player-*.js
+test/testdata/thumbnails/empty.webp
+test/testdata/thumbnails/foo\ %d\ bar/foo_%d.*
# Binary
/youtube-dl
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 6aa52c595..ba23b66dc 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -775,3 +775,12 @@ GeoffreyFrogeye
Pawka
v3DJG6GL
yozel
+brian6932
+iednod55
+maxbin123
+nullpos
+anlar
+eason1478
+ceandreasen
+chauhantirth
+helpimnotdrowning
diff --git a/Changelog.md b/Changelog.md
index 80b72da05..5a5c18cf3 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,103 @@ # Changelog
# To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
-->
+### 2025.06.30
+
+#### Core changes
+- **jsinterp**: [Fix `extract_object`](https://github.com/yt-dlp/yt-dlp/commit/958153a226214c86879e36211ac191bf78289578) ([#13580](https://github.com/yt-dlp/yt-dlp/issues/13580)) by [seproDev](https://github.com/seproDev)
+
+#### Extractor changes
+- **bilibilispacevideo**: [Extract hidden-mode collections as playlists](https://github.com/yt-dlp/yt-dlp/commit/99b85ac102047446e6adf5b62bfc3c8d80b53778) ([#13533](https://github.com/yt-dlp/yt-dlp/issues/13533)) by [c-basalt](https://github.com/c-basalt)
+- **hotstar**
+ - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/b5bd057fe86550f3aa67f2fc8790d1c6a251c57b) ([#13530](https://github.com/yt-dlp/yt-dlp/issues/13530)) by [bashonly](https://github.com/bashonly), [chauhantirth](https://github.com/chauhantirth) (With fixes in [e9f1576](https://github.com/yt-dlp/yt-dlp/commit/e9f157669e24953a88d15ce22053649db7a8e81e) by [bashonly](https://github.com/bashonly))
+ - [Fix metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/0a6b1044899f452cd10b6c7a6b00fa985a9a8b97) ([#13560](https://github.com/yt-dlp/yt-dlp/issues/13560)) by [bashonly](https://github.com/bashonly)
+ - [Raise for login required](https://github.com/yt-dlp/yt-dlp/commit/5e292baad62c749b6c340621ab2d0f904165ddfb) ([#10405](https://github.com/yt-dlp/yt-dlp/issues/10405)) by [bashonly](https://github.com/bashonly)
+ - series: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/4bd9a7ade7e0508b9795b3e72a69eeb40788b62b) ([#13564](https://github.com/yt-dlp/yt-dlp/issues/13564)) by [bashonly](https://github.com/bashonly)
+- **jiocinema**: [Remove extractors](https://github.com/yt-dlp/yt-dlp/commit/7e2504f941a11ea2b0dba00de3f0295cdc253e79) ([#13565](https://github.com/yt-dlp/yt-dlp/issues/13565)) by [bashonly](https://github.com/bashonly)
+- **kick**: [Support subscriber-only content](https://github.com/yt-dlp/yt-dlp/commit/b16722ede83377f77ea8352dcd0a6ca8e83b8f0f) ([#13550](https://github.com/yt-dlp/yt-dlp/issues/13550)) by [helpimnotdrowning](https://github.com/helpimnotdrowning)
+- **niconico**: live: [Fix extractor and downloader](https://github.com/yt-dlp/yt-dlp/commit/06c1a8cdffe14050206683253726875144192ef5) ([#13158](https://github.com/yt-dlp/yt-dlp/issues/13158)) by [doe1080](https://github.com/doe1080)
+- **sauceplus**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/35fc33fbc51c7f5392fb2300f65abf6cf107ef90) ([#13567](https://github.com/yt-dlp/yt-dlp/issues/13567)) by [bashonly](https://github.com/bashonly), [ceandreasen](https://github.com/ceandreasen)
+- **sproutvideo**: [Support browser impersonation](https://github.com/yt-dlp/yt-dlp/commit/11b9416e10cff7513167d76d6c47774fcdd3e26a) ([#13589](https://github.com/yt-dlp/yt-dlp/issues/13589)) by [bashonly](https://github.com/bashonly)
+- **youtube**: [Fix premium formats extraction](https://github.com/yt-dlp/yt-dlp/commit/2ba5391cd68ed4f2415c827d2cecbcbc75ace10b) ([#13586](https://github.com/yt-dlp/yt-dlp/issues/13586)) by [bashonly](https://github.com/bashonly)
+
+#### Misc. changes
+- **ci**: [Add signature tests](https://github.com/yt-dlp/yt-dlp/commit/1b883846347addeab12663fd74317fd544341a1c) ([#13582](https://github.com/yt-dlp/yt-dlp/issues/13582)) by [bashonly](https://github.com/bashonly)
+- **cleanup**: Miscellaneous: [b018784](https://github.com/yt-dlp/yt-dlp/commit/b0187844988e557c7e1e6bb1aabd4c1176768d86) by [bashonly](https://github.com/bashonly)
+
+### 2025.06.25
+
+#### Extractor changes
+- [Add `_search_nuxt_json` helper](https://github.com/yt-dlp/yt-dlp/commit/51887484e46ab6015c041cb1ab626a55f25a03bd) ([#13386](https://github.com/yt-dlp/yt-dlp/issues/13386)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+- **brightcove**: new: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/e6bd4a3da295b760ab20b39c18ce8934d312c2bf) ([#13461](https://github.com/yt-dlp/yt-dlp/issues/13461)) by [doe1080](https://github.com/doe1080)
+- **huya**: live: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/2600849badb0d08c55b58dcc77a13af6ba423da6) ([#13520](https://github.com/yt-dlp/yt-dlp/issues/13520)) by [doe1080](https://github.com/doe1080)
+- **hypergryph**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/1722c55400ff30bb5aee5dd7a262f0b7e9ce2f0e) ([#13415](https://github.com/yt-dlp/yt-dlp/issues/13415)) by [doe1080](https://github.com/doe1080), [eason1478](https://github.com/eason1478)
+- **lsm**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/c57412d1f9cf0124adc972a47858ac42b740c61d) ([#13126](https://github.com/yt-dlp/yt-dlp/issues/13126)) by [Caesim404](https://github.com/Caesim404)
+- **mave**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/1838a1ce5d4ade80770ba9162eaffc9a1607dc70) ([#13380](https://github.com/yt-dlp/yt-dlp/issues/13380)) by [anlar](https://github.com/anlar)
+- **sportdeutschland**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/a4ce4327c9836691d3b6b00e44a90b6741601ed8) ([#13519](https://github.com/yt-dlp/yt-dlp/issues/13519)) by [DTrombett](https://github.com/DTrombett)
+- **sproutvideo**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/5b559d0072b7164daf06bacdc41c6f11283452c8) ([#13544](https://github.com/yt-dlp/yt-dlp/issues/13544)) by [bashonly](https://github.com/bashonly)
+- **tv8.it**: [Support slugless URLs](https://github.com/yt-dlp/yt-dlp/commit/3bd30291601c47fa4a257983473884103ecab0c7) ([#13478](https://github.com/yt-dlp/yt-dlp/issues/13478)) by [DTrombett](https://github.com/DTrombett)
+- **youtube**
+ - [Check any `ios` m3u8 formats prior to download](https://github.com/yt-dlp/yt-dlp/commit/8f94b76cbf7bbd9dfd8762c63cdea04f90f1297f) ([#13524](https://github.com/yt-dlp/yt-dlp/issues/13524)) by [bashonly](https://github.com/bashonly)
+ - [Improve player context payloads](https://github.com/yt-dlp/yt-dlp/commit/ff6f94041aeee19c5559e1c1cd693960a1c1dd14) ([#13539](https://github.com/yt-dlp/yt-dlp/issues/13539)) by [bashonly](https://github.com/bashonly)
+
+#### Misc. changes
+- **test**: `traversal`: [Fix morsel tests for Python 3.14](https://github.com/yt-dlp/yt-dlp/commit/73bf10211668e4a59ccafd790e06ee82d9fea9ea) ([#13471](https://github.com/yt-dlp/yt-dlp/issues/13471)) by [Grub4K](https://github.com/Grub4K)
+
+### 2025.06.09
+
+#### Extractor changes
+- [Improve JSON LD thumbnails extraction](https://github.com/yt-dlp/yt-dlp/commit/85c8a405e3651dc041b758f4744d4fb3c4c55e01) ([#13368](https://github.com/yt-dlp/yt-dlp/issues/13368)) by [bashonly](https://github.com/bashonly), [doe1080](https://github.com/doe1080)
+- **10play**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/6d265388c6e943419ac99e9151cf75a3265f980f) ([#13349](https://github.com/yt-dlp/yt-dlp/issues/13349)) by [bashonly](https://github.com/bashonly)
+- **adobepass**
+ - [Add Fubo MSO](https://github.com/yt-dlp/yt-dlp/commit/eee90acc47d7f8de24afaa8b0271ccaefdf6e88c) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [maxbin123](https://github.com/maxbin123)
+ - [Always add newer user-agent when required](https://github.com/yt-dlp/yt-dlp/commit/0ee1102268cf31b07f8a8318a47424c66b2f7378) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [bashonly](https://github.com/bashonly)
+ - [Fix Philo MSO authentication](https://github.com/yt-dlp/yt-dlp/commit/943083edcd3df45aaa597a6967bc6c95b720f54c) ([#13335](https://github.com/yt-dlp/yt-dlp/issues/13335)) by [Sipherdrakon](https://github.com/Sipherdrakon)
+ - [Rework to require software statement](https://github.com/yt-dlp/yt-dlp/commit/711c5d5d098fee2992a1a624b1c4b30364b91426) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [bashonly](https://github.com/bashonly), [maxbin123](https://github.com/maxbin123)
+ - [Validate login URL before sending credentials](https://github.com/yt-dlp/yt-dlp/commit/89c1b349ad81318d9d3bea76c01c891696e58d38) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [bashonly](https://github.com/bashonly)
+- **aenetworks**
+ - [Fix playlist extractors](https://github.com/yt-dlp/yt-dlp/commit/f37d599a697e82fe68b423865897d55bae34f373) ([#13408](https://github.com/yt-dlp/yt-dlp/issues/13408)) by [Sipherdrakon](https://github.com/Sipherdrakon)
+ - [Fix provider-locked content extraction](https://github.com/yt-dlp/yt-dlp/commit/6693d6603358ae6beca834dbd822a7917498b813) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [maxbin123](https://github.com/maxbin123)
+- **bilibilibangumi**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/13e55162719528d42d2133e16b65ff59a667a6e4) ([#13416](https://github.com/yt-dlp/yt-dlp/issues/13416)) by [c-basalt](https://github.com/c-basalt)
+- **brightcove**: new: [Adapt to new AdobePass requirement](https://github.com/yt-dlp/yt-dlp/commit/98f8eec956e3b16cb66a3d49cc71af3807db795e) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [bashonly](https://github.com/bashonly)
+- **cu.ntv.co.jp**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/aa863ddab9b1d104678e9cf39bb76f5b14fca660) ([#13302](https://github.com/yt-dlp/yt-dlp/issues/13302)) by [doe1080](https://github.com/doe1080), [nullpos](https://github.com/nullpos)
+- **go**: [Fix provider-locked content extraction](https://github.com/yt-dlp/yt-dlp/commit/2e5bf002dad16f5ce35aa2023d392c9e518fcd8f) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [bashonly](https://github.com/bashonly), [maxbin123](https://github.com/maxbin123)
+- **nbc**: [Rework and adapt extractors to new AdobePass flow](https://github.com/yt-dlp/yt-dlp/commit/2d7949d5642bc37d1e71bf00c9a55260e5505d58) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [bashonly](https://github.com/bashonly)
+- **nobelprize**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/97ddfefeb4faba6e61cd80996c16952b8eab16f3) ([#13205](https://github.com/yt-dlp/yt-dlp/issues/13205)) by [doe1080](https://github.com/doe1080)
+- **odnoklassniki**: [Detect and raise when login is required](https://github.com/yt-dlp/yt-dlp/commit/148a1eb4c59e127965396c7a6e6acf1979de459e) ([#13361](https://github.com/yt-dlp/yt-dlp/issues/13361)) by [bashonly](https://github.com/bashonly)
+- **patreon**: [Fix m3u8 formats extraction](https://github.com/yt-dlp/yt-dlp/commit/e0d6c0822930f6e63f574d46d946a58b73ecd10c) ([#13266](https://github.com/yt-dlp/yt-dlp/issues/13266)) by [bashonly](https://github.com/bashonly) (With fixes in [1a8a03e](https://github.com/yt-dlp/yt-dlp/commit/1a8a03ea8d827107319a18076ee3505090667c5a))
+- **podchaser**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/538eb305673c26bff6a2b12f1c96375fe02ce41a) ([#13271](https://github.com/yt-dlp/yt-dlp/issues/13271)) by [bashonly](https://github.com/bashonly)
+- **sr**: mediathek: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/e3c605a61f4cc2de9059f37434fa108c3c20f58e) ([#13294](https://github.com/yt-dlp/yt-dlp/issues/13294)) by [doe1080](https://github.com/doe1080)
+- **stacommu**: [Avoid partial stream formats](https://github.com/yt-dlp/yt-dlp/commit/5d96527be80dc1ed1702d9cd548ff86de570ad70) ([#13412](https://github.com/yt-dlp/yt-dlp/issues/13412)) by [bashonly](https://github.com/bashonly)
+- **startrek**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/a8bf0011bde92b3f1324a98bfbd38932fd3ebe18) ([#13188](https://github.com/yt-dlp/yt-dlp/issues/13188)) by [doe1080](https://github.com/doe1080)
+- **svt**: play: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/e1b6062f8c4a3fa33c65269d48d09ec78de765a2) ([#13329](https://github.com/yt-dlp/yt-dlp/issues/13329)) by [barsnick](https://github.com/barsnick), [bashonly](https://github.com/bashonly)
+- **telecinco**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/03dba2012d9bd3f402fa8c2f122afba89bbd22a4) ([#13379](https://github.com/yt-dlp/yt-dlp/issues/13379)) by [bashonly](https://github.com/bashonly)
+- **theplatform**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/ed108b3ea481c6a4b5215a9302ba92d74baa2425) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [bashonly](https://github.com/bashonly)
+- **toutiao**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/f8051e3a61686c5db1de5f5746366ecfbc3ad20c) ([#13246](https://github.com/yt-dlp/yt-dlp/issues/13246)) by [doe1080](https://github.com/doe1080)
+- **turner**: [Adapt extractors to new AdobePass flow](https://github.com/yt-dlp/yt-dlp/commit/0daddc780d3ac5bebc3a3ec5b884d9243cbc0745) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [bashonly](https://github.com/bashonly)
+- **twitcasting**: [Fix password-protected livestream support](https://github.com/yt-dlp/yt-dlp/commit/52f9729c9a92ad4656d746ff0b1acecb87b3e96d) ([#13097](https://github.com/yt-dlp/yt-dlp/issues/13097)) by [bashonly](https://github.com/bashonly)
+- **twitter**: broadcast: [Support events URLs](https://github.com/yt-dlp/yt-dlp/commit/7794374de8afb20499b023107e2abfd4e6b93ee4) ([#13248](https://github.com/yt-dlp/yt-dlp/issues/13248)) by [doe1080](https://github.com/doe1080)
+- **umg**: de: [Rework extractor](https://github.com/yt-dlp/yt-dlp/commit/4e7c1ea346b510280218b47e8653dbbca3a69870) ([#13373](https://github.com/yt-dlp/yt-dlp/issues/13373)) by [doe1080](https://github.com/doe1080)
+- **vice**: [Mark extractors as broken](https://github.com/yt-dlp/yt-dlp/commit/6121559e027a04574690799c1776bc42bb51af31) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [bashonly](https://github.com/bashonly)
+- **vimeo**: [Extract subtitles from player subdomain](https://github.com/yt-dlp/yt-dlp/commit/c723c4e5e78263df178dbe69844a3d05f3ef9e35) ([#13350](https://github.com/yt-dlp/yt-dlp/issues/13350)) by [bashonly](https://github.com/bashonly)
+- **watchespn**: [Fix provider-locked content extraction](https://github.com/yt-dlp/yt-dlp/commit/b094747e93cfb0a2c53007120e37d0d84d41f030) ([#13131](https://github.com/yt-dlp/yt-dlp/issues/13131)) by [maxbin123](https://github.com/maxbin123)
+- **weverse**: [Support login with oauth refresh tokens](https://github.com/yt-dlp/yt-dlp/commit/3fe72e9eea38d9a58211cde42cfaa577ce020e2c) ([#13284](https://github.com/yt-dlp/yt-dlp/issues/13284)) by [bashonly](https://github.com/bashonly)
+- **youtube**
+ - [Add `tv_simply` player client](https://github.com/yt-dlp/yt-dlp/commit/1fd0e88b67db53ad163393d6965f68e908fa70e3) ([#13389](https://github.com/yt-dlp/yt-dlp/issues/13389)) by [gamer191](https://github.com/gamer191)
+ - [Extract srt subtitles](https://github.com/yt-dlp/yt-dlp/commit/231349786e8c42089c2e079ec94c0ea866c37999) ([#13411](https://github.com/yt-dlp/yt-dlp/issues/13411)) by [gamer191](https://github.com/gamer191)
+ - [Fix `--mark-watched` support](https://github.com/yt-dlp/yt-dlp/commit/b5be29fa58ec98226e11621fd9c58585bcff6879) ([#13222](https://github.com/yt-dlp/yt-dlp/issues/13222)) by [brian6932](https://github.com/brian6932), [iednod55](https://github.com/iednod55)
+ - [Fix automatic captions for some client combinations](https://github.com/yt-dlp/yt-dlp/commit/53ea743a9c158f8ca2d75a09ca44ba68606042d8) ([#13268](https://github.com/yt-dlp/yt-dlp/issues/13268)) by [bashonly](https://github.com/bashonly)
+ - [Improve signature extraction debug output](https://github.com/yt-dlp/yt-dlp/commit/d30a49742cfa22e61c47df4ac0e7334d648fb85d) ([#13327](https://github.com/yt-dlp/yt-dlp/issues/13327)) by [bashonly](https://github.com/bashonly)
+ - [Rework nsig function name extraction](https://github.com/yt-dlp/yt-dlp/commit/9e38b273b7ac942e7e9fc05a651ed810ab7d30ba) ([#13403](https://github.com/yt-dlp/yt-dlp/issues/13403)) by [Grub4K](https://github.com/Grub4K)
+ - [nsig code improvements and cleanup](https://github.com/yt-dlp/yt-dlp/commit/f7bbf5a617f9ab54ef51eaef99be36e175b5e9c3) ([#13280](https://github.com/yt-dlp/yt-dlp/issues/13280)) by [bashonly](https://github.com/bashonly)
+- **zdf**: [Fix language extraction and format sorting](https://github.com/yt-dlp/yt-dlp/commit/db162b76f6bdece50babe2e0cacfe56888c2e125) ([#13313](https://github.com/yt-dlp/yt-dlp/issues/13313)) by [InvalidUsernameException](https://github.com/InvalidUsernameException)
+
+#### Misc. changes
+- **build**
+ - [Exclude `pkg_resources` from being collected](https://github.com/yt-dlp/yt-dlp/commit/cc749a8a3b8b6e5c05318868c72a403f376a1b38) ([#13320](https://github.com/yt-dlp/yt-dlp/issues/13320)) by [bashonly](https://github.com/bashonly)
+ - [Fix macOS requirements caching](https://github.com/yt-dlp/yt-dlp/commit/201812100f315c6727a4418698d5b4e8a79863d4) ([#13328](https://github.com/yt-dlp/yt-dlp/issues/13328)) by [bashonly](https://github.com/bashonly)
+- **cleanup**: Miscellaneous: [339614a](https://github.com/yt-dlp/yt-dlp/commit/339614a173c74b42d63e858c446a9cae262a13af) by [bashonly](https://github.com/bashonly)
+- **test**: postprocessors: [Remove binary thumbnail test data](https://github.com/yt-dlp/yt-dlp/commit/a9b370069838e84d44ac7ad095d657003665885a) ([#13341](https://github.com/yt-dlp/yt-dlp/issues/13341)) by [bashonly](https://github.com/bashonly)
+
### 2025.05.22
#### Core changes
diff --git a/Makefile b/Makefile
index 6c72ead1e..273cb3cc0 100644
--- a/Makefile
+++ b/Makefile
@@ -18,10 +18,11 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \
tar pypi-files lazy-extractors install uninstall
clean-test:
- rm -rf test/testdata/sigs/player-*.js tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
+ rm -rf tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
*.frag.aria2 *.frag.urls *.info.json *.live_chat.json *.meta *.part* *.tmp *.temp *.unknown_video *.ytdl \
*.3gp *.ape *.ass *.avi *.desktop *.f4v *.flac *.flv *.gif *.jpeg *.jpg *.lrc *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 *.mp4 \
- *.mpg *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.ssa *.swf *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
+ *.mpg *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.ssa *.swf *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp \
+ test/testdata/sigs/player-*.js test/testdata/thumbnails/empty.webp "test/testdata/thumbnails/foo %d bar/foo_%d."*
clean-dist:
rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS
diff --git a/README.md b/README.md
index 6e2dc6243..925ebd8c5 100644
--- a/README.md
+++ b/README.md
@@ -1156,15 +1156,15 @@ # CONFIGURATION
* `/etc/yt-dlp/config`
* `/etc/yt-dlp/config.txt`
-E.g. with the following configuration file, yt-dlp will always extract the audio, not copy the mtime, use a proxy and save all videos under `YouTube` directory in your home directory:
+E.g. with the following configuration file, yt-dlp will always extract the audio, copy the mtime, use a proxy and save all videos under `YouTube` directory in your home directory:
```
# Lines starting with # are comments
# Always extract audio
-x
-# Do not copy the mtime
---no-mtime
+# Copy the mtime
+--mtime
# Use this proxy
--proxy 127.0.0.1:3128
@@ -1795,10 +1795,11 @@ # EXTRACTOR ARGUMENTS
The following extractors use this feature:
#### youtube
-* `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube.py](https://github.com/yt-dlp/yt-dlp/blob/c26f9b991a0681fd3ea548d535919cec1fbbd430/yt_dlp/extractor/youtube.py#L381-L390) for list of supported content language codes
+* `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube/_base.py](https://github.com/yt-dlp/yt-dlp/blob/415b4c9f955b1a0391204bd24a7132590e7b3bdb/yt_dlp/extractor/youtube/_base.py#L402-L409) for the list of supported content language codes
* `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
-* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `android`, `android_vr`, `tv` and `tv_embedded`. By default, `tv,ios,web` is used, or `tv,web` is used when authenticating with cookies. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only works if the video is embeddable. The `tv_embedded` and `web_creator` clients are added for age-restricted videos if account age-verification is required. Some clients, such as `web` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-ios`
+* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `android`, `android_vr`, `tv`, `tv_simply` and `tv_embedded`. By default, `tv,ios,web` is used, or `tv,web` is used when authenticating with cookies. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only works if the video is embeddable. The `tv_embedded` and `web_creator` clients are added for age-restricted videos if account age-verification is required. Some clients, such as `web` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-ios`
* `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player), `initial_data` (skip initial data/next ep request). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause issues such as missing formats or metadata. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) and [#12826](https://github.com/yt-dlp/yt-dlp/issues/12826) for more details
+* `webpage_skip`: Skip extraction of embedded webpage data. One or both of `player_response`, `initial_data`. These options are for testing purposes and don't skip any network requests
* `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp.
* `player_js_variant`: The player javascript variant to use for signature and nsig deciphering. The known variants are: `main`, `tce`, `tv`, `tv_es6`, `phone`, `tablet`. Only `main` is recommended as a possible workaround; the others are for debugging purposes. The default is to use what is prescribed by the site, and can be selected with `actual`
* `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
@@ -1900,6 +1901,10 @@ #### sonylivseries
#### tver
* `backend`: Backend API to use for extraction - one of `streaks` (default) or `brightcove` (deprecated)
+#### vimeo
+* `client`: Client to extract video data from. One of `android` (default), `ios` or `web`. The `ios` client only works with previously cached OAuth tokens. The `web` client only works when authenticated with credentials or account cookies
+* `original_format_policy`: Policy for when to try extracting original formats. One of `always`, `never`, or `auto`. The default `auto` policy tries to avoid exceeding the API rate-limit by only making an extra request when Vimeo publicizes the video's downloadability
+
**Note**: These options may be changed/removed in the future without concern for backward compatibility
@@ -2262,6 +2267,7 @@ ### Differences in default behavior
* yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests.
* The sub-modules `swfinterp`, `casefold` are removed.
* Passing `--simulate` (or calling `extract_info` with `download=False`) no longer alters the default format selection. See [#9843](https://github.com/yt-dlp/yt-dlp/issues/9843) for details.
+* yt-dlp no longer applies the server modified time to downloaded files by default. Use `--mtime` or `--compat-options mtime-by-default` to revert this.
For ease of use, a few more compat options are available:
@@ -2271,7 +2277,7 @@ ### Differences in default behavior
* `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization`
* `--compat-options 2022`: Same as `--compat-options 2023,playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx`
* `--compat-options 2023`: Same as `--compat-options 2024,prefer-vp9-sort`
-* `--compat-options 2024`: Currently does nothing. Use this to enable all future compat options
+* `--compat-options 2024`: Same as `--compat-options mtime-by-default`. Use this to enable all future compat options
The following compat options restore vulnerable behavior from before security patches:
diff --git a/devscripts/bash-completion.in b/devscripts/bash-completion.in
index 21f52798e..bb66c2095 100644
--- a/devscripts/bash-completion.in
+++ b/devscripts/bash-completion.in
@@ -10,9 +10,13 @@ __yt_dlp()
diropts="--cache-dir"
if [[ ${prev} =~ ${fileopts} ]]; then
+ local IFS=$'\n'
+ type compopt &>/dev/null && compopt -o filenames
COMPREPLY=( $(compgen -f -- ${cur}) )
return 0
elif [[ ${prev} =~ ${diropts} ]]; then
+ local IFS=$'\n'
+ type compopt &>/dev/null && compopt -o dirnames
COMPREPLY=( $(compgen -d -- ${cur}) )
return 0
fi
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index 269de2c68..d7296bf30 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -254,5 +254,13 @@
{
"action": "remove",
"when": "d596824c2f8428362c072518856065070616e348"
+ },
+ {
+ "action": "remove",
+ "when": "7b81634fb1d15999757e7a9883daa6ef09ea785b"
+ },
+ {
+ "action": "remove",
+ "when": "500761e41acb96953a5064e951d41d190c287e46"
}
]
diff --git a/pyproject.toml b/pyproject.toml
index 3775251e1..41d5ec3b0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -75,7 +75,7 @@ dev = [
]
static-analysis = [
"autopep8~=2.0",
- "ruff~=0.11.0",
+ "ruff~=0.12.0",
]
test = [
"pytest~=8.1",
@@ -210,10 +210,12 @@ ignore = [
"TD001", # invalid-todo-tag
"TD002", # missing-todo-author
"TD003", # missing-todo-link
+ "PLC0415", # import-outside-top-level
"PLE0604", # invalid-all-object (false positives)
"PLE0643", # potential-index-error (false positives)
"PLW0603", # global-statement
"PLW1510", # subprocess-run-without-check
+ "PLW1641", # eq-without-hash
"PLW2901", # redefined-loop-name
"RUF001", # ambiguous-unicode-character-string
"RUF012", # mutable-class-default
diff --git a/supportedsites.md b/supportedsites.md
index c2d7b4555..8e48135d2 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -5,6 +5,8 @@ # Supported sites
Not all sites listed here are guaranteed to work; websites are constantly changing and sometimes this breaks yt-dlp's support for them.
The only reliable way to check if a site is supported is to try it.
+ - **10play**: [*10play*](## "netrc machine")
+ - **10play:season**
- **17live**
- **17live:clip**
- **17live:vod**
@@ -295,7 +297,7 @@ # Supported sites
- **CNNIndonesia**
- **ComedyCentral**
- **ComedyCentralTV**
- - **ConanClassic**
+ - **ConanClassic**: (**Currently broken**)
- **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
- **CONtv**
- **CookingChannel**
@@ -317,7 +319,7 @@ # Supported sites
- **CtsNews**: 華視新聞
- **CTV**
- **CTVNews**
- - **cu.ntv.co.jp**: Nippon Television Network
+ - **cu.ntv.co.jp**: 日テレ無料TADA!
- **CultureUnplugged**
- **curiositystream**: [*curiositystream*](## "netrc machine")
- **curiositystream:collections**: [*curiositystream*](## "netrc machine")
@@ -573,9 +575,7 @@ # Supported sites
- **HollywoodReporterPlaylist**
- **Holodex**
- **HotNewHipHop**: (**Currently broken**)
- - **hotstar**
- - **hotstar:playlist**
- - **hotstar:season**
+ - **hotstar**: JioHotstar
- **hotstar:series**
- **hrfernsehen**
- **HRTi**: [*hrti*](## "netrc machine")
@@ -588,7 +588,7 @@ # Supported sites
- **Hungama**
- **HungamaAlbumPlaylist**
- **HungamaSong**
- - **huya:live**: huya.com
+ - **huya:live**: 虎牙直播
- **huya:video**: 虎牙视频
- **Hypem**
- **Hytale**
@@ -645,8 +645,6 @@ # Supported sites
- **Jamendo**
- **JamendoAlbum**
- **JeuxVideo**: (**Currently broken**)
- - **jiocinema**: [*jiocinema*](## "netrc machine")
- - **jiocinema:series**: [*jiocinema*](## "netrc machine")
- **jiosaavn:album**
- **jiosaavn:artist**
- **jiosaavn:playlist**
@@ -774,6 +772,7 @@ # Supported sites
- **massengeschmack.tv**
- **Masters**
- **MatchTV**
+ - **Mave**
- **MBN**: mbn.co.kr (매일방송)
- **MDR**: MDR.DE
- **MedalTV**
@@ -830,7 +829,7 @@ # Supported sites
- **Mojevideo**: mojevideo.sk
- **Mojvideo**
- **Monstercat**
- - **MonsterSirenHypergryphMusic**
+ - **monstersiren**: 塞壬唱片
- **Motherless**
- **MotherlessGallery**
- **MotherlessGroup**
@@ -882,19 +881,19 @@ # Supported sites
- **Naver**
- **Naver:live**
- **navernow**
- - **nba**
- - **nba:channel**
- - **nba:embed**
- - **nba:watch**
- - **nba:watch:collection**
- - **nba:watch:embed**
+ - **nba**: (**Currently broken**)
+ - **nba:channel**: (**Currently broken**)
+ - **nba:embed**: (**Currently broken**)
+ - **nba:watch**: (**Currently broken**)
+ - **nba:watch:collection**: (**Currently broken**)
+ - **nba:watch:embed**: (**Currently broken**)
- **NBC**
- **NBCNews**
- **nbcolympics**
- - **nbcolympics:stream**
- - **NBCSports**
- - **NBCSportsStream**
- - **NBCSportsVPlayer**
+ - **nbcolympics:stream**: (**Currently broken**)
+ - **NBCSports**: (**Currently broken**)
+ - **NBCSportsStream**: (**Currently broken**)
+ - **NBCSportsVPlayer**: (**Currently broken**)
- **NBCStations**
- **ndr**: NDR.de - Norddeutscher Rundfunk
- **ndr:embed**
@@ -970,7 +969,7 @@ # Supported sites
- **Nitter**
- **njoy**: N-JOY
- **njoy:embed**
- - **NobelPrize**: (**Currently broken**)
+ - **NobelPrize**
- **NoicePodcast**
- **NonkTube**
- **NoodleMagazine**
@@ -1296,6 +1295,7 @@ # Supported sites
- **SampleFocus**
- **Sangiin**: 参議院インターネット審議中継 (archive)
- **Sapo**: SAPO Vídeos
+ - **SaucePlus**: Sauce+
- **SBS**: sbs.com.au
- **sbs.co.kr**
- **sbs.co.kr:allvod_program**
@@ -1393,14 +1393,14 @@ # Supported sites
- **SpreakerShow**
- **SpringboardPlatform**
- **SproutVideo**
- - **sr:mediathek**: Saarländischer Rundfunk (**Currently broken**)
+ - **sr:mediathek**: Saarländischer Rundfunk
- **SRGSSR**
- **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
- **StacommuLive**: [*stacommu*](## "netrc machine")
- **StacommuVOD**: [*stacommu*](## "netrc machine")
- **StagePlusVODConcert**: [*stageplus*](## "netrc machine")
- **stanfordoc**: Stanford Open ClassRoom
- - **StarTrek**: (**Currently broken**)
+ - **startrek**: STAR TREK
- **startv**
- **Steam**
- **SteamCommunityBroadcast**
@@ -1423,12 +1423,11 @@ # Supported sites
- **SunPorno**
- **sverigesradio:episode**
- **sverigesradio:publication**
- - **SVT**
- - **SVTPage**
- - **SVTPlay**: SVT Play and Öppet arkiv
- - **SVTSeries**
+ - **svt:page**
+ - **svt:play**: SVT Play and Öppet arkiv
+ - **svt:play:series**
- **SwearnetEpisode**
- - **Syfy**: (**Currently broken**)
+ - **Syfy**
- **SYVDK**
- **SztvHu**
- **t-online.de**: (**Currently broken**)
@@ -1472,8 +1471,6 @@ # Supported sites
- **Telewebion**: (**Currently broken**)
- **Tempo**
- **TennisTV**: [*tennistv*](## "netrc machine")
- - **TenPlay**: [*10play*](## "netrc machine")
- - **TenPlaySeason**
- **TF1**
- **TFO**
- **theatercomplextown:ppv**: [*theatercomplextown*](## "netrc machine")
@@ -1511,6 +1508,7 @@ # Supported sites
- **tokfm:podcast**
- **ToonGoggles**
- **tou.tv**: [*toutv*](## "netrc machine")
+ - **toutiao**: 今日头条
- **Toypics**: Toypics video (**Currently broken**)
- **ToypicsUser**: Toypics user profile (**Currently broken**)
- **TrailerAddict**: (**Currently broken**)
@@ -1600,7 +1598,7 @@ # Supported sites
- **UKTVPlay**
- **UlizaPlayer**
- **UlizaPortal**: ulizaportal.jp
- - **umg:de**: Universal Music Deutschland (**Currently broken**)
+ - **umg:de**: Universal Music Deutschland
- **Unistra**
- **Unity**: (**Currently broken**)
- **uol.com.br**
@@ -1623,9 +1621,9 @@ # Supported sites
- **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
- **vh1.com**
- **vhx:embed**: [*vimeo*](## "netrc machine")
- - **vice**
- - **vice:article**
- - **vice:show**
+ - **vice**: (**Currently broken**)
+ - **vice:article**: (**Currently broken**)
+ - **vice:show**: (**Currently broken**)
- **Viddler**
- **Videa**
- **video.arnes.si**: Arnes Video
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index c6ff6209a..40dd05e13 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -36,6 +36,18 @@ def do_GET(self):
self.send_header('Content-Type', 'text/html; charset=utf-8')
self.end_headers()
self.wfile.write(TEAPOT_RESPONSE_BODY.encode())
+ elif self.path == '/fake.m3u8':
+ self.send_response(200)
+ self.send_header('Content-Length', '1024')
+ self.end_headers()
+ self.wfile.write(1024 * b'\x00')
+ elif self.path == '/bipbop.m3u8':
+ with open('test/testdata/m3u8/bipbop_16x9.m3u8', 'rb') as f:
+ data = f.read()
+ self.send_response(200)
+ self.send_header('Content-Length', str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
else:
assert False
@@ -314,6 +326,20 @@ def test_search_json_ld_realworld(self):
},
{},
),
+ (
+ # test thumbnail_url key without URL scheme
+ r'''
+''',
+ {
+ 'thumbnails': [{'url': 'https://www.nobelprize.org/images/12693-landscape-medium-gallery.jpg'}],
+ },
+ {},
+ ),
]
for html, expected_dict, search_json_ld_kwargs in _TESTS:
expect_dict(
@@ -1933,6 +1959,208 @@ def test_search_nextjs_data(self):
with self.assertWarns(DeprecationWarning):
self.assertEqual(self.ie._search_nextjs_data('', None, default='{}'), {})
+ def test_search_nextjs_v13_data(self):
+ HTML = R'''
+
+
+
+
+
+
+
+ '''
+ EXPECTED = {
+ '18': {
+ 'foo': 'bar',
+ },
+ '16': {
+ 'meta': {
+ 'dateCreated': 1730489700,
+ 'uuid': '40cac41d-8d29-4ef5-aa11-75047b9f0907',
+ },
+ },
+ '19': {
+ 'duplicated_field_name': {'x': 1},
+ },
+ '20': {
+ 'duplicated_field_name': {'y': 2},
+ },
+ }
+ self.assertEqual(self.ie._search_nextjs_v13_data(HTML, None), EXPECTED)
+ self.assertEqual(self.ie._search_nextjs_v13_data('', None, fatal=False), {})
+ self.assertEqual(self.ie._search_nextjs_v13_data(None, None, fatal=False), {})
+
+ def test_search_nuxt_json(self):
+ HTML_TMPL = ''
+ VALID_DATA = '''
+ ["ShallowReactive",1],
+ {"data":2,"state":21,"once":25,"_errors":28,"_server_errors":30},
+ ["ShallowReactive",3],
+ {"$abcdef123456":4},
+ {"podcast":5,"activeEpisodeData":7},
+ {"podcast":6,"seasons":14},
+ {"title":10,"id":11},
+ ["Reactive",8],
+ {"episode":9,"creators":18,"empty_list":20},
+ {"title":12,"id":13,"refs":34,"empty_refs":35},
+ "Series Title",
+ "podcast-id-01",
+ "Episode Title",
+ "episode-id-99",
+ [15,16,17],
+ 1,
+ 2,
+ 3,
+ [19],
+ "Podcast Creator",
+ [],
+ {"$ssite-config":22},
+ {"env":23,"name":24,"map":26,"numbers":14},
+ "production",
+ "podcast-website",
+ ["Set"],
+ ["Reactive",27],
+ ["Map"],
+ ["ShallowReactive",29],
+ {},
+ ["NuxtError",31],
+ {"status":32,"message":33},
+ 503,
+ "Service Unavailable",
+ [36,37],
+ [38,39],
+ ["Ref",40],
+ ["ShallowRef",41],
+ ["EmptyRef",42],
+ ["EmptyShallowRef",43],
+ "ref",
+ "shallow_ref",
+ "{\\"ref\\":1}",
+ "{\\"shallow_ref\\":2}"
+ '''
+ PAYLOAD = {
+ 'data': {
+ '$abcdef123456': {
+ 'podcast': {
+ 'podcast': {
+ 'title': 'Series Title',
+ 'id': 'podcast-id-01',
+ },
+ 'seasons': [1, 2, 3],
+ },
+ 'activeEpisodeData': {
+ 'episode': {
+ 'title': 'Episode Title',
+ 'id': 'episode-id-99',
+ 'refs': ['ref', 'shallow_ref'],
+ 'empty_refs': [{'ref': 1}, {'shallow_ref': 2}],
+ },
+ 'creators': ['Podcast Creator'],
+ 'empty_list': [],
+ },
+ },
+ },
+ 'state': {
+ '$ssite-config': {
+ 'env': 'production',
+ 'name': 'podcast-website',
+ 'map': [],
+ 'numbers': [1, 2, 3],
+ },
+ },
+ 'once': [],
+ '_errors': {},
+ '_server_errors': {
+ 'status': 503,
+ 'message': 'Service Unavailable',
+ },
+ }
+ PARTIALLY_INVALID = [(
+ '''
+ {"data":1},
+ {"invalid_raw_list":2},
+ [15,16,17]
+ ''',
+ {'data': {'invalid_raw_list': [None, None, None]}},
+ ), (
+ '''
+ {"data":1},
+ ["EmptyRef",2],
+ "not valid JSON"
+ ''',
+ {'data': None},
+ ), (
+ '''
+ {"data":1},
+ ["EmptyShallowRef",2],
+ "not valid JSON"
+ ''',
+ {'data': None},
+ )]
+ INVALID = [
+ '''
+ []
+ ''',
+ '''
+ ["unsupported",1],
+ {"data":2},
+ {}
+ ''',
+ ]
+ DEFAULT = object()
+
+ self.assertEqual(self.ie._search_nuxt_json(HTML_TMPL.format(VALID_DATA), None), PAYLOAD)
+ self.assertEqual(self.ie._search_nuxt_json('', None, fatal=False), {})
+ self.assertIs(self.ie._search_nuxt_json('', None, default=DEFAULT), DEFAULT)
+
+ for data, expected in PARTIALLY_INVALID:
+ self.assertEqual(
+ self.ie._search_nuxt_json(HTML_TMPL.format(data), None, fatal=False), expected)
+
+ for data in INVALID:
+ self.assertIs(
+ self.ie._search_nuxt_json(HTML_TMPL.format(data), None, default=DEFAULT), DEFAULT)
+
+
+class TestInfoExtractorNetwork(unittest.TestCase):
+ def setUp(self, /):
+ self.httpd = http.server.HTTPServer(
+ ('127.0.0.1', 0), InfoExtractorTestRequestHandler)
+ self.port = http_server_port(self.httpd)
+
+ self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+ self.server_thread.daemon = True
+ self.server_thread.start()
+
+ self.called = False
+
+ def require_warning(*args, **kwargs):
+ self.called = True
+
+ self.ydl = FakeYDL()
+ self.ydl.report_warning = require_warning
+ self.ie = DummyIE(self.ydl)
+
+ def tearDown(self, /):
+ self.ydl.close()
+ self.httpd.shutdown()
+ self.httpd.server_close()
+ self.server_thread.join(1)
+
+ def test_extract_m3u8_formats(self):
+ formats, subtitles = self.ie._extract_m3u8_formats_and_subtitles(
+ f'http://127.0.0.1:{self.port}/bipbop.m3u8', None, fatal=False)
+ self.assertFalse(self.called)
+ self.assertTrue(formats)
+ self.assertTrue(subtitles)
+
+ def test_extract_m3u8_formats_warning(self):
+ formats, subtitles = self.ie._extract_m3u8_formats_and_subtitles(
+ f'http://127.0.0.1:{self.port}/fake.m3u8', None, fatal=False)
+ self.assertTrue(self.called, 'Warning was not issued for binary m3u8 file')
+ self.assertFalse(formats)
+ self.assertFalse(subtitles)
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/test_devalue.py b/test/test_devalue.py
new file mode 100644
index 000000000..29eb89e87
--- /dev/null
+++ b/test/test_devalue.py
@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+import datetime as dt
+import json
+import math
+import re
+import unittest
+
+from yt_dlp.utils.jslib import devalue
+
+
+TEST_CASES_EQUALS = [{
+ 'name': 'int',
+ 'unparsed': [-42],
+ 'parsed': -42,
+}, {
+ 'name': 'str',
+ 'unparsed': ['woo!!!'],
+ 'parsed': 'woo!!!',
+}, {
+ 'name': 'Number',
+ 'unparsed': [['Object', 42]],
+ 'parsed': 42,
+}, {
+ 'name': 'String',
+ 'unparsed': [['Object', 'yar']],
+ 'parsed': 'yar',
+}, {
+ 'name': 'Infinity',
+ 'unparsed': -4,
+ 'parsed': math.inf,
+}, {
+ 'name': 'negative Infinity',
+ 'unparsed': -5,
+ 'parsed': -math.inf,
+}, {
+ 'name': 'negative zero',
+ 'unparsed': -6,
+ 'parsed': -0.0,
+}, {
+ 'name': 'RegExp',
+ 'unparsed': [['RegExp', 'regexp', 'gim']], # XXX: flags are ignored
+ 'parsed': re.compile('regexp'),
+}, {
+ 'name': 'Date',
+ 'unparsed': [['Date', '2001-09-09T01:46:40.000Z']],
+ 'parsed': dt.datetime.fromtimestamp(1e9, tz=dt.timezone.utc),
+}, {
+ 'name': 'Array',
+ 'unparsed': [[1, 2, 3], 'a', 'b', 'c'],
+ 'parsed': ['a', 'b', 'c'],
+}, {
+ 'name': 'Array (empty)',
+ 'unparsed': [[]],
+ 'parsed': [],
+}, {
+ 'name': 'Array (sparse)',
+ 'unparsed': [[-2, 1, -2], 'b'],
+ 'parsed': [None, 'b', None],
+}, {
+ 'name': 'Object',
+ 'unparsed': [{'foo': 1, 'x-y': 2}, 'bar', 'z'],
+ 'parsed': {'foo': 'bar', 'x-y': 'z'},
+}, {
+ 'name': 'Set',
+ 'unparsed': [['Set', 1, 2, 3], 1, 2, 3],
+ 'parsed': [1, 2, 3],
+}, {
+ 'name': 'Map',
+ 'unparsed': [['Map', 1, 2], 'a', 'b'],
+ 'parsed': [['a', 'b']],
+}, {
+ 'name': 'BigInt',
+ 'unparsed': [['BigInt', '1']],
+ 'parsed': 1,
+}, {
+ 'name': 'Uint8Array',
+ 'unparsed': [['Uint8Array', 'AQID']],
+ 'parsed': [1, 2, 3],
+}, {
+ 'name': 'ArrayBuffer',
+ 'unparsed': [['ArrayBuffer', 'AQID']],
+ 'parsed': [1, 2, 3],
+}, {
+ 'name': 'str (repetition)',
+ 'unparsed': [[1, 1], 'a string'],
+ 'parsed': ['a string', 'a string'],
+}, {
+ 'name': 'None (repetition)',
+ 'unparsed': [[1, 1], None],
+ 'parsed': [None, None],
+}, {
+ 'name': 'dict (repetition)',
+ 'unparsed': [[1, 1], {}],
+ 'parsed': [{}, {}],
+}, {
+ 'name': 'Object without prototype',
+ 'unparsed': [['null']],
+ 'parsed': {},
+}, {
+ 'name': 'cross-realm POJO',
+ 'unparsed': [{}],
+ 'parsed': {},
+}]
+
+TEST_CASES_IS = [{
+ 'name': 'bool',
+ 'unparsed': [True],
+ 'parsed': True,
+}, {
+ 'name': 'Boolean',
+ 'unparsed': [['Object', False]],
+ 'parsed': False,
+}, {
+ 'name': 'undefined',
+ 'unparsed': -1,
+ 'parsed': None,
+}, {
+ 'name': 'null',
+ 'unparsed': [None],
+ 'parsed': None,
+}, {
+ 'name': 'NaN',
+ 'unparsed': -3,
+ 'parsed': math.nan,
+}]
+
+TEST_CASES_INVALID = [{
+ 'name': 'empty string',
+ 'unparsed': '',
+ 'error': ValueError,
+ 'pattern': r'expected int or list as input',
+}, {
+ 'name': 'hole',
+ 'unparsed': -2,
+ 'error': ValueError,
+ 'pattern': r'invalid integer input',
+}, {
+ 'name': 'string',
+ 'unparsed': 'hello',
+ 'error': ValueError,
+ 'pattern': r'expected int or list as input',
+}, {
+ 'name': 'number',
+ 'unparsed': 42,
+ 'error': ValueError,
+ 'pattern': r'invalid integer input',
+}, {
+ 'name': 'boolean',
+ 'unparsed': True,
+ 'error': ValueError,
+ 'pattern': r'expected int or list as input',
+}, {
+ 'name': 'null',
+ 'unparsed': None,
+ 'error': ValueError,
+ 'pattern': r'expected int or list as input',
+}, {
+ 'name': 'object',
+ 'unparsed': {},
+ 'error': ValueError,
+ 'pattern': r'expected int or list as input',
+}, {
+ 'name': 'empty array',
+ 'unparsed': [],
+ 'error': ValueError,
+ 'pattern': r'expected a non-empty list as input',
+}, {
+ 'name': 'Python negative indexing',
+ 'unparsed': [[1, 2, 3, 4, 5, 6, 7, -7], 1, 2, 3, 4, 5, 6, 7],
+ 'error': IndexError,
+ 'pattern': r'invalid index: -7',
+}]
+
+
+class TestDevalue(unittest.TestCase):
+ def test_devalue_parse_equals(self):
+ for tc in TEST_CASES_EQUALS:
+ self.assertEqual(devalue.parse(tc['unparsed']), tc['parsed'], tc['name'])
+
+ def test_devalue_parse_is(self):
+ for tc in TEST_CASES_IS:
+ self.assertIs(devalue.parse(tc['unparsed']), tc['parsed'], tc['name'])
+
+ def test_devalue_parse_invalid(self):
+ for tc in TEST_CASES_INVALID:
+ with self.assertRaisesRegex(tc['error'], tc['pattern'], msg=tc['name']):
+ devalue.parse(tc['unparsed'])
+
+ def test_devalue_parse_cyclical(self):
+ name = 'Map (cyclical)'
+ result = devalue.parse([['Map', 1, 0], 'self'])
+ self.assertEqual(result[0][0], 'self', name)
+ self.assertIs(result, result[0][1], name)
+
+ name = 'Set (cyclical)'
+ result = devalue.parse([['Set', 0, 1], 42])
+ self.assertEqual(result[1], 42, name)
+ self.assertIs(result, result[0], name)
+
+ result = devalue.parse([[0]])
+ self.assertIs(result, result[0], 'Array (cyclical)')
+
+ name = 'Object (cyclical)'
+ result = devalue.parse([{'self': 0}])
+ self.assertIs(result, result['self'], name)
+
+ name = 'Object with null prototype (cyclical)'
+ result = devalue.parse([['null', 'self', 0]])
+ self.assertIs(result, result['self'], name)
+
+ name = 'Objects (cyclical)'
+ result = devalue.parse([[1, 2], {'second': 2}, {'first': 1}])
+ self.assertIs(result[0], result[1]['first'], name)
+ self.assertIs(result[1], result[0]['second'], name)
+
+ def test_devalue_parse_revivers(self):
+ self.assertEqual(
+ devalue.parse([['indirect', 1], {'a': 2}, 'b'], revivers={'indirect': lambda x: x}),
+ {'a': 'b'}, 'revivers (indirect)')
+
+ self.assertEqual(
+ devalue.parse([['parse', 1], '{"a":0}'], revivers={'parse': lambda x: json.loads(x)}),
+ {'a': 0}, 'revivers (parse)')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/test_download.py b/test/test_download.py
index 3f36869d9..c7842735c 100755
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -14,6 +14,7 @@
from test.helper import (
assertGreaterEqual,
+ assertLessEqual,
expect_info_dict,
expect_warnings,
get_params,
@@ -121,10 +122,13 @@ def print_skipping(reason):
params = get_params(test_case.get('params', {}))
params['outtmpl'] = tname + '_' + params['outtmpl']
if is_playlist and 'playlist' not in test_case:
- params.setdefault('extract_flat', 'in_playlist')
- params.setdefault('playlistend', test_case.get(
- 'playlist_mincount', test_case.get('playlist_count', -2) + 1))
+ params.setdefault('playlistend', max(
+ test_case.get('playlist_mincount', -1),
+ test_case.get('playlist_count', -2) + 1,
+ test_case.get('playlist_maxcount', -2) + 1))
params.setdefault('skip_download', True)
+ if 'playlist_duration_sum' not in test_case:
+ params.setdefault('extract_flat', 'in_playlist')
ydl = YoutubeDL(params, auto_init=False)
ydl.add_default_info_extractors()
@@ -159,6 +163,7 @@ def try_rm_tcs_files(tcs=None):
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
try_rm_tcs_files()
try:
+ test_url = test_case['url']
try_num = 1
while True:
try:
@@ -166,7 +171,7 @@ def try_rm_tcs_files(tcs=None):
# for outside error handling, and returns the exit code
# instead of the result dict.
res_dict = ydl.extract_info(
- test_case['url'],
+ test_url,
force_generic_extractor=params.get('force_generic_extractor', False))
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
@@ -194,23 +199,23 @@ def try_rm_tcs_files(tcs=None):
self.assertTrue('entries' in res_dict)
expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
+ num_entries = len(res_dict.get('entries', []))
if 'playlist_mincount' in test_case:
+ mincount = test_case['playlist_mincount']
assertGreaterEqual(
- self,
- len(res_dict['entries']),
- test_case['playlist_mincount'],
- 'Expected at least %d in playlist %s, but got only %d' % (
- test_case['playlist_mincount'], test_case['url'],
- len(res_dict['entries'])))
+ self, num_entries, mincount,
+ f'Expected at least {mincount} entries in playlist {test_url}, but got only {num_entries}')
if 'playlist_count' in test_case:
+ count = test_case['playlist_count']
+ got = num_entries if num_entries <= count else 'more'
self.assertEqual(
- len(res_dict['entries']),
- test_case['playlist_count'],
- 'Expected %d entries in playlist %s, but got %d.' % (
- test_case['playlist_count'],
- test_case['url'],
- len(res_dict['entries']),
- ))
+ num_entries, count,
+ f'Expected exactly {count} entries in playlist {test_url}, but got {got}')
+ if 'playlist_maxcount' in test_case:
+ maxcount = test_case['playlist_maxcount']
+ assertLessEqual(
+ self, num_entries, maxcount,
+ f'Expected at most {maxcount} entries in playlist {test_url}, but got more')
if 'playlist_duration_sum' in test_case:
got_duration = sum(e['duration'] for e in res_dict['entries'])
self.assertEqual(
diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 2e3cdc2a5..43b1d0fde 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -478,6 +478,10 @@ def test_extract_function_with_global_stack(self):
func = jsi.extract_function('c', {'e': 10}, {'f': 100, 'g': 1000})
self.assertEqual(func([1]), 1111)
+ def test_extract_object(self):
+ jsi = JSInterpreter('var a={};a.xy={};var xy;var zxy={};xy={z:function(){return "abc"}};')
+ self.assertTrue('z' in jsi.extract_object('xy', None))
+
def test_increment_decrement(self):
self._test('function f() { var x = 1; return ++x; }', 2)
self._test('function f() { var x = 1; return x++; }', 1)
@@ -486,6 +490,57 @@ def test_increment_decrement(self):
self._test('function f() { var a = "test--"; return a; }', 'test--')
self._test('function f() { var b = 1; var a = "b--"; return a; }', 'b--')
+ def test_nested_function_scoping(self):
+ self._test(R'''
+ function f() {
+ var g = function() {
+ var P = 2;
+ return P;
+ };
+ var P = 1;
+ g();
+ return P;
+ }
+ ''', 1)
+ self._test(R'''
+ function f() {
+ var x = function() {
+ for (var w = 1, M = []; w < 2; w++) switch (w) {
+ case 1:
+ M.push("a");
+ case 2:
+ M.push("b");
+ }
+ return M
+ };
+ var w = "c";
+ var M = "d";
+ var y = x();
+ y.push(w);
+ y.push(M);
+ return y;
+ }
+ ''', ['a', 'b', 'c', 'd'])
+ self._test(R'''
+ function f() {
+ var P, Q;
+ var z = 100;
+ var g = function() {
+ var P, Q; P = 2; Q = 15;
+ z = 0;
+ return P+Q;
+ };
+ P = 1; Q = 10;
+ var x = g(), y = 3;
+ return P+Q+x+y+z;
+ }
+ ''', 31)
+
+ def test_undefined_varnames(self):
+ jsi = JSInterpreter('function f(){ var a; return [a, b]; }')
+ self._test(jsi, [JS_Undefined, JS_Undefined])
+ self.assertEqual(jsi._undefined_varnames, {'b'})
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/test_networking.py b/test/test_networking.py
index 2f441fced..afdd0c7aa 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -22,7 +22,6 @@
import tempfile
import threading
import time
-import urllib.error
import urllib.request
import warnings
import zlib
@@ -223,10 +222,7 @@ def do_GET(self):
if encoding == 'br' and brotli:
payload = brotli.compress(payload)
elif encoding == 'gzip':
- buf = io.BytesIO()
- with gzip.GzipFile(fileobj=buf, mode='wb') as f:
- f.write(payload)
- payload = buf.getvalue()
+ payload = gzip.compress(payload, mtime=0)
elif encoding == 'deflate':
payload = zlib.compress(payload)
elif encoding == 'unsupported':
@@ -729,6 +725,17 @@ def test_keep_header_casing(self, handler):
assert 'X-test-heaDer: test' in res
+ def test_partial_read_then_full_read(self, handler):
+ with handler() as rh:
+ for encoding in ('', 'gzip', 'deflate'):
+ res = validate_and_send(rh, Request(
+ f'http://127.0.0.1:{self.http_port}/content-encoding',
+ headers={'ytdl-encoding': encoding}))
+ assert res.headers.get('Content-Encoding') == encoding
+ assert res.read(6) == b''
+ assert res.read(0) == b''
+ assert res.read() == b''
+
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
class TestClientCertificate:
diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py
index 603f85c65..ecc73e39e 100644
--- a/test/test_postprocessors.py
+++ b/test/test_postprocessors.py
@@ -8,6 +8,8 @@
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import subprocess
+
from yt_dlp import YoutubeDL
from yt_dlp.utils import shell_quote
from yt_dlp.postprocessor import (
@@ -47,7 +49,18 @@ def test_escaping(self):
print('Skipping: ffmpeg not found')
return
- file = 'test/testdata/thumbnails/foo %d bar/foo_%d.{}'
+ test_data_dir = 'test/testdata/thumbnails'
+ generated_file = f'{test_data_dir}/empty.webp'
+
+ subprocess.check_call([
+ pp.executable, '-y', '-f', 'lavfi', '-i', 'color=c=black:s=320x320',
+ '-c:v', 'libwebp', '-pix_fmt', 'yuv420p', '-vframes', '1', generated_file,
+ ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+ file = test_data_dir + '/foo %d bar/foo_%d.{}'
+ initial_file = file.format('webp')
+ os.replace(generated_file, initial_file)
+
tests = (('webp', 'png'), ('png', 'jpg'))
for inp, out in tests:
@@ -55,11 +68,13 @@ def test_escaping(self):
if os.path.exists(out_file):
os.remove(out_file)
pp.convert_thumbnail(file.format(inp), out)
- assert os.path.exists(out_file)
+ self.assertTrue(os.path.exists(out_file))
for _, out in tests:
os.remove(file.format(out))
+ os.remove(initial_file)
+
class TestExec(unittest.TestCase):
def test_parse_cmd(self):
@@ -610,3 +625,7 @@ def test_quote_for_concat_QuotesAtEnd(self):
self.assertEqual(
r"'special '\'' characters '\'' galore'\'\'\'",
self._pp._quote_for_ffmpeg("special ' characters ' galore'''"))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/test_pot/test_pot_builtin_utils.py b/test/test_pot/test_pot_builtin_utils.py
index a95fc4e15..7645ba601 100644
--- a/test/test_pot/test_pot_builtin_utils.py
+++ b/test/test_pot/test_pot_builtin_utils.py
@@ -11,7 +11,7 @@ class TestGetWebPoContentBinding:
@pytest.mark.parametrize('client_name, context, is_authenticated, expected', [
*[(client, context, is_authenticated, expected) for client in [
- 'WEB', 'MWEB', 'TVHTML5', 'WEB_EMBEDDED_PLAYER', 'WEB_CREATOR', 'TVHTML5_SIMPLY_EMBEDDED_PLAYER']
+ 'WEB', 'MWEB', 'TVHTML5', 'WEB_EMBEDDED_PLAYER', 'WEB_CREATOR', 'TVHTML5_SIMPLY_EMBEDDED_PLAYER', 'TVHTML5_SIMPLY']
for context, is_authenticated, expected in [
(PoTokenContext.GVS, False, ('example-visitor-data', ContentBindingType.VISITOR_DATA)),
(PoTokenContext.PLAYER, False, ('example-video-id', ContentBindingType.VIDEO_ID)),
diff --git a/test/test_pot/test_pot_builtin_webpospec.py b/test/test_pot/test_pot_builtin_webpospec.py
index c5fb6f382..078008415 100644
--- a/test/test_pot/test_pot_builtin_webpospec.py
+++ b/test/test_pot/test_pot_builtin_webpospec.py
@@ -49,7 +49,7 @@ def test_not_supports(self, ie, logger, pot_request, client_name, context, is_au
@pytest.mark.parametrize('client_name, context, is_authenticated, remote_host, source_address, request_proxy, expected', [
*[(client, context, is_authenticated, remote_host, source_address, request_proxy, expected) for client in [
- 'WEB', 'MWEB', 'TVHTML5', 'WEB_EMBEDDED_PLAYER', 'WEB_CREATOR', 'TVHTML5_SIMPLY_EMBEDDED_PLAYER']
+ 'WEB', 'MWEB', 'TVHTML5', 'WEB_EMBEDDED_PLAYER', 'WEB_CREATOR', 'TVHTML5_SIMPLY_EMBEDDED_PLAYER', 'TVHTML5_SIMPLY']
for context, is_authenticated, remote_host, source_address, request_proxy, expected in [
(PoTokenContext.GVS, False, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': '123abcXYZ_-', 'cbt': 'visitor_id'}),
(PoTokenContext.PLAYER, False, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': '123abcXYZ_-', 'cbt': 'video_id'}),
diff --git a/test/test_traversal.py b/test/test_traversal.py
index bc433029d..52215f5a7 100644
--- a/test/test_traversal.py
+++ b/test/test_traversal.py
@@ -416,18 +416,8 @@ def test_traversal_unbranching(self):
'`any` should allow further branching'
def test_traversal_morsel(self):
- values = {
- 'expires': 'a',
- 'path': 'b',
- 'comment': 'c',
- 'domain': 'd',
- 'max-age': 'e',
- 'secure': 'f',
- 'httponly': 'g',
- 'version': 'h',
- 'samesite': 'i',
- }
morsel = http.cookies.Morsel()
+ values = dict(zip(morsel, 'abcdefghijklmnop'))
morsel.set('item_key', 'item_value', 'coded_value')
morsel.update(values)
values['key'] = 'item_key'
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 3f777aed7..456246753 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -133,6 +133,11 @@
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0',
),
+ (
+ 'https://www.youtube.com/s/player/e12fbea4/player_ias.vflset/en_US/base.js',
+ 'gN7a-hudCuAuPH6fByOk1_GNXN0yNMHShjZXS2VOgsEItAJz0tipeavEOmNdYN-wUtcEqD3bCXjc0iyKfAyZxCBGgIARwsSdQfJ2CJtt',
+ 'JC2JfQdSswRAIgGBCxZyAfKyi0cjXCb3DqEctUw-NYdNmOEvaepit0zJAtIEsgOV2SXZjhSHMNy0NXNG_1kOyBf6HPuAuCduh-a',
+ ),
]
_NSIG_TESTS = [
@@ -320,6 +325,58 @@
'https://www.youtube.com/s/player/59b252b9/player_ias.vflset/en_US/base.js',
'D3XWVpYgwhLLKNK4AGX', 'aZrQ1qWJ5yv5h',
),
+ (
+ 'https://www.youtube.com/s/player/fc2a56a5/player_ias.vflset/en_US/base.js',
+ 'qTKWg_Il804jd2kAC', 'OtUAm2W6gyzJjB9u',
+ ),
+ (
+ 'https://www.youtube.com/s/player/fc2a56a5/tv-player-ias.vflset/tv-player-ias.js',
+ 'qTKWg_Il804jd2kAC', 'OtUAm2W6gyzJjB9u',
+ ),
+ (
+ 'https://www.youtube.com/s/player/a74bf670/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', 'hQP7k1hA22OrNTnq',
+ ),
+ (
+ 'https://www.youtube.com/s/player/6275f73c/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', '-I03XF0iyf6I_X0A',
+ ),
+ (
+ 'https://www.youtube.com/s/player/20c72c18/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', '-I03XF0iyf6I_X0A',
+ ),
+ (
+ 'https://www.youtube.com/s/player/9fe2e06e/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', '6r5ekNIiEMPutZy',
+ ),
+ (
+ 'https://www.youtube.com/s/player/680f8c75/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', '0ml9caTwpa55Jf',
+ ),
+ (
+ 'https://www.youtube.com/s/player/14397202/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', 'ozZFAN21okDdJTa',
+ ),
+ (
+ 'https://www.youtube.com/s/player/5dcb2c1f/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', 'p7iTbRZDYAF',
+ ),
+ (
+ 'https://www.youtube.com/s/player/a10d7fcc/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', '9Zue7DDHJSD',
+ ),
+ (
+ 'https://www.youtube.com/s/player/8e20cb06/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', '5-4tTneTROTpMzba',
+ ),
+ (
+ 'https://www.youtube.com/s/player/e12fbea4/player_ias_tce.vflset/en_US/base.js',
+ 'kM5r52fugSZRAKHfo3', 'XkeRfXIPOkSwfg',
+ ),
+ (
+ 'https://www.youtube.com/s/player/ef259203/player_ias_tce.vflset/en_US/base.js',
+ 'rPqBC01nJpqhhi2iA2U', 'hY7dbiKFT51UIA',
+ ),
]
diff --git a/test/testdata/thumbnails/foo %d bar/foo_%d.webp b/test/testdata/thumbnails/foo %d bar/foo_%d.webp
deleted file mode 100644
index d64d0839f..000000000
Binary files a/test/testdata/thumbnails/foo %d bar/foo_%d.webp and /dev/null differ
diff --git a/test/testdata/thumbnails/foo %d bar/placeholder b/test/testdata/thumbnails/foo %d bar/placeholder
new file mode 100644
index 000000000..e69de29bb
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 707578f0d..5d221fd25 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -482,7 +482,8 @@ class YoutubeDL:
The following options do not work when used through the API:
filename, abort-on-error, multistreams, no-live-chat,
format-sort, no-clean-infojson, no-playlist-metafiles,
- no-keep-subs, no-attach-info-json, allow-unsafe-ext, prefer-vp9-sort.
+ no-keep-subs, no-attach-info-json, allow-unsafe-ext, prefer-vp9-sort,
+ mtime-by-default.
Refer __init__.py for their implementation
progress_template: Dictionary of templates for progress outputs.
Allowed keys are 'download', 'postprocess',
@@ -490,7 +491,7 @@ class YoutubeDL:
The template is mapped on a dictionary with keys 'progress' and 'info'
retry_sleep_functions: Dictionary of functions that takes the number of attempts
as argument and returns the time to sleep in seconds.
- Allowed keys are 'http', 'fragment', 'file_access'
+ Allowed keys are 'http', 'fragment', 'file_access', 'extractor'
download_ranges: A callback function that gets called for every video with
the signature (info_dict, ydl) -> Iterable[Section].
Only the returned sections will be downloaded.
@@ -2219,6 +2220,7 @@ def _check_formats(self, formats):
self.report_warning(f'Unable to delete temporary file "{temp_file.name}"')
f['__working'] = success
if success:
+ f.pop('__needs_testing', None)
yield f
else:
self.to_screen('[info] Unable to download format {}. Skipping...'.format(f['format_id']))
@@ -3972,6 +3974,7 @@ def simplified_codec(f, field):
self._format_out('UNSUPPORTED', self.Styles.BAD_FORMAT) if f.get('ext') in ('f4f', 'f4m') else None,
(self._format_out('Maybe DRM', self.Styles.WARNING) if f.get('has_drm') == 'maybe'
else self._format_out('DRM', self.Styles.BAD_FORMAT) if f.get('has_drm') else None),
+ self._format_out('Untested', self.Styles.WARNING) if f.get('__needs_testing') else None,
format_field(f, 'format_note'),
format_field(f, 'container', ignore=(None, f.get('ext'))),
delim=', '), delim=' '),
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 714d9ad5c..2e7646b7e 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -159,6 +159,12 @@ def set_default_compat(compat_name, opt_name, default=True, remove_compat=True):
elif 'prefer-vp9-sort' in opts.compat_opts:
opts.format_sort.extend(FormatSorter._prefer_vp9_sort)
+ if 'mtime-by-default' in opts.compat_opts:
+ if opts.updatetime is None:
+ opts.updatetime = True
+ else:
+ _unused_compat_opt('mtime-by-default')
+
_video_multistreams_set = set_default_compat('multistreams', 'allow_multiple_video_streams', False, remove_compat=False)
_audio_multistreams_set = set_default_compat('multistreams', 'allow_multiple_audio_streams', False, remove_compat=False)
if _video_multistreams_set is False and _audio_multistreams_set is False:
diff --git a/yt_dlp/aes.py b/yt_dlp/aes.py
index 065901d68..600cb12a8 100644
--- a/yt_dlp/aes.py
+++ b/yt_dlp/aes.py
@@ -435,7 +435,7 @@ def sub_bytes_inv(data):
def rotate(data):
- return data[1:] + [data[0]]
+ return [*data[1:], data[0]]
def key_schedule_core(data, rcon_iteration):
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 98784e703..7852ae90d 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -302,7 +302,7 @@ def _finish_frag_download(self, ctx, info_dict):
elif to_file:
self.try_rename(ctx['tmpfilename'], ctx['filename'])
filetime = ctx.get('fragment_filetime')
- if self.params.get('updatetime', True) and filetime:
+ if self.params.get('updatetime') and filetime:
with contextlib.suppress(Exception):
os.utime(ctx['filename'], (time.time(), filetime))
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index 1f36a07f5..225630578 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -94,12 +94,19 @@ def real_download(self, filename, info_dict):
can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
if can_download:
has_ffmpeg = FFmpegFD.available()
- no_crypto = not Cryptodome.AES and '#EXT-X-KEY:METHOD=AES-128' in s
- if no_crypto and has_ffmpeg:
- can_download, message = False, 'The stream has AES-128 encryption and pycryptodomex is not available'
- elif no_crypto:
- message = ('The stream has AES-128 encryption and neither ffmpeg nor pycryptodomex are available; '
- 'Decryption will be performed natively, but will be extremely slow')
+ if not Cryptodome.AES and '#EXT-X-KEY:METHOD=AES-128' in s:
+ # Even if pycryptodomex isn't available, force HlsFD for m3u8s that won't work with ffmpeg
+ ffmpeg_can_dl = not traverse_obj(info_dict, ((
+ 'extra_param_to_segment_url', 'extra_param_to_key_url',
+ 'hls_media_playlist_data', ('hls_aes', ('uri', 'key', 'iv')),
+ ), any))
+ message = 'The stream has AES-128 encryption and {} available'.format(
+ 'neither ffmpeg nor pycryptodomex are' if ffmpeg_can_dl and not has_ffmpeg else
+ 'pycryptodomex is not')
+ if has_ffmpeg and ffmpeg_can_dl:
+ can_download = False
+ else:
+ message += '; decryption will be performed natively, but will be extremely slow'
elif info_dict.get('extractor_key') == 'Generic' and re.search(r'(?m)#EXT-X-MEDIA-SEQUENCE:(?!0$)', s):
install_ffmpeg = '' if has_ffmpeg else 'install ffmpeg and '
message = ('Live HLS streams are not supported by the native downloader. If this is a livestream, '
diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index 9c6dd8b79..90bfcaf55 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -348,7 +348,7 @@ def retry(e):
self.try_rename(ctx.tmpfilename, ctx.filename)
# Update file modification time
- if self.params.get('updatetime', True):
+ if self.params.get('updatetime'):
info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.headers.get('last-modified', None))
self._hook_progress({
diff --git a/yt_dlp/downloader/niconico.py b/yt_dlp/downloader/niconico.py
index 33cf15df8..35a12b555 100644
--- a/yt_dlp/downloader/niconico.py
+++ b/yt_dlp/downloader/niconico.py
@@ -5,47 +5,46 @@
from .common import FileDownloader
from .external import FFmpegFD
from ..networking import Request
-from ..utils import DownloadError, str_or_none, try_get
+from ..networking.websocket import WebSocketResponse
+from ..utils import DownloadError, str_or_none, truncate_string
+from ..utils.traversal import traverse_obj
class NiconicoLiveFD(FileDownloader):
""" Downloads niconico live without being stopped """
def real_download(self, filename, info_dict):
- video_id = info_dict['video_id']
- ws_url = info_dict['url']
- ws_extractor = info_dict['ws']
- ws_origin_host = info_dict['origin']
- live_quality = info_dict.get('live_quality', 'high')
- live_latency = info_dict.get('live_latency', 'high')
+ video_id = info_dict['id']
+ opts = info_dict['downloader_options']
+ quality, ws_extractor, ws_url = opts['max_quality'], opts['ws'], opts['ws_url']
dl = FFmpegFD(self.ydl, self.params or {})
new_info_dict = info_dict.copy()
- new_info_dict.update({
- 'protocol': 'm3u8',
- })
+ new_info_dict['protocol'] = 'm3u8'
def communicate_ws(reconnect):
- if reconnect:
- ws = self.ydl.urlopen(Request(ws_url, headers={'Origin': f'https://{ws_origin_host}'}))
+ # Support --load-info-json as if it is a reconnect attempt
+ if reconnect or not isinstance(ws_extractor, WebSocketResponse):
+ ws = self.ydl.urlopen(Request(
+ ws_url, headers={'Origin': 'https://live.nicovideo.jp'}))
if self.ydl.params.get('verbose', False):
- self.to_screen('[debug] Sending startWatching request')
+ self.write_debug('Sending startWatching request')
ws.send(json.dumps({
- 'type': 'startWatching',
'data': {
+ 'reconnect': True,
+ 'room': {
+ 'commentable': True,
+ 'protocol': 'webSocket',
+ },
'stream': {
- 'quality': live_quality,
- 'protocol': 'hls+fmp4',
- 'latency': live_latency,
'accessRightMethod': 'single_cookie',
'chasePlay': False,
+ 'latency': 'high',
+ 'protocol': 'hls',
+ 'quality': quality,
},
- 'room': {
- 'protocol': 'webSocket',
- 'commentable': True,
- },
- 'reconnect': True,
},
+ 'type': 'startWatching',
}))
else:
ws = ws_extractor
@@ -58,7 +57,6 @@ def communicate_ws(reconnect):
if not data or not isinstance(data, dict):
continue
if data.get('type') == 'ping':
- # pong back
ws.send(r'{"type":"pong"}')
ws.send(r'{"type":"keepSeat"}')
elif data.get('type') == 'disconnect':
@@ -66,12 +64,10 @@ def communicate_ws(reconnect):
return True
elif data.get('type') == 'error':
self.write_debug(data)
- message = try_get(data, lambda x: x['body']['code'], str) or recv
+ message = traverse_obj(data, ('body', 'code', {str_or_none}), default=recv)
return DownloadError(message)
elif self.ydl.params.get('verbose', False):
- if len(recv) > 100:
- recv = recv[:100] + '...'
- self.to_screen(f'[debug] Server said: {recv}')
+ self.write_debug(f'Server response: {truncate_string(recv, 100)}')
def ws_main():
reconnect = False
@@ -81,7 +77,8 @@ def ws_main():
if ret is True:
return
except BaseException as e:
- self.to_screen('[{}] {}: Connection error occured, reconnecting after 10 seconds: {}'.format('niconico:live', video_id, str_or_none(e)))
+ self.to_screen(
+ f'[niconico:live] {video_id}: Connection error occured, reconnecting after 10 seconds: {e}')
time.sleep(10)
continue
finally:
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b0c52e0fc..4d67e1caa 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -201,7 +201,6 @@
BanByeChannelIE,
BanByeIE,
)
-from .bandaichannel import BandaiChannelIE
from .bandcamp import (
BandcampAlbumIE,
BandcampIE,
@@ -229,7 +228,6 @@
from .beatport import BeatportIE
from .beeg import BeegIE
from .behindkink import BehindKinkIE
-from .bellmedia import BellMediaIE
from .berufetv import BerufeTVIE
from .bet import BetIE
from .bfi import BFIPlayerIE
@@ -275,7 +273,10 @@
BitChuteChannelIE,
BitChuteIE,
)
-from .blackboardcollaborate import BlackboardCollaborateIE
+from .blackboardcollaborate import (
+ BlackboardCollaborateIE,
+ BlackboardCollaborateLaunchIE,
+)
from .bleacherreport import (
BleacherReportCMSIE,
BleacherReportIE,
@@ -309,6 +310,7 @@
BrilliantpalaClassesIE,
BrilliantpalaElearnIE,
)
+from .btvplus import BTVPlusIE
from .bundesliga import BundesligaIE
from .bundestag import BundestagIE
from .bunnycdn import BunnyCdnIE
@@ -446,7 +448,6 @@
CSpanIE,
)
from .ctsnews import CtsNewsIE
-from .ctv import CTVIE
from .ctvnews import CTVNewsIE
from .cultureunplugged import CultureUnpluggedIE
from .curiositystream import (
@@ -805,9 +806,7 @@
from .hotnewhiphop import HotNewHipHopIE
from .hotstar import (
HotStarIE,
- HotStarPlaylistIE,
HotStarPrefixIE,
- HotStarSeasonIE,
HotStarSeriesIE,
)
from .hrefli import HrefLiRedirectIE
@@ -921,10 +920,6 @@
ShugiinItvVodIE,
)
from .jeuxvideo import JeuxVideoIE
-from .jiocinema import (
- JioCinemaIE,
- JioCinemaSeriesIE,
-)
from .jiosaavn import (
JioSaavnAlbumIE,
JioSaavnArtistIE,
@@ -934,7 +929,6 @@
JioSaavnSongIE,
)
from .joj import JojIE
-from .joqrag import JoqrAgIE
from .jove import JoveIE
from .jstream import JStreamIE
from .jtbc import (
@@ -1037,11 +1031,6 @@
LikeeIE,
LikeeUserIE,
)
-from .limelight import (
- LimelightChannelIE,
- LimelightChannelListIE,
- LimelightMediaIE,
-)
from .linkedin import (
LinkedInEventsIE,
LinkedInIE,
@@ -1107,6 +1096,7 @@
from .massengeschmacktv import MassengeschmackTVIE
from .masters import MastersIE
from .matchtv import MatchTVIE
+from .mave import MaveIE
from .mbn import MBNIE
from .mdr import MDRIE
from .medaltv import MedalTVIE
@@ -1152,6 +1142,7 @@
MindsIE,
)
from .minoto import MinotoIE
+from .mir24tv import Mir24TvIE
from .mirrativ import (
MirrativIE,
MirrativUserIE,
@@ -1172,6 +1163,10 @@
MixcloudPlaylistIE,
MixcloudUserIE,
)
+from .mixlr import (
+ MixlrIE,
+ MixlrRecoringIE,
+)
from .mlb import (
MLBIE,
MLBTVIE,
@@ -1382,7 +1377,6 @@
from .noice import NoicePodcastIE
from .nonktube import NonkTubeIE
from .noodlemagazine import NoodleMagazineIE
-from .noovo import NoovoIE
from .nosnl import NOSNLArticleIE
from .nova import (
NovaEmbedIE,
@@ -1829,6 +1823,7 @@
from .saitosan import SaitosanIE
from .samplefocus import SampleFocusIE
from .sapo import SapoIE
+from .sauceplus import SaucePlusIE
from .sbs import SBSIE
from .sbscokr import (
SBSCoKrAllvodProgramIE,
@@ -2017,7 +2012,6 @@
SverigesRadioPublicationIE,
)
from .svt import (
- SVTIE,
SVTPageIE,
SVTPlayIE,
SVTSeriesIE,
@@ -2101,6 +2095,7 @@
TheGuardianPodcastIE,
TheGuardianPodcastPlaylistIE,
)
+from .thehighwire import TheHighWireIE
from .theholetv import TheHoleTvIE
from .theintercept import TheInterceptIE
from .theplatform import (
@@ -2289,6 +2284,7 @@
)
from .umg import UMGDeIE
from .unistra import UnistraIE
+from .unitednations import UnitedNationsWebTvIE
from .unity import UnityIE
from .unsupported import (
KnownDRMIE,
diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py
index 91c40b32e..8c2d9d934 100644
--- a/yt_dlp/extractor/adobepass.py
+++ b/yt_dlp/extractor/adobepass.py
@@ -1574,18 +1574,29 @@ def extract_redirect_url(html, url=None, fatal=False):
post_form(mvpd_confirm_page_res, 'Confirming Login')
elif mso_id == 'Philo':
# Philo has very unique authentication method
- self._download_webpage(
- 'https://idp.philo.com/auth/init/login_code', video_id, 'Requesting auth code', data=urlencode_postdata({
+ self._request_webpage(
+ 'https://idp.philo.com/auth/init/login_code', video_id,
+ 'Requesting Philo auth code', data=json.dumps({
'ident': username,
'device': 'web',
'send_confirm_link': False,
'send_token': True,
- }))
+ 'device_ident': f'web-{uuid.uuid4().hex}',
+ 'include_login_link': True,
+ }).encode(), headers={
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ })
+
philo_code = getpass.getpass('Type auth code you have received [Return]: ')
- self._download_webpage(
- 'https://idp.philo.com/auth/update/login_code', video_id, 'Submitting token', data=urlencode_postdata({
- 'token': philo_code,
- }))
+ self._request_webpage(
+ 'https://idp.philo.com/auth/update/login_code', video_id,
+ 'Submitting token', data=json.dumps({'token': philo_code}).encode(),
+ headers={
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ })
+
mvpd_confirm_page_res = self._download_webpage_handle('https://idp.philo.com/idp/submit', video_id, 'Confirming Philo Login')
post_form(mvpd_confirm_page_res, 'Confirming Login')
elif mso_id == 'Verizon':
diff --git a/yt_dlp/extractor/aenetworks.py b/yt_dlp/extractor/aenetworks.py
index c6a1b1509..e5c922b41 100644
--- a/yt_dlp/extractor/aenetworks.py
+++ b/yt_dlp/extractor/aenetworks.py
@@ -1,3 +1,5 @@
+import json
+
from .theplatform import ThePlatformIE
from ..utils import (
ExtractorError,
@@ -6,7 +8,6 @@
remove_start,
traverse_obj,
update_url_query,
- urlencode_postdata,
)
@@ -204,18 +205,19 @@ def _real_extract(self, url):
class AENetworksListBaseIE(AENetworksBaseIE):
def _call_api(self, resource, slug, brand, fields):
return self._download_json(
- 'https://yoga.appsvcs.aetnd.com/graphql',
- slug, query={'brand': brand}, data=urlencode_postdata({
+ 'https://yoga.appsvcs.aetnd.com/graphql', slug,
+ query={'brand': brand}, headers={'Content-Type': 'application/json'},
+ data=json.dumps({
'query': '''{
%s(slug: "%s") {
%s
}
}''' % (resource, slug, fields), # noqa: UP031
- }))['data'][resource]
+ }).encode())['data'][resource]
def _real_extract(self, url):
domain, slug = self._match_valid_url(url).groups()
- _, brand = self._DOMAIN_MAP[domain]
+ _, brand, _ = self._DOMAIN_MAP[domain]
playlist = self._call_api(self._RESOURCE, slug, brand, self._FIELDS)
base_url = f'http://watch.{domain}'
diff --git a/yt_dlp/extractor/archiveorg.py b/yt_dlp/extractor/archiveorg.py
index 2849d9fd5..572bd6bfe 100644
--- a/yt_dlp/extractor/archiveorg.py
+++ b/yt_dlp/extractor/archiveorg.py
@@ -16,6 +16,7 @@
dict_get,
extract_attributes,
get_element_by_id,
+ get_element_text_and_html_by_tag,
int_or_none,
join_nonempty,
js_to_json,
@@ -72,6 +73,7 @@ class ArchiveOrgIE(InfoExtractor):
'display_id': 'Cops-v2.mp4',
'thumbnail': r're:https://archive\.org/download/.*\.jpg',
'duration': 1091.96,
+ 'track': 'Cops-v2',
},
}, {
'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
@@ -86,6 +88,7 @@ class ArchiveOrgIE(InfoExtractor):
'thumbnail': r're:https://archive\.org/download/.*\.jpg',
'duration': 59.77,
'display_id': 'Commercial-JFK1960ElectionAdCampaignJingle.mpg',
+ 'track': 'Commercial-JFK1960ElectionAdCampaignJingle',
},
}, {
'url': 'https://archive.org/details/Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpg',
@@ -102,6 +105,7 @@ class ArchiveOrgIE(InfoExtractor):
'duration': 59.51,
'license': 'http://creativecommons.org/licenses/publicdomain/',
'thumbnail': r're:https://archive\.org/download/.*\.jpg',
+ 'track': 'Commercial-Nixon1960ElectionAdToughonDefense',
},
}, {
'url': 'https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16',
@@ -182,6 +186,7 @@ class ArchiveOrgIE(InfoExtractor):
'duration': 130.46,
'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_01_000117.jpg',
'display_id': 'irelandthemakingofarepublicreel1_01.mov',
+ 'track': 'irelandthemakingofarepublicreel1 01',
},
}, {
'md5': '67335ee3b23a0da930841981c1e79b02',
@@ -192,6 +197,7 @@ class ArchiveOrgIE(InfoExtractor):
'title': 'irelandthemakingofarepublicreel1_02.mov',
'display_id': 'irelandthemakingofarepublicreel1_02.mov',
'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_02_001374.jpg',
+ 'track': 'irelandthemakingofarepublicreel1 02',
},
}, {
'md5': 'e470e86787893603f4a341a16c281eb5',
@@ -202,6 +208,7 @@ class ArchiveOrgIE(InfoExtractor):
'title': 'irelandthemakingofarepublicreel2.mov',
'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel2_001554.jpg',
'display_id': 'irelandthemakingofarepublicreel2.mov',
+ 'track': 'irelandthemakingofarepublicreel2',
},
},
],
@@ -229,15 +236,8 @@ class ArchiveOrgIE(InfoExtractor):
@staticmethod
def _playlist_data(webpage):
- element = re.findall(r'''(?xs)
-
- ''', webpage)[0]
-
- return json.loads(extract_attributes(element)['value'])
+ element = get_element_text_and_html_by_tag('play-av', webpage)[1]
+ return json.loads(extract_attributes(element)['playlist'])
def _real_extract(self, url):
video_id = urllib.parse.unquote_plus(self._match_id(url))
diff --git a/yt_dlp/extractor/bandaichannel.py b/yt_dlp/extractor/bandaichannel.py
deleted file mode 100644
index d7fcf44bd..000000000
--- a/yt_dlp/extractor/bandaichannel.py
+++ /dev/null
@@ -1,33 +0,0 @@
-from .brightcove import BrightcoveNewBaseIE
-from ..utils import extract_attributes
-
-
-class BandaiChannelIE(BrightcoveNewBaseIE):
- IE_NAME = 'bandaichannel'
- _VALID_URL = r'https?://(?:www\.)?b-ch\.com/titles/(?P\d+/\d+)'
- _TESTS = [{
- 'url': 'https://www.b-ch.com/titles/514/001',
- 'md5': 'a0f2d787baa5729bed71108257f613a4',
- 'info_dict': {
- 'id': '6128044564001',
- 'ext': 'mp4',
- 'title': 'メタルファイターMIKU 第1話',
- 'timestamp': 1580354056,
- 'uploader_id': '5797077852001',
- 'upload_date': '20200130',
- 'duration': 1387.733,
- },
- 'params': {
- 'skip_download': True,
- },
- }]
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
- attrs = extract_attributes(self._search_regex(
- r'(]+\bid="bcplayer"[^>]*>)', webpage, 'player'))
- bc = self._download_json(
- 'https://pbifcd.b-ch.com/v1/playbackinfo/ST/70/' + attrs['data-info'],
- video_id, headers={'X-API-KEY': attrs['data-auth'].strip()})['bc']
- return self._parse_brightcove_metadata(bc, bc['id'])
diff --git a/yt_dlp/extractor/bellmedia.py b/yt_dlp/extractor/bellmedia.py
deleted file mode 100644
index ac45dd477..000000000
--- a/yt_dlp/extractor/bellmedia.py
+++ /dev/null
@@ -1,91 +0,0 @@
-from .common import InfoExtractor
-
-
-class BellMediaIE(InfoExtractor):
- _VALID_URL = r'''(?x)https?://(?:www\.)?
- (?P
- (?:
- ctv|
- tsn|
- bnn(?:bloomberg)?|
- thecomedynetwork|
- discovery|
- discoveryvelocity|
- sciencechannel|
- investigationdiscovery|
- animalplanet|
- bravo|
- mtv|
- space|
- etalk|
- marilyn
- )\.ca|
- (?:much|cp24)\.com
- )/.*?(?:\b(?:vid(?:eoid)?|clipId)=|-vid|~|%7E|/(?:episode)?)(?P[0-9]{6,})'''
- _TESTS = [{
- 'url': 'https://www.bnnbloomberg.ca/video/david-cockfield-s-top-picks~1403070',
- 'md5': '3e5b8e38370741d5089da79161646635',
- 'info_dict': {
- 'id': '1403070',
- 'ext': 'flv',
- 'title': 'David Cockfield\'s Top Picks',
- 'description': 'md5:810f7f8c6a83ad5b48677c3f8e5bb2c3',
- 'upload_date': '20180525',
- 'timestamp': 1527288600,
- 'season_id': '73997',
- 'season': '2018',
- 'thumbnail': 'http://images2.9c9media.com/image_asset/2018_5_25_baf30cbd-b28d-4a18-9903-4bb8713b00f5_PNG_956x536.jpg',
- 'tags': [],
- 'categories': ['ETFs'],
- 'season_number': 8,
- 'duration': 272.038,
- 'series': 'Market Call Tonight',
- },
- }, {
- 'url': 'http://www.thecomedynetwork.ca/video/player?vid=923582',
- 'only_matching': True,
- }, {
- 'url': 'http://www.tsn.ca/video/expectations-high-for-milos-raonic-at-us-open~939549',
- 'only_matching': True,
- }, {
- 'url': 'http://www.bnn.ca/video/berman-s-call-part-two-viewer-questions~939654',
- 'only_matching': True,
- }, {
- 'url': 'http://www.ctv.ca/YourMorning/Video/S1E6-Monday-August-29-2016-vid938009',
- 'only_matching': True,
- }, {
- 'url': 'http://www.much.com/shows/atmidnight/episode948007/tuesday-september-13-2016',
- 'only_matching': True,
- }, {
- 'url': 'http://www.much.com/shows/the-almost-impossible-gameshow/928979/episode-6',
- 'only_matching': True,
- }, {
- 'url': 'http://www.ctv.ca/DCs-Legends-of-Tomorrow/Video/S2E11-Turncoat-vid1051430',
- 'only_matching': True,
- }, {
- 'url': 'http://www.etalk.ca/video?videoid=663455',
- 'only_matching': True,
- }, {
- 'url': 'https://www.cp24.com/video?clipId=1982548',
- 'only_matching': True,
- }]
- _DOMAINS = {
- 'thecomedynetwork': 'comedy',
- 'discoveryvelocity': 'discvel',
- 'sciencechannel': 'discsci',
- 'investigationdiscovery': 'invdisc',
- 'animalplanet': 'aniplan',
- 'etalk': 'ctv',
- 'bnnbloomberg': 'bnn',
- 'marilyn': 'ctv_marilyn',
- }
-
- def _real_extract(self, url):
- domain, video_id = self._match_valid_url(url).groups()
- domain = domain.split('.')[0]
- return {
- '_type': 'url_transparent',
- 'id': video_id,
- 'url': f'9c9media:{self._DOMAINS.get(domain, domain)}_web:{video_id}',
- 'ie_key': 'NineCNineMedia',
- }
diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 6508942a4..0c6535fc7 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -816,6 +816,26 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
'upload_date': '20111104',
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
},
+ }, {
+ 'note': 'new playurlSSRData scheme',
+ 'url': 'https://www.bilibili.com/bangumi/play/ep678060',
+ 'info_dict': {
+ 'id': '678060',
+ 'ext': 'mp4',
+ 'series': '去你家吃饭好吗',
+ 'series_id': '6198',
+ 'season': '第二季',
+ 'season_id': '42542',
+ 'season_number': 2,
+ 'episode': '吴老二：你家大公鸡养不熟，能煮熟吗…',
+ 'episode_id': '678060',
+ 'episode_number': 61,
+ 'title': '一只小九九丫 吴老二：你家大公鸡养不熟，能煮熟吗…',
+ 'duration': 266.123,
+ 'timestamp': 1663315904,
+ 'upload_date': '20220916',
+ 'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+ },
}, {
'url': 'https://www.bilibili.com/bangumi/play/ep267851',
'info_dict': {
@@ -879,12 +899,28 @@ def _real_extract(self, url):
'Extracting episode', query={'fnval': 12240, 'ep_id': episode_id},
headers=headers))
+ geo_blocked = traverse_obj(play_info, (
+ ('result', ('raw', 'data')), 'plugins',
+ lambda _, v: v['name'] == 'AreaLimitPanel',
+ 'config', 'is_block', {bool}, any))
premium_only = play_info.get('code') == -10403
- play_info = traverse_obj(play_info, ('result', 'video_info', {dict})) or {}
- formats = self.extract_formats(play_info)
- if not formats and (premium_only or '成为大会员抢先看' in webpage or '开通大会员观看' in webpage):
- self.raise_login_required('This video is for premium members only')
+ video_info = traverse_obj(play_info, (('result', ('raw', 'data')), 'video_info', {dict}, any)) or {}
+ formats = self.extract_formats(video_info)
+
+ if not formats:
+ if geo_blocked:
+ self.raise_geo_restricted()
+ elif premium_only or '成为大会员抢先看' in webpage or '开通大会员观看' in webpage:
+ self.raise_login_required('This video is for premium members only')
+
+ if traverse_obj(play_info, ((
+ ('result', 'play_check', 'play_detail'), # 'PLAY_PREVIEW' vs 'PLAY_WHOLE'
+ (('result', ('raw', 'data')), 'play_video_type'), # 'preview' vs 'whole' vs 'none'
+ ), any, {lambda x: x in ('PLAY_PREVIEW', 'preview')})):
+ self.report_warning(
+ 'Only preview format is available, '
+ f'you have to become a premium member to access full video. {self._login_hint()}')
bangumi_info = self._download_json(
'https://api.bilibili.com/pgc/view/web/season', episode_id, 'Get episode details',
@@ -922,7 +958,7 @@ def _real_extract(self, url):
'season': str_or_none(season_title),
'season_id': str_or_none(season_id),
'season_number': season_number,
- 'duration': float_or_none(play_info.get('timelength'), scale=1000),
+ 'duration': float_or_none(video_info.get('timelength'), scale=1000),
'subtitles': self.extract_subtitles(episode_id, episode_info.get('cid'), aid=aid),
'__post_extractor': self.extract_comments(aid),
'http_headers': {'Referer': url},
@@ -1192,6 +1228,26 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
'id': '313580179',
},
'playlist_mincount': 92,
+ }, {
+ # Hidden-mode collection
+ 'url': 'https://space.bilibili.com/3669403/video',
+ 'info_dict': {
+ 'id': '3669403',
+ },
+ 'playlist': [{
+ 'info_dict': {
+ '_type': 'playlist',
+ 'id': '3669403_3958082',
+ 'title': '合集·直播回放',
+ 'description': '',
+ 'uploader': '月路Yuel',
+ 'uploader_id': '3669403',
+ 'timestamp': int,
+ 'upload_date': str,
+ 'thumbnail': str,
+ },
+ }],
+ 'params': {'playlist_items': '7'},
}]
def _real_extract(self, url):
@@ -1248,8 +1304,14 @@ def get_metadata(page_data):
}
def get_entries(page_data):
- for entry in traverse_obj(page_data, ('list', 'vlist')) or []:
- yield self.url_result(f'https://www.bilibili.com/video/{entry["bvid"]}', BiliBiliIE, entry['bvid'])
+ for entry in traverse_obj(page_data, ('list', 'vlist', ..., {dict})):
+ if traverse_obj(entry, ('meta', 'attribute')) == 156:
+ # hidden-mode collection doesn't show its videos in uploads; extract as playlist instead
+ yield self.url_result(
+ f'https://space.bilibili.com/{entry["mid"]}/lists/{entry["meta"]["id"]}?type=season',
+ BilibiliCollectionListIE, f'{entry["mid"]}_{entry["meta"]["id"]}')
+ else:
+ yield self.url_result(f'https://www.bilibili.com/video/{entry["bvid"]}', BiliBiliIE, entry['bvid'])
metadata, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries)
return self.playlist_result(paged_list, playlist_id)
diff --git a/yt_dlp/extractor/blackboardcollaborate.py b/yt_dlp/extractor/blackboardcollaborate.py
index 535890979..c14ff1f14 100644
--- a/yt_dlp/extractor/blackboardcollaborate.py
+++ b/yt_dlp/extractor/blackboardcollaborate.py
@@ -1,16 +1,27 @@
from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+ UnsupportedError,
+ float_or_none,
+ int_or_none,
+ join_nonempty,
+ jwt_decode_hs256,
+ mimetype2ext,
+ parse_iso8601,
+ parse_qs,
+ url_or_none,
+)
+from ..utils.traversal import traverse_obj
class BlackboardCollaborateIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://
- (?P[a-z-]+)\.bbcollab\.com/
+ (?P[a-z]+)(?:-lti)?\.bbcollab\.com/
(?:
collab/ui/session/playback/load|
recording
)/
- (?P[^/]+)'''
+ (?P[^/?#]+)'''
_TESTS = [
{
'url': 'https://us-lti.bbcollab.com/collab/ui/session/playback/load/0a633b6a88824deb8c918f470b22b256',
@@ -19,9 +30,55 @@ class BlackboardCollaborateIE(InfoExtractor):
'id': '0a633b6a88824deb8c918f470b22b256',
'title': 'HESI A2 Information Session - Thursday, May 6, 2021 - recording_1',
'ext': 'mp4',
- 'duration': 1896000,
- 'timestamp': 1620331399,
+ 'duration': 1896,
+ 'timestamp': 1620333295,
'upload_date': '20210506',
+ 'subtitles': {
+ 'live_chat': 'mincount:1',
+ },
+ },
+ },
+ {
+ 'url': 'https://eu.bbcollab.com/collab/ui/session/playback/load/4bde2dee104f40289a10f8e554270600',
+ 'md5': '108db6a8f83dcb0c2a07793649581865',
+ 'info_dict': {
+ 'id': '4bde2dee104f40289a10f8e554270600',
+ 'title': 'Meeting - Azerbaycanca erize formasi',
+ 'ext': 'mp4',
+ 'duration': 880,
+ 'timestamp': 1671176868,
+ 'upload_date': '20221216',
+ },
+ },
+ {
+ 'url': 'https://eu.bbcollab.com/recording/f83be390ecff46c0bf7dccb9dddcf5f6',
+ 'md5': 'e3b0b88ddf7847eae4b4c0e2d40b83a5',
+ 'info_dict': {
+ 'id': 'f83be390ecff46c0bf7dccb9dddcf5f6',
+ 'title': 'Keynote lecture by Laura Carvalho - recording_1',
+ 'ext': 'mp4',
+ 'duration': 5506,
+ 'timestamp': 1662721705,
+ 'upload_date': '20220909',
+ 'subtitles': {
+ 'live_chat': 'mincount:1',
+ },
+ },
+ },
+ {
+ 'url': 'https://eu.bbcollab.com/recording/c3e1e7c9e83d4cd9981c93c74888d496',
+ 'md5': 'fdb2d8c43d66fbc0b0b74ef5e604eb1f',
+ 'info_dict': {
+ 'id': 'c3e1e7c9e83d4cd9981c93c74888d496',
+ 'title': 'International Ally User Group - recording_18',
+ 'ext': 'mp4',
+ 'duration': 3479,
+ 'timestamp': 1721919621,
+ 'upload_date': '20240725',
+ 'subtitles': {
+ 'en': 'mincount:1',
+ 'live_chat': 'mincount:1',
+ },
},
},
{
@@ -42,22 +99,81 @@ class BlackboardCollaborateIE(InfoExtractor):
},
]
+ def _call_api(self, region, video_id, path=None, token=None, note=None, fatal=False):
+ # Ref: https://github.com/blackboard/BBDN-Collab-Postman-REST
+ return self._download_json(
+ join_nonempty(f'https://{region}.bbcollab.com/collab/api/csa/recordings', video_id, path, delim='/'),
+ video_id, note or 'Downloading JSON metadata', fatal=fatal,
+ headers={'Authorization': f'Bearer {token}'} if token else None)
+
def _real_extract(self, url):
mobj = self._match_valid_url(url)
region = mobj.group('region')
video_id = mobj.group('id')
- info = self._download_json(
- f'https://{region}.bbcollab.com/collab/api/csa/recordings/{video_id}/data', video_id)
- duration = info.get('duration')
- title = info['name']
- upload_date = info.get('created')
- streams = info['streams']
- formats = [{'format_id': k, 'url': url} for k, url in streams.items()]
+ token = parse_qs(url).get('authToken', [None])[-1]
+
+ video_info = self._call_api(region, video_id, path='data/secure', token=token, note='Trying auth token')
+ if video_info:
+ video_extra = self._call_api(region, video_id, token=token, note='Retrieving extra attributes')
+ else:
+ video_info = self._call_api(region, video_id, path='data', note='Trying fallback', fatal=True)
+ video_extra = {}
+
+ formats = traverse_obj(video_info, ('extStreams', lambda _, v: url_or_none(v['streamUrl']), {
+ 'url': 'streamUrl',
+ 'ext': ('contentType', {mimetype2ext}),
+ 'aspect_ratio': ('aspectRatio', {float_or_none}),
+ }))
+
+ if filesize := traverse_obj(video_extra, ('storageSize', {int_or_none})):
+ for fmt in formats:
+ fmt['filesize'] = filesize
+
+ subtitles = {}
+ for subs in traverse_obj(video_info, ('subtitles', lambda _, v: url_or_none(v['url']))):
+ subtitles.setdefault(subs.get('lang') or 'und', []).append({
+ 'name': traverse_obj(subs, ('label', {str})),
+ 'url': subs['url'],
+ })
+
+ for live_chat_url in traverse_obj(video_info, ('chats', ..., 'url', {url_or_none})):
+ subtitles.setdefault('live_chat', []).append({'url': live_chat_url})
return {
- 'duration': duration,
+ **traverse_obj(video_info, {
+ 'title': ('name', {str}),
+ 'timestamp': ('created', {parse_iso8601}),
+ 'duration': ('duration', {int_or_none(scale=1000)}),
+ }),
'formats': formats,
'id': video_id,
- 'timestamp': parse_iso8601(upload_date),
- 'title': title,
+ 'subtitles': subtitles,
}
+
+
+class BlackboardCollaborateLaunchIE(InfoExtractor):
+ _VALID_URL = r'https?://[a-z]+\.bbcollab\.com/launch/(?P[^/?#]+)'
+
+ _TESTS = [
+ {
+ 'url': 'https://au.bbcollab.com/launch/eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJiYkNvbGxhYkFwaSIsInN1YiI6ImJiQ29sbGFiQXBpIiwiZXhwIjoxNzQwNDE2NDgzLCJpYXQiOjE3NDA0MTYxODMsInJlc291cmNlQWNjZXNzVGlja2V0Ijp7InJlc291cmNlSWQiOiI3MzI4YzRjZTNmM2U0ZTcwYmY3MTY3N2RkZTgzMzk2NSIsImNvbnN1bWVySWQiOiJhM2Q3NGM0Y2QyZGU0MGJmODFkMjFlODNlMmEzNzM5MCIsInR5cGUiOiJSRUNPUkRJTkciLCJyZXN0cmljdGlvbiI6eyJ0eXBlIjoiVElNRSIsImV4cGlyYXRpb25Ib3VycyI6MCwiZXhwaXJhdGlvbk1pbnV0ZXMiOjUsIm1heFJlcXVlc3RzIjotMX0sImRpc3Bvc2l0aW9uIjoiTEFVTkNIIiwibGF1bmNoVHlwZSI6bnVsbCwibGF1bmNoQ29tcG9uZW50IjpudWxsLCJsYXVuY2hQYXJhbUtleSI6bnVsbH19.xuELw4EafEwUMoYcCHidGn4Tw9O1QCbYHzYGJUl0kKk',
+ 'only_matching': True,
+ },
+ {
+ 'url': 'https://us.bbcollab.com/launch/eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJiYkNvbGxhYkFwaSIsInN1YiI6ImJiQ29sbGFiQXBpIiwiZXhwIjoxNjk0NDgxOTc3LCJpYXQiOjE2OTQ0ODE2NzcsInJlc291cmNlQWNjZXNzVGlja2V0Ijp7InJlc291cmNlSWQiOiI3YWU0MTFhNTU3NjU0OWFiOTZlYjVmMTM1YmY3MWU5MCIsImNvbnN1bWVySWQiOiJBRUU2MEI4MDI2QzM3ODU2RjMwMzNEN0ZEOTQzMTFFNSIsInR5cGUiOiJSRUNPUkRJTkciLCJyZXN0cmljdGlvbiI6eyJ0eXBlIjoiVElNRSIsImV4cGlyYXRpb25Ib3VycyI6MCwiZXhwaXJhdGlvbk1pbnV0ZXMiOjUsIm1heFJlcXVlc3RzIjotMX0sImRpc3Bvc2l0aW9uIjoiTEFVTkNIIiwibGF1bmNoVHlwZSI6bnVsbCwibGF1bmNoQ29tcG9uZW50IjpudWxsLCJsYXVuY2hQYXJhbUtleSI6bnVsbH19.yOhRZNaIjXYoMYMpcTzgjZJCnIFaYf2cAzbco8OAxlY',
+ 'only_matching': True,
+ },
+ {
+ 'url': 'https://eu.bbcollab.com/launch/eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJiYkNvbGxhYkFwaSIsInN1YiI6ImJiQ29sbGFiQXBpIiwiZXhwIjoxNzUyNjgyODYwLCJpYXQiOjE3NTI2ODI1NjAsInJlc291cmNlQWNjZXNzVGlja2V0Ijp7InJlc291cmNlSWQiOiI4MjQzYjFiODg2Nzk0NTZkYjkwN2NmNDZmZmE1MmFhZiIsImNvbnN1bWVySWQiOiI5ZTY4NzYwZWJiNzM0MzRiYWY3NTQyZjA1YmJkOTMzMCIsInR5cGUiOiJSRUNPUkRJTkciLCJyZXN0cmljdGlvbiI6eyJ0eXBlIjoiVElNRSIsImV4cGlyYXRpb25Ib3VycyI6MCwiZXhwaXJhdGlvbk1pbnV0ZXMiOjUsIm1heFJlcXVlc3RzIjotMX0sImRpc3Bvc2l0aW9uIjoiTEFVTkNIIiwibGF1bmNoVHlwZSI6bnVsbCwibGF1bmNoQ29tcG9uZW50IjpudWxsLCJsYXVuY2hQYXJhbUtleSI6bnVsbH19.Xj4ymojYLwZ1vKPKZ-KxjpqQvFXoJekjRaG0npngwWs',
+ 'only_matching': True,
+ },
+ ]
+
+ def _real_extract(self, url):
+ token = self._match_id(url)
+ video_id = jwt_decode_hs256(token)['resourceAccessTicket']['resourceId']
+
+ redirect_url = self._request_webpage(url, video_id).url
+ if self.suitable(redirect_url):
+ raise UnsupportedError(redirect_url)
+ return self.url_result(redirect_url, BlackboardCollaborateIE, video_id)
diff --git a/yt_dlp/extractor/brightcove.py b/yt_dlp/extractor/brightcove.py
index d4ac7a0c2..c0f2f8b57 100644
--- a/yt_dlp/extractor/brightcove.py
+++ b/yt_dlp/extractor/brightcove.py
@@ -495,8 +495,6 @@ def _real_extract(self, url):
class BrightcoveNewBaseIE(AdobePassIE):
def _parse_brightcove_metadata(self, json_data, video_id, headers={}):
- title = json_data['name'].strip()
-
formats, subtitles = [], {}
sources = json_data.get('sources') or []
for source in sources:
@@ -600,16 +598,18 @@ def build_format_id(kind):
return {
'id': video_id,
- 'title': title,
- 'description': clean_html(json_data.get('description')),
'thumbnails': thumbnails,
'duration': duration,
- 'timestamp': parse_iso8601(json_data.get('published_at')),
- 'uploader_id': json_data.get('account_id'),
'formats': formats,
'subtitles': subtitles,
- 'tags': json_data.get('tags', []),
'is_live': is_live,
+ **traverse_obj(json_data, {
+ 'title': ('name', {clean_html}),
+ 'description': ('description', {clean_html}),
+ 'tags': ('tags', ..., {str}, filter, all, filter),
+ 'timestamp': ('published_at', {parse_iso8601}),
+ 'uploader_id': ('account_id', {str}),
+ }),
}
@@ -645,10 +645,7 @@ class BrightcoveNewIE(BrightcoveNewBaseIE):
'uploader_id': '4036320279001',
'formats': 'mincount:39',
},
- 'params': {
- # m3u8 download
- 'skip_download': True,
- },
+ 'skip': '404 Not Found',
}, {
# playlist stream
'url': 'https://players.brightcove.net/1752604059001/S13cJdUBz_default/index.html?playlistId=5718313430001',
@@ -709,7 +706,6 @@ class BrightcoveNewIE(BrightcoveNewBaseIE):
'ext': 'mp4',
'title': 'TGD_01-032_5',
'thumbnail': r're:^https?://.*\.jpg$',
- 'tags': [],
'timestamp': 1646078943,
'uploader_id': '1569565978001',
'upload_date': '20220228',
@@ -721,7 +717,6 @@ class BrightcoveNewIE(BrightcoveNewBaseIE):
'ext': 'mp4',
'title': 'TGD 01-087 (Airs 05.25.22)_Segment 5',
'thumbnail': r're:^https?://.*\.jpg$',
- 'tags': [],
'timestamp': 1651604591,
'uploader_id': '1569565978001',
'upload_date': '20220503',
diff --git a/yt_dlp/extractor/btvplus.py b/yt_dlp/extractor/btvplus.py
new file mode 100644
index 000000000..531ace147
--- /dev/null
+++ b/yt_dlp/extractor/btvplus.py
@@ -0,0 +1,73 @@
+from .common import InfoExtractor
+from ..utils import (
+ bug_reports_message,
+ clean_html,
+ get_element_by_class,
+ js_to_json,
+ mimetype2ext,
+ strip_or_none,
+ url_or_none,
+ urljoin,
+)
+from ..utils.traversal import traverse_obj
+
+
+class BTVPlusIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?btvplus\.bg/produkt/(?:predavaniya|seriali|novini)/(?P\d+)'
+ _TESTS = [{
+ 'url': 'https://btvplus.bg/produkt/predavaniya/67271/btv-reporterite/btv-reporterite-12-07-2025-g',
+ 'info_dict': {
+ 'ext': 'mp4',
+ 'id': '67271',
+ 'title': 'bTV Репортерите - 12.07.2025 г.',
+ 'thumbnail': 'https://cdn.btv.bg/media/images/940x529/Jul2025/2113606319.jpg',
+ },
+ }, {
+ 'url': 'https://btvplus.bg/produkt/seriali/66942/sezon-2/plen-sezon-2-epizod-55',
+ 'info_dict': {
+ 'ext': 'mp4',
+ 'id': '66942',
+ 'title': 'Плен - сезон 2, епизод 55',
+ 'thumbnail': 'https://cdn.btv.bg/media/images/940x529/Jun2025/2113595104.jpg',
+ },
+ }, {
+ 'url': 'https://btvplus.bg/produkt/novini/67270/btv-novinite-centralna-emisija-12-07-2025',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ player_url = self._search_regex(
+ r'var\s+videoUrl\s*=\s*[\'"]([^\'"]+)[\'"]',
+ webpage, 'player URL')
+
+ player_config = self._download_json(
+ urljoin('https://btvplus.bg', player_url), video_id)['config']
+
+ videojs_data = self._search_json(
+ r'videojs\(["\'][^"\']+["\'],', player_config, 'videojs data',
+ video_id, transform_source=js_to_json)
+ formats = []
+ subtitles = {}
+ for src in traverse_obj(videojs_data, ('sources', lambda _, v: url_or_none(v['src']))):
+ ext = mimetype2ext(src.get('type'))
+ if ext == 'm3u8':
+ fmts, subs = self._extract_m3u8_formats_and_subtitles(
+ src['src'], video_id, 'mp4', m3u8_id='hls', fatal=False)
+ formats.extend(fmts)
+ self._merge_subtitles(subs, target=subtitles)
+ else:
+ self.report_warning(f'Unknown format type {ext}{bug_reports_message()}')
+
+ return {
+ 'id': video_id,
+ 'formats': formats,
+ 'subtitles': subtitles,
+ 'title': (
+ strip_or_none(self._og_search_title(webpage, default=None))
+ or clean_html(get_element_by_class('product-title', webpage))),
+ 'thumbnail': self._og_search_thumbnail(webpage, default=None),
+ 'description': self._og_search_description(webpage, default=None),
+ }
diff --git a/yt_dlp/extractor/cloudycdn.py b/yt_dlp/extractor/cloudycdn.py
index 6e757d79e..a9a539274 100644
--- a/yt_dlp/extractor/cloudycdn.py
+++ b/yt_dlp/extractor/cloudycdn.py
@@ -11,7 +11,7 @@
class CloudyCDNIE(InfoExtractor):
- _VALID_URL = r'(?:https?:)?//embed\.cloudycdn\.services/(?P[^/?#]+)/media/(?P[\w-]+)'
+ _VALID_URL = r'(?:https?:)?//embed\.(?Pcloudycdn\.services|backscreen\.com)/(?P[^/?#]+)/media/(?P[\w-]+)'
_EMBED_REGEX = [rf']+\bsrc=[\'"](?P{_VALID_URL})']
_TESTS = [{
'url': 'https://embed.cloudycdn.services/ltv/media/46k_d23-6000-105?',
@@ -23,7 +23,7 @@ class CloudyCDNIE(InfoExtractor):
'duration': 1442,
'upload_date': '20231121',
'title': 'D23-6000-105_cetstud',
- 'thumbnail': 'https://store.cloudycdn.services/tmsp00060/assets/media/660858/placeholder1700589200.jpg',
+ 'thumbnail': 'https://store.bstrm.net/tmsp00060/assets/media/660858/placeholder1700589200.jpg',
},
}, {
'url': 'https://embed.cloudycdn.services/izm/media/26e_lv-8-5-1',
@@ -33,7 +33,7 @@ class CloudyCDNIE(InfoExtractor):
'ext': 'mp4',
'title': 'LV-8-5-1',
'timestamp': 1669767167,
- 'thumbnail': 'https://store.cloudycdn.services/tmsp00120/assets/media/488306/placeholder1679423604.jpg',
+ 'thumbnail': 'https://store.bstrm.net/tmsp00120/assets/media/488306/placeholder1679423604.jpg',
'duration': 1205,
'upload_date': '20221130',
},
@@ -48,9 +48,21 @@ class CloudyCDNIE(InfoExtractor):
'duration': 1673,
'title': 'D24-6000-074-cetstud',
'timestamp': 1718902233,
- 'thumbnail': 'https://store.cloudycdn.services/tmsp00060/assets/media/788392/placeholder1718903938.jpg',
+ 'thumbnail': 'https://store.bstrm.net/tmsp00060/assets/media/788392/placeholder1718903938.jpg',
},
'params': {'format': 'bv'},
+ }, {
+ 'url': 'https://embed.backscreen.com/ltv/media/32j_z25-0600-127?',
+ 'md5': '9b6fa09ac1a4de53d4f42b94affc3b42',
+ 'info_dict': {
+ 'id': '32j_z25-0600-127',
+ 'ext': 'mp4',
+ 'title': 'Z25-0600-127-DZ',
+ 'duration': 1906,
+ 'thumbnail': 'https://store.bstrm.net/tmsp00060/assets/media/977427/placeholder1746633646.jpg',
+ 'timestamp': 1746632402,
+ 'upload_date': '20250507',
+ },
}]
_WEBPAGE_TESTS = [{
'url': 'https://www.tavaklase.lv/video/es-esmu-mina-um-2/',
@@ -60,17 +72,17 @@ class CloudyCDNIE(InfoExtractor):
'ext': 'mp4',
'upload_date': '20230223',
'duration': 629,
- 'thumbnail': 'https://store.cloudycdn.services/tmsp00120/assets/media/518407/placeholder1678748124.jpg',
+ 'thumbnail': 'https://store.bstrm.net/tmsp00120/assets/media/518407/placeholder1678748124.jpg',
'timestamp': 1677181513,
'title': 'LIB-2',
},
}]
def _real_extract(self, url):
- site_id, video_id = self._match_valid_url(url).group('site_id', 'id')
+ domain, site_id, video_id = self._match_valid_url(url).group('domain', 'site_id', 'id')
data = self._download_json(
- f'https://player.cloudycdn.services/player/{site_id}/media/{video_id}/',
+ f'https://player.{domain}/player/{site_id}/media/{video_id}/',
video_id, data=urlencode_postdata({
'version': '6.4.0',
'referer': url,
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index d5607296d..d601e1751 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -1,5 +1,6 @@
import base64
import collections
+import contextlib
import functools
import getpass
import http.client
@@ -101,6 +102,7 @@
xpath_with_ns,
)
from ..utils._utils import _request_dump_filename
+from ..utils.jslib import devalue
class InfoExtractor:
@@ -262,6 +264,9 @@ class InfoExtractor:
* http_chunk_size Chunk size for HTTP downloads
* ffmpeg_args Extra arguments for ffmpeg downloader (input)
* ffmpeg_args_out Extra arguments for ffmpeg downloader (output)
+ * ws (NiconicoLiveFD only) WebSocketResponse
+ * ws_url (NiconicoLiveFD only) Websockets URL
+ * max_quality (NiconicoLiveFD only) Max stream quality string
* is_dash_periods Whether the format is a result of merging
multiple DASH periods.
RTMP formats can also have the additional fields: page_url,
@@ -1675,9 +1680,9 @@ def extract_video_object(e):
'ext': mimetype2ext(e.get('encodingFormat')),
'title': unescapeHTML(e.get('name')),
'description': unescapeHTML(e.get('description')),
- 'thumbnails': [{'url': unescapeHTML(url)}
- for url in variadic(traverse_obj(e, 'thumbnailUrl', 'thumbnailURL'))
- if url_or_none(url)],
+ 'thumbnails': traverse_obj(e, (('thumbnailUrl', 'thumbnailURL', 'thumbnail_url'), (None, ...), {
+ 'url': ({str}, {unescapeHTML}, {self._proto_relative_url}, {url_or_none}),
+ })),
'duration': parse_duration(e.get('duration')),
'timestamp': unified_timestamp(e.get('uploadDate')),
# author can be an instance of 'Organization' or 'Person' types.
@@ -1778,6 +1783,59 @@ def _search_nextjs_data(self, webpage, video_id, *, fatal=True, default=NO_DEFAU
r']+id=[\'"]__NEXT_DATA__[\'"][^>]*>', webpage, 'next.js data',
video_id, end_pattern='', fatal=fatal, default=default, **kw)
+ def _search_nextjs_v13_data(self, webpage, video_id, fatal=True):
+ """Parses Next.js app router flight data that was introduced in Next.js v13"""
+ nextjs_data = {}
+ if not fatal and not isinstance(webpage, str):
+ return nextjs_data
+
+ def flatten(flight_data):
+ if not isinstance(flight_data, list):
+ return
+ if len(flight_data) == 4 and flight_data[0] == '$':
+ _, name, _, data = flight_data
+ if not isinstance(data, dict):
+ return
+ children = data.pop('children', None)
+ if data and isinstance(name, str) and re.fullmatch(r'\$L[0-9a-f]+', name):
+ # It is useful hydration JSON data
+ nextjs_data[name[2:]] = data
+ flatten(children)
+ return
+ for f in flight_data:
+ flatten(f)
+
+ flight_text = ''
+ # The pattern for the surrounding JS/tag should be strict as it's a hardcoded string in the next.js source
+ # Ref: https://github.com/vercel/next.js/blob/5a4a08fdc/packages/next/src/server/app-render/use-flight-response.tsx#L189
+ for flight_segment in re.findall(r']*>self\.__next_f\.push\((\[.+?\])\)', webpage):
+ segment = self._parse_json(flight_segment, video_id, fatal=fatal, errnote=None if fatal else False)
+ # Some earlier versions of next.js "optimized" away this array structure; this is unsupported
+ # Ref: https://github.com/vercel/next.js/commit/0123a9d5c9a9a77a86f135b7ae30b46ca986d761
+ if not isinstance(segment, list) or len(segment) != 2:
+ self.write_debug(
+ f'{video_id}: Unsupported next.js flight data structure detected', only_once=True)
+ continue
+ # Only use the relevant payload type (1 == data)
+ # Ref: https://github.com/vercel/next.js/blob/5a4a08fdc/packages/next/src/server/app-render/use-flight-response.tsx#L11-L14
+ payload_type, chunk = segment
+ if payload_type == 1:
+ flight_text += chunk
+
+ for f in flight_text.splitlines():
+ prefix, _, body = f.lstrip().partition(':')
+ if not re.fullmatch(r'[0-9a-f]+', prefix):
+ continue
+ # The body still isn't guaranteed to be valid JSON, so parsing should always be non-fatal
+ if body.startswith('[') and body.endswith(']'):
+ flatten(self._parse_json(body, video_id, fatal=False, errnote=False))
+ elif body.startswith('{') and body.endswith('}'):
+ data = self._parse_json(body, video_id, fatal=False, errnote=False)
+ if data is not None:
+ nextjs_data[prefix] = data
+
+ return nextjs_data
+
def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', *, fatal=True, traverse=('data', 0)):
"""Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
rectx = re.escape(context_name)
@@ -1795,6 +1853,63 @@ def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', *, fatal
ret = self._parse_json(js, video_id, transform_source=functools.partial(js_to_json, vars=args), fatal=fatal)
return traverse_obj(ret, traverse) or {}
+ def _resolve_nuxt_array(self, array, video_id, *, fatal=True, default=NO_DEFAULT):
+ """Resolves Nuxt rich JSON payload arrays"""
+ # Ref: https://github.com/nuxt/nuxt/commit/9e503be0f2a24f4df72a3ccab2db4d3e63511f57
+ # https://github.com/nuxt/nuxt/pull/19205
+ if default is not NO_DEFAULT:
+ fatal = False
+
+ if not isinstance(array, list) or not array:
+ error_msg = 'Unable to resolve Nuxt JSON data: invalid input'
+ if fatal:
+ raise ExtractorError(error_msg, video_id=video_id)
+ elif default is NO_DEFAULT:
+ self.report_warning(error_msg, video_id=video_id)
+ return {} if default is NO_DEFAULT else default
+
+ def indirect_reviver(data):
+ return data
+
+ def json_reviver(data):
+ return json.loads(data)
+
+ gen = devalue.parse_iter(array, revivers={
+ 'NuxtError': indirect_reviver,
+ 'EmptyShallowRef': json_reviver,
+ 'EmptyRef': json_reviver,
+ 'ShallowRef': indirect_reviver,
+ 'ShallowReactive': indirect_reviver,
+ 'Ref': indirect_reviver,
+ 'Reactive': indirect_reviver,
+ })
+
+ while True:
+ try:
+ error_msg = f'Error resolving Nuxt JSON: {gen.send(None)}'
+ if fatal:
+ raise ExtractorError(error_msg, video_id=video_id)
+ elif default is NO_DEFAULT:
+ self.report_warning(error_msg, video_id=video_id, only_once=True)
+ else:
+ self.write_debug(f'{video_id}: {error_msg}', only_once=True)
+ except StopIteration as error:
+ return error.value or ({} if default is NO_DEFAULT else default)
+
+ def _search_nuxt_json(self, webpage, video_id, *, fatal=True, default=NO_DEFAULT):
+ """Parses metadata from Nuxt rich JSON payloads embedded in HTML"""
+ passed_default = default is not NO_DEFAULT
+
+ array = self._search_json(
+ r']+\bid="__NUXT_DATA__"[^>]*>', webpage,
+ 'Nuxt JSON data', video_id, contains_pattern=r'\[(?s:.+)\]',
+ fatal=fatal, default=NO_DEFAULT if not passed_default else None)
+
+ if not array:
+ return default if passed_default else {}
+
+ return self._resolve_nuxt_array(array, video_id, fatal=fatal, default=default)
+
@staticmethod
def _hidden_inputs(html):
html = re.sub(r'', '', html)
@@ -2068,21 +2183,33 @@ def _extract_m3u8_formats_and_subtitles(
raise ExtractorError(errnote, video_id=video_id)
self.report_warning(f'{errnote}{bug_reports_message()}')
return [], {}
-
- res = self._download_webpage_handle(
- m3u8_url, video_id,
- note='Downloading m3u8 information' if note is None else note,
- errnote='Failed to download m3u8 information' if errnote is None else errnote,
+ if note is None:
+ note = 'Downloading m3u8 information'
+ if errnote is None:
+ errnote = 'Failed to download m3u8 information'
+ response = self._request_webpage(
+ m3u8_url, video_id, note=note, errnote=errnote,
fatal=fatal, data=data, headers=headers, query=query)
-
- if res is False:
+ if response is False:
return [], {}
- m3u8_doc, urlh = res
- m3u8_url = urlh.url
+ with contextlib.closing(response):
+ prefix = response.read(512)
+ if not prefix.startswith(b'#EXTM3U'):
+ msg = 'Response data has no m3u header'
+ if fatal:
+ raise ExtractorError(msg, video_id=video_id)
+ self.report_warning(f'{msg}{bug_reports_message()}', video_id=video_id)
+ return [], {}
+
+ content = self._webpage_read_content(
+ response, m3u8_url, video_id, note=note, errnote=errnote,
+ fatal=fatal, prefix=prefix, data=data)
+ if content is False:
+ return [], {}
return self._parse_m3u8_formats_and_subtitles(
- m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
+ content, response.url, ext=ext, entry_protocol=entry_protocol,
preference=preference, quality=quality, m3u8_id=m3u8_id,
note=note, errnote=errnote, fatal=fatal, live=live, data=data,
headers=headers, query=query, video_id=video_id)
diff --git a/yt_dlp/extractor/ctv.py b/yt_dlp/extractor/ctv.py
deleted file mode 100644
index a41dab11b..000000000
--- a/yt_dlp/extractor/ctv.py
+++ /dev/null
@@ -1,49 +0,0 @@
-from .common import InfoExtractor
-
-
-class CTVIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?ctv\.ca/(?P(?:show|movie)s/[^/]+/[^/?#&]+)'
- _TESTS = [{
- 'url': 'https://www.ctv.ca/shows/your-morning/wednesday-december-23-2020-s5e88',
- 'info_dict': {
- 'id': '2102249',
- 'ext': 'flv',
- 'title': 'Wednesday, December 23, 2020',
- 'thumbnail': r're:^https?://.*\.jpg$',
- 'description': 'Your Morning delivers original perspectives and unique insights into the headlines of the day.',
- 'timestamp': 1608732000,
- 'upload_date': '20201223',
- 'series': 'Your Morning',
- 'season': '2020-2021',
- 'season_number': 5,
- 'episode_number': 88,
- 'tags': ['Your Morning'],
- 'categories': ['Talk Show'],
- 'duration': 7467.126,
- },
- }, {
- 'url': 'https://www.ctv.ca/movies/adam-sandlers-eight-crazy-nights/adam-sandlers-eight-crazy-nights',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- display_id = self._match_id(url)
- content = self._download_json(
- 'https://www.ctv.ca/space-graphql/graphql', display_id, query={
- 'query': '''{
- resolvedPath(path: "/%s") {
- lastSegment {
- content {
- ... on AxisContent {
- axisId
- videoPlayerDestCode
- }
- }
- }
- }
-}''' % display_id, # noqa: UP031
- })['data']['resolvedPath']['lastSegment']['content']
- video_id = content['axisId']
- return self.url_result(
- '9c9media:{}:{}'.format(content['videoPlayerDestCode'], video_id),
- 'NineCNineMedia', video_id)
diff --git a/yt_dlp/extractor/douyutv.py b/yt_dlp/extractor/douyutv.py
index e36eac919..68ace240c 100644
--- a/yt_dlp/extractor/douyutv.py
+++ b/yt_dlp/extractor/douyutv.py
@@ -206,7 +206,7 @@ def _real_extract(self, url):
'is_live': True,
**traverse_obj(room, {
'display_id': ('url', {str}, {lambda i: i[1:]}),
- 'title': ('room_name', {unescapeHTML}),
+ 'title': ('room_name', {str}, {unescapeHTML}),
'description': ('show_details', {str}),
'uploader': ('nickname', {str}),
'thumbnail': ('room_src', {url_or_none}),
diff --git a/yt_dlp/extractor/dreisat.py b/yt_dlp/extractor/dreisat.py
index edd66e46c..fb8a8e87c 100644
--- a/yt_dlp/extractor/dreisat.py
+++ b/yt_dlp/extractor/dreisat.py
@@ -64,7 +64,7 @@ class DreiSatIE(ZDFBaseIE):
'title': 'dein buch - Das Beste von der Leipziger Buchmesse 2025 - Teil 1',
'description': 'md5:bae51bfc22f15563ce3acbf97d2e8844',
'duration': 5399.0,
- 'thumbnail': 'https://www.3sat.de/assets/buchmesse-kerkeling-100~original?cb=1743329640903',
+ 'thumbnail': 'https://www.3sat.de/assets/buchmesse-kerkeling-100~original?cb=1747256996338',
'chapters': 'count:24',
'episode': 'dein buch - Das Beste von der Leipziger Buchmesse 2025 - Teil 1',
'episode_id': 'POS_1ef236cc-b390-401e-acd0-4fb4b04315fb',
diff --git a/yt_dlp/extractor/floatplane.py b/yt_dlp/extractor/floatplane.py
index b7ee160a4..7dd3b0eb2 100644
--- a/yt_dlp/extractor/floatplane.py
+++ b/yt_dlp/extractor/floatplane.py
@@ -17,8 +17,140 @@
from ..utils.traversal import traverse_obj
-class FloatplaneIE(InfoExtractor):
+class FloatplaneBaseIE(InfoExtractor):
+ def _real_extract(self, url):
+ post_id = self._match_id(url)
+
+ post_data = self._download_json(
+ f'{self._BASE_URL}/api/v3/content/post', post_id, query={'id': post_id},
+ note='Downloading post data', errnote='Unable to download post data',
+ impersonate=self._IMPERSONATE_TARGET)
+
+ if not any(traverse_obj(post_data, ('metadata', ('hasVideo', 'hasAudio')))):
+ raise ExtractorError('Post does not contain a video or audio track', expected=True)
+
+ uploader_url = format_field(
+ post_data, [('creator', 'urlname')], f'{self._BASE_URL}/channel/%s/home') or None
+
+ common_info = {
+ 'uploader_url': uploader_url,
+ 'channel_url': urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname'))),
+ 'availability': self._availability(needs_subscription=True),
+ **traverse_obj(post_data, {
+ 'uploader': ('creator', 'title', {str}),
+ 'uploader_id': ('creator', 'id', {str}),
+ 'channel': ('channel', 'title', {str}),
+ 'channel_id': ('channel', 'id', {str}),
+ 'release_timestamp': ('releaseDate', {parse_iso8601}),
+ }),
+ }
+
+ items = []
+ for media in traverse_obj(post_data, (('videoAttachments', 'audioAttachments'), ...)):
+ media_id = media['id']
+ media_typ = media.get('type') or 'video'
+
+ metadata = self._download_json(
+ f'{self._BASE_URL}/api/v3/content/{media_typ}', media_id, query={'id': media_id},
+ note=f'Downloading {media_typ} metadata', impersonate=self._IMPERSONATE_TARGET)
+
+ stream = self._download_json(
+ f'{self._BASE_URL}/api/v2/cdn/delivery', media_id, query={
+ 'type': 'vod' if media_typ == 'video' else 'aod',
+ 'guid': metadata['guid'],
+ }, note=f'Downloading {media_typ} stream data',
+ impersonate=self._IMPERSONATE_TARGET)
+
+ path_template = traverse_obj(stream, ('resource', 'uri', {str}))
+
+ def format_path(params):
+ path = path_template
+ for i, val in (params or {}).items():
+ path = path.replace(f'{{qualityLevelParams.{i}}}', val)
+ return path
+
+ formats = []
+ for quality in traverse_obj(stream, ('resource', 'data', 'qualityLevels', ...)):
+ url = urljoin(stream['cdn'], format_path(traverse_obj(
+ stream, ('resource', 'data', 'qualityLevelParams', quality['name'], {dict}))))
+ format_id = traverse_obj(quality, ('name', {str}))
+ hls_aes = {}
+ m3u8_data = None
+
+ # If we need impersonation for the API, then we need it for HLS keys too: extract in advance
+ if self._IMPERSONATE_TARGET is not None:
+ m3u8_data = self._download_webpage(
+ url, media_id, fatal=False, impersonate=self._IMPERSONATE_TARGET, headers=self._HEADERS,
+ note=join_nonempty('Downloading', format_id, 'm3u8 information', delim=' '),
+ errnote=join_nonempty('Failed to download', format_id, 'm3u8 information', delim=' '))
+ if not m3u8_data:
+ continue
+
+ key_url = self._search_regex(
+ r'#EXT-X-KEY:METHOD=AES-128,URI="(https?://[^"]+)"',
+ m3u8_data, 'HLS AES key URI', default=None)
+ if key_url:
+ urlh = self._request_webpage(
+ key_url, media_id, fatal=False, impersonate=self._IMPERSONATE_TARGET, headers=self._HEADERS,
+ note=join_nonempty('Downloading', format_id, 'HLS AES key', delim=' '),
+ errnote=join_nonempty('Failed to download', format_id, 'HLS AES key', delim=' '))
+ if urlh:
+ hls_aes['key'] = urlh.read().hex()
+
+ formats.append({
+ **traverse_obj(quality, {
+ 'format_note': ('label', {str}),
+ 'width': ('width', {int}),
+ 'height': ('height', {int}),
+ }),
+ **parse_codecs(quality.get('codecs')),
+ 'url': url,
+ 'ext': determine_ext(url.partition('/chunk.m3u8')[0], 'mp4'),
+ 'format_id': format_id,
+ 'hls_media_playlist_data': m3u8_data,
+ 'hls_aes': hls_aes or None,
+ })
+ items.append({
+ **common_info,
+ 'id': media_id,
+ **traverse_obj(metadata, {
+ 'title': ('title', {str}),
+ 'duration': ('duration', {int_or_none}),
+ 'thumbnail': ('thumbnail', 'path', {url_or_none}),
+ }),
+ 'formats': formats,
+ })
+
+ post_info = {
+ **common_info,
+ 'id': post_id,
+ 'display_id': post_id,
+ **traverse_obj(post_data, {
+ 'title': ('title', {str}),
+ 'description': ('text', {clean_html}),
+ 'like_count': ('likes', {int_or_none}),
+ 'dislike_count': ('dislikes', {int_or_none}),
+ 'comment_count': ('comments', {int_or_none}),
+ 'thumbnail': ('thumbnail', 'path', {url_or_none}),
+ }),
+ 'http_headers': self._HEADERS,
+ }
+
+ if len(items) > 1:
+ return self.playlist_result(items, **post_info)
+
+ post_info.update(items[0])
+ return post_info
+
+
+class FloatplaneIE(FloatplaneBaseIE):
_VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/post/(?P\w+)'
+ _BASE_URL = 'https://www.floatplane.com'
+ _IMPERSONATE_TARGET = None
+ _HEADERS = {
+ 'Origin': _BASE_URL,
+ 'Referer': f'{_BASE_URL}/',
+ }
_TESTS = [{
'url': 'https://www.floatplane.com/post/2Yf3UedF7C',
'info_dict': {
@@ -170,105 +302,9 @@ class FloatplaneIE(InfoExtractor):
}]
def _real_initialize(self):
- if not self._get_cookies('https://www.floatplane.com').get('sails.sid'):
+ if not self._get_cookies(self._BASE_URL).get('sails.sid'):
self.raise_login_required()
- def _real_extract(self, url):
- post_id = self._match_id(url)
-
- post_data = self._download_json(
- 'https://www.floatplane.com/api/v3/content/post', post_id, query={'id': post_id},
- note='Downloading post data', errnote='Unable to download post data')
-
- if not any(traverse_obj(post_data, ('metadata', ('hasVideo', 'hasAudio')))):
- raise ExtractorError('Post does not contain a video or audio track', expected=True)
-
- uploader_url = format_field(
- post_data, [('creator', 'urlname')], 'https://www.floatplane.com/channel/%s/home') or None
-
- common_info = {
- 'uploader_url': uploader_url,
- 'channel_url': urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname'))),
- 'availability': self._availability(needs_subscription=True),
- **traverse_obj(post_data, {
- 'uploader': ('creator', 'title', {str}),
- 'uploader_id': ('creator', 'id', {str}),
- 'channel': ('channel', 'title', {str}),
- 'channel_id': ('channel', 'id', {str}),
- 'release_timestamp': ('releaseDate', {parse_iso8601}),
- }),
- }
-
- items = []
- for media in traverse_obj(post_data, (('videoAttachments', 'audioAttachments'), ...)):
- media_id = media['id']
- media_typ = media.get('type') or 'video'
-
- metadata = self._download_json(
- f'https://www.floatplane.com/api/v3/content/{media_typ}', media_id, query={'id': media_id},
- note=f'Downloading {media_typ} metadata')
-
- stream = self._download_json(
- 'https://www.floatplane.com/api/v2/cdn/delivery', media_id, query={
- 'type': 'vod' if media_typ == 'video' else 'aod',
- 'guid': metadata['guid'],
- }, note=f'Downloading {media_typ} stream data')
-
- path_template = traverse_obj(stream, ('resource', 'uri', {str}))
-
- def format_path(params):
- path = path_template
- for i, val in (params or {}).items():
- path = path.replace(f'{{qualityLevelParams.{i}}}', val)
- return path
-
- formats = []
- for quality in traverse_obj(stream, ('resource', 'data', 'qualityLevels', ...)):
- url = urljoin(stream['cdn'], format_path(traverse_obj(
- stream, ('resource', 'data', 'qualityLevelParams', quality['name'], {dict}))))
- formats.append({
- **traverse_obj(quality, {
- 'format_id': ('name', {str}),
- 'format_note': ('label', {str}),
- 'width': ('width', {int}),
- 'height': ('height', {int}),
- }),
- **parse_codecs(quality.get('codecs')),
- 'url': url,
- 'ext': determine_ext(url.partition('/chunk.m3u8')[0], 'mp4'),
- })
-
- items.append({
- **common_info,
- 'id': media_id,
- **traverse_obj(metadata, {
- 'title': ('title', {str}),
- 'duration': ('duration', {int_or_none}),
- 'thumbnail': ('thumbnail', 'path', {url_or_none}),
- }),
- 'formats': formats,
- })
-
- post_info = {
- **common_info,
- 'id': post_id,
- 'display_id': post_id,
- **traverse_obj(post_data, {
- 'title': ('title', {str}),
- 'description': ('text', {clean_html}),
- 'like_count': ('likes', {int_or_none}),
- 'dislike_count': ('dislikes', {int_or_none}),
- 'comment_count': ('comments', {int_or_none}),
- 'thumbnail': ('thumbnail', 'path', {url_or_none}),
- }),
- }
-
- if len(items) > 1:
- return self.playlist_result(items, **post_info)
-
- post_info.update(items[0])
- return post_info
-
class FloatplaneChannelIE(InfoExtractor):
_VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/channel/(?P[\w-]+)/home(?:/(?P[\w-]+))?'
diff --git a/yt_dlp/extractor/francetv.py b/yt_dlp/extractor/francetv.py
index 5c9f8e36d..54c2c53ac 100644
--- a/yt_dlp/extractor/francetv.py
+++ b/yt_dlp/extractor/francetv.py
@@ -1,4 +1,3 @@
-import json
import re
import urllib.parse
@@ -19,7 +18,11 @@
unsmuggle_url,
url_or_none,
)
-from ..utils.traversal import find_element, traverse_obj
+from ..utils.traversal import (
+ find_element,
+ get_first,
+ traverse_obj,
+)
class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -121,9 +124,10 @@ def _extract_video(self, video_id, hostname=None):
elif code := traverse_obj(dinfo, ('code', {int})):
if code == 2009:
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
- elif code in (2015, 2017):
+ elif code in (2015, 2017, 2019):
# 2015: L'accès à cette vidéo est impossible. (DRM-only)
# 2017: Cette vidéo n'est pas disponible depuis le site web mobile (b/c DRM)
+ # 2019: L'accès à cette vidéo est incompatible avec votre configuration. (DRM-only)
drm_formats = True
continue
self.report_warning(
@@ -258,7 +262,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
_TESTS = [{
'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html',
'info_dict': {
- 'id': 'ec217ecc-0733-48cf-ac06-af1347b849d1', # old: c5bda21d-2c6f-4470-8849-3d8327adb2ba'
+ 'id': 'b2cf9fd8-e971-4757-8651-848f2772df61', # old: ec217ecc-0733-48cf-ac06-af1347b849d1
'ext': 'mp4',
'title': '13h15, le dimanche... - Les mystères de Jésus',
'timestamp': 1502623500,
@@ -269,7 +273,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
'params': {
'skip_download': True,
},
- 'add_ie': [FranceTVIE.ie_key()],
+ 'skip': 'Unfortunately, this video is no longer available',
}, {
# geo-restricted
'url': 'https://www.france.tv/enfants/six-huit-ans/foot2rue/saison-1/3066387-duel-au-vieux-port.html',
@@ -287,7 +291,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 1441,
},
- 'skip': 'No longer available',
+ 'skip': 'Unfortunately, this video is no longer available',
}, {
# geo-restricted livestream (workflow == 'token-akamai')
'url': 'https://www.france.tv/france-4/direct.html',
@@ -308,6 +312,19 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
'live_status': 'is_live',
},
'params': {'skip_download': 'livestream'},
+ }, {
+ # Not geo-restricted
+ 'url': 'https://www.france.tv/france-2/la-maison-des-maternelles/5574051-nous-sommes-amis-et-nous-avons-fait-un-enfant-ensemble.html',
+ 'info_dict': {
+ 'id': 'b448bfe4-9fe7-11ee-97d8-2ba3426fa3df',
+ 'ext': 'mp4',
+ 'title': 'Nous sommes amis et nous avons fait un enfant ensemble - Émission du jeudi 21 décembre 2023',
+ 'duration': 1065,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'timestamp': 1703147921,
+ 'upload_date': '20231221',
+ },
+ 'params': {'skip_download': 'm3u8'},
}, {
# france3
'url': 'https://www.france.tv/france-3/des-chiffres-et-des-lettres/139063-emission-du-mardi-9-mai-2017.html',
@@ -342,30 +359,16 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
'only_matching': True,
}]
- # XXX: For parsing next.js v15+ data; see also yt_dlp.extractor.goplay
- def _find_json(self, s):
- return self._search_json(
- r'\w+\s*:\s*', s, 'next js data', None, contains_pattern=r'\[(?s:.+)\]', default=None)
-
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
+ nextjs_data = self._search_nextjs_v13_data(webpage, display_id)
- nextjs_data = traverse_obj(
- re.findall(r']*>\s*self\.__next_f\.push\(\s*(\[.+?\])\s*\);?\s*', webpage),
- (..., {json.loads}, ..., {self._find_json}, ..., 'children', ..., ..., 'children', ..., ..., 'children'))
-
- if traverse_obj(nextjs_data, (..., ..., 'children', ..., 'isLive', {bool}, any)):
+ if get_first(nextjs_data, ('isLive', {bool})):
# For livestreams we need the id of the stream instead of the currently airing episode id
- video_id = traverse_obj(nextjs_data, (
- ..., ..., 'children', ..., 'children', ..., 'children', ..., 'children', ..., ...,
- 'children', ..., ..., 'children', ..., ..., 'children', (..., (..., ...)),
- 'options', 'id', {str}, any))
+ video_id = get_first(nextjs_data, ('options', 'id', {str}))
else:
- video_id = traverse_obj(nextjs_data, (
- ..., ..., ..., 'children',
- lambda _, v: v['video']['url'] == urllib.parse.urlparse(url).path,
- 'video', ('playerReplayId', 'siId'), {str}, any))
+ video_id = get_first(nextjs_data, ('video', ('playerReplayId', 'siId'), {str}))
if not video_id:
raise ExtractorError('Unable to extract video ID')
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 721d04e31..d9a666f99 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -1481,30 +1481,6 @@ class GenericIE(InfoExtractor):
},
'add_ie': ['SenateISVP'],
},
- {
- # Limelight embeds (1 channel embed + 4 media embeds)
- 'url': 'http://www.sedona.com/FacilitatorTraining2017',
- 'info_dict': {
- 'id': 'FacilitatorTraining2017',
- 'title': 'Facilitator Training 2017',
- },
- 'playlist_mincount': 5,
- },
- {
- # Limelight embed (LimelightPlayerUtil.embed)
- 'url': 'https://tv5.ca/videos?v=xuu8qowr291ri',
- 'info_dict': {
- 'id': '95d035dc5c8a401588e9c0e6bd1e9c92',
- 'ext': 'mp4',
- 'title': '07448641',
- 'timestamp': 1499890639,
- 'upload_date': '20170712',
- },
- 'params': {
- 'skip_download': True,
- },
- 'add_ie': ['LimelightMedia'],
- },
{
'url': 'http://kron4.com/2017/04/28/standoff-with-walnut-creek-murder-suspect-ends-with-arrest/',
'info_dict': {
diff --git a/yt_dlp/extractor/goplay.py b/yt_dlp/extractor/goplay.py
index c654c757c..2e959cead 100644
--- a/yt_dlp/extractor/goplay.py
+++ b/yt_dlp/extractor/goplay.py
@@ -5,16 +5,11 @@
import hmac
import json
import os
-import re
import urllib.parse
from .common import InfoExtractor
-from ..utils import (
- ExtractorError,
- int_or_none,
- remove_end,
- traverse_obj,
-)
+from ..utils import ExtractorError, int_or_none
+from ..utils.traversal import get_first, traverse_obj
class GoPlayIE(InfoExtractor):
@@ -27,10 +22,10 @@ class GoPlayIE(InfoExtractor):
'info_dict': {
'id': '2baa4560-87a0-421b-bffc-359914e3c387',
'ext': 'mp4',
- 'title': 'S22 - Aflevering 1',
+ 'title': 'De Slimste Mens ter Wereld - S22 - Aflevering 1',
'description': r're:In aflevering 1 nemen Daan Alferink, Tess Elst en Xander De Rycke .{66}',
'series': 'De Slimste Mens ter Wereld',
- 'episode': 'Episode 1',
+ 'episode': 'Wordt aangekondigd',
'season_number': 22,
'episode_number': 1,
'season': 'Season 22',
@@ -52,7 +47,7 @@ class GoPlayIE(InfoExtractor):
'info_dict': {
'id': 'ecb79672-92b9-4cd9-a0d7-e2f0250681ee',
'ext': 'mp4',
- 'title': 'S11 - Aflevering 1',
+ 'title': 'De Mol - S11 - Aflevering 1',
'description': r're:Tien kandidaten beginnen aan hun verovering van Amerika en ontmoeten .{102}',
'episode': 'Episode 1',
'series': 'De Mol',
@@ -75,21 +70,13 @@ def _real_initialize(self):
if not self._id_token:
raise self.raise_login_required(method='password')
- # XXX: For parsing next.js v15+ data; see also yt_dlp.extractor.francetv
- def _find_json(self, s):
- return self._search_json(
- r'\w+\s*:\s*', s, 'next js data', None, contains_pattern=r'\[(?s:.+)\]', default=None)
-
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
- nextjs_data = traverse_obj(
- re.findall(r']*>\s*self\.__next_f\.push\(\s*(\[.+?\])\s*\);?\s*', webpage),
- (..., {json.loads}, ..., {self._find_json}, ...))
- meta = traverse_obj(nextjs_data, (
- ..., ..., 'children', ..., ..., 'children',
- lambda _, v: v['video']['path'] == urllib.parse.urlparse(url).path, 'video', any))
+ nextjs_data = self._search_nextjs_v13_data(webpage, display_id)
+ meta = get_first(nextjs_data, (
+ lambda k, v: k in ('video', 'meta') and v['path'] == urllib.parse.urlparse(url).path))
video_id = meta['uuid']
info_dict = traverse_obj(meta, {
@@ -98,19 +85,18 @@ def _real_extract(self, url):
})
if traverse_obj(meta, ('program', 'subtype')) != 'movie':
- for season_data in traverse_obj(nextjs_data, (..., 'children', ..., 'playlists', ...)):
- episode_data = traverse_obj(
- season_data, ('videos', lambda _, v: v['videoId'] == video_id, any))
+ for season_data in traverse_obj(nextjs_data, (..., 'playlists', ..., {dict})):
+ episode_data = traverse_obj(season_data, ('videos', lambda _, v: v['videoId'] == video_id, any))
if not episode_data:
continue
- episode_title = traverse_obj(
- episode_data, 'contextualTitle', 'episodeTitle', expected_type=str)
+ season_number = traverse_obj(season_data, ('season', {int_or_none}))
info_dict.update({
- 'title': episode_title or info_dict.get('title'),
- 'series': remove_end(info_dict.get('title'), f' - {episode_title}'),
- 'season_number': traverse_obj(season_data, ('season', {int_or_none})),
+ 'episode': traverse_obj(episode_data, ('episodeTitle', {str})),
'episode_number': traverse_obj(episode_data, ('episodeNumber', {int_or_none})),
+ 'season_number': season_number,
+ 'series': self._search_regex(
+ fr'^(.+)? - S{season_number} - ', info_dict.get('title'), 'series', default=None),
})
break
diff --git a/yt_dlp/extractor/hotstar.py b/yt_dlp/extractor/hotstar.py
index e97740c90..b280fb53a 100644
--- a/yt_dlp/extractor/hotstar.py
+++ b/yt_dlp/extractor/hotstar.py
@@ -1,3 +1,4 @@
+import functools
import hashlib
import hmac
import json
@@ -9,77 +10,125 @@
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
+ OnDemandPagedList,
determine_ext,
+ filter_dict,
int_or_none,
join_nonempty,
+ jwt_decode_hs256,
+ parse_iso8601,
str_or_none,
- traverse_obj,
url_or_none,
)
+from ..utils.traversal import require, traverse_obj
class HotStarBaseIE(InfoExtractor):
+ _TOKEN_NAME = 'userUP'
_BASE_URL = 'https://www.hotstar.com'
_API_URL = 'https://api.hotstar.com'
+ _API_URL_V2 = 'https://apix.hotstar.com/v2'
_AKAMAI_ENCRYPTION_KEY = b'\x05\xfc\x1a\x01\xca\xc9\x4b\xc4\x12\xfc\x53\x12\x07\x75\xf9\xee'
+ _FREE_HEADERS = {
+ 'user-agent': 'Hotstar;in.startv.hotstar/25.06.30.0.11580 (Android/12)',
+ 'x-hs-client': 'platform:android;app_id:in.startv.hotstar;app_version:25.06.30.0;os:Android;os_version:12;schema_version:0.0.1523',
+ 'x-hs-platform': 'android',
+ }
+ _SUB_HEADERS = {
+ 'user-agent': 'Disney+;in.startv.hotstar.dplus.tv/23.08.14.4.2915 (Android/13)',
+ 'x-hs-client': 'platform:androidtv;app_id:in.startv.hotstar.dplus.tv;app_version:23.08.14.4;os:Android;os_version:13;schema_version:0.0.970',
+ 'x-hs-platform': 'androidtv',
+ }
+
+ def _has_active_subscription(self, cookies, server_time):
+ expiry = traverse_obj(cookies, (
+ self._TOKEN_NAME, 'value', {jwt_decode_hs256}, 'sub', {json.loads},
+ 'subscriptions', 'in', ..., 'expiry', {parse_iso8601}, all, {max})) or 0
+ return expiry > server_time
+
def _call_api_v1(self, path, *args, **kwargs):
return self._download_json(
f'{self._API_URL}/o/v1/{path}', *args, **kwargs,
headers={'x-country-code': 'IN', 'x-platform-code': 'PCTV'})
- def _call_api_impl(self, path, video_id, query, st=None, cookies=None):
+ def _call_api_impl(self, path, video_id, query, cookies=None, st=None):
st = int_or_none(st) or int(time.time())
exp = st + 6000
auth = f'st={st}~exp={exp}~acl=/*'
auth += '~hmac=' + hmac.new(self._AKAMAI_ENCRYPTION_KEY, auth.encode(), hashlib.sha256).hexdigest()
-
- if cookies and cookies.get('userUP'):
- token = cookies.get('userUP').value
- else:
- token = self._download_json(
- f'{self._API_URL}/um/v3/users',
- video_id, note='Downloading token',
- data=json.dumps({'device_ids': [{'id': str(uuid.uuid4()), 'type': 'device_id'}]}).encode(),
- headers={
- 'hotstarauth': auth,
- 'x-hs-platform': 'PCTV', # or 'web'
- 'Content-Type': 'application/json',
- })['user_identity']
-
response = self._download_json(
- f'{self._API_URL}/{path}', video_id, query=query,
- headers={
+ f'{self._API_URL_V2}/{path}', video_id, query=query,
+ headers=filter_dict({
+ **(self._SUB_HEADERS if self._has_active_subscription(cookies, st) else self._FREE_HEADERS),
'hotstarauth': auth,
- 'x-hs-appversion': '6.72.2',
- 'x-hs-platform': 'web',
- 'x-hs-usertoken': token,
- })
+ 'x-hs-usertoken': traverse_obj(cookies, (self._TOKEN_NAME, 'value')),
+ 'x-hs-device-id': traverse_obj(cookies, ('deviceId', 'value')) or str(uuid.uuid4()),
+ 'content-type': 'application/json',
+ }))
- if response['message'] != "Playback URL's fetched successfully":
- raise ExtractorError(
- response['message'], expected=True)
- return response['data']
+ if not traverse_obj(response, ('success', {dict})):
+ raise ExtractorError('API call was unsuccessful')
+ return response['success']
- def _call_api_v2(self, path, video_id, st=None, cookies=None):
- return self._call_api_impl(
- f'{path}/content/{video_id}', video_id, st=st, cookies=cookies, query={
- 'desired-config': 'audio_channel:stereo|container:fmp4|dynamic_range:hdr|encryption:plain|ladder:tv|package:dash|resolution:fhd|subs-tag:HotstarVIP|video_codec:h265',
- 'device-id': cookies.get('device_id').value if cookies.get('device_id') else str(uuid.uuid4()),
- 'os-name': 'Windows',
- 'os-version': '10',
- })
+ def _call_api_v2(self, path, video_id, content_type, cookies=None, st=None):
+ return self._call_api_impl(f'{path}', video_id, query={
+ 'content_id': video_id,
+ 'filters': f'content_type={content_type}',
+ 'client_capabilities': json.dumps({
+ 'package': ['dash', 'hls'],
+ 'container': ['fmp4', 'fmp4br', 'ts'],
+ 'ads': ['non_ssai', 'ssai'],
+ 'audio_channel': ['stereo', 'dolby51', 'atmos'],
+ 'encryption': ['plain', 'widevine'], # wv only so we can raise appropriate error
+ 'video_codec': ['h264', 'h265'],
+ 'video_codec_non_secure': ['h264', 'h265', 'vp9'],
+ 'ladder': ['phone', 'tv', 'full'],
+ 'resolution': ['hd', '4k'],
+ 'true_resolution': ['hd', '4k'],
+ 'dynamic_range': ['sdr', 'hdr'],
+ }, separators=(',', ':')),
+ 'drm_parameters': json.dumps({
+ 'widevine_security_level': ['SW_SECURE_DECODE', 'SW_SECURE_CRYPTO'],
+ 'hdcp_version': ['HDCP_V2_2', 'HDCP_V2_1', 'HDCP_V2', 'HDCP_V1'],
+ }, separators=(',', ':')),
+ }, cookies=cookies, st=st)
- def _playlist_entries(self, path, item_id, root=None, **kwargs):
- results = self._call_api_v1(path, item_id, **kwargs)['body']['results']
- for video in traverse_obj(results, (('assets', None), 'items', ...)):
- if video.get('contentId'):
- yield self.url_result(
- HotStarIE._video_url(video['contentId'], root=root), HotStarIE, video['contentId'])
+ @staticmethod
+ def _parse_metadata_v1(video_data):
+ return traverse_obj(video_data, {
+ 'id': ('contentId', {str}),
+ 'title': ('title', {str}),
+ 'description': ('description', {str}),
+ 'duration': ('duration', {int_or_none}),
+ 'timestamp': (('broadcastDate', 'startDate'), {int_or_none}, any),
+ 'release_year': ('year', {int_or_none}),
+ 'channel': ('channelName', {str}),
+ 'channel_id': ('channelId', {int}, {str_or_none}),
+ 'series': ('showName', {str}),
+ 'season': ('seasonName', {str}),
+ 'season_number': ('seasonNo', {int_or_none}),
+ 'season_id': ('seasonId', {int}, {str_or_none}),
+ 'episode': ('title', {str}),
+ 'episode_number': ('episodeNo', {int_or_none}),
+ })
+
+ def _fetch_page(self, path, item_id, name, query, root, page):
+ results = self._call_api_v1(
+ path, item_id, note=f'Downloading {name} page {page + 1} JSON', query={
+ **query,
+ 'tao': page * self._PAGE_SIZE,
+ 'tas': self._PAGE_SIZE,
+ })['body']['results']
+
+ for video in traverse_obj(results, (('assets', None), 'items', lambda _, v: v['contentId'])):
+ yield self.url_result(
+ HotStarIE._video_url(video['contentId'], root=root), HotStarIE, **self._parse_metadata_v1(video))
class HotStarIE(HotStarBaseIE):
IE_NAME = 'hotstar'
+ IE_DESC = 'JioHotstar'
_VALID_URL = r'''(?x)
https?://(?:www\.)?hotstar\.com(?:/in)?/(?!in/)
(?:
@@ -114,15 +163,16 @@ class HotStarIE(HotStarBaseIE):
'upload_date': '20190501',
'duration': 1219,
'channel': 'StarPlus',
- 'channel_id': '3',
+ 'channel_id': '821',
'series': 'Ek Bhram - Sarvagun Sampanna',
'season': 'Chapter 1',
'season_number': 1,
- 'season_id': '6771',
+ 'season_id': '1260004607',
'episode': 'Janhvi Targets Suman',
'episode_number': 8,
},
- }, {
+ 'params': {'skip_download': 'm3u8'},
+ }, { # Metadata call gets HTTP Error 504 with tas=10000
'url': 'https://www.hotstar.com/in/shows/anupama/1260022017/anupama-anuj-share-a-moment/1000282843',
'info_dict': {
'id': '1000282843',
@@ -134,14 +184,14 @@ class HotStarIE(HotStarBaseIE):
'channel': 'StarPlus',
'series': 'Anupama',
'season_number': 1,
- 'season_id': '7399',
+ 'season_id': '1260022018',
'upload_date': '20230307',
'episode': 'Anupama, Anuj Share a Moment',
'episode_number': 853,
- 'duration': 1272,
- 'channel_id': '3',
+ 'duration': 1266,
+ 'channel_id': '821',
},
- 'skip': 'HTTP Error 504: Gateway Time-out', # XXX: Investigate 504 errors on some episodes
+ 'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.hotstar.com/in/shows/kana-kaanum-kaalangal/1260097087/back-to-school/1260097320',
'info_dict': {
@@ -154,14 +204,15 @@ class HotStarIE(HotStarBaseIE):
'channel': 'Hotstar Specials',
'series': 'Kana Kaanum Kaalangal',
'season_number': 1,
- 'season_id': '9441',
+ 'season_id': '1260097089',
'upload_date': '20220421',
'episode': 'Back To School',
'episode_number': 1,
'duration': 1810,
- 'channel_id': '54',
+ 'channel_id': '1260003991',
},
- }, {
+ 'params': {'skip_download': 'm3u8'},
+ }, { # Metadata call gets HTTP Error 504 with tas=10000
'url': 'https://www.hotstar.com/in/clips/e3-sairat-kahani-pyaar-ki/1000262286',
'info_dict': {
'id': '1000262286',
@@ -173,6 +224,7 @@ class HotStarIE(HotStarBaseIE):
'timestamp': 1622943900,
'duration': 5395,
},
+ 'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.hotstar.com/in/movies/premam/1000091195',
'info_dict': {
@@ -180,12 +232,13 @@ class HotStarIE(HotStarBaseIE):
'ext': 'mp4',
'title': 'Premam',
'release_year': 2015,
- 'description': 'md5:d833c654e4187b5e34757eafb5b72d7f',
+ 'description': 'md5:096cd8aaae8dab56524823dc19dfa9f7',
'timestamp': 1462149000,
'upload_date': '20160502',
'episode': 'Premam',
'duration': 8994,
},
+ 'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.hotstar.com/movies/radha-gopalam/1000057157',
'only_matching': True,
@@ -208,6 +261,13 @@ class HotStarIE(HotStarBaseIE):
None: 'content',
}
+ _CONTENT_TYPE = {
+ 'movie': 'MOVIE',
+ 'episode': 'EPISODE',
+ 'match': 'SPORT',
+ 'content': 'CLIPS',
+ }
+
_IGNORE_MAP = {
'res': 'resolution',
'vcodec': 'video_codec',
@@ -229,38 +289,50 @@ def _video_url(cls, video_id, video_type=None, *, slug='ignore_me', root=None):
def _real_extract(self, url):
video_id, video_type = self._match_valid_url(url).group('id', 'type')
- video_type = self._TYPE.get(video_type, video_type)
+ video_type = self._TYPE[video_type]
cookies = self._get_cookies(url) # Cookies before any request
+ if not cookies or not cookies.get(self._TOKEN_NAME):
+ self.raise_login_required()
video_data = traverse_obj(
- self._call_api_v1(
- f'{video_type}/detail', video_id, fatal=False, query={'tas': 10000, 'contentId': video_id}),
- ('body', 'results', 'item', {dict})) or {}
- if not self.get_param('allow_unplayable_formats') and video_data.get('drmProtected'):
+ self._call_api_v1(f'{video_type}/detail', video_id, fatal=False, query={
+ 'tas': 5, # See https://github.com/yt-dlp/yt-dlp/issues/7946
+ 'contentId': video_id,
+ }), ('body', 'results', 'item', {dict})) or {}
+
+ if video_data.get('drmProtected'):
self.report_drm(video_id)
- # See https://github.com/yt-dlp/yt-dlp/issues/396
- st = self._download_webpage_handle(f'{self._BASE_URL}/in', video_id)[1].headers.get('x-origin-date')
-
geo_restricted = False
- formats, subs = [], {}
+ formats, subs, has_drm = [], {}, False
headers = {'Referer': f'{self._BASE_URL}/in'}
+ content_type = traverse_obj(video_data, ('contentType', {str})) or self._CONTENT_TYPE[video_type]
- # change to v2 in the future
- playback_sets = self._call_api_v2('play/v1/playback', video_id, st=st, cookies=cookies)['playBackSets']
- for playback_set in playback_sets:
- if not isinstance(playback_set, dict):
- continue
- tags = str_or_none(playback_set.get('tagsCombination')) or ''
+ # See https://github.com/yt-dlp/yt-dlp/issues/396
+ st = self._request_webpage(
+ f'{self._BASE_URL}/in', video_id, 'Fetching server time').get_header('x-origin-date')
+ watch = self._call_api_v2('pages/watch', video_id, content_type, cookies, st)
+ player_config = traverse_obj(watch, (
+ 'page', 'spaces', 'player', 'widget_wrappers', lambda _, v: v['template'] == 'PlayerWidget',
+ 'widget', 'data', 'player_config', {dict}, any, {require('player config')}))
+
+ for playback_set in traverse_obj(player_config, (
+ ('media_asset', 'media_asset_v2'),
+ ('primary', 'fallback'),
+ all, lambda _, v: url_or_none(v['content_url']),
+ )):
+ tags = str_or_none(playback_set.get('playback_tags')) or ''
if any(f'{prefix}:{ignore}' in tags
for key, prefix in self._IGNORE_MAP.items()
for ignore in self._configuration_arg(key)):
continue
- format_url = url_or_none(playback_set.get('playbackUrl'))
- if not format_url:
+ tag_dict = dict((*t.split(':', 1), None)[:2] for t in tags.split(';'))
+ if tag_dict.get('encryption') not in ('plain', None):
+ has_drm = True
continue
- format_url = re.sub(r'(?<=//staragvod)(\d)', r'web\1', format_url)
+
+ format_url = re.sub(r'(?<=//staragvod)(\d)', r'web\1', playback_set['content_url'])
ext = determine_ext(format_url)
current_formats, current_subs = [], {}
@@ -280,14 +352,12 @@ def _real_extract(self, url):
'height': int_or_none(playback_set.get('height')),
}]
except ExtractorError as e:
- if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status in (403, 474):
geo_restricted = True
+ else:
+ self.write_debug(e)
continue
- tag_dict = dict((*t.split(':', 1), None)[:2] for t in tags.split(';'))
- if tag_dict.get('encryption') not in ('plain', None):
- for f in current_formats:
- f['has_drm'] = True
for f in current_formats:
for k, v in self._TAG_FIELDS.items():
if not f.get(k):
@@ -299,6 +369,11 @@ def _real_extract(self, url):
'stereo': 2,
'dolby51': 6,
}.get(tag_dict.get('audio_channel'))
+ if (
+ 'Audio_Description' in f['format_id']
+ or 'Audio Description' in (f.get('format_note') or '')
+ ):
+ f['source_preference'] = -99 + (f.get('source_preference') or -1)
f['format_note'] = join_nonempty(
tag_dict.get('ladder'),
tag_dict.get('audio_channel') if f.get('acodec') != 'none' else None,
@@ -308,29 +383,22 @@ def _real_extract(self, url):
formats.extend(current_formats)
subs = self._merge_subtitles(subs, current_subs)
- if not formats and geo_restricted:
- self.raise_geo_restricted(countries=['IN'], metadata_available=True)
+ if not formats:
+ if geo_restricted:
+ self.raise_geo_restricted(countries=['IN'], metadata_available=True)
+ elif has_drm:
+ self.report_drm(video_id)
+ elif not self._has_active_subscription(cookies, st):
+ self.raise_no_formats('Your account does not have access to this content', expected=True)
self._remove_duplicate_formats(formats)
for f in formats:
f.setdefault('http_headers', {}).update(headers)
return {
+ **self._parse_metadata_v1(video_data),
'id': video_id,
- 'title': video_data.get('title'),
- 'description': video_data.get('description'),
- 'duration': int_or_none(video_data.get('duration')),
- 'timestamp': int_or_none(traverse_obj(video_data, 'broadcastDate', 'startDate')),
- 'release_year': int_or_none(video_data.get('year')),
'formats': formats,
'subtitles': subs,
- 'channel': video_data.get('channelName'),
- 'channel_id': str_or_none(video_data.get('channelId')),
- 'series': video_data.get('showName'),
- 'season': video_data.get('seasonName'),
- 'season_number': int_or_none(video_data.get('seasonNo')),
- 'season_id': str_or_none(video_data.get('seasonId')),
- 'episode': video_data.get('title'),
- 'episode_number': int_or_none(video_data.get('episodeNo')),
}
@@ -371,64 +439,6 @@ def _real_extract(self, url):
return self.url_result(HotStarIE._video_url(video_id, video_type), HotStarIE, video_id)
-class HotStarPlaylistIE(HotStarBaseIE):
- IE_NAME = 'hotstar:playlist'
- _VALID_URL = r'https?://(?:www\.)?hotstar\.com(?:/in)?/(?:tv|shows)(?:/[^/]+){2}/list/[^/]+/t-(?P\w+)'
- _TESTS = [{
- 'url': 'https://www.hotstar.com/tv/savdhaan-india/s-26/list/popular-clips/t-3_2_26',
- 'info_dict': {
- 'id': '3_2_26',
- },
- 'playlist_mincount': 20,
- }, {
- 'url': 'https://www.hotstar.com/shows/savdhaan-india/s-26/list/popular-clips/t-3_2_26',
- 'only_matching': True,
- }, {
- 'url': 'https://www.hotstar.com/tv/savdhaan-india/s-26/list/extras/t-2480',
- 'only_matching': True,
- }, {
- 'url': 'https://www.hotstar.com/in/tv/karthika-deepam/15457/list/popular-clips/t-3_2_1272',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- id_ = self._match_id(url)
- return self.playlist_result(
- self._playlist_entries('tray/find', id_, query={'tas': 10000, 'uqId': id_}), id_)
-
-
-class HotStarSeasonIE(HotStarBaseIE):
- IE_NAME = 'hotstar:season'
- _VALID_URL = r'(?Phttps?://(?:www\.)?hotstar\.com(?:/in)?/(?:tv|shows)/[^/]+/\w+)/seasons/[^/]+/ss-(?P\w+)'
- _TESTS = [{
- 'url': 'https://www.hotstar.com/tv/radhakrishn/1260000646/seasons/season-2/ss-8028',
- 'info_dict': {
- 'id': '8028',
- },
- 'playlist_mincount': 35,
- }, {
- 'url': 'https://www.hotstar.com/in/tv/ishqbaaz/9567/seasons/season-2/ss-4357',
- 'info_dict': {
- 'id': '4357',
- },
- 'playlist_mincount': 30,
- }, {
- 'url': 'https://www.hotstar.com/in/tv/bigg-boss/14714/seasons/season-4/ss-8208/',
- 'info_dict': {
- 'id': '8208',
- },
- 'playlist_mincount': 19,
- }, {
- 'url': 'https://www.hotstar.com/in/shows/bigg-boss/14714/seasons/season-4/ss-8208/',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- url, season_id = self._match_valid_url(url).groups()
- return self.playlist_result(self._playlist_entries(
- 'season/asset', season_id, url, query={'tao': 0, 'tas': 0, 'size': 10000, 'id': season_id}), season_id)
-
-
class HotStarSeriesIE(HotStarBaseIE):
IE_NAME = 'hotstar:series'
_VALID_URL = r'(?Phttps?://(?:www\.)?hotstar\.com(?:/in)?/(?:tv|shows)/[^/]+/(?P\d+))/?(?:[#?]|$)'
@@ -443,25 +453,29 @@ class HotStarSeriesIE(HotStarBaseIE):
'info_dict': {
'id': '1260050431',
},
- 'playlist_mincount': 43,
+ 'playlist_mincount': 42,
}, {
'url': 'https://www.hotstar.com/in/tv/mahabharat/435/',
'info_dict': {
'id': '435',
},
'playlist_mincount': 267,
- }, {
+ }, { # HTTP Error 504 with tas=10000 (possibly because total size is over 1000 items?)
'url': 'https://www.hotstar.com/in/shows/anupama/1260022017/',
'info_dict': {
'id': '1260022017',
},
- 'playlist_mincount': 940,
+ 'playlist_mincount': 1601,
}]
+ _PAGE_SIZE = 100
def _real_extract(self, url):
- url, series_id = self._match_valid_url(url).groups()
- id_ = self._call_api_v1(
+ url, series_id = self._match_valid_url(url).group('url', 'id')
+ eid = self._call_api_v1(
'show/detail', series_id, query={'contentId': series_id})['body']['results']['item']['id']
- return self.playlist_result(self._playlist_entries(
- 'tray/g/1/items', series_id, url, query={'tao': 0, 'tas': 10000, 'etid': 0, 'eid': id_}), series_id)
+ entries = OnDemandPagedList(functools.partial(
+ self._fetch_page, 'tray/g/1/items', series_id,
+ 'series', {'etid': 0, 'eid': eid}, url), self._PAGE_SIZE)
+
+ return self.playlist_result(entries, series_id)
diff --git a/yt_dlp/extractor/huya.py b/yt_dlp/extractor/huya.py
index f79e032e4..65099b062 100644
--- a/yt_dlp/extractor/huya.py
+++ b/yt_dlp/extractor/huya.py
@@ -7,12 +7,13 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
+ clean_html,
int_or_none,
parse_duration,
str_or_none,
try_get,
unescapeHTML,
- unified_strdate,
+ update_url,
update_url_query,
url_or_none,
)
@@ -22,8 +23,8 @@
class HuyaLiveIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.|m\.)?huya\.com/(?!(?:video/play/))(?P[^/#?&]+)(?:\D|$)'
IE_NAME = 'huya:live'
- IE_DESC = 'huya.com'
- TESTS = [{
+ IE_DESC = '虎牙直播'
+ _TESTS = [{
'url': 'https://www.huya.com/572329',
'info_dict': {
'id': '572329',
@@ -149,63 +150,94 @@ class HuyaVideoIE(InfoExtractor):
'id': '1002412640',
'ext': 'mp4',
'title': '8月3日',
- 'thumbnail': r're:https?://.*\.jpg',
- 'duration': 14,
+ 'categories': ['主机游戏'],
+ 'duration': 14.0,
'uploader': '虎牙-ATS欧卡车队青木',
'uploader_id': '1564376151',
'upload_date': '20240803',
'view_count': int,
'comment_count': int,
'like_count': int,
+ 'thumbnail': r're:https?://.+\.jpg',
+ 'timestamp': 1722675433,
},
- },
- {
+ }, {
'url': 'https://www.huya.com/video/play/556054543.html',
'info_dict': {
'id': '556054543',
'ext': 'mp4',
'title': '我不挑事 也不怕事',
- 'thumbnail': r're:https?://.*\.jpg',
- 'duration': 1864,
+ 'categories': ['英雄联盟'],
+ 'description': 'md5:58184869687d18ce62dc7b4b2ad21201',
+ 'duration': 1864.0,
'uploader': '卡尔',
'uploader_id': '367138632',
'upload_date': '20210811',
'view_count': int,
'comment_count': int,
'like_count': int,
+ 'tags': 'count:4',
+ 'thumbnail': r're:https?://.+\.jpg',
+ 'timestamp': 1628675950,
+ },
+ }, {
+ # Only m3u8 available
+ 'url': 'https://www.huya.com/video/play/1063345618.html',
+ 'info_dict': {
+ 'id': '1063345618',
+ 'ext': 'mp4',
+ 'title': '峡谷第一中！黑铁上钻石顶级教学对抗elo',
+ 'categories': ['英雄联盟'],
+ 'comment_count': int,
+ 'duration': 21603.0,
+ 'like_count': int,
+ 'thumbnail': r're:https?://.+\.jpg',
+ 'timestamp': 1749668803,
+ 'upload_date': '20250611',
+ 'uploader': '北枫CC',
+ 'uploader_id': '2183525275',
+ 'view_count': int,
},
}]
def _real_extract(self, url: str):
video_id = self._match_id(url)
- video_data = self._download_json(
- 'https://liveapi.huya.com/moment/getMomentContent', video_id,
- query={'videoId': video_id})['data']['moment']['videoInfo']
+ moment = self._download_json(
+ 'https://liveapi.huya.com/moment/getMomentContent',
+ video_id, query={'videoId': video_id})['data']['moment']
formats = []
- for definition in traverse_obj(video_data, ('definitions', lambda _, v: url_or_none(v['url']))):
- formats.append({
- 'url': definition['url'],
- **traverse_obj(definition, {
- 'format_id': ('defName', {str}),
- 'width': ('width', {int_or_none}),
- 'height': ('height', {int_or_none}),
+ for definition in traverse_obj(moment, (
+ 'videoInfo', 'definitions', lambda _, v: url_or_none(v['m3u8']),
+ )):
+ fmts = self._extract_m3u8_formats(definition['m3u8'], video_id, 'mp4', fatal=False)
+ for fmt in fmts:
+ fmt.update(**traverse_obj(definition, {
'filesize': ('size', {int_or_none}),
- }),
- })
+ 'format_id': ('defName', {str}),
+ 'height': ('height', {int_or_none}),
+ 'quality': ('definition', {int_or_none}),
+ 'width': ('width', {int_or_none}),
+ }))
+ formats.extend(fmts)
return {
'id': video_id,
'formats': formats,
- **traverse_obj(video_data, {
+ **traverse_obj(moment, {
+ 'comment_count': ('commentCount', {int_or_none}),
+ 'description': ('content', {clean_html}, filter),
+ 'like_count': ('favorCount', {int_or_none}),
+ 'timestamp': ('cTime', {int_or_none}),
+ }),
+ **traverse_obj(moment, ('videoInfo', {
'title': ('videoTitle', {str}),
- 'thumbnail': ('videoCover', {url_or_none}),
+ 'categories': ('category', {str}, filter, all, filter),
'duration': ('videoDuration', {parse_duration}),
+ 'tags': ('tags', ..., {str}, filter, all, filter),
+ 'thumbnail': (('videoBigCover', 'videoCover'), {url_or_none}, {update_url(query=None)}, any),
'uploader': ('nickName', {str}),
'uploader_id': ('uid', {str_or_none}),
- 'upload_date': ('videoUploadTime', {unified_strdate}),
'view_count': ('videoPlayNum', {int_or_none}),
- 'comment_count': ('videoCommentNum', {int_or_none}),
- 'like_count': ('favorCount', {int_or_none}),
- }),
+ })),
}
diff --git a/yt_dlp/extractor/hypergryph.py b/yt_dlp/extractor/hypergryph.py
index 1fb2e9a98..f405d14b5 100644
--- a/yt_dlp/extractor/hypergryph.py
+++ b/yt_dlp/extractor/hypergryph.py
@@ -1,32 +1,66 @@
from .common import InfoExtractor
-from ..utils import js_to_json, traverse_obj
+from ..utils import (
+ ExtractorError,
+ clean_html,
+ url_or_none,
+)
+from ..utils.traversal import subs_list_to_dict, traverse_obj
class MonsterSirenHypergryphMusicIE(InfoExtractor):
+ IE_NAME = 'monstersiren'
+ IE_DESC = '塞壬唱片'
+ _API_BASE = 'https://monster-siren.hypergryph.com/api'
_VALID_URL = r'https?://monster-siren\.hypergryph\.com/music/(?P\d+)'
_TESTS = [{
'url': 'https://monster-siren.hypergryph.com/music/514562',
'info_dict': {
'id': '514562',
'ext': 'wav',
- 'artists': ['塞壬唱片-MSR'],
- 'album': 'Flame Shadow',
'title': 'Flame Shadow',
+ 'album': 'Flame Shadow',
+ 'artists': ['塞壬唱片-MSR'],
+ 'description': 'md5:19e2acfcd1b65b41b29e8079ab948053',
+ 'thumbnail': r're:https?://web\.hycdn\.cn/siren/pic/.+\.jpg',
+ },
+ }, {
+ 'url': 'https://monster-siren.hypergryph.com/music/514518',
+ 'info_dict': {
+ 'id': '514518',
+ 'ext': 'wav',
+ 'title': 'Heavenly Me (Instrumental)',
+ 'album': 'Heavenly Me',
+ 'artists': ['塞壬唱片-MSR', 'AIYUE blessed : 理名'],
+ 'description': 'md5:ce790b41c932d1ad72eb791d1d8ae598',
+ 'thumbnail': r're:https?://web\.hycdn\.cn/siren/pic/.+\.jpg',
},
}]
def _real_extract(self, url):
audio_id = self._match_id(url)
- webpage = self._download_webpage(url, audio_id)
- json_data = self._search_json(
- r'window\.g_initialProps\s*=', webpage, 'data', audio_id, transform_source=js_to_json)
+ song = self._download_json(f'{self._API_BASE}/song/{audio_id}', audio_id)
+ if traverse_obj(song, 'code') != 0:
+ msg = traverse_obj(song, ('msg', {str}, filter))
+ raise ExtractorError(
+ msg or 'API returned an error response', expected=bool(msg))
+
+ album = None
+ if album_id := traverse_obj(song, ('data', 'albumCid', {str})):
+ album = self._download_json(
+ f'{self._API_BASE}/album/{album_id}/detail', album_id, fatal=False)
return {
'id': audio_id,
- 'title': traverse_obj(json_data, ('player', 'songDetail', 'name')),
- 'url': traverse_obj(json_data, ('player', 'songDetail', 'sourceUrl')),
- 'ext': 'wav',
'vcodec': 'none',
- 'artists': traverse_obj(json_data, ('player', 'songDetail', 'artists', ...)),
- 'album': traverse_obj(json_data, ('musicPlay', 'albumDetail', 'name')),
+ **traverse_obj(song, ('data', {
+ 'title': ('name', {str}),
+ 'artists': ('artists', ..., {str}),
+ 'subtitles': ({'url': 'lyricUrl'}, all, {subs_list_to_dict(lang='en')}),
+ 'url': ('sourceUrl', {url_or_none}),
+ })),
+ **traverse_obj(album, ('data', {
+ 'album': ('name', {str}),
+ 'description': ('intro', {clean_html}),
+ 'thumbnail': ('coverUrl', {url_or_none}),
+ })),
}
diff --git a/yt_dlp/extractor/jiocinema.py b/yt_dlp/extractor/jiocinema.py
deleted file mode 100644
index 94c85064e..000000000
--- a/yt_dlp/extractor/jiocinema.py
+++ /dev/null
@@ -1,408 +0,0 @@
-import base64
-import itertools
-import json
-import random
-import re
-import string
-import time
-
-from .common import InfoExtractor
-from ..utils import (
- ExtractorError,
- float_or_none,
- int_or_none,
- jwt_decode_hs256,
- parse_age_limit,
- try_call,
- url_or_none,
-)
-from ..utils.traversal import traverse_obj
-
-
-class JioCinemaBaseIE(InfoExtractor):
- _NETRC_MACHINE = 'jiocinema'
- _GEO_BYPASS = False
- _ACCESS_TOKEN = None
- _REFRESH_TOKEN = None
- _GUEST_TOKEN = None
- _USER_ID = None
- _DEVICE_ID = None
- _API_HEADERS = {'Origin': 'https://www.jiocinema.com', 'Referer': 'https://www.jiocinema.com/'}
- _APP_NAME = {'appName': 'RJIL_JioCinema'}
- _APP_VERSION = {'appVersion': '5.0.0'}
- _API_SIGNATURES = 'o668nxgzwff'
- _METADATA_API_BASE = 'https://content-jiovoot.voot.com/psapi'
- _ACCESS_HINT = 'the `accessToken` from your browser local storage'
- _LOGIN_HINT = (
- 'Log in with "-u phone -p " to authenticate with OTP, '
- f'or use "-u token -p " to log in with {_ACCESS_HINT}. '
- 'If you have previously logged in with yt-dlp and your session '
- 'has been cached, you can use "-u device -p "')
-
- def _cache_token(self, token_type):
- assert token_type in ('access', 'refresh', 'all')
- if token_type in ('access', 'all'):
- self.cache.store(
- JioCinemaBaseIE._NETRC_MACHINE, f'{JioCinemaBaseIE._DEVICE_ID}-access', JioCinemaBaseIE._ACCESS_TOKEN)
- if token_type in ('refresh', 'all'):
- self.cache.store(
- JioCinemaBaseIE._NETRC_MACHINE, f'{JioCinemaBaseIE._DEVICE_ID}-refresh', JioCinemaBaseIE._REFRESH_TOKEN)
-
- def _call_api(self, url, video_id, note='Downloading API JSON', headers={}, data={}):
- return self._download_json(
- url, video_id, note, data=json.dumps(data, separators=(',', ':')).encode(), headers={
- 'Content-Type': 'application/json',
- 'Accept': 'application/json',
- **self._API_HEADERS,
- **headers,
- }, expected_status=(400, 403, 474))
-
- def _call_auth_api(self, service, endpoint, note, headers={}, data={}):
- return self._call_api(
- f'https://auth-jiocinema.voot.com/{service}service/apis/v4/{endpoint}',
- None, note=note, headers=headers, data=data)
-
- def _refresh_token(self):
- if not JioCinemaBaseIE._REFRESH_TOKEN or not JioCinemaBaseIE._DEVICE_ID:
- raise ExtractorError('User token has expired', expected=True)
- response = self._call_auth_api(
- 'token', 'refreshtoken', 'Refreshing token',
- headers={'accesstoken': self._ACCESS_TOKEN}, data={
- **self._APP_NAME,
- 'deviceId': self._DEVICE_ID,
- 'refreshToken': self._REFRESH_TOKEN,
- **self._APP_VERSION,
- })
- refresh_token = response.get('refreshTokenId')
- if refresh_token and refresh_token != JioCinemaBaseIE._REFRESH_TOKEN:
- JioCinemaBaseIE._REFRESH_TOKEN = refresh_token
- self._cache_token('refresh')
- JioCinemaBaseIE._ACCESS_TOKEN = response['authToken']
- self._cache_token('access')
-
- def _fetch_guest_token(self):
- JioCinemaBaseIE._DEVICE_ID = ''.join(random.choices(string.digits, k=10))
- guest_token = self._call_auth_api(
- 'token', 'guest', 'Downloading guest token', data={
- **self._APP_NAME,
- 'deviceType': 'phone',
- 'os': 'ios',
- 'deviceId': self._DEVICE_ID,
- 'freshLaunch': False,
- 'adId': self._DEVICE_ID,
- **self._APP_VERSION,
- })
- self._GUEST_TOKEN = guest_token['authToken']
- self._USER_ID = guest_token['userId']
-
- def _call_login_api(self, endpoint, guest_token, data, note):
- return self._call_auth_api(
- 'user', f'loginotp/{endpoint}', note, headers={
- **self.geo_verification_headers(),
- 'accesstoken': self._GUEST_TOKEN,
- **self._APP_NAME,
- **traverse_obj(guest_token, 'data', {
- 'deviceType': ('deviceType', {str}),
- 'os': ('os', {str}),
- })}, data=data)
-
- def _is_token_expired(self, token):
- return (try_call(lambda: jwt_decode_hs256(token)['exp']) or 0) <= int(time.time() - 180)
-
- def _perform_login(self, username, password):
- if self._ACCESS_TOKEN and not self._is_token_expired(self._ACCESS_TOKEN):
- return
-
- UUID_RE = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}'
-
- if username.lower() == 'token':
- if try_call(lambda: jwt_decode_hs256(password)):
- JioCinemaBaseIE._ACCESS_TOKEN = password
- refresh_hint = 'the `refreshToken` UUID from your browser local storage'
- refresh_token = self._configuration_arg('refresh_token', [''], ie_key=JioCinemaIE)[0]
- if not refresh_token:
- self.to_screen(
- 'To extend the life of your login session, in addition to your access token, '
- 'you can pass --extractor-args "jiocinema:refresh_token=REFRESH_TOKEN" '
- f'where REFRESH_TOKEN is {refresh_hint}')
- elif re.fullmatch(UUID_RE, refresh_token):
- JioCinemaBaseIE._REFRESH_TOKEN = refresh_token
- else:
- self.report_warning(f'Invalid refresh_token value. Use {refresh_hint}')
- else:
- raise ExtractorError(
- f'The password given could not be decoded as a token; use {self._ACCESS_HINT}', expected=True)
-
- elif username.lower() == 'device' and re.fullmatch(rf'(?:{UUID_RE}|\d+)', password):
- JioCinemaBaseIE._REFRESH_TOKEN = self.cache.load(JioCinemaBaseIE._NETRC_MACHINE, f'{password}-refresh')
- JioCinemaBaseIE._ACCESS_TOKEN = self.cache.load(JioCinemaBaseIE._NETRC_MACHINE, f'{password}-access')
- if not JioCinemaBaseIE._REFRESH_TOKEN or not JioCinemaBaseIE._ACCESS_TOKEN:
- raise ExtractorError(f'Failed to load cached tokens for device ID "{password}"', expected=True)
-
- elif username.lower() == 'phone' and re.fullmatch(r'\+?\d+', password):
- self._fetch_guest_token()
- guest_token = jwt_decode_hs256(self._GUEST_TOKEN)
- initial_data = {
- 'number': base64.b64encode(password.encode()).decode(),
- **self._APP_VERSION,
- }
- response = self._call_login_api('send', guest_token, initial_data, 'Requesting OTP')
- if not traverse_obj(response, ('OTPInfo', {dict})):
- raise ExtractorError('There was a problem with the phone number login attempt')
-
- is_iphone = guest_token.get('os') == 'ios'
- response = self._call_login_api('verify', guest_token, {
- 'deviceInfo': {
- 'consumptionDeviceName': 'iPhone' if is_iphone else 'Android',
- 'info': {
- 'platform': {'name': 'iPhone OS' if is_iphone else 'Android'},
- 'androidId': self._DEVICE_ID,
- 'type': 'iOS' if is_iphone else 'Android',
- },
- },
- **initial_data,
- 'otp': self._get_tfa_info('the one-time password sent to your phone'),
- }, 'Submitting OTP')
- if traverse_obj(response, 'code') == 1043:
- raise ExtractorError('Wrong OTP', expected=True)
- JioCinemaBaseIE._REFRESH_TOKEN = response['refreshToken']
- JioCinemaBaseIE._ACCESS_TOKEN = response['authToken']
-
- else:
- raise ExtractorError(self._LOGIN_HINT, expected=True)
-
- user_token = jwt_decode_hs256(JioCinemaBaseIE._ACCESS_TOKEN)['data']
- JioCinemaBaseIE._USER_ID = user_token['userId']
- JioCinemaBaseIE._DEVICE_ID = user_token['deviceId']
- if JioCinemaBaseIE._REFRESH_TOKEN and username != 'device':
- self._cache_token('all')
- if self.get_param('cachedir') is not False:
- self.to_screen(
- f'NOTE: For subsequent logins you can use "-u device -p {JioCinemaBaseIE._DEVICE_ID}"')
- elif not JioCinemaBaseIE._REFRESH_TOKEN:
- JioCinemaBaseIE._REFRESH_TOKEN = self.cache.load(
- JioCinemaBaseIE._NETRC_MACHINE, f'{JioCinemaBaseIE._DEVICE_ID}-refresh')
- if JioCinemaBaseIE._REFRESH_TOKEN:
- self._cache_token('access')
- self.to_screen(f'Logging in as device ID "{JioCinemaBaseIE._DEVICE_ID}"')
- if self._is_token_expired(JioCinemaBaseIE._ACCESS_TOKEN):
- self._refresh_token()
-
-
-class JioCinemaIE(JioCinemaBaseIE):
- IE_NAME = 'jiocinema'
- _VALID_URL = r'https?://(?:www\.)?jiocinema\.com/?(?:movies?/[^/?#]+/|tv-shows/(?:[^/?#]+/){3})(?P\d{3,})'
- _TESTS = [{
- 'url': 'https://www.jiocinema.com/tv-shows/agnisakshi-ek-samjhauta/1/pradeep-to-stop-the-wedding/3759931',
- 'info_dict': {
- 'id': '3759931',
- 'ext': 'mp4',
- 'title': 'Pradeep to stop the wedding?',
- 'description': 'md5:75f72d1d1a66976633345a3de6d672b1',
- 'episode': 'Pradeep to stop the wedding?',
- 'episode_number': 89,
- 'season': 'Agnisakshi…Ek Samjhauta-S1',
- 'season_number': 1,
- 'series': 'Agnisakshi Ek Samjhauta',
- 'duration': 1238.0,
- 'thumbnail': r're:https?://.+\.jpg',
- 'age_limit': 13,
- 'season_id': '3698031',
- 'upload_date': '20230606',
- 'timestamp': 1686009600,
- 'release_date': '20230607',
- 'genres': ['Drama'],
- },
- 'params': {'skip_download': 'm3u8'},
- }, {
- 'url': 'https://www.jiocinema.com/movies/bhediya/3754021/watch',
- 'info_dict': {
- 'id': '3754021',
- 'ext': 'mp4',
- 'title': 'Bhediya',
- 'description': 'md5:a6bf2900371ac2fc3f1447401a9f7bb0',
- 'episode': 'Bhediya',
- 'duration': 8500.0,
- 'thumbnail': r're:https?://.+\.jpg',
- 'age_limit': 13,
- 'upload_date': '20230525',
- 'timestamp': 1685026200,
- 'release_date': '20230524',
- 'genres': ['Comedy'],
- },
- 'params': {'skip_download': 'm3u8'},
- }]
-
- def _extract_formats_and_subtitles(self, playback, video_id):
- m3u8_url = traverse_obj(playback, (
- 'data', 'playbackUrls', lambda _, v: v['streamtype'] == 'hls', 'url', {url_or_none}, any))
- if not m3u8_url: # DRM-only content only serves dash urls
- self.report_drm(video_id)
- formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, m3u8_id='hls')
- self._remove_duplicate_formats(formats)
-
- return {
- # '/_definst_/smil:vod/' m3u8 manifests claim to have 720p+ formats but max out at 480p
- 'formats': traverse_obj(formats, (
- lambda _, v: '/_definst_/smil:vod/' not in v['url'] or v['height'] <= 480)),
- 'subtitles': subtitles,
- }
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
- if not self._ACCESS_TOKEN and self._is_token_expired(self._GUEST_TOKEN):
- self._fetch_guest_token()
- elif self._ACCESS_TOKEN and self._is_token_expired(self._ACCESS_TOKEN):
- self._refresh_token()
-
- playback = self._call_api(
- f'https://apis-jiovoot.voot.com/playbackjv/v3/{video_id}', video_id,
- 'Downloading playback JSON', headers={
- **self.geo_verification_headers(),
- 'accesstoken': self._ACCESS_TOKEN or self._GUEST_TOKEN,
- **self._APP_NAME,
- 'deviceid': self._DEVICE_ID,
- 'uniqueid': self._USER_ID,
- 'x-apisignatures': self._API_SIGNATURES,
- 'x-platform': 'androidweb',
- 'x-platform-token': 'web',
- }, data={
- '4k': False,
- 'ageGroup': '18+',
- 'appVersion': '3.4.0',
- 'bitrateProfile': 'xhdpi',
- 'capability': {
- 'drmCapability': {
- 'aesSupport': 'yes',
- 'fairPlayDrmSupport': 'none',
- 'playreadyDrmSupport': 'none',
- 'widevineDRMSupport': 'none',
- },
- 'frameRateCapability': [{
- 'frameRateSupport': '30fps',
- 'videoQuality': '1440p',
- }],
- },
- 'continueWatchingRequired': False,
- 'dolby': False,
- 'downloadRequest': False,
- 'hevc': False,
- 'kidsSafe': False,
- 'manufacturer': 'Windows',
- 'model': 'Windows',
- 'multiAudioRequired': True,
- 'osVersion': '10',
- 'parentalPinValid': True,
- 'x-apisignatures': self._API_SIGNATURES,
- })
-
- status_code = traverse_obj(playback, ('code', {int}))
- if status_code == 474:
- self.raise_geo_restricted(countries=['IN'])
- elif status_code == 1008:
- error_msg = 'This content is only available for premium users'
- if self._ACCESS_TOKEN:
- raise ExtractorError(error_msg, expected=True)
- self.raise_login_required(f'{error_msg}. {self._LOGIN_HINT}', method=None)
- elif status_code == 400:
- raise ExtractorError('The requested content is not available', expected=True)
- elif status_code is not None and status_code != 200:
- raise ExtractorError(
- f'JioCinema says: {traverse_obj(playback, ("message", {str})) or status_code}')
-
- metadata = self._download_json(
- f'{self._METADATA_API_BASE}/voot/v1/voot-web/content/query/asset-details',
- video_id, fatal=False, query={
- 'ids': f'include:{video_id}',
- 'responseType': 'common',
- 'devicePlatformType': 'desktop',
- })
-
- return {
- 'id': video_id,
- 'http_headers': self._API_HEADERS,
- **self._extract_formats_and_subtitles(playback, video_id),
- **traverse_obj(playback, ('data', {
- # fallback metadata
- 'title': ('name', {str}),
- 'description': ('fullSynopsis', {str}),
- 'series': ('show', 'name', {str}, filter),
- 'season': ('tournamentName', {str}, {lambda x: x if x != 'Season 0' else None}),
- 'season_number': ('episode', 'season', {int_or_none}, filter),
- 'episode': ('fullTitle', {str}),
- 'episode_number': ('episode', 'episodeNo', {int_or_none}, filter),
- 'age_limit': ('ageNemonic', {parse_age_limit}),
- 'duration': ('totalDuration', {float_or_none}),
- 'thumbnail': ('images', {url_or_none}),
- })),
- **traverse_obj(metadata, ('result', 0, {
- 'title': ('fullTitle', {str}),
- 'description': ('fullSynopsis', {str}),
- 'series': ('showName', {str}, filter),
- 'season': ('seasonName', {str}, filter),
- 'season_number': ('season', {int_or_none}),
- 'season_id': ('seasonId', {str}, filter),
- 'episode': ('fullTitle', {str}),
- 'episode_number': ('episode', {int_or_none}),
- 'timestamp': ('uploadTime', {int_or_none}),
- 'release_date': ('telecastDate', {str}),
- 'age_limit': ('ageNemonic', {parse_age_limit}),
- 'duration': ('duration', {float_or_none}),
- 'genres': ('genres', ..., {str}),
- 'thumbnail': ('seo', 'ogImage', {url_or_none}),
- })),
- }
-
-
-class JioCinemaSeriesIE(JioCinemaBaseIE):
- IE_NAME = 'jiocinema:series'
- _VALID_URL = r'https?://(?:www\.)?jiocinema\.com/tv-shows/(?P[\w-]+)/(?P\d{3,})'
- _TESTS = [{
- 'url': 'https://www.jiocinema.com/tv-shows/naagin/3499917',
- 'info_dict': {
- 'id': '3499917',
- 'title': 'naagin',
- },
- 'playlist_mincount': 120,
- }, {
- 'url': 'https://www.jiocinema.com/tv-shows/mtv-splitsvilla-x5/3499820',
- 'info_dict': {
- 'id': '3499820',
- 'title': 'mtv-splitsvilla-x5',
- },
- 'playlist_mincount': 310,
- }]
-
- def _entries(self, series_id):
- seasons = traverse_obj(self._download_json(
- f'{self._METADATA_API_BASE}/voot/v1/voot-web/view/show/{series_id}', series_id,
- 'Downloading series metadata JSON', query={'responseType': 'common'}), (
- 'trays', lambda _, v: v['trayId'] == 'season-by-show-multifilter',
- 'trayTabs', lambda _, v: v['id']))
-
- for season_num, season in enumerate(seasons, start=1):
- season_id = season['id']
- label = season.get('label') or season_num
- for page_num in itertools.count(1):
- episodes = traverse_obj(self._download_json(
- f'{self._METADATA_API_BASE}/voot/v1/voot-web/content/generic/series-wise-episode',
- season_id, f'Downloading season {label} page {page_num} JSON', query={
- 'sort': 'episode:asc',
- 'id': season_id,
- 'responseType': 'common',
- 'page': page_num,
- }), ('result', lambda _, v: v['id'] and url_or_none(v['slug'])))
- if not episodes:
- break
- for episode in episodes:
- yield self.url_result(
- episode['slug'], JioCinemaIE, **traverse_obj(episode, {
- 'video_id': 'id',
- 'video_title': ('fullTitle', {str}),
- 'season_number': ('season', {int_or_none}),
- 'episode_number': ('episode', {int_or_none}),
- }))
-
- def _real_extract(self, url):
- slug, series_id = self._match_valid_url(url).group('slug', 'id')
- return self.playlist_result(self._entries(series_id), series_id, slug)
diff --git a/yt_dlp/extractor/joqrag.py b/yt_dlp/extractor/joqrag.py
deleted file mode 100644
index 7a91d4a23..000000000
--- a/yt_dlp/extractor/joqrag.py
+++ /dev/null
@@ -1,112 +0,0 @@
-import datetime as dt
-import urllib.parse
-
-from .common import InfoExtractor
-from ..utils import (
- clean_html,
- datetime_from_str,
- unified_timestamp,
- urljoin,
-)
-
-
-class JoqrAgIE(InfoExtractor):
- IE_DESC = '超!A&G+ 文化放送 (f.k.a. AGQR) Nippon Cultural Broadcasting, Inc. (JOQR)'
- _VALID_URL = [r'https?://www\.uniqueradio\.jp/agplayer5/(?:player|inc-player-hls)\.php',
- r'https?://(?:www\.)?joqr\.co\.jp/ag/',
- r'https?://(?:www\.)?joqr\.co\.jp/qr/ag(?:daily|regular)program/?(?:$|[#?])']
- _TESTS = [{
- 'url': 'https://www.uniqueradio.jp/agplayer5/player.php',
- 'info_dict': {
- 'id': 'live',
- 'title': str,
- 'channel': '超!A&G+',
- 'description': str,
- 'live_status': 'is_live',
- 'release_timestamp': int,
- },
- 'params': {
- 'skip_download': True,
- 'ignore_no_formats_error': True,
- },
- }, {
- 'url': 'https://www.uniqueradio.jp/agplayer5/inc-player-hls.php',
- 'only_matching': True,
- }, {
- 'url': 'https://www.joqr.co.jp/ag/article/103760/',
- 'only_matching': True,
- }, {
- 'url': 'http://www.joqr.co.jp/qr/agdailyprogram/',
- 'only_matching': True,
- }, {
- 'url': 'http://www.joqr.co.jp/qr/agregularprogram/',
- 'only_matching': True,
- }]
-
- def _extract_metadata(self, variable, html):
- return clean_html(urllib.parse.unquote_plus(self._search_regex(
- rf'var\s+{variable}\s*=\s*(["\'])(?P(?:(?!\1).)+)\1',
- html, 'metadata', group='value', default=''))) or None
-
- def _extract_start_timestamp(self, video_id, is_live):
- def extract_start_time_from(date_str):
- dt_ = datetime_from_str(date_str) + dt.timedelta(hours=9)
- date = dt_.strftime('%Y%m%d')
- start_time = self._search_regex(
- r']+\bclass="dailyProgram-itemHeaderTime"[^>]*>[\s\d:]+–\s*(\d{1,2}:\d{1,2})',
- self._download_webpage(
- f'https://www.joqr.co.jp/qr/agdailyprogram/?date={date}', video_id,
- note=f'Downloading program list of {date}', fatal=False,
- errnote=f'Failed to download program list of {date}') or '',
- 'start time', default=None)
- if start_time:
- return unified_timestamp(f'{dt_.strftime("%Y/%m/%d")} {start_time} +09:00')
- return None
-
- start_timestamp = extract_start_time_from('today')
- if not start_timestamp:
- return None
-
- if not is_live or start_timestamp < datetime_from_str('now').timestamp():
- return start_timestamp
- else:
- return extract_start_time_from('yesterday')
-
- def _real_extract(self, url):
- video_id = 'live'
-
- metadata = self._download_webpage(
- 'https://www.uniqueradio.jp/aandg', video_id,
- note='Downloading metadata', errnote='Failed to download metadata')
- title = self._extract_metadata('Program_name', metadata)
-
- if not title or title == '放送休止':
- formats = []
- live_status = 'is_upcoming'
- release_timestamp = self._extract_start_timestamp(video_id, False)
- msg = 'This stream is not currently live'
- if release_timestamp:
- msg += (' and will start at '
- + dt.datetime.fromtimestamp(release_timestamp).strftime('%Y-%m-%d %H:%M:%S'))
- self.raise_no_formats(msg, expected=True)
- else:
- m3u8_path = self._search_regex(
- r']*\bsrc="([^"]+)"',
- self._download_webpage(
- 'https://www.uniqueradio.jp/agplayer5/inc-player-hls.php', video_id,
- note='Downloading player data', errnote='Failed to download player data'),
- 'm3u8 url')
- formats = self._extract_m3u8_formats(
- urljoin('https://www.uniqueradio.jp/', m3u8_path), video_id)
- live_status = 'is_live'
- release_timestamp = self._extract_start_timestamp(video_id, True)
-
- return {
- 'id': video_id,
- 'title': title,
- 'channel': '超!A&G+',
- 'description': self._extract_metadata('Program_text', metadata),
- 'formats': formats,
- 'live_status': live_status,
- 'release_timestamp': release_timestamp,
- }
diff --git a/yt_dlp/extractor/kick.py b/yt_dlp/extractor/kick.py
index 1f001d421..8049e1e34 100644
--- a/yt_dlp/extractor/kick.py
+++ b/yt_dlp/extractor/kick.py
@@ -1,12 +1,12 @@
+import functools
+import urllib.parse
from .common import InfoExtractor
-from ..networking import HEADRequest
from ..utils import (
UserNotLive,
determine_ext,
float_or_none,
int_or_none,
- merge_dicts,
parse_iso8601,
str_or_none,
traverse_obj,
@@ -16,21 +16,17 @@
class KickBaseIE(InfoExtractor):
- def _real_initialize(self):
- self._request_webpage(
- HEADRequest('https://kick.com/'), None, 'Setting up session', fatal=False, impersonate=True)
- xsrf_token = self._get_cookies('https://kick.com/').get('XSRF-TOKEN')
- if not xsrf_token:
- self.write_debug('kick.com did not set XSRF-TOKEN cookie')
- KickBaseIE._API_HEADERS = {
- 'Authorization': f'Bearer {xsrf_token.value}',
- 'X-XSRF-TOKEN': xsrf_token.value,
- } if xsrf_token else {}
+ @functools.cached_property
+ def _api_headers(self):
+ token = traverse_obj(
+ self._get_cookies('https://kick.com/'),
+ ('session_token', 'value', {urllib.parse.unquote}))
+ return {'Authorization': f'Bearer {token}'} if token else {}
def _call_api(self, path, display_id, note='Downloading API JSON', headers={}, **kwargs):
return self._download_json(
f'https://kick.com/api/{path}', display_id, note=note,
- headers=merge_dicts(headers, self._API_HEADERS), impersonate=True, **kwargs)
+ headers={**self._api_headers, **headers}, impersonate=True, **kwargs)
class KickIE(KickBaseIE):
diff --git a/yt_dlp/extractor/limelight.py b/yt_dlp/extractor/limelight.py
deleted file mode 100644
index 763a01448..000000000
--- a/yt_dlp/extractor/limelight.py
+++ /dev/null
@@ -1,358 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..networking.exceptions import HTTPError
-from ..utils import (
- ExtractorError,
- determine_ext,
- float_or_none,
- int_or_none,
- smuggle_url,
- try_get,
- unsmuggle_url,
-)
-
-
-class LimelightBaseIE(InfoExtractor):
- _PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
-
- @classmethod
- def _extract_embed_urls(cls, url, webpage):
- lm = {
- 'Media': 'media',
- 'Channel': 'channel',
- 'ChannelList': 'channel_list',
- }
-
- def smuggle(url):
- return smuggle_url(url, {'source_url': url})
-
- entries = []
- for kind, video_id in re.findall(
- r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P[a-z0-9]{32})',
- webpage):
- entries.append(cls.url_result(
- smuggle(f'limelight:{lm[kind]}:{video_id}'),
- f'Limelight{kind}', video_id))
- for mobj in re.finditer(
- # As per [1] class attribute should be exactly equal to
- # LimelightEmbeddedPlayerFlash but numerous examples seen
- # that don't exactly match it (e.g. [2]).
- # 1. http://support.3playmedia.com/hc/en-us/articles/227732408-Limelight-Embedding-the-Captions-Plugin-with-the-Limelight-Player-on-Your-Webpage
- # 2. http://www.sedona.com/FacilitatorTraining2017
- r'''(?sx)
- ]+class=(["\'])(?:(?!\1).)*\bLimelightEmbeddedPlayerFlash\b(?:(?!\1).)*\1[^>]*>.*?
- ]+
- name=(["\'])flashVars\2[^>]+
- value=(["\'])(?:(?!\3).)*(?Pmedia|channel(?:List)?)Id=(?P[a-z0-9]{32})
- ''', webpage):
- kind, video_id = mobj.group('kind'), mobj.group('id')
- entries.append(cls.url_result(
- smuggle(f'limelight:{kind}:{video_id}'),
- f'Limelight{kind.capitalize()}', video_id))
- # http://support.3playmedia.com/hc/en-us/articles/115009517327-Limelight-Embedding-the-Audio-Description-Plugin-with-the-Limelight-Player-on-Your-Web-Page)
- for video_id in re.findall(
- r'(?s)LimelightPlayerUtil\.embed\s*\(\s*{.*?\bmediaId["\']\s*:\s*["\'](?P[a-z0-9]{32})',
- webpage):
- entries.append(cls.url_result(
- smuggle(f'limelight:media:{video_id}'),
- LimelightMediaIE.ie_key(), video_id))
- return entries
-
- def _call_playlist_service(self, item_id, method, fatal=True, referer=None):
- headers = {}
- if referer:
- headers['Referer'] = referer
- try:
- return self._download_json(
- self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
- item_id, f'Downloading PlaylistService {method} JSON',
- fatal=fatal, headers=headers)
- except ExtractorError as e:
- if isinstance(e.cause, HTTPError) and e.cause.status == 403:
- error = self._parse_json(e.cause.response.read().decode(), item_id)['detail']['contentAccessPermission']
- if error == 'CountryDisabled':
- self.raise_geo_restricted()
- raise ExtractorError(error, expected=True)
- raise
-
- def _extract(self, item_id, pc_method, mobile_method, referer=None):
- pc = self._call_playlist_service(item_id, pc_method, referer=referer)
- mobile = self._call_playlist_service(
- item_id, mobile_method, fatal=False, referer=referer)
- return pc, mobile
-
- def _extract_info(self, pc, mobile, i, referer):
- get_item = lambda x, y: try_get(x, lambda x: x[y][i], dict) or {}
- pc_item = get_item(pc, 'playlistItems')
- mobile_item = get_item(mobile, 'mediaList')
- video_id = pc_item.get('mediaId') or mobile_item['mediaId']
- title = pc_item.get('title') or mobile_item['title']
-
- formats = []
- urls = []
- for stream in pc_item.get('streams', []):
- stream_url = stream.get('url')
- if not stream_url or stream_url in urls:
- continue
- if not self.get_param('allow_unplayable_formats') and stream.get('drmProtected'):
- continue
- urls.append(stream_url)
- ext = determine_ext(stream_url)
- if ext == 'f4m':
- formats.extend(self._extract_f4m_formats(
- stream_url, video_id, f4m_id='hds', fatal=False))
- else:
- fmt = {
- 'url': stream_url,
- 'abr': float_or_none(stream.get('audioBitRate')),
- 'fps': float_or_none(stream.get('videoFrameRate')),
- 'ext': ext,
- }
- width = int_or_none(stream.get('videoWidthInPixels'))
- height = int_or_none(stream.get('videoHeightInPixels'))
- vbr = float_or_none(stream.get('videoBitRate'))
- if width or height or vbr:
- fmt.update({
- 'width': width,
- 'height': height,
- 'vbr': vbr,
- })
- else:
- fmt['vcodec'] = 'none'
- rtmp = re.search(r'^(?Prtmpe?://(?P[^/]+)/(?P.+))/(?Pmp[34]:.+)$', stream_url)
- if rtmp:
- format_id = 'rtmp'
- if stream.get('videoBitRate'):
- format_id += '-%d' % int_or_none(stream['videoBitRate'])
- http_format_id = format_id.replace('rtmp', 'http')
-
- CDN_HOSTS = (
- ('delvenetworks.com', 'cpl.delvenetworks.com'),
- ('video.llnw.net', 's2.content.video.llnw.net'),
- )
- for cdn_host, http_host in CDN_HOSTS:
- if cdn_host not in rtmp.group('host').lower():
- continue
- http_url = 'http://{}/{}'.format(http_host, rtmp.group('playpath')[4:])
- urls.append(http_url)
- if self._is_valid_url(http_url, video_id, http_format_id):
- http_fmt = fmt.copy()
- http_fmt.update({
- 'url': http_url,
- 'format_id': http_format_id,
- })
- formats.append(http_fmt)
- break
-
- fmt.update({
- 'url': rtmp.group('url'),
- 'play_path': rtmp.group('playpath'),
- 'app': rtmp.group('app'),
- 'ext': 'flv',
- 'format_id': format_id,
- })
- formats.append(fmt)
-
- for mobile_url in mobile_item.get('mobileUrls', []):
- media_url = mobile_url.get('mobileUrl')
- format_id = mobile_url.get('targetMediaPlatform')
- if not media_url or media_url in urls:
- continue
- if (format_id in ('Widevine', 'SmoothStreaming')
- and not self.get_param('allow_unplayable_formats', False)):
- continue
- urls.append(media_url)
- ext = determine_ext(media_url)
- if ext == 'm3u8':
- formats.extend(self._extract_m3u8_formats(
- media_url, video_id, 'mp4', 'm3u8_native',
- m3u8_id=format_id, fatal=False))
- elif ext == 'f4m':
- formats.extend(self._extract_f4m_formats(
- stream_url, video_id, f4m_id=format_id, fatal=False))
- else:
- formats.append({
- 'url': media_url,
- 'format_id': format_id,
- 'quality': -10,
- 'ext': ext,
- })
-
- subtitles = {}
- for flag in mobile_item.get('flags'):
- if flag == 'ClosedCaptions':
- closed_captions = self._call_playlist_service(
- video_id, 'getClosedCaptionsDetailsByMediaId',
- False, referer) or []
- for cc in closed_captions:
- cc_url = cc.get('webvttFileUrl')
- if not cc_url:
- continue
- lang = cc.get('languageCode') or self._search_regex(r'/([a-z]{2})\.vtt', cc_url, 'lang', default='en')
- subtitles.setdefault(lang, []).append({
- 'url': cc_url,
- })
- break
-
- get_meta = lambda x: pc_item.get(x) or mobile_item.get(x)
-
- return {
- 'id': video_id,
- 'title': title,
- 'description': get_meta('description'),
- 'formats': formats,
- 'duration': float_or_none(get_meta('durationInMilliseconds'), 1000),
- 'thumbnail': get_meta('previewImageUrl') or get_meta('thumbnailImageUrl'),
- 'subtitles': subtitles,
- }
-
-
-class LimelightMediaIE(LimelightBaseIE):
- IE_NAME = 'limelight'
- _VALID_URL = r'''(?x)
- (?:
- limelight:media:|
- https?://
- (?:
- link\.videoplatform\.limelight\.com/media/|
- assets\.delvenetworks\.com/player/loader\.swf
- )
- \?.*?\bmediaId=
- )
- (?P[a-z0-9]{32})
- '''
- _TESTS = [{
- 'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
- 'info_dict': {
- 'id': '3ffd040b522b4485b6d84effc750cd86',
- 'ext': 'mp4',
- 'title': 'HaP and the HB Prince Trailer',
- 'description': 'md5:8005b944181778e313d95c1237ddb640',
- 'thumbnail': r're:^https?://.*\.jpeg$',
- 'duration': 144.23,
- },
- 'params': {
- # m3u8 download
- 'skip_download': True,
- },
- }, {
- # video with subtitles
- 'url': 'limelight:media:a3e00274d4564ec4a9b29b9466432335',
- 'md5': '2fa3bad9ac321e23860ca23bc2c69e3d',
- 'info_dict': {
- 'id': 'a3e00274d4564ec4a9b29b9466432335',
- 'ext': 'mp4',
- 'title': '3Play Media Overview Video',
- 'thumbnail': r're:^https?://.*\.jpeg$',
- 'duration': 78.101,
- # TODO: extract all languages that were accessible via API
- # 'subtitles': 'mincount:9',
- 'subtitles': 'mincount:1',
- },
- }, {
- 'url': 'https://assets.delvenetworks.com/player/loader.swf?mediaId=8018a574f08d416e95ceaccae4ba0452',
- 'only_matching': True,
- }]
- _PLAYLIST_SERVICE_PATH = 'media'
-
- def _real_extract(self, url):
- url, smuggled_data = unsmuggle_url(url, {})
- video_id = self._match_id(url)
- source_url = smuggled_data.get('source_url')
- self._initialize_geo_bypass({
- 'countries': smuggled_data.get('geo_countries'),
- })
-
- pc, mobile = self._extract(
- video_id, 'getPlaylistByMediaId',
- 'getMobilePlaylistByMediaId', source_url)
-
- return self._extract_info(pc, mobile, 0, source_url)
-
-
-class LimelightChannelIE(LimelightBaseIE):
- IE_NAME = 'limelight:channel'
- _VALID_URL = r'''(?x)
- (?:
- limelight:channel:|
- https?://
- (?:
- link\.videoplatform\.limelight\.com/media/|
- assets\.delvenetworks\.com/player/loader\.swf
- )
- \?.*?\bchannelId=
- )
- (?P[a-z0-9]{32})
- '''
- _TESTS = [{
- 'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
- 'info_dict': {
- 'id': 'ab6a524c379342f9b23642917020c082',
- 'title': 'Javascript Sample Code',
- 'description': 'Javascript Sample Code - http://www.delvenetworks.com/sample-code/playerCode-demo.html',
- },
- 'playlist_mincount': 3,
- }, {
- 'url': 'http://assets.delvenetworks.com/player/loader.swf?channelId=ab6a524c379342f9b23642917020c082',
- 'only_matching': True,
- }]
- _PLAYLIST_SERVICE_PATH = 'channel'
-
- def _real_extract(self, url):
- url, smuggled_data = unsmuggle_url(url, {})
- channel_id = self._match_id(url)
- source_url = smuggled_data.get('source_url')
-
- pc, mobile = self._extract(
- channel_id, 'getPlaylistByChannelId',
- 'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1',
- source_url)
-
- entries = [
- self._extract_info(pc, mobile, i, source_url)
- for i in range(len(pc['playlistItems']))]
-
- return self.playlist_result(
- entries, channel_id, pc.get('title'), mobile.get('description'))
-
-
-class LimelightChannelListIE(LimelightBaseIE):
- IE_NAME = 'limelight:channel_list'
- _VALID_URL = r'''(?x)
- (?:
- limelight:channel_list:|
- https?://
- (?:
- link\.videoplatform\.limelight\.com/media/|
- assets\.delvenetworks\.com/player/loader\.swf
- )
- \?.*?\bchannelListId=
- )
- (?P[a-z0-9]{32})
- '''
- _TESTS = [{
- 'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
- 'info_dict': {
- 'id': '301b117890c4465c8179ede21fd92e2b',
- 'title': 'Website - Hero Player',
- },
- 'playlist_mincount': 2,
- }, {
- 'url': 'https://assets.delvenetworks.com/player/loader.swf?channelListId=301b117890c4465c8179ede21fd92e2b',
- 'only_matching': True,
- }]
- _PLAYLIST_SERVICE_PATH = 'channel_list'
-
- def _real_extract(self, url):
- channel_list_id = self._match_id(url)
-
- channel_list = self._call_playlist_service(
- channel_list_id, 'getMobileChannelListById')
-
- entries = [
- self.url_result('limelight:channel:{}'.format(channel['id']), 'LimelightChannel')
- for channel in channel_list['channelList']]
-
- return self.playlist_result(
- entries, channel_list_id, channel_list['title'])
diff --git a/yt_dlp/extractor/lrt.py b/yt_dlp/extractor/lrt.py
index caff9125e..34c9ece2d 100644
--- a/yt_dlp/extractor/lrt.py
+++ b/yt_dlp/extractor/lrt.py
@@ -134,7 +134,7 @@ class LRTRadioIE(LRTBaseIE):
def _real_extract(self, url):
video_id, path = self._match_valid_url(url).group('id', 'path')
media = self._download_json(
- 'https://www.lrt.lt/radioteka/api/media', video_id,
+ 'https://www.lrt.lt/rest-api/media', video_id,
query={'url': f'/mediateka/irasas/{video_id}/{path}'})
return {
diff --git a/yt_dlp/extractor/lsm.py b/yt_dlp/extractor/lsm.py
index 56c06d745..93ea2e35d 100644
--- a/yt_dlp/extractor/lsm.py
+++ b/yt_dlp/extractor/lsm.py
@@ -167,11 +167,11 @@ class LSMLTVEmbedIE(InfoExtractor):
'duration': 1442,
'upload_date': '20231121',
'title': 'D23-6000-105_cetstud',
- 'thumbnail': 'https://store.cloudycdn.services/tmsp00060/assets/media/660858/placeholder1700589200.jpg',
+ 'thumbnail': 'https://store.bstrm.net/tmsp00060/assets/media/660858/placeholder1700589200.jpg',
},
}, {
'url': 'https://ltv.lsm.lv/embed?enablesdkjs=1&c=eyJpdiI6IncwVzZmUFk2MU12enVWK1I3SUcwQ1E9PSIsInZhbHVlIjoid3FhV29vamc3T2sxL1RaRmJ5Rm1GTXozU0o2dVczdUtLK0cwZEZJMDQ2a3ZIRG5DK2pneGlnbktBQy9uazVleHN6VXhxdWIweWNvcHRDSnlISlNYOHlVZ1lpcTUrcWZSTUZPQW14TVdkMW9aOUtRWVNDcFF4eWpHNGcrT0VZbUNFQStKQk91cGpndW9FVjJIa0lpbkh3PT0iLCJtYWMiOiIyZGI1NDJlMWRlM2QyMGNhOGEwYTM2MmNlN2JlOGRhY2QyYjdkMmEzN2RlOTEzYTVkNzI1ODlhZDlhZjU4MjQ2IiwidGFnIjoiIn0=',
- 'md5': 'a1711e190fe680fdb68fd8413b378e87',
+ 'md5': 'f236cef2fd5953612754e4e66be51e7a',
'info_dict': {
'id': 'wUnFArIPDSY',
'ext': 'mp4',
@@ -198,6 +198,8 @@ class LSMLTVEmbedIE(InfoExtractor):
'uploader_url': 'https://www.youtube.com/@LTV16plus',
'like_count': int,
'description': 'md5:7ff0c42ba971e3c13e4b8a2ff03b70b5',
+ 'media_type': 'livestream',
+ 'timestamp': 1652550741,
},
}]
@@ -208,7 +210,7 @@ def _real_extract(self, url):
r'window\.ltvEmbedPayload\s*=', webpage, 'embed json', video_id)
embed_type = traverse_obj(data, ('source', 'name', {str}))
- if embed_type == 'telia':
+ if embed_type in ('backscreen', 'telia'): # 'telia' only for backwards compat
ie_key = 'CloudyCDN'
embed_url = traverse_obj(data, ('source', 'embed_url', {url_or_none}))
elif embed_type == 'youtube':
@@ -226,9 +228,9 @@ def _real_extract(self, url):
class LSMReplayIE(InfoExtractor):
- _VALID_URL = r'https?://replay\.lsm\.lv/[^/?#]+/(?:ieraksts|statja)/[^/?#]+/(?P\d+)'
+ _VALID_URL = r'https?://replay\.lsm\.lv/[^/?#]+/(?:skaties/|klausies/)?(?:ieraksts|statja)/[^/?#]+/(?P\d+)'
_TESTS = [{
- 'url': 'https://replay.lsm.lv/lv/ieraksts/ltv/311130/4-studija-zolitudes-tragedija-un-incupes-stacija',
+ 'url': 'https://replay.lsm.lv/lv/skaties/ieraksts/ltv/311130/4-studija-zolitudes-tragedija-un-incupes-stacija',
'md5': '64f72a360ca530d5ed89c77646c9eee5',
'info_dict': {
'id': '46k_d23-6000-105',
@@ -241,20 +243,23 @@ class LSMReplayIE(InfoExtractor):
'thumbnail': 'https://ltv.lsm.lv/storage/media/8/7/large/5/1f9604e1.jpg',
},
}, {
- 'url': 'https://replay.lsm.lv/lv/ieraksts/lr/183522/138-nepilniga-kompensejamo-zalu-sistema-pat-menesiem-dzena-pacientus-pa-aptiekam',
- 'md5': '719b33875cd1429846eeeaeec6df2830',
+ 'url': 'https://replay.lsm.lv/lv/klausies/ieraksts/lr/183522/138-nepilniga-kompensejamo-zalu-sistema-pat-menesiem-dzena-pacientus-pa-aptiekam',
+ 'md5': '84feb80fd7e6ec07744726a9f01cda4d',
'info_dict': {
- 'id': 'a342781',
- 'ext': 'mp3',
+ 'id': '183522',
+ 'ext': 'm4a',
'duration': 1823,
'title': '#138 Nepilnīgā kompensējamo zāļu sistēma pat mēnešiem dzenā pacientus pa aptiekām',
'thumbnail': 'https://pic.latvijasradio.lv/public/assets/media/9/d/large_fd4675ac.jpg',
'upload_date': '20231102',
- 'timestamp': 1698921060,
+ 'timestamp': 1698913860,
'description': 'md5:7bac3b2dd41e44325032943251c357b1',
},
}, {
- 'url': 'https://replay.lsm.lv/ru/statja/ltv/311130/4-studija-zolitudes-tragedija-un-incupes-stacija',
+ 'url': 'https://replay.lsm.lv/ru/skaties/statja/ltv/355067/v-kengaragse-nacalas-ukladka-relsov',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://replay.lsm.lv/lv/ieraksts/ltv/311130/4-studija-zolitudes-tragedija-un-incupes-stacija',
'only_matching': True,
}]
@@ -267,12 +272,24 @@ def _real_extract(self, url):
data = self._search_nuxt_data(
self._fix_nuxt_data(webpage), video_id, context_name='__REPLAY__')
+ playback_type = data['playback']['type']
+
+ if playback_type == 'playable_audio_lr':
+ playback_data = {
+ 'formats': self._extract_m3u8_formats(data['playback']['service']['hls_url'], video_id),
+ }
+ elif playback_type == 'embed':
+ playback_data = {
+ '_type': 'url_transparent',
+ 'url': data['playback']['service']['url'],
+ }
+ else:
+ raise ExtractorError(f'Unsupported playback type "{playback_type}"')
return {
- '_type': 'url_transparent',
'id': video_id,
+ **playback_data,
**traverse_obj(data, {
- 'url': ('playback', 'service', 'url', {url_or_none}),
'title': ('mediaItem', 'title'),
'description': ('mediaItem', ('lead', 'body')),
'duration': ('mediaItem', 'duration', {int_or_none}),
diff --git a/yt_dlp/extractor/mave.py b/yt_dlp/extractor/mave.py
new file mode 100644
index 000000000..86d8d8b7c
--- /dev/null
+++ b/yt_dlp/extractor/mave.py
@@ -0,0 +1,107 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ clean_html,
+ int_or_none,
+ parse_iso8601,
+ urljoin,
+)
+from ..utils.traversal import require, traverse_obj
+
+
+class MaveIE(InfoExtractor):
+ _VALID_URL = r'https?://(?P[\w-]+)\.mave\.digital/(?Pep-\d+)'
+ _TESTS = [{
+ 'url': 'https://ochenlichnoe.mave.digital/ep-25',
+ 'md5': 'aa3e513ef588b4366df1520657cbc10c',
+ 'info_dict': {
+ 'id': '4035f587-914b-44b6-aa5a-d76685ad9bc2',
+ 'ext': 'mp3',
+ 'display_id': 'ochenlichnoe-ep-25',
+ 'title': 'Между мной и миром: психология самооценки',
+ 'description': 'md5:4b7463baaccb6982f326bce5c700382a',
+ 'uploader': 'Самарский университет',
+ 'channel': 'Очень личное',
+ 'channel_id': 'ochenlichnoe',
+ 'channel_url': 'https://ochenlichnoe.mave.digital/',
+ 'view_count': int,
+ 'like_count': int,
+ 'dislike_count': int,
+ 'duration': 3744,
+ 'thumbnail': r're:https://.+/storage/podcasts/.+\.jpg',
+ 'series': 'Очень личное',
+ 'series_id': '2e0c3749-6df2-4946-82f4-50691419c065',
+ 'season': 'Season 3',
+ 'season_number': 3,
+ 'episode': 'Episode 3',
+ 'episode_number': 3,
+ 'timestamp': 1747817300,
+ 'upload_date': '20250521',
+ },
+ }, {
+ 'url': 'https://budem.mave.digital/ep-12',
+ 'md5': 'e1ce2780fcdb6f17821aa3ca3e8c919f',
+ 'info_dict': {
+ 'id': '41898bb5-ff57-4797-9236-37a8e537aa21',
+ 'ext': 'mp3',
+ 'display_id': 'budem-ep-12',
+ 'title': 'Екатерина Михайлова: "Горе от ума" не про женщин написана',
+ 'description': 'md5:fa3bdd59ee829dfaf16e3efcb13f1d19',
+ 'uploader': 'Полина Цветкова+Евгения Акопова',
+ 'channel': 'Все там будем',
+ 'channel_id': 'budem',
+ 'channel_url': 'https://budem.mave.digital/',
+ 'view_count': int,
+ 'like_count': int,
+ 'dislike_count': int,
+ 'age_limit': 18,
+ 'duration': 3664,
+ 'thumbnail': r're:https://.+/storage/podcasts/.+\.jpg',
+ 'series': 'Все там будем',
+ 'series_id': 'fe9347bf-c009-4ebd-87e8-b06f2f324746',
+ 'season': 'Season 2',
+ 'season_number': 2,
+ 'episode': 'Episode 5',
+ 'episode_number': 5,
+ 'timestamp': 1735538400,
+ 'upload_date': '20241230',
+ },
+ }]
+ _API_BASE_URL = 'https://api.mave.digital/'
+
+ def _real_extract(self, url):
+ channel_id, slug = self._match_valid_url(url).group('channel', 'id')
+ display_id = f'{channel_id}-{slug}'
+ webpage = self._download_webpage(url, display_id)
+ data = traverse_obj(
+ self._search_nuxt_json(webpage, display_id),
+ ('data', lambda _, v: v['activeEpisodeData'], any, {require('podcast data')}))
+
+ return {
+ 'display_id': display_id,
+ 'channel_id': channel_id,
+ 'channel_url': f'https://{channel_id}.mave.digital/',
+ 'vcodec': 'none',
+ 'thumbnail': re.sub(r'_\d+(?=\.(?:jpg|png))', '', self._og_search_thumbnail(webpage, default='')) or None,
+ **traverse_obj(data, ('activeEpisodeData', {
+ 'url': ('audio', {urljoin(self._API_BASE_URL)}),
+ 'id': ('id', {str}),
+ 'title': ('title', {str}),
+ 'description': ('description', {clean_html}),
+ 'duration': ('duration', {int_or_none}),
+ 'season_number': ('season', {int_or_none}),
+ 'episode_number': ('number', {int_or_none}),
+ 'view_count': ('listenings', {int_or_none}),
+ 'like_count': ('reactions', lambda _, v: v['type'] == 'like', 'count', {int_or_none}, any),
+ 'dislike_count': ('reactions', lambda _, v: v['type'] == 'dislike', 'count', {int_or_none}, any),
+ 'age_limit': ('is_explicit', {bool}, {lambda x: 18 if x else None}),
+ 'timestamp': ('publish_date', {parse_iso8601}),
+ })),
+ **traverse_obj(data, ('podcast', 'podcast', {
+ 'series_id': ('id', {str}),
+ 'series': ('title', {str}),
+ 'channel': ('title', {str}),
+ 'uploader': ('author', {str}),
+ })),
+ }
diff --git a/yt_dlp/extractor/mir24tv.py b/yt_dlp/extractor/mir24tv.py
new file mode 100644
index 000000000..5832901bf
--- /dev/null
+++ b/yt_dlp/extractor/mir24tv.py
@@ -0,0 +1,37 @@
+from .common import InfoExtractor
+from ..utils import parse_qs, url_or_none
+from ..utils.traversal import require, traverse_obj
+
+
+class Mir24TvIE(InfoExtractor):
+ IE_NAME = 'mir24.tv'
+ _VALID_URL = r'https?://(?:www\.)?mir24\.tv/news/(?P[0-9]+)/[^/?#]+'
+ _TESTS = [{
+ 'url': 'https://mir24.tv/news/16635210/dni-kultury-rossii-otkrylis-v-uzbekistane.-na-prazdnichnom-koncerte-vystupili-zvezdy-rossijskoj-estrada',
+ 'info_dict': {
+ 'id': '16635210',
+ 'title': 'Дни культуры России открылись в Узбекистане. На праздничном концерте выступили звезды российской эстрады',
+ 'ext': 'mp4',
+ 'thumbnail': r're:https://images\.mir24\.tv/.+\.jpg',
+ },
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id, impersonate=True)
+
+ iframe_url = self._search_regex(
+ r']+\bsrc=["\'](https?://mir24\.tv/players/[^"\']+)',
+ webpage, 'iframe URL')
+
+ m3u8_url = traverse_obj(iframe_url, (
+ {parse_qs}, 'source', -1, {self._proto_relative_url}, {url_or_none}, {require('m3u8 URL')}))
+ formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
+
+ return {
+ 'id': video_id,
+ 'title': self._og_search_title(webpage, default=None) or self._html_extract_title(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage, default=None),
+ 'formats': formats,
+ 'subtitles': subtitles,
+ }
diff --git a/yt_dlp/extractor/mitele.py b/yt_dlp/extractor/mitele.py
index 55fa83b51..0dded38c6 100644
--- a/yt_dlp/extractor/mitele.py
+++ b/yt_dlp/extractor/mitele.py
@@ -1,7 +1,5 @@
from .telecinco import TelecincoBaseIE
-from ..networking.exceptions import HTTPError
from ..utils import (
- ExtractorError,
int_or_none,
parse_iso8601,
)
@@ -81,17 +79,7 @@ class MiTeleIE(TelecincoBaseIE):
def _real_extract(self, url):
display_id = self._match_id(url)
-
- try: # yt-dlp's default user-agents are too old and blocked by akamai
- webpage = self._download_webpage(url, display_id, headers={
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:136.0) Gecko/20100101 Firefox/136.0',
- })
- except ExtractorError as e:
- if not isinstance(e.cause, HTTPError) or e.cause.status != 403:
- raise
- # Retry with impersonation if hardcoded UA is insufficient to bypass akamai
- webpage = self._download_webpage(url, display_id, impersonate=True)
-
+ webpage = self._download_akamai_webpage(url, display_id)
pre_player = self._search_json(
r'window\.\$REACTBASE_STATE\.prePlayer_mtweb\s*=',
webpage, 'Pre Player', display_id)['prePlayer']
diff --git a/yt_dlp/extractor/mixlr.py b/yt_dlp/extractor/mixlr.py
new file mode 100644
index 000000000..53f3ffe6f
--- /dev/null
+++ b/yt_dlp/extractor/mixlr.py
@@ -0,0 +1,134 @@
+from .common import InfoExtractor
+from ..networking import HEADRequest
+from ..utils import int_or_none, parse_iso8601, url_or_none, urlhandle_detect_ext
+from ..utils.traversal import traverse_obj
+
+
+class MixlrIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?(?P[\w-]+)\.mixlr\.com/events/(?P\d+)'
+ _TESTS = [{
+ 'url': 'https://suncity-104-9fm.mixlr.com/events/4387115',
+ 'info_dict': {
+ 'id': '4387115',
+ 'ext': 'mp3',
+ 'title': r're:SUNCITY 104.9FM\'s live audio \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
+ 'uploader': 'suncity-104-9fm',
+ 'like_count': int,
+ 'thumbnail': r're:https://imagecdn\.mixlr\.com/cdn-cgi/image/[^/?#]+/cd5b34d05fa2cee72d80477724a2f02e.png',
+ 'timestamp': 1751943773,
+ 'upload_date': '20250708',
+ 'release_timestamp': 1751943764,
+ 'release_date': '20250708',
+ 'live_status': 'is_live',
+ },
+ }, {
+ 'url': 'https://brcountdown.mixlr.com/events/4395480',
+ 'info_dict': {
+ 'id': '4395480',
+ 'ext': 'aac',
+ 'title': r're:Beats Revolution Countdown Episodio 461 \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
+ 'description': 'md5:5cacd089723f7add3f266bd588315bb3',
+ 'uploader': 'brcountdown',
+ 'like_count': int,
+ 'thumbnail': r're:https://imagecdn\.mixlr\.com/cdn-cgi/image/[^/?#]+/c48727a59f690b87a55d47d123ba0d6d.jpg',
+ 'timestamp': 1752354007,
+ 'upload_date': '20250712',
+ 'release_timestamp': 1752354000,
+ 'release_date': '20250712',
+ 'live_status': 'is_live',
+ },
+ }, {
+ 'url': 'https://www.brcountdown.mixlr.com/events/4395480',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ username, event_id = self._match_valid_url(url).group('username', 'id')
+
+ broadcast_info = self._download_json(
+ f'https://api.mixlr.com/v3/channels/{username}/events/{event_id}', event_id)
+
+ formats = []
+ format_url = traverse_obj(
+ broadcast_info, ('included', 0, 'attributes', 'progressive_stream_url', {url_or_none}))
+ if format_url:
+ urlh = self._request_webpage(
+ HEADRequest(format_url), event_id, fatal=False, note='Checking stream')
+ if urlh and urlh.status == 200:
+ ext = urlhandle_detect_ext(urlh)
+ if ext == 'octet-stream':
+ self.report_warning(
+ 'The server did not return a valid file extension for the stream URL. '
+ 'Assuming an mp3 stream; postprocessing may fail if this is incorrect')
+ ext = 'mp3'
+ formats.append({
+ 'url': format_url,
+ 'ext': ext,
+ 'vcodec': 'none',
+ })
+
+ release_timestamp = traverse_obj(
+ broadcast_info, ('data', 'attributes', 'starts_at', {str}))
+ if not formats and release_timestamp:
+ self.raise_no_formats(f'This event will start at {release_timestamp}', expected=True)
+
+ return {
+ 'id': event_id,
+ 'uploader': username,
+ 'formats': formats,
+ 'release_timestamp': parse_iso8601(release_timestamp),
+ **traverse_obj(broadcast_info, ('included', 0, 'attributes', {
+ 'title': ('title', {str}),
+ 'timestamp': ('started_at', {parse_iso8601}),
+ 'concurrent_view_count': ('concurrent_view_count', {int_or_none}),
+ 'like_count': ('heart_count', {int_or_none}),
+ 'is_live': ('live', {bool}),
+ })),
+ **traverse_obj(broadcast_info, ('data', 'attributes', {
+ 'title': ('title', {str}),
+ 'description': ('description', {str}),
+ 'timestamp': ('started_at', {parse_iso8601}),
+ 'concurrent_view_count': ('concurrent_view_count', {int_or_none}),
+ 'like_count': ('heart_count', {int_or_none}),
+ 'thumbnail': ('artwork_url', {url_or_none}),
+ 'uploader_id': ('broadcaster_id', {str}),
+ })),
+ }
+
+
+class MixlrRecoringIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?(?P[\w-]+)\.mixlr\.com/recordings/(?P\d+)'
+ _TESTS = [{
+ 'url': 'https://biblewayng.mixlr.com/recordings/2375193',
+ 'info_dict': {
+ 'id': '2375193',
+ 'ext': 'mp3',
+ 'title': "God's Jewels and Their Resting Place Bro. Adeniji",
+ 'description': 'Preached February 21, 2024 in the evening',
+ 'uploader_id': '8659190',
+ 'duration': 10968,
+ 'thumbnail': r're:https://imagecdn\.mixlr\.com/cdn-cgi/image/[^/?#]+/ceca120ef707f642abeea6e29cd74238.jpg',
+ 'timestamp': 1708544542,
+ 'upload_date': '20240221',
+ },
+ }]
+
+ def _real_extract(self, url):
+ username, recording_id = self._match_valid_url(url).group('username', 'id')
+
+ recording_info = self._download_json(
+ f'https://api.mixlr.com/v3/channels/{username}/recordings/{recording_id}', recording_id)
+
+ return {
+ 'id': recording_id,
+ **traverse_obj(recording_info, ('data', 'attributes', {
+ 'ext': ('file_format', {str}),
+ 'url': ('url', {url_or_none}),
+ 'title': ('title', {str}),
+ 'description': ('description', {str}),
+ 'timestamp': ('created_at', {parse_iso8601}),
+ 'duration': ('duration', {int_or_none}),
+ 'thumbnail': ('artwork_url', {url_or_none}),
+ 'uploader_id': ('user_id', {str}),
+ })),
+ }
diff --git a/yt_dlp/extractor/newspicks.py b/yt_dlp/extractor/newspicks.py
index 4a1cb0a73..5f19eed98 100644
--- a/yt_dlp/extractor/newspicks.py
+++ b/yt_dlp/extractor/newspicks.py
@@ -1,53 +1,72 @@
-import re
-
from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+ clean_html,
+ parse_iso8601,
+ parse_qs,
+ url_or_none,
+)
+from ..utils.traversal import require, traverse_obj
class NewsPicksIE(InfoExtractor):
- _VALID_URL = r'https?://newspicks\.com/movie-series/(?P\d+)\?movieId=(?P\d+)'
-
+ _VALID_URL = r'https?://newspicks\.com/movie-series/(?P[^?/#]+)'
_TESTS = [{
- 'url': 'https://newspicks.com/movie-series/11?movieId=1813',
+ 'url': 'https://newspicks.com/movie-series/11/?movieId=1813',
'info_dict': {
'id': '1813',
- 'title': '日本の課題を破壊せよ【ゲスト：成田悠輔】',
- 'description': 'md5:09397aad46d6ded6487ff13f138acadf',
- 'channel': 'HORIE ONE',
- 'channel_id': '11',
- 'release_date': '20220117',
- 'thumbnail': r're:https://.+jpg',
'ext': 'mp4',
+ 'title': '日本の課題を破壊せよ【ゲスト：成田悠輔】',
+ 'cast': 'count:4',
+ 'description': 'md5:09397aad46d6ded6487ff13f138acadf',
+ 'duration': 2940,
+ 'release_date': '20220117',
+ 'release_timestamp': 1642424400,
+ 'series': 'HORIE ONE',
+ 'series_id': '11',
+ 'thumbnail': r're:https?://resources\.newspicks\.com/.+\.(?:jpe?g|png)',
+ 'timestamp': 1642424420,
+ 'upload_date': '20220117',
+ },
+ }, {
+ 'url': 'https://newspicks.com/movie-series/158/?movieId=3932',
+ 'info_dict': {
+ 'id': '3932',
+ 'ext': 'mp4',
+ 'title': '【検証】専門家は、KADOKAWAをどう見るか',
+ 'cast': 'count:3',
+ 'description': 'md5:2c2d4bf77484a4333ec995d676f9a91d',
+ 'duration': 1320,
+ 'release_date': '20240622',
+ 'release_timestamp': 1719088080,
+ 'series': 'NPレポート',
+ 'series_id': '158',
+ 'thumbnail': r're:https?://resources\.newspicks\.com/.+\.(?:jpe?g|png)',
+ 'timestamp': 1719086400,
+ 'upload_date': '20240622',
},
}]
def _real_extract(self, url):
- video_id, channel_id = self._match_valid_url(url).group('id', 'channel_id')
+ series_id = self._match_id(url)
+ video_id = traverse_obj(parse_qs(url), ('movieId', -1, {str}, {require('movie ID')}))
webpage = self._download_webpage(url, video_id)
- entries = self._parse_html5_media_entries(
- url, webpage.replace('movie-for-pc', 'movie'), video_id, 'hls')
- if not entries:
- raise ExtractorError('No HTML5 media elements found')
- info = entries[0]
- title = self._html_search_meta('og:title', webpage, fatal=False)
- description = self._html_search_meta(
- ('og:description', 'twitter:title'), webpage, fatal=False)
- channel = self._html_search_regex(
- r'value="11".+?(.+?)\s*(\d+)年(\d+)月(\d+)日\s*',
- webpage, 'release date', fatal=False, group=(1, 2, 3))
-
- info.update({
+ return {
'id': video_id,
- 'title': title,
- 'description': description,
- 'channel': channel,
- 'channel_id': channel_id,
- 'release_date': ('%04d%02d%02d' % tuple(map(int, release_date))) if release_date else None,
- })
- return info
+ 'formats': formats,
+ 'series': traverse_obj(fragment, ('series', 'title', {str})),
+ 'series_id': series_id,
+ 'subtitles': subtitles,
+ **traverse_obj(fragment, ('movie', {
+ 'title': ('title', {str}),
+ 'cast': ('relatedUsers', ..., 'displayName', {str}, filter, all, filter),
+ 'description': ('explanation', {clean_html}),
+ 'release_timestamp': ('onAirStartDate', {parse_iso8601}),
+ 'thumbnail': (('image', 'coverImageUrl'), {url_or_none}, any),
+ 'timestamp': ('published', {parse_iso8601}),
+ })),
+ }
diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py
index 0bd6edfcb..14fbd6ce8 100644
--- a/yt_dlp/extractor/nhk.py
+++ b/yt_dlp/extractor/nhk.py
@@ -8,6 +8,8 @@
get_element_by_class,
int_or_none,
join_nonempty,
+ make_archive_id,
+ orderedSet,
parse_duration,
remove_end,
traverse_obj,
@@ -16,6 +18,7 @@
unified_timestamp,
url_or_none,
urljoin,
+ variadic,
)
@@ -495,7 +498,7 @@ def _real_extract(self, url):
chapters = None
if chapter_durations and chapter_titles and len(chapter_durations) == len(chapter_titles):
start_time = chapter_durations
- end_time = chapter_durations[1:] + [duration]
+ end_time = [*chapter_durations[1:], duration]
chapters = [{
'start_time': s,
'end_time': e,
@@ -591,102 +594,179 @@ class NhkRadiruIE(InfoExtractor):
IE_DESC = 'NHK らじる (Radiru/Rajiru)'
_VALID_URL = r'https?://www\.nhk\.or\.jp/radio/(?:player/ondemand|ondemand/detail)\.html\?p=(?P[\da-zA-Z]+)_(?P[\da-zA-Z]+)(?:_(?P[\da-zA-Z]+))?'
_TESTS = [{
- 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=0449_01_4003239',
- 'skip': 'Episode expired on 2024-06-09',
+ 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=LG96ZW5KZ4_01_4251382',
+ 'skip': 'Episode expires on 2025-07-14',
'info_dict': {
- 'title': 'ジャズ・トゥナイト ジャズ「Ｎｉｇｈｔ ａｎｄ Ｄａｙ」特集',
- 'id': '0449_01_4003239',
+ 'title': 'クラシックの庭\u3000特集「ドボルザークを聴く」（１）交響曲を中心に',
+ 'id': 'LG96ZW5KZ4_01_4251382',
'ext': 'm4a',
- 'uploader': 'NHK FM 東京',
- 'description': 'md5:ad05f3c3f3f6e99b2e69f9b5e49551dc',
- 'series': 'ジャズ・トゥナイト',
- 'channel': 'NHK FM 東京',
- 'thumbnail': 'https://www.nhk.or.jp/prog/img/449/g449.jpg',
- 'upload_date': '20240601',
- 'series_id': '0449_01',
- 'release_date': '20240601',
- 'timestamp': 1717257600,
- 'release_timestamp': 1717250400,
+ 'description': 'md5:652d3c38a25b77959c716421eba1617a',
+ 'uploader': 'NHK FM・東京',
+ 'channel': 'NHK FM・東京',
+ 'duration': 6597.0,
+ 'thumbnail': 'https://www.nhk.jp/static/assets/images/radioseries/rs/LG96ZW5KZ4/LG96ZW5KZ4-eyecatch_a67c6e949325016c0724f2ed3eec8a2f.jpg',
+ 'categories': ['音楽', 'クラシック・オペラ'],
+ 'cast': ['田添菜穂子'],
+ 'series': 'クラシックの庭',
+ 'series_id': 'LG96ZW5KZ4',
+ 'episode': '特集「ドボルザークを聴く」(1)交響曲を中心に',
+ 'episode_id': 'QP1Q2ZXZY3',
+ 'timestamp': 1751871000,
+ 'upload_date': '20250707',
+ 'release_timestamp': 1751864403,
+ 'release_date': '20250707',
},
}, {
# playlist, airs every weekday so it should _hopefully_ be okay forever
- 'url': 'https://www.nhk.or.jp/radio/ondemand/detail.html?p=0458_01',
+ 'url': 'https://www.nhk.or.jp/radio/ondemand/detail.html?p=Z9L1V2M24L_01',
'info_dict': {
- 'id': '0458_01',
+ 'id': 'Z9L1V2M24L_01',
'title': 'ベストオブクラシック',
'description': '世界中の上質な演奏会をじっくり堪能する本格派クラシック番組。',
- 'thumbnail': 'https://www.nhk.or.jp/prog/img/458/g458.jpg',
- 'series_id': '0458_01',
+ 'thumbnail': 'https://www.nhk.jp/static/assets/images/radioseries/rs/Z9L1V2M24L/Z9L1V2M24L-eyecatch_83ed28b4782907998875965fee60a351.jpg',
+ 'series_id': 'Z9L1V2M24L_01',
'uploader': 'NHK FM',
'channel': 'NHK FM',
'series': 'ベストオブクラシック',
},
'playlist_mincount': 3,
- }, {
- # one with letters in the id
- 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=F683_01_3910688',
- 'note': 'Expires on 2025-03-31',
- 'info_dict': {
- 'id': 'F683_01_3910688',
- 'ext': 'm4a',
- 'title': '夏目漱石「文鳥」第1回',
- 'series': '【らじる文庫】夏目漱石「文鳥」（全4回）',
- 'series_id': 'F683_01',
- 'description': '朗読：浅井理アナウンサー',
- 'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F683/img/roudoku_05_rod_640.jpg',
- 'upload_date': '20240106',
- 'release_date': '20240106',
- 'uploader': 'NHK R1',
- 'release_timestamp': 1704511800,
- 'channel': 'NHK R1',
- 'timestamp': 1704512700,
- },
- 'expected_warnings': ['Unable to download JSON metadata',
- 'Failed to get extended metadata. API returned Error 1: Invalid parameters'],
}, {
# news
- 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=F261_01_4012173',
+ 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=18439M2W42_02_4251212',
+ 'skip': 'Expires on 2025-07-15',
'info_dict': {
- 'id': 'F261_01_4012173',
+ 'id': '18439M2W42_02_4251212',
'ext': 'm4a',
- 'channel': 'NHKラジオ第1',
+ 'title': 'マイあさ! 午前5時のNHKニュース 2025年7月8日',
'uploader': 'NHKラジオ第1',
+ 'channel': 'NHKラジオ第1',
+ 'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/18439M2W42/img/series_945_thumbnail.jpg',
'series': 'NHKラジオニュース',
- 'title': '午前０時のNHKニュース',
- 'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F261/img/RADIONEWS_640.jpg',
- 'release_timestamp': 1718290800,
- 'release_date': '20240613',
- 'timestamp': 1718291400,
- 'upload_date': '20240613',
+ 'timestamp': 1751919420,
+ 'upload_date': '20250707',
+ 'release_timestamp': 1751918400,
+ 'release_date': '20250707',
},
}, {
# fallback when extended metadata fails
- 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=2834_01_4009298',
- 'skip': 'Expires on 2024-06-07',
+ 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=J8792PY43V_20_4253945',
+ 'skip': 'Expires on 2025-09-01',
'info_dict': {
- 'id': '2834_01_4009298',
- 'title': 'まち☆キラ！開成町特集',
+ 'id': 'J8792PY43V_20_4253945',
'ext': 'm4a',
- 'release_date': '20240531',
- 'upload_date': '20240531',
- 'series': 'はま☆キラ！',
- 'thumbnail': 'https://www.nhk.or.jp/prog/img/2834/g2834.jpg',
- 'channel': 'NHK R1,FM',
- 'description': '',
- 'timestamp': 1717123800,
- 'uploader': 'NHK R1,FM',
- 'release_timestamp': 1717120800,
- 'series_id': '2834_01',
+ 'title': '「後絶たない筋肉増強剤の使用」ワールドリポート',
+ 'description': '大濱 敦（ソウル支局）',
+ 'uploader': 'NHK R1',
+ 'channel': 'NHK R1',
+ 'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/J8792PY43V/img/corner/box_31_thumbnail.jpg',
+ 'series': 'マイあさ！ ワールドリポート',
+ 'series_id': 'J8792PY43V_20',
+ 'timestamp': 1751837100,
+ 'upload_date': '20250706',
+ 'release_timestamp': 1751835600,
+ 'release_date': '20250706',
+
},
- 'expected_warnings': ['Failed to get extended metadata. API returned empty list.'],
+ 'expected_warnings': ['Failed to download extended metadata: HTTP Error 404: Not Found'],
}]
_API_URL_TMPL = None
+ # The `_format_*` and `_make_*` functions are ported from: https://www.nhk.or.jp/radio/assets/js/timetable_detail_new.js
+
+ def _format_act_list(self, act_list):
+ role_groups = {}
+ for act in traverse_obj(act_list, (..., {dict})):
+ role = act.get('role')
+ if role not in role_groups:
+ role_groups[role] = []
+ role_groups[role].append(act)
+
+ formatted_roles = []
+ for role, acts in role_groups.items():
+ for i, act in enumerate(acts):
+ res = f'【{role}】' if i == 0 and role is not None else ''
+ if title := act.get('title'):
+ res += f'{title}…'
+ formatted_roles.append(join_nonempty(res, act.get('name'), delim=''))
+ return join_nonempty(*formatted_roles, delim='，')
+
+ def _make_artists(self, track, key):
+ artists = []
+ for artist in traverse_obj(track, (key, ..., {dict})):
+ if res := join_nonempty(*traverse_obj(artist, ((
+ ('role', filter, {'{}…'.format}),
+ ('part', filter, {'（{}）'.format}),
+ ('name', filter),
+ ), {str})), delim=''):
+ artists.append(res)
+
+ return '、'.join(artists) or None
+
+ def _make_duration(self, track, key):
+ d = traverse_obj(track, (key, {parse_duration}))
+ if d is None:
+ return None
+ hours, remainder = divmod(d, 3600)
+ minutes, seconds = divmod(remainder, 60)
+ res = '（'
+ if hours > 0:
+ res += f'{int(hours)}時間'
+ if minutes > 0:
+ res += f'{int(minutes)}分'
+ res += f'{int(seconds):02}秒）'
+ return res
+
+ def _format_music_list(self, music_list):
+ tracks = []
+ for track in traverse_obj(music_list, (..., {dict})):
+ track_details = traverse_obj(track, ((
+ ('name', filter, {'「{}」'.format}),
+ ('lyricist', filter, {'{}:作詞'.format}),
+ ('composer', filter, {'{}:作曲'.format}),
+ ('arranger', filter, {'{}:編曲'.format}),
+ ), {str}))
+
+ track_details.append(self._make_artists(track, 'byArtist'))
+ track_details.append(self._make_duration(track, 'duration'))
+
+ if label := join_nonempty('label', 'code', delim=' ', from_dict=track):
+ track_details.append(f'＜{label}＞')
+ if location := traverse_obj(track, ('location', {str})):
+ track_details.append(f'～{location}～')
+ tracks.append(join_nonempty(*track_details, delim='\n'))
+ return '\n\n'.join(tracks)
+
+ def _format_description(self, response):
+ detailed_description = traverse_obj(response, ('detailedDescription', {dict})) or {}
+ return join_nonempty(
+ join_nonempty('epg80', 'epg200', delim='\n\n', from_dict=detailed_description),
+ traverse_obj(response, ('misc', 'actList', {self._format_act_list})),
+ traverse_obj(response, ('misc', 'musicList', {self._format_music_list})),
+ delim='\n\n')
+
+ def _get_thumbnails(self, data, keys, name=None, preference=-1):
+ thumbnails = []
+ for size, thumb in traverse_obj(data, (
+ *variadic(keys, (str, bytes, dict, set)), {dict.items},
+ lambda _, v: v[0] != 'copyright' and url_or_none(v[1]['url']),
+ )):
+ thumbnails.append({
+ 'url': thumb['url'],
+ 'width': int_or_none(thumb.get('width')),
+ 'height': int_or_none(thumb.get('height')),
+ 'preference': preference,
+ 'id': join_nonempty(name, size),
+ })
+ preference -= 1
+ return thumbnails
+
def _extract_extended_metadata(self, episode_id, aa_vinfo):
service, _, area = traverse_obj(aa_vinfo, (2, {str}, {lambda x: (x or '').partition(',')}))
+ date_id = aa_vinfo[3]
+
detail_url = try_call(
- lambda: self._API_URL_TMPL.format(area=area, service=service, dateid=aa_vinfo[3]))
+ lambda: self._API_URL_TMPL.format(broadcastEventId=join_nonempty(service, area, date_id)))
if not detail_url:
return {}
@@ -699,36 +779,37 @@ def _extract_extended_metadata(self, episode_id, aa_vinfo):
if error := traverse_obj(response, ('error', {dict})):
self.report_warning(
'Failed to get extended metadata. API returned '
- f'Error {join_nonempty("code", "message", from_dict=error, delim=": ")}')
+ f'Error {join_nonempty("statuscode", "message", from_dict=error, delim=": ")}')
return {}
- full_meta = traverse_obj(response, ('list', service, 0, {dict}))
- if not full_meta:
- self.report_warning('Failed to get extended metadata. API returned empty list.')
- return {}
+ station = traverse_obj(response, ('publishedOn', 'broadcastDisplayName', {str}))
- station = ' '.join(traverse_obj(full_meta, (('service', 'area'), 'name', {str}))) or None
- thumbnails = [{
- 'id': str(id_),
- 'preference': 1 if id_.startswith('thumbnail') else -2 if id_.startswith('logo') else -1,
- **traverse_obj(thumb, {
- 'url': 'url',
- 'width': ('width', {int_or_none}),
- 'height': ('height', {int_or_none}),
- }),
- } for id_, thumb in traverse_obj(full_meta, ('images', {dict.items}, lambda _, v: v[1]['url']))]
+ thumbnails = []
+ thumbnails.extend(self._get_thumbnails(response, ('about', 'eyecatch')))
+ for num, dct in enumerate(traverse_obj(response, ('about', 'eyecatchList', ...))):
+ thumbnails.extend(self._get_thumbnails(dct, None, join_nonempty('list', num), -2))
+ thumbnails.extend(
+ self._get_thumbnails(response, ('about', 'partOfSeries', 'eyecatch'), 'series', -3))
return filter_dict({
+ 'description': self._format_description(response),
+ 'cast': traverse_obj(response, ('misc', 'actList', ..., 'name', {str})),
+ 'thumbnails': thumbnails,
+ **traverse_obj(response, {
+ 'title': ('name', {str}),
+ 'timestamp': ('endDate', {unified_timestamp}),
+ 'release_timestamp': ('startDate', {unified_timestamp}),
+ 'duration': ('duration', {parse_duration}),
+ }),
+ **traverse_obj(response, ('identifierGroup', {
+ 'series': ('radioSeriesName', {str}),
+ 'series_id': ('radioSeriesId', {str}),
+ 'episode': ('radioEpisodeName', {str}),
+ 'episode_id': ('radioEpisodeId', {str}),
+ 'categories': ('genre', ..., ['name1', 'name2'], {str}, all, {orderedSet}),
+ })),
'channel': station,
'uploader': station,
- 'description': join_nonempty(
- 'subtitle', 'content', 'act', 'music', delim='\n\n', from_dict=full_meta),
- 'thumbnails': thumbnails,
- **traverse_obj(full_meta, {
- 'title': ('title', {str}),
- 'timestamp': ('end_time', {unified_timestamp}),
- 'release_timestamp': ('start_time', {unified_timestamp}),
- }),
})
def _extract_episode_info(self, episode, programme_id, series_meta):
@@ -782,7 +863,9 @@ def _real_extract(self, url):
site_id, corner_id, headline_id = self._match_valid_url(url).group('site', 'corner', 'headline')
programme_id = f'{site_id}_{corner_id}'
- if site_id == 'F261': # XXX: News programmes use old API (for now?)
+ # XXX: News programmes use the old API
+ # Can't move this to NhkRadioNewsPageIE because news items still use the normal URL format
+ if site_id == '18439M2W42':
meta = self._download_json(
'https://www.nhk.or.jp/s-media/news/news-site/list/v1/all.json', programme_id)['main']
series_meta = traverse_obj(meta, {
@@ -843,8 +926,8 @@ class NhkRadioNewsPageIE(InfoExtractor):
'url': 'https://www.nhk.or.jp/radionews/',
'playlist_mincount': 5,
'info_dict': {
- 'id': 'F261_01',
- 'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F261/img/RADIONEWS_640.jpg',
+ 'id': '18439M2W42_01',
+ 'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/18439M2W42/img/series_945_thumbnail.jpg',
'description': 'md5:bf2c5b397e44bc7eb26de98d8f15d79d',
'channel': 'NHKラジオ第1',
'uploader': 'NHKラジオ第1',
@@ -853,7 +936,7 @@ class NhkRadioNewsPageIE(InfoExtractor):
}]
def _real_extract(self, url):
- return self.url_result('https://www.nhk.or.jp/radio/ondemand/detail.html?p=F261_01', NhkRadiruIE)
+ return self.url_result('https://www.nhk.or.jp/radio/ondemand/detail.html?p=18439M2W42_01', NhkRadiruIE)
class NhkRadiruLiveIE(InfoExtractor):
@@ -863,11 +946,12 @@ class NhkRadiruLiveIE(InfoExtractor):
# radio 1, no area specified
'url': 'https://www.nhk.or.jp/radio/player/?ch=r1',
'info_dict': {
- 'id': 'r1-tokyo',
- 'title': 're:^ＮＨＫネットラジオ第1 東京.+$',
+ 'id': 'bs-r1-130',
+ 'title': 're:^NHKラジオ第1・東京.+$',
'ext': 'm4a',
- 'thumbnail': 'https://www.nhk.or.jp/common/img/media/r1-200x200.png',
+ 'thumbnail': 'https://www.nhk.jp/assets/images/broadcastservice/bs/r1/r1-logo.svg',
'live_status': 'is_live',
+ '_old_archive_ids': ['nhkradirulive r1-tokyo'],
},
}, {
# radio 2, area specified
@@ -875,26 +959,28 @@ class NhkRadiruLiveIE(InfoExtractor):
'url': 'https://www.nhk.or.jp/radio/player/?ch=r2',
'params': {'extractor_args': {'nhkradirulive': {'area': ['fukuoka']}}},
'info_dict': {
- 'id': 'r2-fukuoka',
- 'title': 're:^ＮＨＫネットラジオ第2 福岡.+$',
+ 'id': 'bs-r2-400',
+ 'title': 're:^NHKラジオ第2.+$',
'ext': 'm4a',
- 'thumbnail': 'https://www.nhk.or.jp/common/img/media/r2-200x200.png',
+ 'thumbnail': 'https://www.nhk.jp/assets/images/broadcastservice/bs/r2/r2-logo.svg',
'live_status': 'is_live',
+ '_old_archive_ids': ['nhkradirulive r2-fukuoka'],
},
}, {
# fm, area specified
'url': 'https://www.nhk.or.jp/radio/player/?ch=fm',
'params': {'extractor_args': {'nhkradirulive': {'area': ['sapporo']}}},
'info_dict': {
- 'id': 'fm-sapporo',
- 'title': 're:^ＮＨＫネットラジオＦＭ 札幌.+$',
+ 'id': 'bs-r3-010',
+ 'title': 're:^NHK FM・札幌.+$',
'ext': 'm4a',
- 'thumbnail': 'https://www.nhk.or.jp/common/img/media/fm-200x200.png',
+ 'thumbnail': 'https://www.nhk.jp/assets/images/broadcastservice/bs/r3/r3-logo.svg',
'live_status': 'is_live',
+ '_old_archive_ids': ['nhkradirulive fm-sapporo'],
},
}]
- _NOA_STATION_IDS = {'r1': 'n1', 'r2': 'n2', 'fm': 'n3'}
+ _NOA_STATION_IDS = {'r1': 'r1', 'r2': 'r2', 'fm': 'r3'}
def _real_extract(self, url):
station = self._match_id(url)
@@ -911,12 +997,15 @@ def _real_extract(self, url):
noa_info = self._download_json(
f'https:{config.find(".//url_program_noa").text}'.format(area=data.find('areakey').text),
station, note=f'Downloading {area} station metadata', fatal=False)
- present_info = traverse_obj(noa_info, ('nowonair_list', self._NOA_STATION_IDS.get(station), 'present'))
+ broadcast_service = traverse_obj(noa_info, (self._NOA_STATION_IDS.get(station), 'publishedOn'))
return {
- 'title': ' '.join(traverse_obj(present_info, (('service', 'area'), 'name', {str}))),
- 'id': join_nonempty(station, area),
- 'thumbnails': traverse_obj(present_info, ('service', 'images', ..., {
+ **traverse_obj(broadcast_service, {
+ 'title': ('broadcastDisplayName', {str}),
+ 'id': ('id', {str}),
+ }),
+ '_old_archive_ids': [make_archive_id(self, join_nonempty(station, area))],
+ 'thumbnails': traverse_obj(broadcast_service, ('logo', ..., {
'url': 'url',
'width': ('width', {int_or_none}),
'height': ('height', {int_or_none}),
diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py
index 0d0f7ceef..a20e570e6 100644
--- a/yt_dlp/extractor/niconico.py
+++ b/yt_dlp/extractor/niconico.py
@@ -4,16 +4,15 @@
import json
import re
import time
-import urllib.parse
from .common import InfoExtractor, SearchInfoExtractor
-from ..networking import Request
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
OnDemandPagedList,
clean_html,
determine_ext,
+ extract_attributes,
float_or_none,
int_or_none,
parse_bitrate,
@@ -22,9 +21,8 @@
parse_qs,
parse_resolution,
qualities,
- remove_start,
str_or_none,
- unescapeHTML,
+ truncate_string,
unified_timestamp,
update_url_query,
url_basename,
@@ -32,7 +30,11 @@
urlencode_postdata,
urljoin,
)
-from ..utils.traversal import find_element, require, traverse_obj
+from ..utils.traversal import (
+ find_element,
+ require,
+ traverse_obj,
+)
class NiconicoBaseIE(InfoExtractor):
@@ -806,41 +808,39 @@ class NiconicoLiveIE(NiconicoBaseIE):
def _real_extract(self, url):
video_id = self._match_id(url)
- webpage, urlh = self._download_webpage_handle(f'https://live.nicovideo.jp/watch/{video_id}', video_id)
+ webpage = self._download_webpage(url, video_id, expected_status=404)
+ if err_msg := traverse_obj(webpage, ({find_element(cls='message')}, {clean_html})):
+ raise ExtractorError(err_msg, expected=True)
- embedded_data = self._parse_json(unescapeHTML(self._search_regex(
- r' 100:
- recv = recv[:100] + '...'
- self.write_debug(f'Server said: {recv}')
+ self.write_debug(f'Server response: {truncate_string(recv, 100)}')
title = traverse_obj(embedded_data, ('program', 'title')) or self._html_search_meta(
('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
- raw_thumbs = traverse_obj(embedded_data, ('program', 'thumbnail')) or {}
+ raw_thumbs = traverse_obj(embedded_data, ('program', 'thumbnail', {dict})) or {}
thumbnails = []
for name, value in raw_thumbs.items():
if not isinstance(value, dict):
@@ -897,31 +895,30 @@ def _real_extract(self, url):
cookie['domain'], cookie['name'], cookie['value'],
expire_time=unified_timestamp(cookie.get('expires')), path=cookie['path'], secure=cookie['secure'])
- fmt_common = {
- 'live_latency': 'high',
- 'origin': hostname,
- 'protocol': 'niconico_live',
- 'video_id': video_id,
- 'ws': ws,
- }
q_iter = (q for q in qualities[1:] if not q.startswith('audio_')) # ignore initial 'abr'
a_map = {96: 'audio_low', 192: 'audio_high'}
formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4', live=True)
for fmt in formats:
+ fmt['protocol'] = 'niconico_live'
if fmt.get('acodec') == 'none':
fmt['format_id'] = next(q_iter, fmt['format_id'])
elif fmt.get('vcodec') == 'none':
abr = parse_bitrate(fmt['url'].lower())
fmt.update({
'abr': abr,
+ 'acodec': 'mp4a.40.2',
'format_id': a_map.get(abr, fmt['format_id']),
})
- fmt.update(fmt_common)
return {
'id': video_id,
'title': title,
+ 'downloader_options': {
+ 'max_quality': traverse_obj(embedded_data, ('program', 'stream', 'maxQuality', {str})) or 'normal',
+ 'ws': ws,
+ 'ws_url': ws_url,
+ },
**traverse_obj(embedded_data, {
'view_count': ('program', 'statistics', 'watchCount'),
'comment_count': ('program', 'statistics', 'commentCount'),
diff --git a/yt_dlp/extractor/ninegag.py b/yt_dlp/extractor/ninegag.py
index 2979f3a50..1b88e9c54 100644
--- a/yt_dlp/extractor/ninegag.py
+++ b/yt_dlp/extractor/ninegag.py
@@ -1,6 +1,5 @@
from .common import InfoExtractor
from ..utils import (
- ExtractorError,
determine_ext,
int_or_none,
traverse_obj,
@@ -61,10 +60,10 @@ def _real_extract(self, url):
post = self._download_json(
'https://9gag.com/v1/post', post_id, query={
'id': post_id,
- })['data']['post']
+ }, impersonate=True)['data']['post']
if post.get('type') != 'Animated':
- raise ExtractorError(
+ self.raise_no_formats(
'The given url does not contain a video',
expected=True)
diff --git a/yt_dlp/extractor/ninenow.py b/yt_dlp/extractor/ninenow.py
index 7b0cb77a7..2f3a4ed28 100644
--- a/yt_dlp/extractor/ninenow.py
+++ b/yt_dlp/extractor/ninenow.py
@@ -1,6 +1,3 @@
-import json
-import re
-
from .brightcove import BrightcoveNewIE
from .common import InfoExtractor
from ..utils import (
@@ -11,7 +8,12 @@
str_or_none,
url_or_none,
)
-from ..utils.traversal import require, traverse_obj, value
+from ..utils.traversal import (
+ get_first,
+ require,
+ traverse_obj,
+ value,
+)
class NineNowIE(InfoExtractor):
@@ -101,20 +103,11 @@ class NineNowIE(InfoExtractor):
}]
BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4460760524001/default_default/index.html?videoId={}'
- # XXX: For parsing next.js v15+ data; see also yt_dlp.extractor.francetv and yt_dlp.extractor.goplay
- def _find_json(self, s):
- return self._search_json(
- r'\w+\s*:\s*', s, 'next js data', None, contains_pattern=r'\[(?s:.+)\]', default=None)
-
def _real_extract(self, url):
display_id, video_type = self._match_valid_url(url).group('id', 'type')
webpage = self._download_webpage(url, display_id)
- common_data = traverse_obj(
- re.findall(r']*>\s*self\.__next_f\.push\(\s*(\[.+?\])\s*\);?\s*', webpage),
- (..., {json.loads}, ..., {self._find_json},
- lambda _, v: v['payload'][video_type]['slug'] == display_id,
- 'payload', any, {require('video data')}))
+ common_data = get_first(self._search_nextjs_v13_data(webpage, display_id), ('payload', {dict}))
if traverse_obj(common_data, (video_type, 'video', 'drm', {bool})):
self.report_drm(display_id)
diff --git a/yt_dlp/extractor/nobelprize.py b/yt_dlp/extractor/nobelprize.py
index 536ca27f7..833bab094 100644
--- a/yt_dlp/extractor/nobelprize.py
+++ b/yt_dlp/extractor/nobelprize.py
@@ -1,59 +1,57 @@
from .common import InfoExtractor
from ..utils import (
- determine_ext,
- get_element_by_attribute,
+ UnsupportedError,
+ clean_html,
int_or_none,
- js_to_json,
- mimetype2ext,
- update_url_query,
+ parse_duration,
+ parse_qs,
+ str_or_none,
+ update_url,
)
+from ..utils.traversal import find_element, traverse_obj
class NobelPrizeIE(InfoExtractor):
- _WORKING = False
- _VALID_URL = r'https?://(?:www\.)?nobelprize\.org/mediaplayer.*?\bid=(?P\d+)'
- _TEST = {
- 'url': 'http://www.nobelprize.org/mediaplayer/?id=2636',
- 'md5': '04c81e5714bb36cc4e2232fee1d8157f',
+ _VALID_URL = r'https?://(?:(?:mediaplayer|www)\.)?nobelprize\.org/mediaplayer/'
+ _TESTS = [{
+ 'url': 'https://www.nobelprize.org/mediaplayer/?id=2636',
'info_dict': {
'id': '2636',
'ext': 'mp4',
'title': 'Announcement of the 2016 Nobel Prize in Physics',
- 'description': 'md5:05beba57f4f5a4bbd4cf2ef28fcff739',
+ 'description': 'md5:1a2d8a6ca80c88fb3b9a326e0b0e8e43',
+ 'duration': 1560.0,
+ 'thumbnail': r're:https?://www\.nobelprize\.org/images/.+\.jpg',
+ 'timestamp': 1504883793,
+ 'upload_date': '20170908',
},
- }
+ }, {
+ 'url': 'https://mediaplayer.nobelprize.org/mediaplayer/?qid=12693',
+ 'info_dict': {
+ 'id': '12693',
+ 'ext': 'mp4',
+ 'title': 'Nobel Lecture by Peter Higgs',
+ 'description': 'md5:9b12e275dbe3a8138484e70e00673a05',
+ 'duration': 1800.0,
+ 'thumbnail': r're:https?://www\.nobelprize\.org/images/.+\.jpg',
+ 'timestamp': 1504883793,
+ 'upload_date': '20170908',
+ },
+ }]
def _real_extract(self, url):
- video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
- media = self._parse_json(self._search_regex(
- r'(?s)var\s*config\s*=\s*({.+?});', webpage,
- 'config'), video_id, js_to_json)['media']
- title = media['title']
-
- formats = []
- for source in media.get('source', []):
- source_src = source.get('src')
- if not source_src:
- continue
- ext = mimetype2ext(source.get('type')) or determine_ext(source_src)
- if ext == 'm3u8':
- formats.extend(self._extract_m3u8_formats(
- source_src, video_id, 'mp4', 'm3u8_native',
- m3u8_id='hls', fatal=False))
- elif ext == 'f4m':
- formats.extend(self._extract_f4m_formats(
- update_url_query(source_src, {'hdcore': '3.7.0'}),
- video_id, f4m_id='hds', fatal=False))
- else:
- formats.append({
- 'url': source_src,
- })
+ video_id = traverse_obj(parse_qs(url), (
+ ('id', 'qid'), -1, {int_or_none}, {str_or_none}, any))
+ if not video_id:
+ raise UnsupportedError(url)
+ webpage = self._download_webpage(
+ update_url(url, netloc='mediaplayer.nobelprize.org'), video_id)
return {
+ **self._search_json_ld(webpage, video_id),
'id': video_id,
- 'title': title,
- 'description': get_element_by_attribute('itemprop', 'description', webpage),
- 'duration': int_or_none(media.get('duration')),
- 'formats': formats,
+ 'title': self._html_search_meta('caption', webpage),
+ 'description': traverse_obj(webpage, (
+ {find_element(tag='span', attr='itemprop', value='description')}, {clean_html})),
+ 'duration': parse_duration(self._html_search_meta('duration', webpage)),
}
diff --git a/yt_dlp/extractor/noovo.py b/yt_dlp/extractor/noovo.py
deleted file mode 100644
index 772d4ed9e..000000000
--- a/yt_dlp/extractor/noovo.py
+++ /dev/null
@@ -1,100 +0,0 @@
-from .brightcove import BrightcoveNewIE
-from .common import InfoExtractor
-from ..utils import (
- int_or_none,
- js_to_json,
- smuggle_url,
- try_get,
-)
-
-
-class NoovoIE(InfoExtractor):
- _VALID_URL = r'https?://(?:[^/]+\.)?noovo\.ca/videos/(?P[^/]+/[^/?#&]+)'
- _TESTS = [{
- # clip
- 'url': 'http://noovo.ca/videos/rpm-plus/chrysler-imperial',
- 'info_dict': {
- 'id': '5386045029001',
- 'ext': 'mp4',
- 'title': 'Chrysler Imperial',
- 'description': 'md5:de3c898d1eb810f3e6243e08c8b4a056',
- 'timestamp': 1491399228,
- 'upload_date': '20170405',
- 'uploader_id': '618566855001',
- 'series': 'RPM+',
- },
- 'params': {
- 'skip_download': True,
- },
- }, {
- # episode
- 'url': 'http://noovo.ca/videos/l-amour-est-dans-le-pre/episode-13-8',
- 'info_dict': {
- 'id': '5395865725001',
- 'title': 'Épisode 13 : Les retrouvailles',
- 'description': 'md5:888c3330f0c1b4476c5bc99a1c040473',
- 'ext': 'mp4',
- 'timestamp': 1492019320,
- 'upload_date': '20170412',
- 'uploader_id': '618566855001',
- 'series': "L'amour est dans le pré",
- 'season_number': 5,
- 'episode': 'Épisode 13',
- 'episode_number': 13,
- },
- 'params': {
- 'skip_download': True,
- },
- }]
- BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/618566855001/default_default/index.html?videoId=%s'
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
-
- webpage = self._download_webpage(url, video_id)
-
- brightcove_id = self._search_regex(
- r'data-video-id=["\'](\d+)', webpage, 'brightcove id')
-
- data = self._parse_json(
- self._search_regex(
- r'(?s)dataLayer\.push\(\s*({.+?})\s*\);', webpage, 'data',
- default='{}'),
- video_id, transform_source=js_to_json, fatal=False)
-
- title = try_get(
- data, lambda x: x['video']['nom'],
- str) or self._html_search_meta(
- 'dcterms.Title', webpage, 'title', fatal=True)
-
- description = self._html_search_meta(
- ('dcterms.Description', 'description'), webpage, 'description')
-
- series = try_get(
- data, lambda x: x['emission']['nom']) or self._search_regex(
- r']+class="banner-card__subtitle h4"[^>]*>([^<]+)',
- webpage, 'series', default=None)
-
- season_el = try_get(data, lambda x: x['emission']['saison'], dict) or {}
- season = try_get(season_el, lambda x: x['nom'], str)
- season_number = int_or_none(try_get(season_el, lambda x: x['numero']))
-
- episode_el = try_get(season_el, lambda x: x['episode'], dict) or {}
- episode = try_get(episode_el, lambda x: x['nom'], str)
- episode_number = int_or_none(try_get(episode_el, lambda x: x['numero']))
-
- return {
- '_type': 'url_transparent',
- 'ie_key': BrightcoveNewIE.ie_key(),
- 'url': smuggle_url(
- self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
- {'geo_countries': ['CA']}),
- 'id': brightcove_id,
- 'title': title,
- 'description': description,
- 'series': series,
- 'season': season,
- 'season_number': season_number,
- 'episode': episode,
- 'episode_number': episode_number,
- }
diff --git a/yt_dlp/extractor/ntvcojp.py b/yt_dlp/extractor/ntvcojp.py
index 422ec6eb0..76c5936ba 100644
--- a/yt_dlp/extractor/ntvcojp.py
+++ b/yt_dlp/extractor/ntvcojp.py
@@ -1,55 +1,82 @@
-from .common import InfoExtractor
+from .streaks import StreaksBaseIE
from ..utils import (
- ExtractorError,
- smuggle_url,
- traverse_obj,
+ int_or_none,
+ parse_iso8601,
+ str_or_none,
+ url_or_none,
)
+from ..utils.traversal import require, traverse_obj
-class NTVCoJpCUIE(InfoExtractor):
+class NTVCoJpCUIE(StreaksBaseIE):
IE_NAME = 'cu.ntv.co.jp'
- IE_DESC = 'Nippon Television Network'
- _VALID_URL = r'https?://cu\.ntv\.co\.jp/(?!program)(?P[^/?&#]+)'
- _TEST = {
- 'url': 'https://cu.ntv.co.jp/televiva-chill-gohan_181031/',
+ IE_DESC = '日テレ無料TADA!'
+ _VALID_URL = r'https?://cu\.ntv\.co\.jp/(?!program-list|search)(?P[\w-]+)/?(?:[?#]|$)'
+ _TESTS = [{
+ 'url': 'https://cu.ntv.co.jp/gaki_20250525/',
'info_dict': {
- 'id': '5978891207001',
+ 'id': 'gaki_20250525',
'ext': 'mp4',
- 'title': '桜エビと炒り卵がポイント！ 「中華風 エビチリおにぎり」──『美虎』五十嵐美幸',
- 'upload_date': '20181213',
- 'description': 'md5:1985b51a9abc285df0104d982a325f2a',
- 'uploader_id': '3855502814001',
- 'timestamp': 1544669941,
+ 'title': '放送開始36年!方正ココリコが選ぶ神回&地獄回!',
+ 'cast': 'count:2',
+ 'description': 'md5:1e1db556224d627d4d2f74370c650927',
+ 'display_id': 'ref:gaki_20250525',
+ 'duration': 1450,
+ 'episode': '放送開始36年!方正ココリコが選ぶ神回&地獄回!',
+ 'episode_id': '000000010172808',
+ 'episode_number': 255,
+ 'genres': ['variety'],
+ 'live_status': 'not_live',
+ 'modified_date': '20250525',
+ 'modified_timestamp': 1748145537,
+ 'release_date': '20250525',
+ 'release_timestamp': 1748145539,
+ 'series': 'ダウンタウンのガキの使いやあらへんで！',
+ 'series_id': 'gaki',
+ 'thumbnail': r're:https?://.+\.jpg',
+ 'timestamp': 1748145197,
+ 'upload_date': '20250525',
+ 'uploader': '日本テレビ放送網',
+ 'uploader_id': '0x7FE2',
},
- 'params': {
- # m3u8 download
- 'skip_download': True,
- },
- }
-
- BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
+ }]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
- player_config = self._search_nuxt_data(webpage, display_id)
- video_id = traverse_obj(player_config, ('movie', 'video_id'))
- if not video_id:
- raise ExtractorError('Failed to extract video ID for Brightcove')
- account_id = traverse_obj(player_config, ('player', 'account')) or '3855502814001'
- title = traverse_obj(player_config, ('movie', 'name'))
- if not title:
- og_title = self._og_search_title(webpage, fatal=False) or traverse_obj(player_config, ('player', 'title'))
- if og_title:
- title = og_title.split('(', 1)[0].strip()
- description = (traverse_obj(player_config, ('movie', 'description'))
- or self._html_search_meta(['description', 'og:description'], webpage))
+
+ info = self._search_json(
+ r'window\.app\s*=', webpage, 'video info',
+ display_id)['falcorCache']['catalog']['episode'][display_id]['value']
+ media_id = traverse_obj(info, (
+ 'streaks_data', 'mediaid', {str_or_none}, {require('Streaks media ID')}))
+ non_phonetic = (lambda _, v: v['is_phonetic'] is False, 'value', {str})
+
return {
- '_type': 'url_transparent',
- 'id': video_id,
- 'display_id': display_id,
- 'title': title,
- 'description': description,
- 'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, video_id), {'geo_countries': ['JP']}),
- 'ie_key': 'BrightcoveNew',
+ **self._extract_from_streaks_api('ntv-tada', media_id, headers={
+ 'X-Streaks-Api-Key': 'df497719056b44059a0483b8faad1f4a',
+ }),
+ **traverse_obj(info, {
+ 'id': ('content_id', {str_or_none}),
+ 'title': ('title', *non_phonetic, any),
+ 'age_limit': ('is_adult_only_content', {lambda x: 18 if x else None}),
+ 'cast': ('credit', ..., 'name', *non_phonetic),
+ 'genres': ('genre', ..., {str}),
+ 'release_timestamp': ('pub_date', {parse_iso8601}),
+ 'tags': ('tags', ..., {str}),
+ 'thumbnail': ('artwork', ..., 'url', any, {url_or_none}),
+ }),
+ **traverse_obj(info, ('tv_episode_info', {
+ 'duration': ('duration', {int_or_none}),
+ 'episode_number': ('episode_number', {int}),
+ 'series': ('parent_show_title', *non_phonetic, any),
+ 'series_id': ('show_content_id', {str}),
+ })),
+ **traverse_obj(info, ('custom_data', {
+ 'description': ('program_detail', {str}),
+ 'episode': ('episode_title', {str}),
+ 'episode_id': ('episode_id', {str_or_none}),
+ 'uploader': ('network_name', {str}),
+ 'uploader_id': ('network_id', {str}),
+ })),
}
diff --git a/yt_dlp/extractor/odnoklassniki.py b/yt_dlp/extractor/odnoklassniki.py
index d27d1c3f0..18eba42e6 100644
--- a/yt_dlp/extractor/odnoklassniki.py
+++ b/yt_dlp/extractor/odnoklassniki.py
@@ -273,6 +273,8 @@ def _extract_desktop(self, url):
return self._extract_desktop(smuggle_url(url, {'referrer': 'https://boosty.to'}))
elif error:
raise ExtractorError(error, expected=True)
+ elif '>Access to this video is restricted' in webpage:
+ self.raise_login_required()
player = self._parse_json(
unescapeHTML(self._search_regex(
@@ -429,7 +431,7 @@ def _extract_mobile(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(
- f'http://m.ok.ru/video/{video_id}', video_id,
+ f'https://m.ok.ru/video/{video_id}', video_id,
note='Downloading mobile webpage')
error = self._search_regex(
diff --git a/yt_dlp/extractor/qqmusic.py b/yt_dlp/extractor/qqmusic.py
index fb46e0d12..56a8e7300 100644
--- a/yt_dlp/extractor/qqmusic.py
+++ b/yt_dlp/extractor/qqmusic.py
@@ -15,7 +15,6 @@
str_or_none,
strip_jsonp,
traverse_obj,
- unescapeHTML,
url_or_none,
urljoin,
)
@@ -425,7 +424,7 @@ def _real_extract(self, url):
return self.playlist_result(entries, list_id, **traverse_obj(list_json, ('cdlist', 0, {
'title': ('dissname', {str}),
- 'description': ('desc', {unescapeHTML}, {clean_html}),
+ 'description': ('desc', {clean_html}),
})))
diff --git a/yt_dlp/extractor/rai.py b/yt_dlp/extractor/rai.py
index c489dc731..027f7a7b6 100644
--- a/yt_dlp/extractor/rai.py
+++ b/yt_dlp/extractor/rai.py
@@ -765,7 +765,7 @@ class RaiCulturaIE(RaiNewsIE): # XXX: Do not subclass from concrete IE
class RaiSudtirolIE(RaiBaseIE):
- _VALID_URL = r'https?://raisudtirol\.rai\.it/.+media=(?P\w+)'
+ _VALID_URL = r'https?://rai(?:bz|sudtirol)\.rai\.it/.+media=(?P\w+)'
_TESTS = [{
# mp4 file
'url': 'https://raisudtirol.rai.it/la/index.php?media=Ptv1619729460',
@@ -791,6 +791,9 @@ class RaiSudtirolIE(RaiBaseIE):
'formats': 'count:6',
},
'params': {'skip_download': True},
+ }, {
+ 'url': 'https://raibz.rai.it/de/index.php?media=Ptv1751660400',
+ 'only_matching': True,
}]
def _real_extract(self, url):
diff --git a/yt_dlp/extractor/sauceplus.py b/yt_dlp/extractor/sauceplus.py
new file mode 100644
index 000000000..75d7022d3
--- /dev/null
+++ b/yt_dlp/extractor/sauceplus.py
@@ -0,0 +1,41 @@
+from .floatplane import FloatplaneBaseIE
+
+
+class SaucePlusIE(FloatplaneBaseIE):
+ IE_DESC = 'Sauce+'
+ _VALID_URL = r'https?://(?:(?:www|beta)\.)?sauceplus\.com/post/(?P\w+)'
+ _BASE_URL = 'https://www.sauceplus.com'
+ _HEADERS = {
+ 'Origin': _BASE_URL,
+ 'Referer': f'{_BASE_URL}/',
+ }
+ _IMPERSONATE_TARGET = True
+ _TESTS = [{
+ 'url': 'https://www.sauceplus.com/post/YbBwIa2A5g',
+ 'info_dict': {
+ 'id': 'eit4Ugu5TL',
+ 'ext': 'mp4',
+ 'display_id': 'YbBwIa2A5g',
+ 'title': 'Scare the Coyote - Episode 3',
+ 'description': '',
+ 'thumbnail': r're:^https?://.*\.jpe?g$',
+ 'duration': 2975,
+ 'comment_count': int,
+ 'like_count': int,
+ 'dislike_count': int,
+ 'release_date': '20250627',
+ 'release_timestamp': 1750993500,
+ 'uploader': 'Scare The Coyote',
+ 'uploader_id': '683e0a3269688656a5a49a44',
+ 'uploader_url': 'https://www.sauceplus.com/channel/ScareTheCoyote/home',
+ 'channel': 'Scare The Coyote',
+ 'channel_id': '683e0a326968866ceba49a45',
+ 'channel_url': 'https://www.sauceplus.com/channel/ScareTheCoyote/home/main',
+ 'availability': 'subscriber_only',
+ },
+ 'params': {'skip_download': 'm3u8'},
+ }]
+
+ def _real_initialize(self):
+ if not self._get_cookies(self._BASE_URL).get('__Host-sp-sess'):
+ self.raise_login_required()
diff --git a/yt_dlp/extractor/skyit.py b/yt_dlp/extractor/skyit.py
index 0013d2621..fe45be774 100644
--- a/yt_dlp/extractor/skyit.py
+++ b/yt_dlp/extractor/skyit.py
@@ -213,7 +213,7 @@ class CieloTVItIE(SkyItIE): # XXX: Do not subclass from concrete IE
class TV8ItIE(SkyItVideoIE): # XXX: Do not subclass from concrete IE
IE_NAME = 'tv8.it'
- _VALID_URL = r'https?://(?:www\.)?tv8\.it/(?:show)?video/[0-9a-z-]+-(?P\d+)'
+ _VALID_URL = r'https?://(?:www\.)?tv8\.it/(?:show)?video/(?:[0-9a-z-]+-)?(?P\d+)'
_TESTS = [{
'url': 'https://www.tv8.it/video/ogni-mattina-ucciso-asino-di-andrea-lo-cicero-630529',
'md5': '9ab906a3f75ea342ed928442f9dabd21',
@@ -227,6 +227,19 @@ class TV8ItIE(SkyItVideoIE): # XXX: Do not subclass from concrete IE
'thumbnail': 'https://videoplatform.sky.it/still/2020/11/18/1605717753954_ogni-mattina-ucciso-asino-di-andrea-lo-cicero_videostill_1.jpg',
},
'params': {'skip_download': 'm3u8'},
+ }, {
+ 'url': 'https://www.tv8.it/video/964361',
+ 'md5': '1e58e807154658a16edc29e45be38107',
+ 'info_dict': {
+ 'id': '964361',
+ 'ext': 'mp4',
+ 'title': 'GialappaShow - S.4 Ep.2',
+ 'description': 'md5:60bb4ff5af18bbeeaedabc1de5f9e1e2',
+ 'duration': 8030,
+ 'thumbnail': 'https://videoplatform.sky.it/captures/494/2024/11/06/964361/964361_1730888412914_thumb_494.jpg',
+ 'timestamp': 1730821499,
+ 'upload_date': '20241105',
+ },
}]
_DOMAIN = 'mtv8'
diff --git a/yt_dlp/extractor/soundcloud.py b/yt_dlp/extractor/soundcloud.py
index 3496a08ef..404e29897 100644
--- a/yt_dlp/extractor/soundcloud.py
+++ b/yt_dlp/extractor/soundcloud.py
@@ -242,7 +242,7 @@ def _extract_info_dict(self, info, full_title=None, secret_token=None, extract_f
format_urls.add(format_url)
formats.append({
'format_id': 'download',
- 'ext': urlhandle_detect_ext(urlh, default='mp3'),
+ 'ext': urlhandle_detect_ext(urlh),
'filesize': int_or_none(urlh.headers.get('Content-Length')),
'url': format_url,
'quality': 10,
diff --git a/yt_dlp/extractor/sportdeutschland.py b/yt_dlp/extractor/sportdeutschland.py
index 2d6acb876..8349d9604 100644
--- a/yt_dlp/extractor/sportdeutschland.py
+++ b/yt_dlp/extractor/sportdeutschland.py
@@ -25,6 +25,7 @@ class SportDeutschlandIE(InfoExtractor):
'upload_date': '20230114',
'timestamp': 1673733618,
},
+ 'skip': 'not found',
}, {
'url': 'https://sportdeutschland.tv/deutscherbadmintonverband/bwf-tour-1-runde-feld-1-yonex-gainward-german-open-2022-0',
'info_dict': {
@@ -41,6 +42,7 @@ class SportDeutschlandIE(InfoExtractor):
'upload_date': '20220309',
'timestamp': 1646860727.0,
},
+ 'skip': 'not found',
}, {
'url': 'https://sportdeutschland.tv/ggcbremen/formationswochenende-latein-2023',
'info_dict': {
@@ -68,6 +70,7 @@ class SportDeutschlandIE(InfoExtractor):
'live_status': 'was_live',
},
}],
+ 'skip': 'not found',
}, {
'url': 'https://sportdeutschland.tv/dtb/gymnastik-international-tag-1',
'info_dict': {
@@ -82,13 +85,30 @@ class SportDeutschlandIE(InfoExtractor):
'live_status': 'is_live',
},
'skip': 'live',
+ }, {
+ 'url': 'https://sportdeutschland.tv/rostock-griffins/gfl2-rostock-griffins-vs-elmshorn-fighting-pirates',
+ 'md5': '35c11a19395c938cdd076b93bda54cde',
+ 'info_dict': {
+ 'id': '9f27a97d-1544-4d0b-aa03-48d92d17a03a',
+ 'ext': 'mp4',
+ 'title': 'GFL2: Rostock Griffins vs. Elmshorn Fighting Pirates',
+ 'display_id': 'rostock-griffins/gfl2-rostock-griffins-vs-elmshorn-fighting-pirates',
+ 'channel': 'Rostock Griffins',
+ 'channel_url': 'https://sportdeutschland.tv/rostock-griffins',
+ 'live_status': 'was_live',
+ 'description': 'md5:60cb00067e55dafa27b0933a43d72862',
+ 'channel_id': '9635f21c-3f67-4584-9ce4-796e9a47276b',
+ 'timestamp': 1749913117,
+ 'upload_date': '20250614',
+ },
}]
def _process_video(self, asset_id, video):
is_live = video['type'] == 'mux_live'
token = self._download_json(
- f'https://api.sportdeutschland.tv/api/frontend/asset-token/{asset_id}',
- video['id'], query={'type': video['type'], 'playback_id': video['src']})['token']
+ f'https://api.sportdeutschland.tv/api/web/personal/asset-token/{asset_id}',
+ video['id'], query={'type': video['type'], 'playback_id': video['src']},
+ headers={'Referer': 'https://sportdeutschland.tv/'})['token']
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
f'https://stream.mux.com/{video["src"]}.m3u8?token={token}', video['id'], live=is_live)
diff --git a/yt_dlp/extractor/sproutvideo.py b/yt_dlp/extractor/sproutvideo.py
index c0923594e..494042738 100644
--- a/yt_dlp/extractor/sproutvideo.py
+++ b/yt_dlp/extractor/sproutvideo.py
@@ -41,6 +41,7 @@ class SproutVideoIE(InfoExtractor):
'duration': 703,
'thumbnail': r're:https?://images\.sproutvideo\.com/.+\.jpg',
},
+ 'skip': 'Account Disabled',
}, {
# http formats 'sd' and 'hd' are available
'url': 'https://videos.sproutvideo.com/embed/119cd6bc1a18e6cd98/30751a1761ae5b90',
@@ -98,10 +99,17 @@ def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {})
video_id = self._match_id(url)
webpage = self._download_webpage(
- url, video_id, headers=traverse_obj(smuggled_data, {'Referer': 'referer'}))
+ url, video_id, headers=traverse_obj(smuggled_data, {'Referer': 'referer'}), impersonate=True)
data = self._search_json(
- r'var\s+dat\s*=\s*["\']', webpage, 'data', video_id, contains_pattern=r'[A-Za-z0-9+/=]+',
- end_pattern=r'["\'];', transform_source=lambda x: base64.b64decode(x).decode())
+ r'(?:var|const|let)\s+(?:dat|playerInfo)\s*=\s*["\']', webpage, 'player info', video_id,
+ contains_pattern=r'[A-Za-z0-9+/=]+', end_pattern=r'["\'];',
+ transform_source=lambda x: base64.b64decode(x).decode())
+
+ # SproutVideo may send player info for 'SMPTE Color Monitor Test' [a791d7b71b12ecc52e]
+ # e.g. if the user-agent we used with the webpage request is too old
+ video_uid = data['videoUid']
+ if video_id != video_uid:
+ raise ExtractorError(f'{self.IE_NAME} sent the wrong video data ({video_uid})')
formats, subtitles = [], {}
headers = {
diff --git a/yt_dlp/extractor/srmediathek.py b/yt_dlp/extractor/srmediathek.py
index fc63d9b1a..d6cab6ae7 100644
--- a/yt_dlp/extractor/srmediathek.py
+++ b/yt_dlp/extractor/srmediathek.py
@@ -1,57 +1,102 @@
from .ard import ARDMediathekBaseIE
from ..utils import (
ExtractorError,
- get_element_by_attribute,
+ clean_html,
+ extract_attributes,
+ parse_duration,
+ parse_qs,
+ unified_strdate,
+)
+from ..utils.traversal import (
+ find_element,
+ require,
+ traverse_obj,
)
class SRMediathekIE(ARDMediathekBaseIE):
- _WORKING = False
IE_NAME = 'sr:mediathek'
IE_DESC = 'Saarländischer Rundfunk'
- _VALID_URL = r'https?://sr-mediathek(?:\.sr-online)?\.de/index\.php\?.*?&id=(?P[0-9]+)'
+ _CLS_COMMON = 'teaser__image__caption__text teaser__image__caption__text--'
+ _VALID_URL = r'https?://(?:www\.)?sr-mediathek\.de/index\.php\?.*?&id=(?P\d+)'
_TESTS = [{
- 'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
+ 'url': 'https://www.sr-mediathek.de/index.php?seite=7&id=141317',
'info_dict': {
- 'id': '28455',
+ 'id': '141317',
'ext': 'mp4',
- 'title': 'sportarena (26.10.2014)',
- 'description': 'Ringen: KSV Köllerbach gegen Aachen-Walheim; Frauen-Fußball: 1. FC Saarbrücken gegen Sindelfingen; Motorsport: Rallye in Losheim; dazu: Interview mit Timo Bernhard; Turnen: TG Saar; Reitsport: Deutscher Voltigier-Pokal; Badminton: Interview mit Michael Fuchs ',
- 'thumbnail': r're:^https?://.*\.jpg$',
- },
- 'skip': 'no longer available',
- }, {
- 'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=37682',
- 'info_dict': {
- 'id': '37682',
- 'ext': 'mp4',
- 'title': 'Love, Cakes and Rock\'n\'Roll',
- 'description': 'md5:18bf9763631c7d326c22603681e1123d',
- },
- 'params': {
- # m3u8 download
- 'skip_download': True,
+ 'title': 'Kärnten, da will ich hin!',
+ 'channel': 'SR Fernsehen',
+ 'description': 'md5:7732e71e803379a499732864a572a456',
+ 'duration': 1788.0,
+ 'release_date': '20250525',
+ 'series': 'da will ich hin!',
+ 'series_id': 'DWIH',
+ 'thumbnail': r're:https?://.+\.jpg',
},
}, {
- 'url': 'http://sr-mediathek.de/index.php?seite=7&id=7480',
- 'only_matching': True,
+ 'url': 'https://www.sr-mediathek.de/index.php?seite=7&id=153853',
+ 'info_dict': {
+ 'id': '153853',
+ 'ext': 'mp3',
+ 'title': 'Kappes, Klöße, Kokosmilch: Bruschetta mit Nduja',
+ 'channel': 'SR 3',
+ 'description': 'md5:3935798de3562b10c4070b408a15e225',
+ 'duration': 139.0,
+ 'release_date': '20250523',
+ 'series': 'Kappes, Klöße, Kokosmilch',
+ 'series_id': 'SR3_KKK_A',
+ 'thumbnail': r're:https?://.+\.jpg',
+ },
+ }, {
+ 'url': 'https://www.sr-mediathek.de/index.php?seite=7&id=31406&pnr=&tbl=pf',
+ 'info_dict': {
+ 'id': '31406',
+ 'ext': 'mp3',
+ 'title': 'Das Leben schwer nehmen, ist einfach zu anstrengend',
+ 'channel': 'SR 1',
+ 'description': 'md5:3e03fd556af831ad984d0add7175fb0c',
+ 'duration': 1769.0,
+ 'release_date': '20230717',
+ 'series': 'Abendrot',
+ 'series_id': 'SR1_AB_P',
+ 'thumbnail': r're:https?://.+\.jpg',
+ },
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
+ description = self._og_search_description(webpage)
- if '>Der gewünschte Beitrag ist leider nicht mehr verfügbar.<' in webpage:
+ if description == 'Der gewünschte Beitrag ist leider nicht mehr vorhanden.':
raise ExtractorError(f'Video {video_id} is no longer available', expected=True)
- media_collection_url = self._search_regex(
- r'data-mediacollection-ardplayer="([^"]+)"', webpage, 'media collection url')
- info = self._extract_media_info(media_collection_url, webpage, video_id)
- info.update({
+ player_url = traverse_obj(webpage, (
+ {find_element(tag='div', id=f'player{video_id}', html=True)},
+ {extract_attributes}, 'data-mediacollection-ardplayer',
+ {self._proto_relative_url}, {require('player URL')}))
+ article = traverse_obj(webpage, (
+ {find_element(cls='article__content')},
+ {find_element(tag='p')}, {clean_html}))
+
+ return {
+ **self._extract_media_info(player_url, webpage, video_id),
'id': video_id,
- 'title': get_element_by_attribute('class', 'ardplayer-title', webpage),
- 'description': self._og_search_description(webpage),
+ 'title': traverse_obj(webpage, (
+ {find_element(cls='ardplayer-title')}, {clean_html})),
+ 'channel': traverse_obj(webpage, (
+ {find_element(cls=f'{self._CLS_COMMON}subheadline')},
+ {lambda x: x.split('|')[0]}, {clean_html})),
+ 'description': description,
+ 'duration': parse_duration(self._search_regex(
+ r'(\d{2}:\d{2}:\d{2})', article, 'duration')),
+ 'release_date': unified_strdate(self._search_regex(
+ r'(\d{2}\.\d{2}\.\d{4})', article, 'release_date')),
+ 'series': traverse_obj(webpage, (
+ {find_element(cls=f'{self._CLS_COMMON}headline')}, {clean_html})),
+ 'series_id': traverse_obj(webpage, (
+ {find_element(cls='teaser__link', html=True)},
+ {extract_attributes}, 'href', {parse_qs}, 'sen', ..., {str}, any)),
'thumbnail': self._og_search_thumbnail(webpage),
- })
- return info
+ }
diff --git a/yt_dlp/extractor/stacommu.py b/yt_dlp/extractor/stacommu.py
index 830018518..e6866f151 100644
--- a/yt_dlp/extractor/stacommu.py
+++ b/yt_dlp/extractor/stacommu.py
@@ -4,6 +4,7 @@
from ..utils import (
int_or_none,
traverse_obj,
+ url_basename,
url_or_none,
)
@@ -65,9 +66,19 @@ def _extract_ppv(self, url):
hls_info, decrypt = self._call_encrypted_api(
video_id, ':watchArchive', 'stream information', data={'method': 1})
+ formats = self._get_formats(hls_info, ('hls', 'urls', ..., {url_or_none}), video_id)
+ for f in formats:
+ # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
+ if f.get('tbr'):
+ f['tbr'] = int(f['tbr'] / 2.5)
+ # prefer variants with the same basename as the master playlist to avoid partial streams
+ f['format_id'] = url_basename(f['url']).partition('.')[0]
+ if not f['format_id'].startswith(url_basename(f['manifest_url']).partition('.')[0]):
+ f['preference'] = -10
+
return {
'id': video_id,
- 'formats': self._get_formats(hls_info, ('hls', 'urls', ..., {url_or_none}), video_id),
+ 'formats': formats,
'hls_aes': self._extract_hls_key(hls_info, 'hls', decrypt),
**traverse_obj(video_info, {
'title': ('displayName', {str}),
diff --git a/yt_dlp/extractor/startrek.py b/yt_dlp/extractor/startrek.py
index c59187173..802702d44 100644
--- a/yt_dlp/extractor/startrek.py
+++ b/yt_dlp/extractor/startrek.py
@@ -1,76 +1,76 @@
from .common import InfoExtractor
-from ..utils import int_or_none, urljoin
+from .youtube import YoutubeIE
+from ..utils import (
+ clean_html,
+ parse_iso8601,
+ update_url,
+ url_or_none,
+)
+from ..utils.traversal import subs_list_to_dict, traverse_obj
class StarTrekIE(InfoExtractor):
- _WORKING = False
- _VALID_URL = r'(?Phttps?://(?:intl|www)\.startrek\.com)/videos/(?P[^/]+)'
+ IE_NAME = 'startrek'
+ IE_DESC = 'STAR TREK'
+ _VALID_URL = r'https?://(?:www\.)?startrek\.com(?:/en-(?:ca|un))?/videos/(?P[^/?#]+)'
_TESTS = [{
- 'url': 'https://intl.startrek.com/videos/watch-welcoming-jess-bush-to-the-ready-room',
- 'md5': '491df5035c9d4dc7f63c79caaf9c839e',
+ 'url': 'https://www.startrek.com/en-un/videos/official-trailer-star-trek-lower-decks-season-4',
'info_dict': {
- 'id': 'watch-welcoming-jess-bush-to-the-ready-room',
+ 'id': 'official-trailer-star-trek-lower-decks-season-4',
'ext': 'mp4',
- 'title': 'WATCH: Welcoming Jess Bush to The Ready Room',
- 'duration': 1888,
- 'timestamp': 1655388000,
- 'upload_date': '20220616',
- 'description': 'md5:1ffee884e3920afbdd6dd04e926a1221',
- 'thumbnail': r're:https://(?:intl|www)\.startrek\.com/sites/default/files/styles/video_1920x1080/public/images/2022-06/pp_14794_rr_thumb_107_yt_16x9\.jpg(?:\?.+)?',
- 'subtitles': {'en-US': [{
- 'url': r're:https://(?:intl|www)\.startrek\.com/sites/default/files/video/captions/2022-06/TRR_SNW_107_v4\.vtt',
- }, {
- 'url': 'https://media.startrek.com/2022/06/16/2043801155561/1069981_hls/trr_snw_107_v4-c4bfc25d/stream_vtt.m3u8',
- }]},
+ 'title': 'Official Trailer | Star Trek: Lower Decks - Season 4',
+ 'alt_title': 'md5:dd7e3191aaaf9e95db16fc3abd5ef68b',
+ 'categories': ['TRAILERS'],
+ 'description': 'md5:563d7856ddab99bee7a5e50f45531757',
+ 'release_date': '20230722',
+ 'release_timestamp': 1690033200,
+ 'series': 'Star Trek: Lower Decks',
+ 'series_id': 'star-trek-lower-decks',
+ 'thumbnail': r're:https?://.+\.(?:jpg|png)',
},
}, {
- 'url': 'https://www.startrek.com/videos/watch-ethan-peck-and-gia-sandhu-beam-down-to-the-ready-room',
- 'md5': 'f5ad74fbb86e91e0882fc0a333178d1d',
+ 'url': 'https://www.startrek.com/en-ca/videos/my-first-contact-senator-cory-booker',
'info_dict': {
- 'id': 'watch-ethan-peck-and-gia-sandhu-beam-down-to-the-ready-room',
+ 'id': 'my-first-contact-senator-cory-booker',
'ext': 'mp4',
- 'title': 'WATCH: Ethan Peck and Gia Sandhu Beam Down to The Ready Room',
- 'duration': 1986,
- 'timestamp': 1654221600,
- 'upload_date': '20220603',
- 'description': 'md5:b3aa0edacfe119386567362dec8ed51b',
- 'thumbnail': r're:https://www\.startrek\.com/sites/default/files/styles/video_1920x1080/public/images/2022-06/pp_14792_rr_thumb_105_yt_16x9_1.jpg(?:\?.+)?',
- 'subtitles': {'en-US': [{
- 'url': r're:https://(?:intl|www)\.startrek\.com/sites/default/files/video/captions/2022-06/TRR_SNW_105_v5\.vtt',
- }]},
+ 'title': 'My First Contact: Senator Cory Booker',
+ 'alt_title': 'md5:fe74a8bdb0afab421c6e159a7680db4d',
+ 'categories': ['MY FIRST CONTACT'],
+ 'description': 'md5:a3992ab3b3e0395925d71156bbc018ce',
+ 'release_date': '20250401',
+ 'release_timestamp': 1743512400,
+ 'series': 'Star Trek: The Original Series',
+ 'series_id': 'star-trek-the-original-series',
+ 'thumbnail': r're:https?://.+\.(?:jpg|png)',
},
}]
def _real_extract(self, url):
- urlbase, video_id = self._match_valid_url(url).group('base', 'id')
+ video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
- player = self._search_regex(
- r'(<\s*div\s+id\s*=\s*"cvp-player-[^<]+<\s*/div\s*>)', webpage, 'player')
+ page_props = self._search_nextjs_data(webpage, video_id)['props']['pageProps']
+ video_data = page_props['video']['data']
+ if youtube_id := video_data.get('youtube_video_id'):
+ return self.url_result(youtube_id, YoutubeIE)
- hls = self._html_search_regex(r'\bdata-hls\s*=\s*"([^"]+)"', player, 'HLS URL')
- formats, subtitles = self._extract_m3u8_formats_and_subtitles(hls, video_id, 'mp4')
-
- captions = self._html_search_regex(
- r'\bdata-captions-url\s*=\s*"([^"]+)"', player, 'captions URL', fatal=False)
- if captions:
- subtitles.setdefault('en-US', [])[:0] = [{'url': urljoin(urlbase, captions)}]
-
- # NB: Most of the data in the json_ld is undesirable
- json_ld = self._search_json_ld(webpage, video_id, fatal=False)
+ series_id = traverse_obj(video_data, (
+ 'series_and_movies', ..., 'series_or_movie', 'slug', {str}, any))
return {
'id': video_id,
- 'title': self._html_search_regex(
- r'\bdata-title\s*=\s*"([^"]+)"', player, 'title', json_ld.get('title')),
- 'description': self._html_search_regex(
- r'(?s)<\s*div\s+class\s*=\s*"header-body"\s*>(.+?)<\s*/div\s*>',
- webpage, 'description', fatal=False),
- 'duration': int_or_none(self._html_search_regex(
- r'\bdata-duration\s*=\s*"(\d+)"', player, 'duration', fatal=False)),
- 'formats': formats,
- 'subtitles': subtitles,
- 'thumbnail': urljoin(urlbase, self._html_search_regex(
- r'\bdata-poster-url\s*=\s*"([^"]+)"', player, 'thumbnail', fatal=False)),
- 'timestamp': json_ld.get('timestamp'),
+ 'series': traverse_obj(page_props, (
+ 'queried', 'header', 'tab3', 'slices', ..., 'items',
+ lambda _, v: v['link']['slug'] == series_id, 'link_copy', {str}, any)),
+ 'series_id': series_id,
+ **traverse_obj(video_data, {
+ 'title': ('title', ..., 'text', {clean_html}, any),
+ 'alt_title': ('subhead', ..., 'text', {clean_html}, any),
+ 'categories': ('category', 'data', 'category_name', {str.upper}, filter, all),
+ 'description': ('slices', ..., 'primary', 'content', ..., 'text', {clean_html}, any),
+ 'release_timestamp': ('published', {parse_iso8601}),
+ 'subtitles': ({'url': 'legacy_subtitle_file'}, all, {subs_list_to_dict(lang='en')}),
+ 'thumbnail': ('poster_frame', 'url', {url_or_none}, {update_url(query=None)}),
+ 'url': ('legacy_video_url', {url_or_none}),
+ }),
}
diff --git a/yt_dlp/extractor/svt.py b/yt_dlp/extractor/svt.py
index 6a72f8d42..a48d7858d 100644
--- a/yt_dlp/extractor/svt.py
+++ b/yt_dlp/extractor/svt.py
@@ -6,10 +6,13 @@
determine_ext,
dict_get,
int_or_none,
- traverse_obj,
try_get,
unified_timestamp,
)
+from ..utils.traversal import (
+ require,
+ traverse_obj,
+)
class SVTBaseIE(InfoExtractor):
@@ -97,40 +100,8 @@ def _extract_video(self, video_info, video_id):
}
-class SVTIE(SVTBaseIE):
- _VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.*?&)?widgetId=(?P\d+)&.*?\barticleId=(?P\d+)'
- _EMBED_REGEX = [rf'(?: