Release 2023.11.16

Created by: bashonly

:ci skip all :ci run dl

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -61,19 +61,18 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        [debug] Request Handlers: urllib, requests
-        Latest version: 2023.09.24, Current version: 2023.09.24
+        [debug] Loaded 1893 extractors
-        yt-dlp is up to date (2023.09.24)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -73,19 +73,18 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        [debug] Request Handlers: urllib, requests
-        Latest version: 2023.09.24, Current version: 2023.09.24
+        [debug] Loaded 1893 extractors
-        yt-dlp is up to date (2023.09.24)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -69,19 +69,18 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        [debug] Request Handlers: urllib, requests
-        Latest version: 2023.09.24, Current version: 2023.09.24
+        [debug] Loaded 1893 extractors
-        yt-dlp is up to date (2023.09.24)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -54,19 +54,18 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        [debug] Request Handlers: urllib, requests
-        Latest version: 2023.09.24, Current version: 2023.09.24
+        [debug] Loaded 1893 extractors
-        yt-dlp is up to date (2023.09.24)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -50,18 +50,17 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        [debug] Request Handlers: urllib, requests
-        Latest version: 2023.09.24, Current version: 2023.09.24
+        [debug] Loaded 1893 extractors
-        yt-dlp is up to date (2023.09.24)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell

.github/ISSUE_TEMPLATE/6_question.yml

@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -56,18 +56,17 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        [debug] Request Handlers: urllib, requests
-        Latest version: 2023.09.24, Current version: 2023.09.24
+        [debug] Loaded 1893 extractors
-        yt-dlp is up to date (2023.09.24)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell

.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml

@@ -12,7 +12,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true

.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml

@@ -12,7 +12,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true

.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml

@@ -12,7 +12,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true

.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml

@@ -12,7 +12,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true

.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml

@@ -14,7 +14,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true

.github/ISSUE_TEMPLATE_tmpl/6_question.yml

@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true

.github/PULL_REQUEST_TEMPLATE.md

@@ -40,10 +40,4 @@ Fixes #
 - [ ] Core bug fix/improvement
 - [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes))
 
-
-<!-- Do NOT edit/remove anything below this! -->
-</details><details><summary>Copilot Summary</summary>  
-
-copilot:all
-
 </details>

.github/workflows/build.yml

@@ -30,6 +30,10 @@ on:
       meta_files:
         default: true
         type: boolean
+      origin:
+        required: false
+        default: ''
+        type: string
     secrets:
       GPG_SIGNING_KEY:
         required: false
@@ -37,11 +41,13 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: Version tag (YYYY.MM.DD[.REV])
+        description: |
+          VERSION: yyyy.mm.dd[.rev] or rev
         required: true
         type: string
       channel:
-        description: Update channel (stable/nightly/...)
+        description: |
+          SOURCE of this build's updates: stable/nightly/master/<repo>
         required: true
         default: stable
         type: string
@@ -73,16 +79,34 @@ on:
         description: SHA2-256SUMS, SHA2-512SUMS, _update_spec
         default: true
         type: boolean
+      origin:
+        description: .
+        required: false
+        default: ''
+        type: choice
+        options:
+        - ''
 
 permissions:
   contents: read
 
 jobs:
+  process:
+    runs-on: ubuntu-latest
+    outputs:
+      origin: ${{ steps.process_origin.outputs.origin }}
+    steps:
+      - name: Process origin
+        id: process_origin
+        run: |
+          echo "origin=${{ inputs.origin || github.repository }}" >> "$GITHUB_OUTPUT"
+
   unix:
+    needs: process
     if: inputs.unix
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
@@ -96,22 +120,21 @@ jobs:
           auto-activate-base: false
       - name: Install Requirements
         run: |
-          sudo apt-get -y install zip pandoc man sed
+          sudo apt -y install zip pandoc man sed
-          python -m pip install -U pip setuptools wheel
-          python -m pip install -U Pyinstaller -r requirements.txt
           reqs=$(mktemp)
-          cat > $reqs << EOF
+          cat > "$reqs" << EOF
           python=3.10.*
           pyinstaller
           cffi
           brotli-python
+          secretstorage
           EOF
-          sed '/^brotli.*/d' requirements.txt >> $reqs
+          sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
-          mamba create -n build --file $reqs
+          mamba create -n build --file "$reqs"
 
       - name: Prepare
         run: |
-          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python devscripts/make_lazy_extractors.py
       - name: Build Unix platform-independent binary
         run: |
@@ -150,6 +173,7 @@ jobs:
             yt-dlp_linux.zip
 
   linux_arm:
+    needs: process
     if: inputs.linux_arm
     permissions:
       contents: read
@@ -162,7 +186,7 @@ jobs:
           - aarch64
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           path: ./repo
       - name: Virtualized Install, Prepare & Build
@@ -180,12 +204,12 @@ jobs:
             apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
             python3.8 -m pip install -U pip setuptools wheel
             # Cannot access requirements.txt from the repo directory at this stage
-            python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi
+            python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage
 
           run: |
             cd repo
-            python3.8 -m pip install -U Pyinstaller -r requirements.txt  # Cached version may be out of date
+            python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt  # Cached version may be out of date
-            python3.8 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+            python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
             python3.8 devscripts/make_lazy_extractors.py
             python3.8 pyinst.py
 
@@ -206,11 +230,12 @@ jobs:
             repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
 
   macos:
+    needs: process
     if: inputs.macos
     runs-on: macos-11
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       # NB: Building universal2 does not work with python from actions/setup-python
       - name: Install Requirements
         run: |
@@ -221,7 +246,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -247,11 +272,12 @@ jobs:
             dist/yt-dlp_macos.zip
 
   macos_legacy:
+    needs: process
     if: inputs.macos_legacy
     runs-on: macos-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Python
         # We need the official Python, because the GA ones only support newer macOS versions
         env:
@@ -272,7 +298,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -296,11 +322,12 @@ jobs:
             dist/yt-dlp_macos_legacy
 
   windows:
+    needs: process
     if: inputs.windows
     runs-on: windows-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with: # 3.8 is used for Win7 support
           python-version: "3.8"
@@ -311,7 +338,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -343,11 +370,12 @@ jobs:
             dist/yt-dlp_win.zip
 
   windows32:
+    needs: process
     if: inputs.windows32
     runs-on: windows-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390
           python-version: "3.7"
@@ -359,7 +387,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -387,6 +415,7 @@ jobs:
   meta_files:
     if: inputs.meta_files && always() && !cancelled()
     needs:
+      - process
       - unix
       - linux_arm
       - macos

.github/workflows/codeql.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/core.yml

@@ -13,7 +13,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         # CPython 3.11 is in quick-test
-        python-version: ['3.8', '3.9', '3.10', '3.12-dev', pypy-3.7, pypy-3.8, pypy-3.10]
+        python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.7, pypy-3.8, pypy-3.10]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
@@ -21,19 +21,19 @@ jobs:
           python-version: '3.7'
           run-tests-ext: bat
         - os: windows-latest
-          python-version: '3.12-dev'
+          python-version: '3.12'
           run-tests-ext: bat
         - os: windows-latest
           python-version: pypy-3.9
           run-tests-ext: bat
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install pytest
+    - name: Install dependencies
-      run: pip install pytest
+      run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: False
       run: |

.github/workflows/download.yml

@@ -9,7 +9,7 @@ jobs:
     if: "contains(github.event.head_commit.message, 'ci run dl')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
@@ -28,7 +28,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.7', '3.10', 3.11-dev, pypy-3.7, pypy-3.8]
+        python-version: ['3.7', '3.10', '3.12', pypy-3.7, pypy-3.8, pypy-3.10]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
@@ -39,7 +39,7 @@ jobs:
           python-version: pypy-3.9
           run-tests-ext: bat
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:

.github/workflows/publish.yml

@@ -1,97 +0,0 @@
-name: Publish
-on:
-  workflow_call:
-    inputs:
-      channel:
-        default: stable
-        required: true
-        type: string
-      version:
-        required: true
-        type: string
-      target_commitish:
-        required: true
-        type: string
-      prerelease:
-        default: false
-        required: true
-        type: boolean
-    secrets:
-      ARCHIVE_REPO_TOKEN:
-        required: false
-
-permissions:
-  contents: write
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/download-artifact@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-
-      - name: Generate release notes
-        run: |
-          printf '%s' \
-            '[![Installation](https://img.shields.io/badge/-Which%20file%20should%20I%20download%3F-white.svg?style=for-the-badge)]' \
-              '(https://github.com/yt-dlp/yt-dlp#installation "Installation instructions") ' \
-            '[![Documentation](https://img.shields.io/badge/-Docs-brightgreen.svg?style=for-the-badge&logo=GitBook&labelColor=555555)]' \
-              '(https://github.com/yt-dlp/yt-dlp/tree/2023.03.04#readme "Documentation") ' \
-            '[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)]' \
-              '(https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators "Donate") ' \
-            '[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)]' \
-              '(https://discord.gg/H5MNcFW63r "Discord") ' \
-            ${{ inputs.channel != 'nightly' && '"[![Nightly](https://img.shields.io/badge/Get%20nightly%20builds-purple.svg?style=for-the-badge)]" \
-              "(https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest \"Nightly builds\")"' || '' }} \
-            > ./RELEASE_NOTES
-          printf '\n\n' >> ./RELEASE_NOTES
-          cat >> ./RELEASE_NOTES << EOF
-          #### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files)
-          ---
-          $(python ./devscripts/make_changelog.py -vv --collapsible)
-          EOF
-          printf '%s\n\n' '**This is an automated nightly pre-release build**' >> ./NIGHTLY_NOTES
-          cat ./RELEASE_NOTES >> ./NIGHTLY_NOTES
-          printf '%s\n\n' 'Generated from: https://github.com/${{ github.repository }}/commit/${{ inputs.target_commitish }}' >> ./ARCHIVE_NOTES
-          cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES
-
-      - name: Archive nightly release
-        env:
-          GH_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
-          GH_REPO: ${{ vars.ARCHIVE_REPO }}
-        if: |
-          inputs.channel == 'nightly' && env.GH_TOKEN != '' && env.GH_REPO != ''
-        run: |
-          gh release create \
-            --notes-file ARCHIVE_NOTES \
-            --title "yt-dlp nightly ${{ inputs.version }}" \
-            ${{ inputs.version }} \
-            artifact/*
-
-      - name: Prune old nightly release
-        if: inputs.channel == 'nightly' && !vars.ARCHIVE_REPO
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          gh release delete --yes --cleanup-tag "nightly" || true
-          git tag --delete "nightly" || true
-          sleep 5  # Enough time to cover deletion race condition
-
-      - name: Publish release${{ inputs.channel == 'nightly' && ' (nightly)' || '' }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-        if: (inputs.channel == 'nightly' && !vars.ARCHIVE_REPO) || inputs.channel != 'nightly'
-        run: |
-          gh release create \
-            --notes-file ${{ inputs.channel == 'nightly' && 'NIGHTLY_NOTES' || 'RELEASE_NOTES' }} \
-            --target ${{ inputs.target_commitish }} \
-            --title "yt-dlp ${{ inputs.channel == 'nightly' && 'nightly ' || '' }}${{ inputs.version }}" \
-            ${{ inputs.prerelease && '--prerelease' || '' }} \
-            ${{ inputs.channel == 'nightly' && '"nightly"' || inputs.version }} \
-            artifact/*

.github/workflows/quick-test.yml

@@ -9,7 +9,7 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'ci skip all')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python 3.11
       uses: actions/setup-python@v4
       with:
@@ -25,7 +25,7 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'ci skip all')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
     - name: Install flake8
       run: pip install flake8

.github/workflows/release-master.yml

@@ -0,0 +1,28 @@
+name: Release (master)
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "yt_dlp/**.py"
+      - "!yt_dlp/version.py"
+      - "setup.py"
+      - "pyinst.py"
+concurrency:
+  group: release-master
+  cancel-in-progress: true
+permissions:
+  contents: read
+
+jobs:
+  release:
+    if: vars.BUILD_MASTER != ''
+    uses: ./.github/workflows/release.yml
+    with:
+      prerelease: true
+      source: master
+    permissions:
+      contents: write
+      packages: write
+      id-token: write  # mandatory for trusted publishing
+    secrets: inherit

.github/workflows/release-nightly.yml

@@ -1,52 +1,35 @@
 name: Release (nightly)
 on:
-  push:
+  schedule:
-    branches:
+    - cron: '23 23 * * *'
-      - master
-    paths:
-      - "yt_dlp/**.py"
-      - "!yt_dlp/version.py"
-concurrency:
-  group: release-nightly
-  cancel-in-progress: true
 permissions:
   contents: read
 
 jobs:
-  prepare:
+  check_nightly:
     if: vars.BUILD_NIGHTLY != ''
     runs-on: ubuntu-latest
     outputs:
-      version: ${{ steps.get_version.outputs.version }}
+      commit: ${{ steps.check_for_new_commits.outputs.commit }}
-
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - name: Get version
+        with:
-        id: get_version
+          fetch-depth: 0
+      - name: Check for new commits
+        id: check_for_new_commits
         run: |
-          python devscripts/update-version.py "$(date -u +"%H%M%S")" | grep -Po "version=\d+(\.\d+){3}" >> "$GITHUB_OUTPUT"
+          relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "setup.py" "pyinst.py")
+          echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT"
 
-  build:
+  release:
-    needs: prepare
+    needs: [check_nightly]
-    uses: ./.github/workflows/build.yml
+    if: ${{ needs.check_nightly.outputs.commit }}
+    uses: ./.github/workflows/release.yml
     with:
-      version: ${{ needs.prepare.outputs.version }}
+      prerelease: true
-      channel: nightly
+      source: nightly
-    permissions:
-      contents: read
-      packages: write # For package cache
-    secrets:
-      GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
-
-  publish:
-    needs: [prepare, build]
-    uses: ./.github/workflows/publish.yml
-    secrets:
-      ARCHIVE_REPO_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
     permissions:
       contents: write
-    with:
+      packages: write
-      channel: nightly
+      id-token: write  # mandatory for trusted publishing
-      prerelease: true
+    secrets: inherit
-      version: ${{ needs.prepare.outputs.version }}
-      target_commitish: ${{ github.sha }}

.github/workflows/release.yml

@@ -1,14 +1,45 @@
 name: Release
 on:
-  workflow_dispatch:
+  workflow_call:
     inputs:
-      version:
+      prerelease:
-        description: Version tag (YYYY.MM.DD[.REV])
+        required: false
+        default: true
+        type: boolean
+      source:
         required: false
         default: ''
         type: string
-      channel:
+      target:
-        description: Update channel (stable/nightly/...)
+        required: false
+        default: ''
+        type: string
+      version:
+        required: false
+        default: ''
+        type: string
+  workflow_dispatch:
+    inputs:
+      source:
+        description: |
+          SOURCE of this release's updates:
+          channel, repo, tag, or channel/repo@tag
+          (default: <current_repo>)
+        required: false
+        default: ''
+        type: string
+      target:
+        description: |
+          TARGET to publish this release to:
+          channel, tag, or channel@tag
+          (default: <source> if writable else <current_repo>[@source_tag])
+        required: false
+        default: ''
+        type: string
+      version:
+        description: |
+          VERSION: yyyy.mm.dd[.rev] or rev
+          (default: auto-generated)
         required: false
         default: ''
         type: string
@@ -26,12 +57,18 @@ jobs:
       contents: write
     runs-on: ubuntu-latest
     outputs:
-      channel: ${{ steps.set_channel.outputs.channel }}
+      channel: ${{ steps.setup_variables.outputs.channel }}
-      version: ${{ steps.update_version.outputs.version }}
+      version: ${{ steps.setup_variables.outputs.version }}
+      target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+      target_repo_token: ${{ steps.setup_variables.outputs.target_repo_token }}
+      target_tag: ${{ steps.setup_variables.outputs.target_tag }}
+      pypi_project: ${{ steps.setup_variables.outputs.pypi_project }}
+      pypi_suffix: ${{ steps.setup_variables.outputs.pypi_suffix }}
+      pypi_token: ${{ steps.setup_variables.outputs.pypi_token }}
       head_sha: ${{ steps.get_target.outputs.head_sha }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -39,25 +76,133 @@ jobs:
         with:
           python-version: "3.10"
 
-      - name: Set channel
+      - name: Process inputs
-        id: set_channel
+        id: process_inputs
         run: |
-          CHANNEL="${{ github.repository == 'yt-dlp/yt-dlp' && 'stable' || github.repository }}"
+          cat << EOF
-          echo "channel=${{ inputs.channel || '$CHANNEL' }}" > "$GITHUB_OUTPUT"
+          ::group::Inputs
+          prerelease=${{ inputs.prerelease }}
+          source=${{ inputs.source }}
+          target=${{ inputs.target }}
+          version=${{ inputs.version }}
+          ::endgroup::
+          EOF
+          IFS='@' read -r source_repo source_tag <<<"${{ inputs.source }}"
+          IFS='@' read -r target_repo target_tag <<<"${{ inputs.target }}"
+          cat << EOF >> "$GITHUB_OUTPUT"
+          source_repo=${source_repo}
+          source_tag=${source_tag}
+          target_repo=${target_repo}
+          target_tag=${target_tag}
+          EOF
 
-      - name: Update version
+      - name: Setup variables
-        id: update_version
+        id: setup_variables
+        env:
+          source_repo: ${{ steps.process_inputs.outputs.source_repo }}
+          source_tag: ${{ steps.process_inputs.outputs.source_tag }}
+          target_repo: ${{ steps.process_inputs.outputs.target_repo }}
+          target_tag: ${{ steps.process_inputs.outputs.target_tag }}
         run: |
-          REVISION="${{ vars.PUSH_VERSION_COMMIT == '' && '$(date -u +"%H%M%S")' || '' }}"
+          # unholy bash monstrosity (sincere apologies)
-          REVISION="${{ inputs.prerelease && '$(date -u +"%H%M%S")' || '$REVISION' }}"
+          fallback_token () {
-          python devscripts/update-version.py ${{ inputs.version || '$REVISION' }} | \
+            if ${{ !secrets.ARCHIVE_REPO_TOKEN }}; then
-            grep -Po "version=\d+\.\d+\.\d+(\.\d+)?" >> "$GITHUB_OUTPUT"
+              echo "::error::Repository access secret ${target_repo_token^^} not found"
+              exit 1
+            fi
+            target_repo_token=ARCHIVE_REPO_TOKEN
+            return 0
+          }
+
+          source_is_channel=0
+          [[ "${source_repo}" == 'stable' ]] && source_repo='yt-dlp/yt-dlp'
+          if [[ -z "${source_repo}" ]]; then
+            source_repo='${{ github.repository }}'
+          elif [[ '${{ vars[format('{0}_archive_repo', env.source_repo)] }}' ]]; then
+            source_is_channel=1
+            source_channel='${{ vars[format('{0}_archive_repo', env.source_repo)] }}'
+          elif [[ -z "${source_tag}" && "${source_repo}" != */* ]]; then
+            source_tag="${source_repo}"
+            source_repo='${{ github.repository }}'
+          fi
+          resolved_source="${source_repo}"
+          if [[ "${source_tag}" ]]; then
+            resolved_source="${resolved_source}@${source_tag}"
+          elif [[ "${source_repo}" == 'yt-dlp/yt-dlp' ]]; then
+            resolved_source='stable'
+          fi
+
+          revision="${{ (inputs.prerelease || !vars.PUSH_VERSION_COMMIT) && '$(date -u +"%H%M%S")' || '' }}"
+          version="$(
+            python devscripts/update-version.py \
+            -c "${resolved_source}" -r "${{ github.repository }}" ${{ inputs.version || '$revision' }} | \
+            grep -Po "version=\K\d+\.\d+\.\d+(\.\d+)?")"
+
+          if [[ "${target_repo}" ]]; then
+            if [[ -z "${target_tag}" ]]; then
+              if [[ '${{ vars[format('{0}_archive_repo', env.target_repo)] }}' ]]; then
+                target_tag="${source_tag:-${version}}"
+              else
+                target_tag="${target_repo}"
+                target_repo='${{ github.repository }}'
+              fi
+            fi
+            if [[ "${target_repo}" != '${{ github.repository}}' ]]; then
+              target_repo='${{ vars[format('{0}_archive_repo', env.target_repo)] }}'
+              target_repo_token='${{ env.target_repo }}_archive_repo_token'
+              ${{ !!secrets[format('{0}_archive_repo_token', env.target_repo)] }} || fallback_token
+              pypi_project='${{ vars[format('{0}_pypi_project', env.target_repo)] }}'
+              pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.target_repo)] }}'
+              ${{ !secrets[format('{0}_pypi_token', env.target_repo)] }} || pypi_token='${{ env.target_repo }}_pypi_token'
+            fi
+          else
+            target_tag="${source_tag:-${version}}"
+            if ((source_is_channel)); then
+              target_repo="${source_channel}"
+              target_repo_token='${{ env.source_repo }}_archive_repo_token'
+              ${{ !!secrets[format('{0}_archive_repo_token', env.source_repo)] }} || fallback_token
+              pypi_project='${{ vars[format('{0}_pypi_project', env.source_repo)] }}'
+              pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.source_repo)] }}'
+              ${{ !secrets[format('{0}_pypi_token', env.source_repo)] }} || pypi_token='${{ env.source_repo }}_pypi_token'
+            else
+              target_repo='${{ github.repository }}'
+            fi
+          fi
+
+          if [[ "${target_repo}" == '${{ github.repository }}' ]] && ${{ !inputs.prerelease }}; then
+            pypi_project='${{ vars.PYPI_PROJECT }}'
+          fi
+          if [[ -z "${pypi_token}" && "${pypi_project}" ]]; then
+            if ${{ !secrets.PYPI_TOKEN }}; then
+              pypi_token=OIDC
+            else
+              pypi_token=PYPI_TOKEN
+            fi
+          fi
+
+          echo "::group::Output variables"
+          cat << EOF | tee -a "$GITHUB_OUTPUT"
+          channel=${resolved_source}
+          version=${version}
+          target_repo=${target_repo}
+          target_repo_token=${target_repo_token}
+          target_tag=${target_tag}
+          pypi_project=${pypi_project}
+          pypi_suffix=${pypi_suffix}
+          pypi_token=${pypi_token}
+          EOF
+          echo "::endgroup::"
 
       - name: Update documentation
+        env:
+          version: ${{ steps.setup_variables.outputs.version }}
+          target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+        if: |
+          !inputs.prerelease && env.target_repo == github.repository
         run: |
           make doc
           sed '/### /Q' Changelog.md >> ./CHANGELOG
-          echo '### ${{ steps.update_version.outputs.version }}' >> ./CHANGELOG
+          echo '### ${{ env.version }}' >> ./CHANGELOG
           python ./devscripts/make_changelog.py -vv -c >> ./CHANGELOG
           echo >> ./CHANGELOG
           grep -Poz '(?s)### \d+\.\d+\.\d+.+' 'Changelog.md' | head -n -1 >> ./CHANGELOG
@@ -65,12 +210,16 @@ jobs:
 
       - name: Push to release
         id: push_release
-        if: ${{ !inputs.prerelease }}
+        env:
+          version: ${{ steps.setup_variables.outputs.version }}
+          target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+        if: |
+          !inputs.prerelease && env.target_repo == github.repository
         run: |
-          git config --global user.name github-actions
+          git config --global user.name "github-actions[bot]"
-          git config --global user.email github-actions@example.com
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git add -u
-          git commit -m "Release ${{ steps.update_version.outputs.version }}" \
+          git commit -m "Release ${{ env.version }}" \
             -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
           git push origin --force ${{ github.event.ref }}:release
 
@@ -80,7 +229,10 @@ jobs:
           echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
 
       - name: Update master
-        if: vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease
+        env:
+          target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+        if: |
+          vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease && env.target_repo == github.repository
         run: git push origin ${{ github.event.ref }}
 
   build:
@@ -89,75 +241,159 @@ jobs:
     with:
       version: ${{ needs.prepare.outputs.version }}
       channel: ${{ needs.prepare.outputs.channel }}
+      origin: ${{ needs.prepare.outputs.target_repo }}
     permissions:
       contents: read
       packages: write # For package cache
     secrets:
       GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
 
-  publish_pypi_homebrew:
+  publish_pypi:
     needs: [prepare, build]
+    if: ${{ needs.prepare.outputs.pypi_project }}
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # mandatory for trusted publishing
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
       - name: Install Requirements
         run: |
-          sudo apt-get -y install pandoc man
+          sudo apt -y install pandoc man
           python -m pip install -U pip setuptools wheel twine
           python -m pip install -U -r requirements.txt
 
       - name: Prepare
-        run: |
-          python devscripts/update-version.py ${{ needs.prepare.outputs.version }}
-          python devscripts/make_lazy_extractors.py
-
-      - name: Build and publish on PyPI
         env:
-          TWINE_USERNAME: __token__
+          version: ${{ needs.prepare.outputs.version }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+          suffix: ${{ needs.prepare.outputs.pypi_suffix }}
-        if: env.TWINE_PASSWORD != '' && !inputs.prerelease
+          channel: ${{ needs.prepare.outputs.channel }}
+          target_repo: ${{ needs.prepare.outputs.target_repo }}
+          pypi_project: ${{ needs.prepare.outputs.pypi_project }}
+        run: |
+          python devscripts/update-version.py -c "${{ env.channel }}" -r "${{ env.target_repo }}" -s "${{ env.suffix }}" "${{ env.version }}"
+          python devscripts/make_lazy_extractors.py
+          sed -i -E "s/(name=')[^']+(',  # package name)/\1${{ env.pypi_project }}\2/" setup.py
+
+      - name: Build
         run: |
           rm -rf dist/*
           make pypi-files
           python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
           python setup.py sdist bdist_wheel
+
+      - name: Publish to PyPI via token
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets[needs.prepare.outputs.pypi_token] }}
+        if: |
+          needs.prepare.outputs.pypi_token != 'OIDC' && env.TWINE_PASSWORD
+        run: |
           twine upload dist/*
 
-      - name: Checkout Homebrew repository
+      - name: Publish to PyPI via trusted publishing
-        env:
+        if: |
-          BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
+          needs.prepare.outputs.pypi_token == 'OIDC'
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        uses: pypa/gh-action-pypi-publish@release/v1
-        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' && !inputs.prerelease
-        uses: actions/checkout@v3
         with:
-          repository: yt-dlp/homebrew-taps
+          verbose: true
-          path: taps
-          ssh-key: ${{ secrets.BREW_TOKEN }}
-
-      - name: Update Homebrew Formulae
-        env:
-          BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' && !inputs.prerelease
-        run: |
-          python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.version }}"
-          git -C taps/ config user.name github-actions
-          git -C taps/ config user.email github-actions@example.com
-          git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.version }}'
-          git -C taps/ push
 
   publish:
     needs: [prepare, build]
-    uses: ./.github/workflows/publish.yml
     permissions:
       contents: write
-    with:
+    runs-on: ubuntu-latest
-      channel: ${{ needs.prepare.outputs.channel }}
+
-      prerelease: ${{ inputs.prerelease }}
+    steps:
-      version: ${{ needs.prepare.outputs.version }}
+      - uses: actions/checkout@v4
-      target_commitish: ${{ needs.prepare.outputs.head_sha }}
+        with:
+          fetch-depth: 0
+      - uses: actions/download-artifact@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Generate release notes
+        env:
+          head_sha: ${{ needs.prepare.outputs.head_sha }}
+          target_repo: ${{ needs.prepare.outputs.target_repo }}
+          target_tag: ${{ needs.prepare.outputs.target_tag }}
+        run: |
+          printf '%s' \
+            '[![Installation](https://img.shields.io/badge/-Which%20file%20should%20I%20download%3F-white.svg?style=for-the-badge)]' \
+              '(https://github.com/${{ github.repository }}#installation "Installation instructions") ' \
+            '[![Documentation](https://img.shields.io/badge/-Docs-brightgreen.svg?style=for-the-badge&logo=GitBook&labelColor=555555)]' \
+              '(https://github.com/${{ github.repository }}' \
+              '${{ env.target_repo == github.repository && format('/tree/{0}', env.target_tag) || '' }}#readme "Documentation") ' \
+            '[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)]' \
+              '(https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators "Donate") ' \
+            '[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)]' \
+              '(https://discord.gg/H5MNcFW63r "Discord") ' \
+            ${{ env.target_repo == 'yt-dlp/yt-dlp' && '\
+              "[![Nightly](https://img.shields.io/badge/Get%20nightly%20builds-purple.svg?style=for-the-badge)]" \
+              "(https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest \"Nightly builds\") " \
+              "[![Master](https://img.shields.io/badge/Get%20master%20builds-lightblue.svg?style=for-the-badge)]" \
+              "(https://github.com/yt-dlp/yt-dlp-master-builds/releases/latest \"Master builds\")"' || '' }} > ./RELEASE_NOTES
+          printf '\n\n' >> ./RELEASE_NOTES
+          cat >> ./RELEASE_NOTES << EOF
+          #### A description of the various files are in the [README](https://github.com/${{ github.repository }}#release-files)
+          ---
+          $(python ./devscripts/make_changelog.py -vv --collapsible)
+          EOF
+          printf '%s\n\n' '**This is a pre-release build**' >> ./PRERELEASE_NOTES
+          cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES
+          printf '%s\n\n' 'Generated from: https://github.com/${{ github.repository }}/commit/${{ env.head_sha }}' >> ./ARCHIVE_NOTES
+          cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES
+
+      - name: Publish to archive repo
+        env:
+          GH_TOKEN: ${{ secrets[needs.prepare.outputs.target_repo_token] }}
+          GH_REPO: ${{ needs.prepare.outputs.target_repo }}
+          version: ${{ needs.prepare.outputs.version }}
+          channel: ${{ needs.prepare.outputs.channel }}
+        if: |
+          inputs.prerelease && env.GH_TOKEN != '' && env.GH_REPO != '' && env.GH_REPO != github.repository
+        run: |
+          title="${{ startswith(env.GH_REPO, 'yt-dlp/') && 'yt-dlp ' || '' }}${{ env.channel }}"
+          gh release create \
+            --notes-file ARCHIVE_NOTES \
+            --title "${title} ${{ env.version }}" \
+            ${{ env.version }} \
+            artifact/*
+
+      - name: Prune old release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          version: ${{ needs.prepare.outputs.version }}
+          target_repo: ${{ needs.prepare.outputs.target_repo }}
+          target_tag: ${{ needs.prepare.outputs.target_tag }}
+        if: |
+          env.target_repo == github.repository && env.target_tag != env.version
+        run: |
+          gh release delete --yes --cleanup-tag "${{ env.target_tag }}" || true
+          git tag --delete "${{ env.target_tag }}" || true
+          sleep 5  # Enough time to cover deletion race condition
+
+      - name: Publish release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          version: ${{ needs.prepare.outputs.version }}
+          target_repo: ${{ needs.prepare.outputs.target_repo }}
+          target_tag: ${{ needs.prepare.outputs.target_tag }}
+          head_sha: ${{ needs.prepare.outputs.head_sha }}
+        if: |
+          env.target_repo == github.repository
+        run: |
+          title="${{ github.repository == 'yt-dlp/yt-dlp' && 'yt-dlp ' || '' }}"
+          title+="${{ env.target_tag != env.version && format('{0} ', env.target_tag) || '' }}"
+          gh release create \
+            --notes-file ${{ inputs.prerelease && 'PRERELEASE_NOTES' || 'RELEASE_NOTES' }} \
+            --target ${{ env.head_sha }} \
+            --title "${title}${{ env.version }}" \
+            ${{ inputs.prerelease && '--prerelease' || '' }} \
+            ${{ env.target_tag }} \
+            artifact/*

CONTRIBUTORS

@@ -503,3 +503,28 @@ Yalab7
 zhallgato
 zhong-yiyu
 Zprokkel
+AS6939
+drzraf
+handlerug
+jiru
+madewokherd
+xofe
+awalgarg
+midnightveil
+naginatana
+Riteo
+1100101
+aniolpages
+bartbroere
+CrendKing
+Esokrates
+HitomaruKonpaku
+LoserFox
+peci1
+saintliao
+shubhexists
+SirElderling
+almx
+elivinsky
+starius
+TravisDupes

Changelog.md

@@ -4,6 +4,144 @@
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.11.16
+
+#### Extractor changes
+- **abc.net.au**: iview, showseries: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/15cb3528cbda7b6198f49a6b5953c226d701696b) ([#8586](https://github.com/yt-dlp/yt-dlp/issues/8586)) by [bashonly](https://github.com/bashonly)
+- **beatbump**: [Update `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/21dc069bea2d4d99345dd969e098f4535c751d45) ([#8576](https://github.com/yt-dlp/yt-dlp/issues/8576)) by [seproDev](https://github.com/seproDev)
+- **dailymotion**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/a489f071508ec5caf5f32052d142afe86c28df7a) ([#7692](https://github.com/yt-dlp/yt-dlp/issues/7692)) by [TravisDupes](https://github.com/TravisDupes)
+- **drtv**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/0783fd558ed0d3a8bc754beb75a406256f8b97b2) ([#8484](https://github.com/yt-dlp/yt-dlp/issues/8484)) by [almx](https://github.com/almx), [seproDev](https://github.com/seproDev)
+- **eltrecetv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/dcfad52812aa8ce007cefbfbe63f58b49f6b1046) ([#8216](https://github.com/yt-dlp/yt-dlp/issues/8216)) by [elivinsky](https://github.com/elivinsky)
+- **jiosaavn**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/b530118e7f48232cacf8050d79a6b20bdfcf5468) ([#8307](https://github.com/yt-dlp/yt-dlp/issues/8307)) by [awalgarg](https://github.com/awalgarg)
+- **njpwworld**: [Remove](https://github.com/yt-dlp/yt-dlp/commit/e569c2d1f4b665795a2b64f0aaf7f76930664233) ([#8570](https://github.com/yt-dlp/yt-dlp/issues/8570)) by [aarubui](https://github.com/aarubui)
+- **tv5mondeplus**: [Extract subtitles](https://github.com/yt-dlp/yt-dlp/commit/0f634dba3afdc429ece8839b02f6d56c27b7973a) ([#4209](https://github.com/yt-dlp/yt-dlp/issues/4209)) by [FrankZ85](https://github.com/FrankZ85)
+- **twitcasting**: [Fix livestream detection](https://github.com/yt-dlp/yt-dlp/commit/2325d03aa7bb80f56ba52cd6992258e44727b424) ([#8574](https://github.com/yt-dlp/yt-dlp/issues/8574)) by [JC-Chung](https://github.com/JC-Chung)
+- **zenyandex**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/5efe68b73cbf6e907c2e6a3aa338664385084184) ([#8454](https://github.com/yt-dlp/yt-dlp/issues/8454)) by [starius](https://github.com/starius)
+
+#### Misc. changes
+- **build**: [Make `secretstorage` an optional dependency](https://github.com/yt-dlp/yt-dlp/commit/24f827875c6ba513f12ed09a3aef2bbed223760d) ([#8585](https://github.com/yt-dlp/yt-dlp/issues/8585)) by [bashonly](https://github.com/bashonly)
+
+### 2023.11.14
+
+#### Important changes
+- **The release channels have been adjusted!**
+    * [`master`](https://github.com/yt-dlp/yt-dlp-master-builds) builds are made after each push, containing the latest fixes (but also possibly bugs). This was previously the `nightly` channel.
+    * [`nightly`](https://github.com/yt-dlp/yt-dlp-nightly-builds) builds are now made once a day, if there were any changes.
+- Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)
+    - Disallow smuggling of arbitrary `http_headers`; extractors now only use specific headers
+
+#### Core changes
+- [Add `--compat-option manifest-filesize-approx`](https://github.com/yt-dlp/yt-dlp/commit/10025b715ea01489557eb2c5a3cc04d361fcdb52) ([#8356](https://github.com/yt-dlp/yt-dlp/issues/8356)) by [bashonly](https://github.com/bashonly)
+- [Fix format sorting with `--load-info-json`](https://github.com/yt-dlp/yt-dlp/commit/595ea4a99b726b8fe9463e7853b7053978d0544e) ([#8521](https://github.com/yt-dlp/yt-dlp/issues/8521)) by [bashonly](https://github.com/bashonly)
+- [Include build origin in verbose output](https://github.com/yt-dlp/yt-dlp/commit/20314dd46f25e0e0a7e985a7804049aefa8b909f) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+- [Only ensure playlist thumbnail dir if writing thumbs](https://github.com/yt-dlp/yt-dlp/commit/a40e0b37dfc8c26916b0e01aa3f29f3bc42250b6) ([#8373](https://github.com/yt-dlp/yt-dlp/issues/8373)) by [bashonly](https://github.com/bashonly)
+- **update**: [Overhaul self-updater](https://github.com/yt-dlp/yt-dlp/commit/0b6ad22e6a432006a75df968f0283e6c6b3cfae6) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+
+#### Extractor changes
+- [Do not smuggle `http_headers`](https://github.com/yt-dlp/yt-dlp/commit/f04b5bedad7b281bee9814686bba1762bae092eb) by [coletdjnz](https://github.com/coletdjnz)
+- [Do not test truth value of `xml.etree.ElementTree.Element`](https://github.com/yt-dlp/yt-dlp/commit/d4f14a72dc1dd79396e0e80980268aee902b61e4) ([#8582](https://github.com/yt-dlp/yt-dlp/issues/8582)) by [bashonly](https://github.com/bashonly)
+- **brilliantpala**: [Fix cookies support](https://github.com/yt-dlp/yt-dlp/commit/9b5bedf13a3323074daceb0ec6ebb3cc6e0b9684) ([#8352](https://github.com/yt-dlp/yt-dlp/issues/8352)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **generic**: [Improve direct video link ext detection](https://github.com/yt-dlp/yt-dlp/commit/4ce2f29a50fcfb9920e6f2ffe42192945a2bad7e) ([#8340](https://github.com/yt-dlp/yt-dlp/issues/8340)) by [bashonly](https://github.com/bashonly)
+- **laxarxames**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/312a2d1e8bc247264f9d85c5ec764e33aa0133b5) ([#8412](https://github.com/yt-dlp/yt-dlp/issues/8412)) by [aniolpages](https://github.com/aniolpages)
+- **n-tv.de**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/8afd9468b0c822843bc480d366d1c86698daabfb) ([#8414](https://github.com/yt-dlp/yt-dlp/issues/8414)) by [1100101](https://github.com/1100101)
+- **neteasemusic**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/46acc418a53470b7f32581b3309c3cb87aa8488d) ([#8531](https://github.com/yt-dlp/yt-dlp/issues/8531)) by [LoserFox](https://github.com/LoserFox)
+- **nhk**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/54579be4364e148277c32e20a5c3efc2c3f52f5b) ([#8388](https://github.com/yt-dlp/yt-dlp/issues/8388)) by [garret1317](https://github.com/garret1317)
+- **novaembed**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/3ff494f6f41c27549420fa88be27555bd449ffdc) ([#8368](https://github.com/yt-dlp/yt-dlp/issues/8368)) by [peci1](https://github.com/peci1)
+- **npo**: [Send `POST` request to streams API endpoint](https://github.com/yt-dlp/yt-dlp/commit/8e02a4dcc800f9444e9d461edc41edd7b662f435) ([#8413](https://github.com/yt-dlp/yt-dlp/issues/8413)) by [bartbroere](https://github.com/bartbroere)
+- **ondemandkorea**: [Overhaul extractor](https://github.com/yt-dlp/yt-dlp/commit/05adfd883a4f2ecae0267e670a62a2e45c351aeb) ([#8386](https://github.com/yt-dlp/yt-dlp/issues/8386)) by [seproDev](https://github.com/seproDev)
+- **orf**: podcast: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/6ba3085616652cbf05d1858efc321fdbfc4c6119) ([#8486](https://github.com/yt-dlp/yt-dlp/issues/8486)) by [Esokrates](https://github.com/Esokrates)
+- **polskieradio**: audition: [Fix playlist extraction](https://github.com/yt-dlp/yt-dlp/commit/464327acdb353ceb91d2115163a5a9621b22fe0d) ([#8459](https://github.com/yt-dlp/yt-dlp/issues/8459)) by [shubhexists](https://github.com/shubhexists)
+- **qdance**: [Update `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/177f0d963e4b9db749805c482e6f288354c8be84) ([#8426](https://github.com/yt-dlp/yt-dlp/issues/8426)) by [bashonly](https://github.com/bashonly)
+- **radiocomercial**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/ef12dbdcd3e7264bd3d744c1e3107597bd23ad35) ([#8508](https://github.com/yt-dlp/yt-dlp/issues/8508)) by [SirElderling](https://github.com/SirElderling)
+- **sbs.co.kr**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/25a4bd345a0dcfece6fef752d4537eb403da94d9) ([#8326](https://github.com/yt-dlp/yt-dlp/issues/8326)) by [seproDev](https://github.com/seproDev)
+- **theatercomplextown**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/2863fcf2b6876d0c7965ff7d6d9242eea653dc6b) ([#8560](https://github.com/yt-dlp/yt-dlp/issues/8560)) by [bashonly](https://github.com/bashonly)
+- **thisav**: [Remove](https://github.com/yt-dlp/yt-dlp/commit/cb480e390d85fb3a598c1b6d5eef3438ce729fc9) ([#8346](https://github.com/yt-dlp/yt-dlp/issues/8346)) by [bashonly](https://github.com/bashonly)
+- **thisoldhouse**: [Add login support](https://github.com/yt-dlp/yt-dlp/commit/c76c96677ff6a056f5844a568ef05ee22c46d6f4) ([#8561](https://github.com/yt-dlp/yt-dlp/issues/8561)) by [bashonly](https://github.com/bashonly)
+- **twitcasting**: [Fix livestream extraction](https://github.com/yt-dlp/yt-dlp/commit/7b8b1cf5eb8bf44ce70bc24e1f56f0dba2737e98) ([#8427](https://github.com/yt-dlp/yt-dlp/issues/8427)) by [JC-Chung](https://github.com/JC-Chung), [saintliao](https://github.com/saintliao)
+- **twitter**
+    - broadcast
+        - [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/7d337ca977d73a0a6c07ab481ed8faa8f6ff8726) ([#8383](https://github.com/yt-dlp/yt-dlp/issues/8383)) by [HitomaruKonpaku](https://github.com/HitomaruKonpaku)
+        - [Support `--wait-for-video`](https://github.com/yt-dlp/yt-dlp/commit/f6e97090d2ed9e05441ab0f4bec3559b816d7a00) ([#8475](https://github.com/yt-dlp/yt-dlp/issues/8475)) by [bashonly](https://github.com/bashonly)
+- **weibo**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/15b252dfd2c6807fe57afc5a95e59abadb32ccd2) ([#8463](https://github.com/yt-dlp/yt-dlp/issues/8463)) by [c-basalt](https://github.com/c-basalt)
+- **weverse**: [Fix login error handling](https://github.com/yt-dlp/yt-dlp/commit/4a601c9eff9fb42e24a4c8da3fa03628e035b35b) ([#8458](https://github.com/yt-dlp/yt-dlp/issues/8458)) by [seproDev](https://github.com/seproDev)
+- **youtube**: [Check newly uploaded iOS HLS formats](https://github.com/yt-dlp/yt-dlp/commit/ef79d20dc9d27ac002a7196f073b37f2f2721aed) ([#8336](https://github.com/yt-dlp/yt-dlp/issues/8336)) by [bashonly](https://github.com/bashonly)
+- **zoom**: [Extract combined view formats](https://github.com/yt-dlp/yt-dlp/commit/3906de07551fedb00b789345bf24cc27d6ddf128) ([#7847](https://github.com/yt-dlp/yt-dlp/issues/7847)) by [Mipsters](https://github.com/Mipsters)
+
+#### Downloader changes
+- **aria2c**: [Remove duplicate `--file-allocation=none`](https://github.com/yt-dlp/yt-dlp/commit/21b25281c51523620706b11bfc1c4a889858e1f2) ([#8332](https://github.com/yt-dlp/yt-dlp/issues/8332)) by [CrendKing](https://github.com/CrendKing)
+- **dash**: [Force native downloader for `--live-from-start`](https://github.com/yt-dlp/yt-dlp/commit/2622c804d1a5accc3045db398e0fc52074f4bdb3) ([#8339](https://github.com/yt-dlp/yt-dlp/issues/8339)) by [bashonly](https://github.com/bashonly)
+
+#### Networking changes
+- **Request Handler**: requests: [Add handler for `requests` HTTP library (#3668)](https://github.com/yt-dlp/yt-dlp/commit/8a8b54523addf46dfd50ef599761a81bc22362e6) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz), [Grub4K](https://github.com/Grub4K) (With fixes in [4e38e2a](https://github.com/yt-dlp/yt-dlp/commit/4e38e2ae9d7380015349e6aee59c78bb3938befd))
+
+    Adds support for HTTPS proxies and persistent connections (keep-alive)
+
+#### Misc. changes
+- **build**
+    - [Include secretstorage in Linux builds](https://github.com/yt-dlp/yt-dlp/commit/9970d74c8383432c6c8779aa47d3253dcf412b14) by [bashonly](https://github.com/bashonly)
+    - [Overhaul and unify release workflow](https://github.com/yt-dlp/yt-dlp/commit/1d03633c5a1621b9f3a756f0a4f9dc61fab3aeaa) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+- **ci**
+    - [Bump `actions/checkout` to v4](https://github.com/yt-dlp/yt-dlp/commit/5438593a35b7b042fc48fe29cad0b9039f07c9bb) by [bashonly](https://github.com/bashonly)
+    - [Run core tests with dependencies](https://github.com/yt-dlp/yt-dlp/commit/700444c23ddb65f618c2abd942acdc0c58c650b1) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz)
+- **cleanup**
+    - [Fix changelog typo](https://github.com/yt-dlp/yt-dlp/commit/a9d3f4b20a3533d2a40104c85bc2cc6c2564c800) by [bashonly](https://github.com/bashonly)
+    - [Update documentation for master and nightly channels](https://github.com/yt-dlp/yt-dlp/commit/a00af29853b8c7350ce086f4cab8c2c9cf2fcf1d) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+    - Miscellaneous: [b012271](https://github.com/yt-dlp/yt-dlp/commit/b012271d01b59759e4eefeab0308698cd9e7224c) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz), [dirkf](https://github.com/dirkf), [gamer191](https://github.com/gamer191), [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev)
+- **test**: update: [Implement simple updater unit tests](https://github.com/yt-dlp/yt-dlp/commit/87264d4fdadcddd91289b968dd0e4bf58d449267) by [bashonly](https://github.com/bashonly)
+
+### 2023.10.13
+
+#### Core changes
+- [Ensure thumbnail output directory exists](https://github.com/yt-dlp/yt-dlp/commit/2acd1d555ef89851c73773776715d3de9a0e30b9) ([#7985](https://github.com/yt-dlp/yt-dlp/issues/7985)) by [Riteo](https://github.com/Riteo)
+- **utils**
+    - `js_to_json`: [Fix `Date` constructor parsing](https://github.com/yt-dlp/yt-dlp/commit/9d7ded6419089c1bf252496073f73ad90ed71004) ([#8295](https://github.com/yt-dlp/yt-dlp/issues/8295)) by [awalgarg](https://github.com/awalgarg), [Grub4K](https://github.com/Grub4K)
+    - `write_xattr`: [Use `os.setxattr` if available](https://github.com/yt-dlp/yt-dlp/commit/84e26038d4002e763ea51ca1bdce4f7e63c540bf) ([#8205](https://github.com/yt-dlp/yt-dlp/issues/8205)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+
+#### Extractor changes
+- **artetv**: [Support age-restricted content](https://github.com/yt-dlp/yt-dlp/commit/09f815ad52843219a7ee3f2a0dddf6c250c91f0c) ([#8301](https://github.com/yt-dlp/yt-dlp/issues/8301)) by [StefanLobbenmeier](https://github.com/StefanLobbenmeier)
+- **jtbc**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/b286ec68f1f28798b3e371f888a2ed97d399cf77) ([#8314](https://github.com/yt-dlp/yt-dlp/issues/8314)) by [seproDev](https://github.com/seproDev)
+- **mbn**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/e030b6b6fba7b2f4614ad2ab9f7649d40a2dd305) ([#8312](https://github.com/yt-dlp/yt-dlp/issues/8312)) by [seproDev](https://github.com/seproDev)
+- **nhk**: [Fix Japanese-language VOD extraction](https://github.com/yt-dlp/yt-dlp/commit/4de94b9e165bfd6421a692f5f2eabcdb08edcb71) ([#8309](https://github.com/yt-dlp/yt-dlp/issues/8309)) by [garret1317](https://github.com/garret1317)
+- **radiko**: [Fix bug with `downloader_options`](https://github.com/yt-dlp/yt-dlp/commit/b9316642313bbc9e209ac0d2276d37ba60bceb49) by [bashonly](https://github.com/bashonly)
+- **tenplay**: [Add support for seasons](https://github.com/yt-dlp/yt-dlp/commit/88a99c87b680ae59002534a517e191f46c42cbd4) ([#7939](https://github.com/yt-dlp/yt-dlp/issues/7939)) by [midnightveil](https://github.com/midnightveil)
+- **youku**: [Improve tudou.com support](https://github.com/yt-dlp/yt-dlp/commit/b7098d46b552a9322c6cea39ba80be5229f922de) ([#8160](https://github.com/yt-dlp/yt-dlp/issues/8160)) by [naginatana](https://github.com/naginatana)
+- **youtube**: [Fix bug with `--extractor-retries inf`](https://github.com/yt-dlp/yt-dlp/commit/feebf6d02fc9651331eee2af5e08e6112288163b) ([#8328](https://github.com/yt-dlp/yt-dlp/issues/8328)) by [Grub4K](https://github.com/Grub4K)
+
+#### Downloader changes
+- **fragment**: [Improve progress calculation](https://github.com/yt-dlp/yt-dlp/commit/1c51c520f7b511ebd9e4eb7322285a8c31eedbbd) ([#8241](https://github.com/yt-dlp/yt-dlp/issues/8241)) by [Grub4K](https://github.com/Grub4K)
+
+#### Misc. changes
+- **cleanup**: Miscellaneous: [b634ba7](https://github.com/yt-dlp/yt-dlp/commit/b634ba742d8f38ce9ecfa0546485728b0c6c59d1) by [bashonly](https://github.com/bashonly), [gamer191](https://github.com/gamer191)
+
+### 2023.10.07
+
+#### Extractor changes
+- **abc.net.au**: iview: [Improve `episode` extraction](https://github.com/yt-dlp/yt-dlp/commit/a9efb4b8d74f3583450ffda0ee57259a47d39c70) ([#8201](https://github.com/yt-dlp/yt-dlp/issues/8201)) by [xofe](https://github.com/xofe)
+- **erocast**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/47c598783c98c179e04dd12c2a3fee0f3dc53087) ([#8264](https://github.com/yt-dlp/yt-dlp/issues/8264)) by [madewokherd](https://github.com/madewokherd)
+- **gofile**: [Fix token cookie bug](https://github.com/yt-dlp/yt-dlp/commit/0730d5a966fa8a937d84bfb7f68be5198acb039b) by [bashonly](https://github.com/bashonly)
+- **iq.com**: [Fix extraction and subtitles](https://github.com/yt-dlp/yt-dlp/commit/35d9cbaf9638ccc9daf8a863063b2e7c135bc664) ([#8260](https://github.com/yt-dlp/yt-dlp/issues/8260)) by [AS6939](https://github.com/AS6939)
+- **lbry**
+    - [Add playlist support](https://github.com/yt-dlp/yt-dlp/commit/48cceec1ddb8649b5e771df8df79eb9c39c82b90) ([#8213](https://github.com/yt-dlp/yt-dlp/issues/8213)) by [bashonly](https://github.com/bashonly), [drzraf](https://github.com/drzraf), [Grub4K](https://github.com/Grub4K)
+    - [Extract `uploader_id`](https://github.com/yt-dlp/yt-dlp/commit/0e722f2f3ca42e634fd7b06ee70b16bf833ce132) ([#8244](https://github.com/yt-dlp/yt-dlp/issues/8244)) by [drzraf](https://github.com/drzraf)
+- **litv**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/91a670a4f7babe9c8aa2018f57d8c8952a6f49d8) ([#7785](https://github.com/yt-dlp/yt-dlp/issues/7785)) by [jiru](https://github.com/jiru)
+- **neteasemusic**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/f980df734cf5c0eaded2f7b38c6c60bccfeebb48) ([#8181](https://github.com/yt-dlp/yt-dlp/issues/8181)) by [c-basalt](https://github.com/c-basalt)
+- **nhk**: [Fix VOD extraction](https://github.com/yt-dlp/yt-dlp/commit/e831c80e8b2fc025b3b67d82974cc59e3526fdc8) ([#8249](https://github.com/yt-dlp/yt-dlp/issues/8249)) by [garret1317](https://github.com/garret1317)
+- **radiko**: [Improve extraction](https://github.com/yt-dlp/yt-dlp/commit/2ad3873f0dfa9285c91d2160e36c039e69d597c7) ([#8221](https://github.com/yt-dlp/yt-dlp/issues/8221)) by [garret1317](https://github.com/garret1317)
+- **substack**
+    - [Fix download cookies bug](https://github.com/yt-dlp/yt-dlp/commit/2f2dda3a7e85148773da3cdbc03ac9949ec1bc45) ([#8219](https://github.com/yt-dlp/yt-dlp/issues/8219)) by [handlerug](https://github.com/handlerug)
+    - [Fix embed extraction](https://github.com/yt-dlp/yt-dlp/commit/fbcc299bd8a19cf8b3c8805d6c268a9110230973) ([#8218](https://github.com/yt-dlp/yt-dlp/issues/8218)) by [handlerug](https://github.com/handlerug)
+- **theta**: [Remove extractors](https://github.com/yt-dlp/yt-dlp/commit/792f1e64f6a2beac51e85408d142b3118115c4fd) ([#8251](https://github.com/yt-dlp/yt-dlp/issues/8251)) by [alerikaisattera](https://github.com/alerikaisattera)
+- **wrestleuniversevod**: [Call API with device ID](https://github.com/yt-dlp/yt-dlp/commit/b095fd3fa9d58a65dc9b830bd63b9d909422aa86) ([#8272](https://github.com/yt-dlp/yt-dlp/issues/8272)) by [bashonly](https://github.com/bashonly)
+- **xhamster**: user: [Support creator urls](https://github.com/yt-dlp/yt-dlp/commit/cc8d8441524ec3442d7c0d3f8f33f15b66aa06f3) ([#8232](https://github.com/yt-dlp/yt-dlp/issues/8232)) by [Grub4K](https://github.com/Grub4K)
+- **youtube**
+    - [Fix `heatmap` extraction](https://github.com/yt-dlp/yt-dlp/commit/03e85ea99db76a2fddb65bf46f8819bda780aaf3) ([#8299](https://github.com/yt-dlp/yt-dlp/issues/8299)) by [bashonly](https://github.com/bashonly)
+    - [Raise a warning for `Incomplete Data` instead of an error](https://github.com/yt-dlp/yt-dlp/commit/eb5bdbfa70126c7d5355cc0954b63720522e462c) ([#8238](https://github.com/yt-dlp/yt-dlp/issues/8238)) by [coletdjnz](https://github.com/coletdjnz)
+
+#### Misc. changes
+- **cleanup**
+    - [Update extractor tests](https://github.com/yt-dlp/yt-dlp/commit/19c90e405b4137c06dfe6f9aaa02396df0da93e5) ([#7718](https://github.com/yt-dlp/yt-dlp/issues/7718)) by [trainman261](https://github.com/trainman261)
+    - Miscellaneous: [377e85a](https://github.com/yt-dlp/yt-dlp/commit/377e85a1797db9e98b78b38203ed9d4ded229991) by [dirkf](https://github.com/dirkf), [gamer191](https://github.com/gamer191), [Grub4K](https://github.com/Grub4K)
+
 ### 2023.09.24
 
 #### Important changes

README.md

@@ -89,7 +89,6 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
     * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326) **\***
     * Supports some (but not all) age-gated content without cookies
     * Download livestreams from the start using `--live-from-start` (*experimental*)
-    * `255kbps` audio is extracted (if available) from YouTube Music when premium cookies are given
     * Channel URLs download all uploads of the channel, including shorts and live
 
 * **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[+KEYRING][:PROFILE][::CONTAINER]`
@@ -122,7 +121,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 * **Self updater**: The releases can be updated using `yt-dlp -U`, and downgraded using `--update-to` if required
 
-* **Nightly builds**: [Automated nightly builds](#update-channels) can be used with `--update-to nightly`
+* **Automated builds**: [Nightly/master builds](#update-channels) can be used with `--update-to nightly` and `--update-to master`
 
 See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/commits) for the full list of changes
 
@@ -158,14 +157,16 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 * yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
 * yt-dlp tries to parse the external downloader outputs into the standard progress output if possible (Currently implemented: [~~aria2c~~](https://github.com/yt-dlp/yt-dlp/issues/5931)). You can use `--compat-options no-external-downloader-progress` to get the downloader output as-is
 * yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filter` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this
+* yt-dlp versions between 2021.11.10 and 2023.06.21 estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values
+* yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests.
 
 For ease of use, a few more compat options are available:
 
 * `--compat-options all`: Use all compat options (Do NOT use)
-* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter`
+* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx`
-* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter`
+* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx`
 * `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date`
-* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress`. Use this to enable all future compat options
+* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx`. Use this to enable all future compat options
 
 
 # INSTALLATION
@@ -192,9 +193,11 @@ For other third-party package managers, see [the wiki](https://github.com/yt-dlp
 
 <a id="update-channels"/>
 
-There are currently two release channels for binaries, `stable` and `nightly`.
+There are currently three release channels for binaries: `stable`, `nightly` and `master`.
-`stable` is the default channel, and many of its changes have been tested by users of the nightly channel.
+
-The `nightly` channel has releases built after each push to the master branch, and will have the most recent fixes and additions, but also have more risk of regressions. They are available in [their own repo](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases).
+* `stable` is the default channel, and many of its changes have been tested by users of the `nightly` and `master` channels.
+* The `nightly` channel has releases scheduled to build every day around midnight UTC, for a snapshot of the project's new patches and changes. This is the **recommended channel for regular users** of yt-dlp. The `nightly` releases are available from [yt-dlp/yt-dlp-nightly-builds](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases) or as development releases of the `yt-dlp` PyPI package (which can be installed with pip's `--pre` flag).
+* The `master` channel features releases that are built after each push to the master branch, and these will have the very latest fixes and additions, but may also be more prone to regressions. They are available from [yt-dlp/yt-dlp-master-builds](https://github.com/yt-dlp/yt-dlp-master-builds/releases).
 
 When using `--update`/`-U`, a release binary will only update to its current channel.
 `--update-to CHANNEL` can be used to switch to a different channel when a newer version is available. `--update-to [CHANNEL@]TAG` can also be used to upgrade or downgrade to specific tags from a channel.
@@ -202,10 +205,19 @@ When using `--update`/`-U`, a release binary will only update to its current cha
 You may also use `--update-to <repository>` (`<owner>/<repository>`) to update to a channel on a completely different repository. Be careful with what repository you are updating to though, there is no verification done for binaries from different repositories.
 
 Example usage:
-* `yt-dlp --update-to nightly` change to `nightly` channel and update to its latest release
+* `yt-dlp --update-to master` switch to the `master` channel and update to its latest release
-* `yt-dlp --update-to stable@2023.02.17` upgrade/downgrade to release to `stable` channel tag `2023.02.17`
+* `yt-dlp --update-to stable@2023.07.06` upgrade/downgrade to release to `stable` channel tag `2023.07.06`
-* `yt-dlp --update-to 2023.01.06` upgrade/downgrade to tag `2023.01.06` if it exists on the current channel
+* `yt-dlp --update-to 2023.10.07` upgrade/downgrade to tag `2023.10.07` if it exists on the current channel
-* `yt-dlp --update-to example/yt-dlp@2023.03.01` upgrade/downgrade to the release from the `example/yt-dlp` repository, tag `2023.03.01`
+* `yt-dlp --update-to example/yt-dlp@2023.09.24` upgrade/downgrade to the release from the `example/yt-dlp` repository, tag `2023.09.24`
+
+**Important**: Any user experiencing an issue with the `stable` release should install or update to the `nightly` release before submitting a bug report:
+```
+# To update to nightly from stable executable/binary:
+yt-dlp --update-to nightly
+
+# To install nightly with pip:
+python -m pip install -U --pre yt-dlp
+```
 
 <!-- MANPAGE: BEGIN EXCLUDED SECTION -->
 ## RELEASE FILES
@@ -275,12 +287,13 @@ While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly
 * [**certifi**](https://github.com/certifi/python-certifi)\* - Provides Mozilla's root certificate bundle. Licensed under [MPLv2](https://github.com/certifi/python-certifi/blob/master/LICENSE)
 * [**brotli**](https://github.com/google/brotli)\* or [**brotlicffi**](https://github.com/python-hyper/brotlicffi) - [Brotli](https://en.wikipedia.org/wiki/Brotli) content encoding support. Both licensed under MIT <sup>[1](https://github.com/google/brotli/blob/master/LICENSE) [2](https://github.com/python-hyper/brotlicffi/blob/master/LICENSE) </sup>
 * [**websockets**](https://github.com/aaugustin/websockets)\* - For downloading over websocket. Licensed under [BSD-3-Clause](https://github.com/aaugustin/websockets/blob/main/LICENSE)
+* [**requests**](https://github.com/psf/requests)\* - HTTP library. For HTTPS proxy and persistent connections support. Licensed under [Apache-2.0](https://github.com/psf/requests/blob/main/LICENSE)
 
 ### Metadata
 
 * [**mutagen**](https://github.com/quodlibet/mutagen)\* - For `--embed-thumbnail` in certain formats. Licensed under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
 * [**AtomicParsley**](https://github.com/wez/atomicparsley) - For `--embed-thumbnail` in `mp4`/`m4a` files when `mutagen`/`ffmpeg` cannot. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
-* [**xattr**](https://github.com/xattr/xattr), [**pyxattr**](https://github.com/iustin/pyxattr) or [**setfattr**](http://savannah.nongnu.org/projects/attr) - For writing xattr metadata (`--xattr`) on **Linux**. Licensed under [MIT](https://github.com/xattr/xattr/blob/master/LICENSE.txt), [LGPL2.1](https://github.com/iustin/pyxattr/blob/master/COPYING) and [GPLv2+](http://git.savannah.nongnu.org/cgit/attr.git/tree/doc/COPYING) respectively
+* [**xattr**](https://github.com/xattr/xattr), [**pyxattr**](https://github.com/iustin/pyxattr) or [**setfattr**](http://savannah.nongnu.org/projects/attr) - For writing xattr metadata (`--xattr`) on **Mac** and **BSD**. Licensed under [MIT](https://github.com/xattr/xattr/blob/master/LICENSE.txt), [LGPL2.1](https://github.com/iustin/pyxattr/blob/master/COPYING) and [GPLv2+](http://git.savannah.nongnu.org/cgit/attr.git/tree/doc/COPYING) respectively
 
 ### Misc
 
@@ -367,7 +380,8 @@ If you fork the project on GitHub, you can run your fork's [build workflow](.git
                                     CHANNEL can be a repository as well. CHANNEL
                                     and TAG default to "stable" and "latest"
                                     respectively if omitted; See "UPDATE" for
-                                    details. Supported channels: stable, nightly
+                                    details. Supported channels: stable,
+                                    nightly, master
     -i, --ignore-errors             Ignore download and postprocessing errors.
                                     The download will be considered successful
                                     even if the postprocessing fails
@@ -913,7 +927,7 @@ If you fork the project on GitHub, you can run your fork's [build workflow](.git
                                     Defaults to ~/.netrc
     --netrc-cmd NETRC_CMD           Command to execute to get the credentials
                                     for an extractor.
-    --video-password PASSWORD       Video password (vimeo, youku)
+    --video-password PASSWORD       Video-specific password
     --ap-mso MSO                    Adobe Pass multiple-system operator (TV
                                     provider) identifier, use --ap-list-mso for
                                     a list of available MSOs
@@ -1809,6 +1823,7 @@ The following extractors use this feature:
 * `formats`: Change the types of formats to return. `dashy` (convert HTTP to DASH), `duplicate` (identical content but different URLs or protocol; includes `dashy`), `incomplete` (cannot be downloaded completely - live dash and post-live m3u8)
 * `innertube_host`: Innertube API host to use for all API requests; e.g. `studio.youtube.com`, `youtubei.googleapis.com`. Note that cookies exported from one subdomain will not work on others
 * `innertube_key`: Innertube API key to use for all API requests
+* `raise_incomplete_data`: `Incomplete Data Received` raises an error instead of reporting a warning
 
 #### youtubetab (YouTube playlists, channels, feeds, etc.)
 * `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details)

devscripts/changelog_override.json

@@ -98,5 +98,21 @@
         "action": "add",
         "when": "61bdf15fc7400601c3da1aa7a43917310a5bf391",
         "short": "[priority] Security: [[CVE-2023-40581](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-40581)] [Prevent RCE when using `--exec` with `%q` on Windows](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-42h4-v29r-42qg)\n    - The shell escape function is now using `\"\"` instead of `\\\"`.\n    - `utils.Popen` has been patched to properly quote commands."
+    },
+    {
+        "action": "change",
+        "when": "8a8b54523addf46dfd50ef599761a81bc22362e6",
+        "short": "[rh:requests] Add handler for `requests` HTTP library (#3668)\n\n\tAdds support for HTTPS proxies and persistent connections (keep-alive)",
+        "authors": ["bashonly", "coletdjnz", "Grub4K"]
+    },
+    {
+        "action": "add",
+        "when": "1d03633c5a1621b9f3a756f0a4f9dc61fab3aeaa",
+        "short": "[priority] **The release channels have been adjusted!**\n\t* [`master`](https://github.com/yt-dlp/yt-dlp-master-builds) builds are made after each push, containing the latest fixes (but also possibly bugs). This was previously the `nightly` channel.\n\t* [`nightly`](https://github.com/yt-dlp/yt-dlp-nightly-builds) builds are now made once a day, if there were any changes."
+    },
+    {
+        "action": "add",
+        "when": "f04b5bedad7b281bee9814686bba1762bae092eb",
+        "short": "[priority] Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)\n\t- Disallow smuggling of arbitrary `http_headers`; extractors now only use specific headers"
     }
 ]

devscripts/make_changelog.py

@@ -56,6 +56,7 @@ class CommitGroup(enum.Enum):
                 },
                 cls.MISC: {
                     'build',
+                    'ci',
                     'cleanup',
                     'devscripts',
                     'docs',

devscripts/make_issue_template.py

@@ -12,7 +12,6 @@ import re
 from devscripts.utils import (
     get_filename_args,
     read_file,
-    read_version,
     write_file,
 )
 
@@ -35,19 +34,18 @@ VERBOSE_TMPL = '''
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version %(version)s [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        [debug] Request Handlers: urllib, requests
-        Latest version: %(version)s, Current version: %(version)s
+        [debug] Loaded 1893 extractors
-        yt-dlp is up to date (%(version)s)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:
@@ -66,7 +64,7 @@ NO_SKIP = '''
 
 
 def main():
-    fields = {'version': read_version(), 'no_skip': NO_SKIP}
+    fields = {'no_skip': NO_SKIP}
     fields['verbose'] = VERBOSE_TMPL % fields
     fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose'])
 

devscripts/update-formulae.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-Usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
-version can be either 0-aligned (yt-dlp version) or normalized (PyPi version)
-"""
-
-# Allow direct execution
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-
-import json
-import re
-import urllib.request
-
-from devscripts.utils import read_file, write_file
-
-filename, version = sys.argv[1:]
-
-normalized_version = '.'.join(str(int(x)) for x in version.split('.'))
-
-pypi_release = json.loads(urllib.request.urlopen(
-    'https://pypi.org/pypi/yt-dlp/%s/json' % normalized_version
-).read().decode())
-
-tarball_file = next(x for x in pypi_release['urls'] if x['filename'].endswith('.tar.gz'))
-
-sha256sum = tarball_file['digests']['sha256']
-url = tarball_file['url']
-
-formulae_text = read_file(filename)
-
-formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text, count=1)
-formulae_text = re.sub(r'url "[^"]*?"', 'url "%s"' % url, formulae_text, count=1)
-
-write_file(filename, formulae_text)

devscripts/update-version.py

@@ -20,7 +20,7 @@ def get_new_version(version, revision):
         version = datetime.now(timezone.utc).strftime('%Y.%m.%d')
 
     if revision:
-        assert revision.isdigit(), 'Revision must be a number'
+        assert revision.isdecimal(), 'Revision must be a number'
     else:
         old_version = read_version().split('.')
         if version.split('.') == old_version[:3]:
@@ -46,6 +46,10 @@ VARIANT = None
 UPDATE_HINT = None
 
 CHANNEL = {channel!r}
+
+ORIGIN = {origin!r}
+
+_pkg_version = {package_version!r}
 '''
 
 if __name__ == '__main__':
@@ -53,6 +57,12 @@ if __name__ == '__main__':
     parser.add_argument(
         '-c', '--channel', default='stable',
         help='Select update channel (default: %(default)s)')
+    parser.add_argument(
+        '-r', '--origin', default='local',
+        help='Select origin/repository (default: %(default)s)')
+    parser.add_argument(
+        '-s', '--suffix', default='',
+        help='Add an alphanumeric suffix to the package version, e.g. "dev"')
     parser.add_argument(
         '-o', '--output', default='yt_dlp/version.py',
         help='The output file to write to (default: %(default)s)')
@@ -66,6 +76,7 @@ if __name__ == '__main__':
         args.version if args.version and '.' in args.version
         else get_new_version(None, args.version))
     write_file(args.output, VERSION_TEMPLATE.format(
-        version=version, git_head=git_head, channel=args.channel))
+        version=version, git_head=git_head, channel=args.channel, origin=args.origin,
+        package_version=f'{version}{args.suffix}'))
 
     print(f'version={version} ({args.channel}), head={git_head}')

devscripts/utils.py

@@ -13,10 +13,11 @@ def write_file(fname, content, mode='w'):
         return f.write(content)
 
 
-def read_version(fname='yt_dlp/version.py'):
+def read_version(fname='yt_dlp/version.py', varname='__version__'):
     """Get the version without importing the package"""
-    exec(compile(read_file(fname), fname, 'exec'))
+    items = {}
-    return locals()['__version__']
+    exec(compile(read_file(fname), fname, 'exec'), items)
+    return items[varname]
 
 
 def get_filename_args(has_infile=False, default_outfile=None):

requirements.txt

@@ -1,6 +1,8 @@
 mutagen
 pycryptodomex
 websockets
-brotli; platform_python_implementation=='CPython'
+brotli; implementation_name=='cpython'
-brotlicffi; platform_python_implementation!='CPython'
+brotlicffi; implementation_name!='cpython'
 certifi
+requests>=2.31.0,<3
+urllib3>=1.26.17,<3

setup.py

@@ -18,7 +18,7 @@ except ImportError:
 
 from devscripts.utils import read_file, read_version
 
-VERSION = read_version()
+VERSION = read_version(varname='_pkg_version')
 
 DESCRIPTION = 'A youtube-dl fork with additional features and patches'
 
@@ -62,7 +62,14 @@ def py2exe_params():
             'compressed': 1,
             'optimize': 2,
             'dist_dir': './dist',
-            'excludes': ['Crypto', 'Cryptodome'],  # py2exe cannot import Crypto
+            'excludes': [
+                # py2exe cannot import Crypto
+                'Crypto',
+                'Cryptodome',
+                # py2exe appears to confuse this with our socks library.
+                # We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
+                'urllib3.contrib.socks'
+            ],
             'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
             # Modules that are only imported dynamically must be added here
             'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
@@ -135,7 +142,7 @@ def main():
         params = build_params()
 
     setup(
-        name='yt-dlp',
+        name='yt-dlp',  # package name (do not change/remove comment)
         version=VERSION,
         maintainer='pukkandan',
         maintainer_email='pukkandan.ytdlp@gmail.com',

supportedsites.md

@@ -414,6 +414,7 @@
  - **EllenTubeVideo**
  - **Elonet**
  - **ElPais**: El País
+ - **ElTreceTV**: El Trece TV (Argentina)
  - **Embedly**
  - **EMPFlix**
  - **Engadget**
@@ -422,6 +423,7 @@
  - **eplus:inbound**: e+ (イープラス) overseas
  - **Epoch**
  - **Eporner**
+ - **Erocast**
  - **EroProfile**: [*eroprofile*](## "netrc machine")
  - **EroProfile:album**
  - **ertflix**: ERTFLIX videos
@@ -653,9 +655,13 @@
  - **Jamendo**
  - **JamendoAlbum**
  - **JeuxVideo**
+ - **JioSaavnAlbum**
+ - **JioSaavnSong**
  - **Joj**
  - **Jove**
  - **JStream**
+ - **JTBC**: jtbc.co.kr
+ - **JTBC:program**
  - **JWPlatform**
  - **Kakao**
  - **Kaltura**
@@ -697,8 +703,10 @@
  - **LastFM**
  - **LastFMPlaylist**
  - **LastFMUser**
+ - **LaXarxaMes**: [*laxarxames*](## "netrc machine")
  - **lbry**
  - **lbry:channel**
+ - **lbry:playlist**
  - **LCI**
  - **Lcp**
  - **LcpPlay**
@@ -764,6 +772,7 @@
  - **massengeschmack.tv**
  - **Masters**
  - **MatchTV**
+ - **MBN**: mbn.co.kr (매일방송)
  - **MDR**: MDR.DE and KiKA
  - **MedalTV**
  - **media.ccc.de**
@@ -970,7 +979,6 @@
  - **Nitter**
  - **njoy**: N-JOY
  - **njoy:embed**
- - **NJPWWorld**: [*njpwworld*](## "netrc machine") 新日本プロレスワールド
  - **NobelPrize**
  - **NoicePodcast**
  - **NonkTube**
@@ -1021,6 +1029,7 @@
  - **on24**: ON24
  - **OnDemandChinaEpisode**
  - **OnDemandKorea**
+ - **OnDemandKoreaProgram**
  - **OneFootball**
  - **OnePlacePodcast**
  - **onet.pl**
@@ -1038,6 +1047,7 @@
  - **OraTV**
  - **orf:​fm4:story**: fm4.orf.at stories
  - **orf:iptv**: iptv.ORF.at
+ - **orf:podcast**
  - **orf:radio**
  - **orf:tvthek**: ORF TVthek
  - **OsnatelTV**: [*osnateltv*](## "netrc machine")
@@ -1175,6 +1185,8 @@
  - **radiobremen**
  - **radiocanada**
  - **radiocanada:audiovideo**
+ - **RadioComercial**
+ - **RadioComercialPlaylist**
  - **radiofrance**
  - **RadioFranceLive**
  - **RadioFrancePodcast**
@@ -1301,6 +1313,9 @@
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
+ - **sbs.co.kr**
+ - **sbs.co.kr:allvod_program**
+ - **sbs.co.kr:programs_vod**
  - **schooltv**
  - **ScienceChannel**
  - **screen.yahoo:search**: Yahoo screen search; "yvsearch:" prefix
@@ -1466,20 +1481,20 @@
  - **Tempo**
  - **TennisTV**: [*tennistv*](## "netrc machine")
  - **TenPlay**: [*10play*](## "netrc machine")
+ - **TenPlaySeason**
  - **TF1**
  - **TFO**
+ - **theatercomplextown:ppv**: [*theatercomplextown*](## "netrc machine")
+ - **theatercomplextown:vod**: [*theatercomplextown*](## "netrc machine")
  - **TheHoleTv**
  - **TheIntercept**
  - **ThePlatform**
  - **ThePlatformFeed**
  - **TheStar**
  - **TheSun**
- - **ThetaStream**
- - **ThetaVideo**
  - **TheWeatherChannel**
  - **ThisAmericanLife**
- - **ThisAV**
+ - **ThisOldHouse**: [*thisoldhouse*](## "netrc machine")
- - **ThisOldHouse**
  - **ThisVid**
  - **ThisVidMember**
  - **ThisVidPlaylist**

test/test_networking.py

@@ -28,7 +28,7 @@ from http.cookiejar import CookieJar
 
 from test.helper import FakeYDL, http_server_port
 from yt_dlp.cookies import YoutubeDLCookieJar
-from yt_dlp.dependencies import brotli
+from yt_dlp.dependencies import brotli, requests, urllib3
 from yt_dlp.networking import (
     HEADRequest,
     PUTRequest,
@@ -43,6 +43,7 @@ from yt_dlp.networking.exceptions import (
     HTTPError,
     IncompleteRead,
     NoSupportingHandlers,
+    ProxyError,
     RequestError,
     SSLError,
     TransportError,
@@ -305,7 +306,7 @@ class TestRequestHandlerBase:
 
 
 class TestHTTPRequestHandler(TestRequestHandlerBase):
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_verify_cert(self, handler):
         with handler() as rh:
             with pytest.raises(CertificateVerifyError):
@@ -316,7 +317,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert r.status == 200
             r.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_ssl_error(self, handler):
         # HTTPS server with too old TLS version
         # XXX: is there a better way to test this than to create a new server?
@@ -334,7 +335,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
                 validate_and_send(rh, Request(f'https://127.0.0.1:{https_port}/headers'))
             assert not issubclass(exc_info.type, CertificateVerifyError)
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_percent_encode(self, handler):
         with handler() as rh:
             # Unicode characters should be encoded with uppercase percent-encoding
@@ -346,7 +347,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert res.status == 200
             res.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_remove_dot_segments(self, handler):
         with handler() as rh:
             # This isn't a comprehensive test,
@@ -361,14 +362,14 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert res.url == f'http://127.0.0.1:{self.http_port}/headers'
             res.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_unicode_path_redirection(self, handler):
         with handler() as rh:
             r = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/302-non-ascii-redirect'))
             assert r.url == f'http://127.0.0.1:{self.http_port}/%E4%B8%AD%E6%96%87.html'
             r.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_raise_http_error(self, handler):
         with handler() as rh:
             for bad_status in (400, 500, 599, 302):
@@ -378,7 +379,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             # Should not raise an error
             validate_and_send(rh, Request('http://127.0.0.1:%d/gen_200' % self.http_port)).close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_response_url(self, handler):
         with handler() as rh:
             # Response url should be that of the last url in redirect chain
@@ -389,7 +390,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert res2.url == f'http://127.0.0.1:{self.http_port}/gen_200'
             res2.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_redirect(self, handler):
         with handler() as rh:
             def do_req(redirect_status, method, assert_no_content=False):
@@ -444,7 +445,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
                 with pytest.raises(HTTPError):
                     do_req(code, 'GET')
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_request_cookie_header(self, handler):
         # We should accept a Cookie header being passed as in normal headers and handle it appropriately.
         with handler() as rh:
@@ -476,19 +477,19 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert b'Cookie: test=ytdlp' not in data
             assert b'Cookie: test=test' in data
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_redirect_loop(self, handler):
         with handler() as rh:
             with pytest.raises(HTTPError, match='redirect loop'):
                 validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/redirect_loop'))
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_incompleteread(self, handler):
         with handler(timeout=2) as rh:
             with pytest.raises(IncompleteRead):
                 validate_and_send(rh, Request('http://127.0.0.1:%d/incompleteread' % self.http_port)).read()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_cookies(self, handler):
         cookiejar = YoutubeDLCookieJar()
         cookiejar.set_cookie(http.cookiejar.Cookie(
@@ -505,7 +506,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
                 rh, Request(f'http://127.0.0.1:{self.http_port}/headers', extensions={'cookiejar': cookiejar})).read()
             assert b'Cookie: test=ytdlp' in data
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_headers(self, handler):
 
         with handler(headers=HTTPHeaderDict({'test1': 'test', 'test2': 'test2'})) as rh:
@@ -521,7 +522,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert b'Test2: test2' not in data
             assert b'Test3: test3' in data
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_timeout(self, handler):
         with handler() as rh:
             # Default timeout is 20 seconds, so this should go through
@@ -537,7 +538,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             validate_and_send(
                 rh, Request(f'http://127.0.0.1:{self.http_port}/timeout_1', extensions={'timeout': 4}))
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_source_address(self, handler):
         source_address = f'127.0.0.{random.randint(5, 255)}'
         with handler(source_address=source_address) as rh:
@@ -545,13 +546,13 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
                 rh, Request(f'http://127.0.0.1:{self.http_port}/source_address')).read().decode()
             assert source_address == data
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_gzip_trailing_garbage(self, handler):
         with handler() as rh:
             data = validate_and_send(rh, Request(f'http://localhost:{self.http_port}/trailing_garbage')).read().decode()
             assert data == '<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     @pytest.mark.skipif(not brotli, reason='brotli support is not installed')
     def test_brotli(self, handler):
         with handler() as rh:
@@ -562,7 +563,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert res.headers.get('Content-Encoding') == 'br'
             assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_deflate(self, handler):
         with handler() as rh:
             res = validate_and_send(
@@ -572,7 +573,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert res.headers.get('Content-Encoding') == 'deflate'
             assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_gzip(self, handler):
         with handler() as rh:
             res = validate_and_send(
@@ -582,7 +583,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert res.headers.get('Content-Encoding') == 'gzip'
             assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_multiple_encodings(self, handler):
         with handler() as rh:
             for pair in ('gzip,deflate', 'deflate, gzip', 'gzip, gzip', 'deflate, deflate'):
@@ -593,7 +594,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
                 assert res.headers.get('Content-Encoding') == pair
                 assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_unsupported_encoding(self, handler):
         with handler() as rh:
             res = validate_and_send(
@@ -603,7 +604,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert res.headers.get('Content-Encoding') == 'unsupported'
             assert res.read() == b'raw'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_read(self, handler):
         with handler() as rh:
             res = validate_and_send(
@@ -633,7 +634,7 @@ class TestHTTPProxy(TestRequestHandlerBase):
         cls.geo_proxy_thread.daemon = True
         cls.geo_proxy_thread.start()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_http_proxy(self, handler):
         http_proxy = f'http://127.0.0.1:{self.proxy_port}'
         geo_proxy = f'http://127.0.0.1:{self.geo_port}'
@@ -659,7 +660,7 @@ class TestHTTPProxy(TestRequestHandlerBase):
             assert res != f'normal: {real_url}'
             assert 'Accept' in res
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_noproxy(self, handler):
         with handler(proxies={'proxy': f'http://127.0.0.1:{self.proxy_port}'}) as rh:
             # NO_PROXY
@@ -669,7 +670,7 @@ class TestHTTPProxy(TestRequestHandlerBase):
                     'utf-8')
                 assert 'Accept' in nop_response
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_allproxy(self, handler):
         url = 'http://foo.com/bar'
         with handler() as rh:
@@ -677,7 +678,7 @@ class TestHTTPProxy(TestRequestHandlerBase):
                 'utf-8')
             assert response == f'normal: {url}'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_http_proxy_with_idn(self, handler):
         with handler(proxies={
             'http': f'http://127.0.0.1:{self.proxy_port}',
@@ -715,27 +716,27 @@ class TestClientCertificate:
         ) as rh:
             validate_and_send(rh, Request(f'https://127.0.0.1:{self.port}/video.html')).read().decode()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_certificate_combined_nopass(self, handler):
         self._run_test(handler, client_cert={
             'client_certificate': os.path.join(self.certdir, 'clientwithkey.crt'),
         })
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_certificate_nocombined_nopass(self, handler):
         self._run_test(handler, client_cert={
             'client_certificate': os.path.join(self.certdir, 'client.crt'),
             'client_certificate_key': os.path.join(self.certdir, 'client.key'),
         })
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_certificate_combined_pass(self, handler):
         self._run_test(handler, client_cert={
             'client_certificate': os.path.join(self.certdir, 'clientwithencryptedkey.crt'),
             'client_certificate_password': 'foobar',
         })
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_certificate_nocombined_pass(self, handler):
         self._run_test(handler, client_cert={
             'client_certificate': os.path.join(self.certdir, 'client.crt'),
@@ -819,6 +820,75 @@ class TestUrllibRequestHandler(TestRequestHandlerBase):
             assert not isinstance(exc_info.value, TransportError)
 
 
+class TestRequestsRequestHandler(TestRequestHandlerBase):
+    @pytest.mark.parametrize('raised,expected', [
+        (lambda: requests.exceptions.ConnectTimeout(), TransportError),
+        (lambda: requests.exceptions.ReadTimeout(), TransportError),
+        (lambda: requests.exceptions.Timeout(), TransportError),
+        (lambda: requests.exceptions.ConnectionError(), TransportError),
+        (lambda: requests.exceptions.ProxyError(), ProxyError),
+        (lambda: requests.exceptions.SSLError('12[CERTIFICATE_VERIFY_FAILED]34'), CertificateVerifyError),
+        (lambda: requests.exceptions.SSLError(), SSLError),
+        (lambda: requests.exceptions.InvalidURL(), RequestError),
+        (lambda: requests.exceptions.InvalidHeader(), RequestError),
+        # catch-all: https://github.com/psf/requests/blob/main/src/requests/adapters.py#L535
+        (lambda: urllib3.exceptions.HTTPError(), TransportError),
+        (lambda: requests.exceptions.RequestException(), RequestError)
+        #  (lambda: requests.exceptions.TooManyRedirects(), HTTPError) - Needs a response object
+    ])
+    @pytest.mark.parametrize('handler', ['Requests'], indirect=True)
+    def test_request_error_mapping(self, handler, monkeypatch, raised, expected):
+        with handler() as rh:
+            def mock_get_instance(*args, **kwargs):
+                class MockSession:
+                    def request(self, *args, **kwargs):
+                        raise raised()
+                return MockSession()
+
+            monkeypatch.setattr(rh, '_get_instance', mock_get_instance)
+
+            with pytest.raises(expected) as exc_info:
+                rh.send(Request('http://fake'))
+
+            assert exc_info.type is expected
+
+    @pytest.mark.parametrize('raised,expected,match', [
+        (lambda: urllib3.exceptions.SSLError(), SSLError, None),
+        (lambda: urllib3.exceptions.TimeoutError(), TransportError, None),
+        (lambda: urllib3.exceptions.ReadTimeoutError(None, None, None), TransportError, None),
+        (lambda: urllib3.exceptions.ProtocolError(), TransportError, None),
+        (lambda: urllib3.exceptions.DecodeError(), TransportError, None),
+        (lambda: urllib3.exceptions.HTTPError(), TransportError, None),  # catch-all
+        (
+            lambda: urllib3.exceptions.ProtocolError('error', http.client.IncompleteRead(partial=b'abc', expected=4)),
+            IncompleteRead,
+            '3 bytes read, 4 more expected'
+        ),
+        (
+            lambda: urllib3.exceptions.ProtocolError('error', urllib3.exceptions.IncompleteRead(partial=3, expected=5)),
+            IncompleteRead,
+            '3 bytes read, 5 more expected'
+        ),
+    ])
+    @pytest.mark.parametrize('handler', ['Requests'], indirect=True)
+    def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
+        from urllib3.response import HTTPResponse as Urllib3Response
+        from requests.models import Response as RequestsResponse
+        from yt_dlp.networking._requests import RequestsResponseAdapter
+        requests_res = RequestsResponse()
+        requests_res.raw = Urllib3Response(body=b'', status=200)
+        res = RequestsResponseAdapter(requests_res)
+
+        def mock_read(*args, **kwargs):
+            raise raised()
+        monkeypatch.setattr(res.fp, 'read', mock_read)
+
+        with pytest.raises(expected, match=match) as exc_info:
+            res.read()
+
+        assert exc_info.type is expected
+
+
 def run_validation(handler, error, req, **handler_kwargs):
     with handler(**handler_kwargs) as rh:
         if error:
@@ -855,6 +925,10 @@ class TestRequestHandlerValidation:
             ('file', UnsupportedRequest, {}),
             ('file', False, {'enable_file_urls': True}),
         ]),
+        ('Requests', [
+            ('http', False, {}),
+            ('https', False, {}),
+        ]),
         (NoCheckRH, [('http', False, {})]),
         (ValidationRH, [('http', UnsupportedRequest, {})])
     ]
@@ -870,6 +944,14 @@ class TestRequestHandlerValidation:
             ('socks5h', False),
             ('socks', UnsupportedRequest),
         ]),
+        ('Requests', [
+            ('http', False),
+            ('https', False),
+            ('socks4', False),
+            ('socks4a', False),
+            ('socks5', False),
+            ('socks5h', False),
+        ]),
         (NoCheckRH, [('http', False)]),
         (HTTPSupportedRH, [('http', UnsupportedRequest)]),
     ]
@@ -880,6 +962,10 @@ class TestRequestHandlerValidation:
             ('all', False),
             ('unrelated', False),
         ]),
+        ('Requests', [
+            ('all', False),
+            ('unrelated', False),
+        ]),
         (NoCheckRH, [('all', False)]),
         (HTTPSupportedRH, [('all', UnsupportedRequest)]),
         (HTTPSupportedRH, [('no', UnsupportedRequest)]),
@@ -894,6 +980,13 @@ class TestRequestHandlerValidation:
             ({'timeout': 'notatimeout'}, AssertionError),
             ({'unsupported': 'value'}, UnsupportedRequest),
         ]),
+        ('Requests', [
+            ({'cookiejar': 'notacookiejar'}, AssertionError),
+            ({'cookiejar': YoutubeDLCookieJar()}, False),
+            ({'timeout': 1}, False),
+            ({'timeout': 'notatimeout'}, AssertionError),
+            ({'unsupported': 'value'}, UnsupportedRequest),
+        ]),
         (NoCheckRH, [
             ({'cookiejar': 'notacookiejar'}, False),
             ({'somerandom': 'test'}, False),  # but any extension is allowed through
@@ -909,7 +1002,7 @@ class TestRequestHandlerValidation:
     def test_url_scheme(self, handler, scheme, fail, handler_kwargs):
         run_validation(handler, fail, Request(f'{scheme}://'), **(handler_kwargs or {}))
 
-    @pytest.mark.parametrize('handler,fail', [('Urllib', False)], indirect=['handler'])
+    @pytest.mark.parametrize('handler,fail', [('Urllib', False), ('Requests', False)], indirect=['handler'])
     def test_no_proxy(self, handler, fail):
         run_validation(handler, fail, Request('http://', proxies={'no': '127.0.0.1,github.com'}))
         run_validation(handler, fail, Request('http://'), proxies={'no': '127.0.0.1,github.com'})
@@ -932,13 +1025,13 @@ class TestRequestHandlerValidation:
         run_validation(handler, fail, Request('http://', proxies={'http': f'{scheme}://example.com'}))
         run_validation(handler, fail, Request('http://'), proxies={'http': f'{scheme}://example.com'})
 
-    @pytest.mark.parametrize('handler', ['Urllib', HTTPSupportedRH], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', HTTPSupportedRH, 'Requests'], indirect=True)
     def test_empty_proxy(self, handler):
         run_validation(handler, False, Request('http://', proxies={'http': None}))
         run_validation(handler, False, Request('http://'), proxies={'http': None})
 
     @pytest.mark.parametrize('proxy_url', ['//example.com', 'example.com', '127.0.0.1', '/a/b/c'])
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_invalid_proxy_url(self, handler, proxy_url):
         run_validation(handler, UnsupportedRequest, Request('http://', proxies={'http': proxy_url}))
 
@@ -1200,6 +1293,10 @@ class TestYoutubeDLNetworking:
             assert 'Youtubedl-no-compression' not in rh.headers
             assert rh.headers.get('Accept-Encoding') == 'identity'
 
+        with FakeYDL({'http_headers': {'Ytdl-socks-proxy': 'socks://localhost:1080'}}) as ydl:
+            rh = self.build_handler(ydl)
+            assert 'Ytdl-socks-proxy' not in rh.headers
+
     def test_build_handler_params(self):
         with FakeYDL({
             'http_headers': {'test': 'testtest'},
@@ -1242,6 +1339,13 @@ class TestYoutubeDLNetworking:
             rh = self.build_handler(ydl, UrllibRH)
             assert rh.enable_file_urls is True
 
+    def test_compat_opt_prefer_urllib(self):
+        # This assumes urllib only has a preference when this compat opt is given
+        with FakeYDL({'compat_opts': ['prefer-legacy-http-handler']}) as ydl:
+            director = ydl.build_request_director([UrllibRH])
+            assert len(director.preferences) == 1
+            assert director.preferences.pop()(UrllibRH, None)
+
 
 class TestRequest:
 

test/test_socks.py

@@ -263,7 +263,7 @@ def ctx(request):
 
 
 class TestSocks4Proxy:
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks4_no_auth(self, handler, ctx):
         with handler() as rh:
             with ctx.socks_server(Socks4ProxyHandler) as server_address:
@@ -271,7 +271,7 @@ class TestSocks4Proxy:
                     rh, proxies={'all': f'socks4://{server_address}'})
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks4_auth(self, handler, ctx):
         with handler() as rh:
             with ctx.socks_server(Socks4ProxyHandler, user_id='user') as server_address:
@@ -281,7 +281,7 @@ class TestSocks4Proxy:
                     rh, proxies={'all': f'socks4://user:@{server_address}'})
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks4a_ipv4_target(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
@@ -289,7 +289,7 @@ class TestSocks4Proxy:
                 assert response['version'] == 4
                 assert (response['ipv4_address'] == '127.0.0.1') != (response['domain_address'] == '127.0.0.1')
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks4a_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
@@ -298,7 +298,7 @@ class TestSocks4Proxy:
                 assert response['ipv4_address'] is None
                 assert response['domain_address'] == 'localhost'
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
@@ -308,7 +308,7 @@ class TestSocks4Proxy:
                 assert response['client_address'][0] == source_address
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     @pytest.mark.parametrize('reply_code', [
         Socks4CD.REQUEST_REJECTED_OR_FAILED,
         Socks4CD.REQUEST_REJECTED_CANNOT_CONNECT_TO_IDENTD,
@@ -320,7 +320,7 @@ class TestSocks4Proxy:
                 with pytest.raises(ProxyError):
                     ctx.socks_info_request(rh)
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_ipv6_socks4_proxy(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler, bind_ip='::1') as server_address:
             with handler(proxies={'all': f'socks4://{server_address}'}) as rh:
@@ -329,7 +329,7 @@ class TestSocks4Proxy:
                 assert response['ipv4_address'] == '127.0.0.1'
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_timeout(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler, sleep=2) as server_address:
             with handler(proxies={'all': f'socks4://{server_address}'}, timeout=0.5) as rh:
@@ -339,7 +339,7 @@ class TestSocks4Proxy:
 
 class TestSocks5Proxy:
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_no_auth(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -347,7 +347,7 @@ class TestSocks5Proxy:
                 assert response['auth_methods'] == [0x0]
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_user_pass(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler, auth=('test', 'testpass')) as server_address:
             with handler() as rh:
@@ -360,7 +360,7 @@ class TestSocks5Proxy:
                 assert response['auth_methods'] == [Socks5Auth.AUTH_NONE, Socks5Auth.AUTH_USER_PASS]
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_ipv4_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -368,7 +368,7 @@ class TestSocks5Proxy:
                 assert response['ipv4_address'] == '127.0.0.1'
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -376,7 +376,7 @@ class TestSocks5Proxy:
                 assert (response['ipv4_address'] == '127.0.0.1') != (response['ipv6_address'] == '::1')
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5h_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
@@ -385,7 +385,7 @@ class TestSocks5Proxy:
                 assert response['domain_address'] == 'localhost'
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5h_ip_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
@@ -394,7 +394,7 @@ class TestSocks5Proxy:
                 assert response['domain_address'] is None
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_ipv6_destination(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -402,7 +402,7 @@ class TestSocks5Proxy:
                 assert response['ipv6_address'] == '::1'
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_ipv6_socks5_proxy(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler, bind_ip='::1') as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -413,7 +413,7 @@ class TestSocks5Proxy:
 
     # XXX: is there any feasible way of testing IPv6 source addresses?
     # Same would go for non-proxy source_address test...
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
@@ -422,7 +422,7 @@ class TestSocks5Proxy:
                 assert response['client_address'][0] == source_address
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     @pytest.mark.parametrize('reply_code', [
         Socks5Reply.GENERAL_FAILURE,
         Socks5Reply.CONNECTION_NOT_ALLOWED,

test/test_update.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+
+# Allow direct execution
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from test.helper import FakeYDL, report_warning
+from yt_dlp.update import Updater, UpdateInfo
+
+TEST_API_DATA = {
+    'yt-dlp/yt-dlp/latest': {
+        'tag_name': '2023.12.31',
+        'target_commitish': 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
+        'name': 'yt-dlp 2023.12.31',
+        'body': 'BODY',
+    },
+    'yt-dlp/yt-dlp-nightly-builds/latest': {
+        'tag_name': '2023.12.31.123456',
+        'target_commitish': 'master',
+        'name': 'yt-dlp nightly 2023.12.31.123456',
+        'body': 'Generated from: https://github.com/yt-dlp/yt-dlp/commit/cccccccccccccccccccccccccccccccccccccccc',
+    },
+    'yt-dlp/yt-dlp-master-builds/latest': {
+        'tag_name': '2023.12.31.987654',
+        'target_commitish': 'master',
+        'name': 'yt-dlp master 2023.12.31.987654',
+        'body': 'Generated from: https://github.com/yt-dlp/yt-dlp/commit/dddddddddddddddddddddddddddddddddddddddd',
+    },
+    'yt-dlp/yt-dlp/tags/testing': {
+        'tag_name': 'testing',
+        'target_commitish': '9999999999999999999999999999999999999999',
+        'name': 'testing',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp/latest': {
+        'tag_name': '2050.12.31',
+        'target_commitish': 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
+        'name': '2050.12.31',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp/tags/pr0000': {
+        'tag_name': 'pr0000',
+        'target_commitish': 'ffffffffffffffffffffffffffffffffffffffff',
+        'name': 'pr1234 2023.11.11.000000',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp/tags/pr1234': {
+        'tag_name': 'pr1234',
+        'target_commitish': '0000000000000000000000000000000000000000',
+        'name': 'pr1234 2023.12.31.555555',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp/tags/pr9999': {
+        'tag_name': 'pr9999',
+        'target_commitish': '1111111111111111111111111111111111111111',
+        'name': 'pr9999',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp-satellite/tags/pr987': {
+        'tag_name': 'pr987',
+        'target_commitish': 'master',
+        'name': 'pr987',
+        'body': 'Generated from: https://github.com/yt-dlp/yt-dlp/commit/2222222222222222222222222222222222222222',
+    },
+}
+
+TEST_LOCKFILE_V1 = '''# This file is used for regulating self-update
+lock 2022.08.18.36 .+ Python 3.6
+lock 2023.11.13 .+ Python 3.7
+'''
+
+TEST_LOCKFILE_V2 = '''# This file is used for regulating self-update
+lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3.6
+lockV2 yt-dlp/yt-dlp 2023.11.13 .+ Python 3.7
+'''
+
+TEST_LOCKFILE_V1_V2 = '''# This file is used for regulating self-update
+lock 2022.08.18.36 .+ Python 3.6
+lock 2023.11.13 .+ Python 3.7
+lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3.6
+lockV2 yt-dlp/yt-dlp 2023.11.13 .+ Python 3.7
+lockV2 fork/yt-dlp pr0000 .+ Python 3.6
+lockV2 fork/yt-dlp pr1234 .+ Python 3.7
+lockV2 fork/yt-dlp pr9999 .+ Python 3.11
+'''
+
+
+class FakeUpdater(Updater):
+    current_version = '2022.01.01'
+    current_commit = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+
+    _channel = 'stable'
+    _origin = 'yt-dlp/yt-dlp'
+
+    def _download_update_spec(self, *args, **kwargs):
+        return TEST_LOCKFILE_V1_V2
+
+    def _call_api(self, tag):
+        tag = f'tags/{tag}' if tag != 'latest' else tag
+        return TEST_API_DATA[f'{self.requested_repo}/{tag}']
+
+    def _report_error(self, msg, *args, **kwargs):
+        report_warning(msg)
+
+
+class TestUpdate(unittest.TestCase):
+    maxDiff = None
+
+    def test_update_spec(self):
+        ydl = FakeYDL()
+        updater = FakeUpdater(ydl, 'stable@latest')
+
+        def test(lockfile, identifier, input_tag, expect_tag, exact=False, repo='yt-dlp/yt-dlp'):
+            updater._identifier = identifier
+            updater._exact = exact
+            updater.requested_repo = repo
+            result = updater._process_update_spec(lockfile, input_tag)
+            self.assertEqual(
+                result, expect_tag,
+                f'{identifier!r} requesting {repo}@{input_tag} (exact={exact}) '
+                f'returned {result!r} instead of {expect_tag!r}')
+
+        test(TEST_LOCKFILE_V1, 'zip Python 3.11.0', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1, 'zip stable Python 3.11.0', '2023.11.13', '2023.11.13', exact=True)
+        test(TEST_LOCKFILE_V1, 'zip Python 3.6.0', '2023.11.13', '2022.08.18.36')
+        test(TEST_LOCKFILE_V1, 'zip stable Python 3.6.0', '2023.11.13', None, exact=True)
+        test(TEST_LOCKFILE_V1, 'zip Python 3.7.0', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1, 'zip stable Python 3.7.1', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1, 'zip Python 3.7.1', '2023.12.31', '2023.11.13')
+        test(TEST_LOCKFILE_V1, 'zip stable Python 3.7.1', '2023.12.31', '2023.11.13')
+
+        test(TEST_LOCKFILE_V2, 'zip Python 3.11.1', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V2, 'zip stable Python 3.11.1', '2023.12.31', '2023.12.31')
+        test(TEST_LOCKFILE_V2, 'zip Python 3.6.1', '2023.11.13', '2022.08.18.36')
+        test(TEST_LOCKFILE_V2, 'zip stable Python 3.7.2', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V2, 'zip Python 3.7.2', '2023.12.31', '2023.11.13')
+
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.2', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.11.2', '2023.12.31', '2023.12.31')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.2', '2023.11.13', '2022.08.18.36')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.7.3', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.3', '2023.12.31', '2023.11.13')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.3', 'pr0000', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.7.4', 'pr0000', 'pr0000', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.4', 'pr0000', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.4', 'pr1234', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.8.1', 'pr1234', 'pr1234', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.5', 'pr1234', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.3', 'pr9999', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.12.0', 'pr9999', 'pr9999', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.4', 'pr9999', None, repo='fork/yt-dlp')
+
+    def test_query_update(self):
+        ydl = FakeYDL()
+
+        def test(target, expected, current_version=None, current_commit=None, identifier=None):
+            updater = FakeUpdater(ydl, target)
+            if current_version:
+                updater.current_version = current_version
+            if current_commit:
+                updater.current_commit = current_commit
+            updater._identifier = identifier or 'zip'
+            update_info = updater.query_update(_output=True)
+            self.assertDictEqual(
+                update_info.__dict__ if update_info else {}, expected.__dict__ if expected else {})
+
+        test('yt-dlp/yt-dlp@latest', UpdateInfo(
+            '2023.12.31', version='2023.12.31', requested_version='2023.12.31', commit='b' * 40))
+        test('yt-dlp/yt-dlp-nightly-builds@latest', UpdateInfo(
+            '2023.12.31.123456', version='2023.12.31.123456', requested_version='2023.12.31.123456', commit='c' * 40))
+        test('yt-dlp/yt-dlp-master-builds@latest', UpdateInfo(
+            '2023.12.31.987654', version='2023.12.31.987654', requested_version='2023.12.31.987654', commit='d' * 40))
+        test('fork/yt-dlp@latest', UpdateInfo(
+            '2050.12.31', version='2050.12.31', requested_version='2050.12.31', commit='e' * 40))
+        test('fork/yt-dlp@pr0000', UpdateInfo(
+            'pr0000', version='2023.11.11.000000', requested_version='2023.11.11.000000', commit='f' * 40))
+        test('fork/yt-dlp@pr1234', UpdateInfo(
+            'pr1234', version='2023.12.31.555555', requested_version='2023.12.31.555555', commit='0' * 40))
+        test('fork/yt-dlp@pr9999', UpdateInfo(
+            'pr9999', version=None, requested_version=None, commit='1' * 40))
+        test('fork/yt-dlp-satellite@pr987', UpdateInfo(
+            'pr987', version=None, requested_version=None, commit='2' * 40))
+        test('yt-dlp/yt-dlp', None, current_version='2024.01.01')
+        test('stable', UpdateInfo(
+            '2023.12.31', version='2023.12.31', requested_version='2023.12.31', commit='b' * 40))
+        test('nightly', UpdateInfo(
+            '2023.12.31.123456', version='2023.12.31.123456', requested_version='2023.12.31.123456', commit='c' * 40))
+        test('master', UpdateInfo(
+            '2023.12.31.987654', version='2023.12.31.987654', requested_version='2023.12.31.987654', commit='d' * 40))
+        test('testing', None, current_commit='9' * 40)
+        test('testing', UpdateInfo('testing', commit='9' * 40))
+
+
+if __name__ == '__main__':
+    unittest.main()

test/test_update.py.disabled

@@ -1,30 +0,0 @@
-#!/usr/bin/env python3
-
-# Allow direct execution
-import os
-import sys
-import unittest
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-
-import json
-
-from yt_dlp.update import rsa_verify
-
-
-class TestUpdate(unittest.TestCase):
-    def test_rsa_verify(self):
-        UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
-        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'versions.json'), 'rb') as f:
-            versions_info = f.read().decode()
-        versions_info = json.loads(versions_info)
-        signature = versions_info['signature']
-        del versions_info['signature']
-        self.assertTrue(rsa_verify(
-            json.dumps(versions_info, sort_keys=True).encode(),
-            signature, UPDATES_RSA_KEY))
-
-
-if __name__ == '__main__':
-    unittest.main()

test/test_utils.py

@@ -1209,6 +1209,9 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('\'"\\""\'')
         self.assertEqual(json.loads(on), '"""', msg='Unnecessary quote escape should be escaped')
 
+        on = js_to_json('[new Date("spam"), \'("eggs")\']')
+        self.assertEqual(json.loads(on), ['spam', '("eggs")'], msg='Date regex should match a single string')
+
     def test_js_to_json_malformed(self):
         self.assertEqual(js_to_json('42a1'), '42"a1"')
         self.assertEqual(js_to_json('42a-1'), '42"a"-1')
@@ -1220,11 +1223,13 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(js_to_json('`${name}"${name}"`', {'name': '5'}), '"5\\"5\\""')
         self.assertEqual(js_to_json('`${name}`', {}), '"name"')
 
-    def test_js_to_json_map_array_constructors(self):
+    def test_js_to_json_common_constructors(self):
         self.assertEqual(json.loads(js_to_json('new Map([["a", 5]])')), {'a': 5})
         self.assertEqual(json.loads(js_to_json('Array(5, 10)')), [5, 10])
         self.assertEqual(json.loads(js_to_json('new Array(15,5)')), [15, 5])
         self.assertEqual(json.loads(js_to_json('new Map([Array(5, 10),new Array(15,5)])')), {'5': 10, '15': 5})
+        self.assertEqual(json.loads(js_to_json('new Date("123")')), "123")
+        self.assertEqual(json.loads(js_to_json('new Date(\'2023-10-19\')')), "2023-10-19")
 
     def test_extract_attributes(self):
         self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})

test/versions.json

@@ -1,34 +0,0 @@
-{
-    "latest": "2013.01.06", 
-    "signature": "72158cdba391628569ffdbea259afbcf279bbe3d8aeb7492690735dc1cfa6afa754f55c61196f3871d429599ab22f2667f1fec98865527b32632e7f4b3675a7ef0f0fbe084d359256ae4bba68f0d33854e531a70754712f244be71d4b92e664302aa99653ee4df19800d955b6c4149cd2b3f24288d6e4b40b16126e01f4c8ce6", 
-    "versions": {
-        "2013.01.02": {
-            "bin": [
-                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl", 
-                "f5b502f8aaa77675c4884938b1e4871ebca2611813a0c0e74f60c0fbd6dcca6b"
-            ], 
-            "exe": [
-                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl.exe", 
-                "75fa89d2ce297d102ff27675aa9d92545bbc91013f52ec52868c069f4f9f0422"
-            ], 
-            "tar": [
-                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl-2013.01.02.tar.gz", 
-                "6a66d022ac8e1c13da284036288a133ec8dba003b7bd3a5179d0c0daca8c8196"
-            ]
-        }, 
-        "2013.01.06": {
-            "bin": [
-                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl", 
-                "64b6ed8865735c6302e836d4d832577321b4519aa02640dc508580c1ee824049"
-            ], 
-            "exe": [
-                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl.exe", 
-                "58609baf91e4389d36e3ba586e21dab882daaaee537e4448b1265392ae86ff84"
-            ], 
-            "tar": [
-                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl-2013.01.06.tar.gz", 
-                "fe77ab20a95d980ed17a659aa67e371fdd4d656d19c4c7950e7b720b0c2f1a86"
-            ]
-        }
-    }
-}

yt_dlp/YoutubeDL.py

@@ -60,7 +60,7 @@ from .postprocessor import (
     get_postprocessor,
 )
 from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
-from .update import REPOSITORY, _get_system_deprecation, current_git_head, detect_variant
+from .update import REPOSITORY, _get_system_deprecation, _make_label, current_git_head, detect_variant
 from .utils import (
     DEFAULT_OUTTMPL,
     IDENTITY,
@@ -158,7 +158,7 @@ from .utils.networking import (
     clean_proxies,
     std_headers,
 )
-from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
+from .version import CHANNEL, ORIGIN, RELEASE_GIT_HEAD, VARIANT, __version__
 
 if compat_os_name == 'nt':
     import ctypes
@@ -2338,7 +2338,7 @@ class YoutubeDL:
                 return
 
             for f in formats:
-                if f.get('has_drm'):
+                if f.get('has_drm') or f.get('__needs_testing'):
                     yield from self._check_formats([f])
                 else:
                     yield f
@@ -2764,7 +2764,8 @@ class YoutubeDL:
                 format['dynamic_range'] = 'SDR'
             if format.get('aspect_ratio') is None:
                 format['aspect_ratio'] = try_call(lambda: round(format['width'] / format['height'], 2))
-            if (not format.get('manifest_url')  # For fragmented formats, "tbr" is often max bitrate and not average
+            # For fragmented formats, "tbr" is often max bitrate and not average
+            if (('manifest-filesize-approx' in self.params['compat_opts'] or not format.get('manifest_url'))
                     and info_dict.get('duration') and format.get('tbr')
                     and not format.get('filesize') and not format.get('filesize_approx')):
                 format['filesize_approx'] = int(info_dict['duration'] * format['tbr'] * (1024 / 8))
@@ -3543,14 +3544,14 @@ class YoutubeDL:
             'version': __version__,
             'current_git_head': current_git_head(),
             'release_git_head': RELEASE_GIT_HEAD,
-            'repository': REPOSITORY,
+            'repository': ORIGIN,
         })
 
         if remove_private_keys:
             reject = lambda k, v: v is None or k.startswith('__') or k in {
                 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
                 'entries', 'filepath', '_filename', 'filename', 'infojson_filename', 'original_url',
-                'playlist_autonumber', '_format_sort_fields',
+                'playlist_autonumber',
             }
         else:
             reject = lambda k, v: False
@@ -3926,8 +3927,8 @@ class YoutubeDL:
             source += '*'
         klass = type(self)
         write_debug(join_nonempty(
-            f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
+            f'{REPOSITORY.rpartition("/")[2]} version',
-            f'{CHANNEL}@{__version__}',
+            _make_label(ORIGIN, CHANNEL.partition('@')[2] or __version__, __version__),
             f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '',
             '' if source == 'unknown' else f'({source})',
             '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}',
@@ -3968,7 +3969,7 @@ class YoutubeDL:
         })) or 'none'))
 
         write_debug(f'Proxy map: {self.proxies}')
-        # write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}')
+        write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}')
         for plugin_type, plugins in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}.items():
             display_list = ['%s%s' % (
                 klass.__name__, '' if klass.__name__ == name else f' as {name}')
@@ -4057,6 +4058,9 @@ class YoutubeDL:
                     raise RequestError(
                         'file:// URLs are disabled by default in yt-dlp for security reasons. '
                         'Use --enable-file-urls to enable at your own risk.', cause=ue) from ue
+                if 'unsupported proxy type: "https"' in ue.msg.lower():
+                    raise RequestError(
+                        'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests')
             raise
         except SSLError as e:
             if 'UNSAFE_LEGACY_RENEGOTIATION_DISABLED' in str(e):
@@ -4099,6 +4103,8 @@ class YoutubeDL:
                 }),
             ))
         director.preferences.update(preferences or [])
+        if 'prefer-legacy-http-handler' in self.params['compat_opts']:
+            director.preferences.add(lambda rh, _: 500 if rh.RH_KEY == 'Urllib' else 0)
         return director
 
     def encode(self, s):
@@ -4221,7 +4227,7 @@ class YoutubeDL:
         return ret
 
     def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None):
-        ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename) '''
+        ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename); or None if error '''
         write_all = self.params.get('write_all_thumbnails', False)
         thumbnails, ret = [], []
         if write_all or self.params.get('writethumbnail', False):
@@ -4237,6 +4243,9 @@ class YoutubeDL:
             self.write_debug(f'Skipping writing {label} thumbnail')
             return ret
 
+        if thumbnails and not self._ensure_dir_exists(filename):
+            return None
+
         for idx, t in list(enumerate(thumbnails))[::-1]:
             thumb_ext = (f'{t["id"]}.' if multiple else '') + determine_ext(t['url'], 'jpg')
             thumb_display_id = f'{label} thumbnail {t["id"]}'

yt_dlp/__pyinstaller/hook-yt_dlp.py

@@ -21,9 +21,11 @@ def get_hidden_imports():
     yield from ('yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated')
     yield from ('yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated')
     yield pycryptodome_module()
-    yield from collect_submodules('websockets')
+    # Only `websockets` is required, others are collected just in case
+    for module in ('websockets', 'requests', 'urllib3'):
+        yield from collect_submodules(module)
     # These are auto-detected, but explicitly add them just in case
-    yield from ('mutagen', 'brotli', 'certifi')
+    yield from ('mutagen', 'brotli', 'certifi', 'secretstorage')
 
 
 hiddenimports = list(get_hidden_imports())

yt_dlp/dependencies/__init__.py

@@ -58,6 +58,15 @@ except (ImportError, SyntaxError):
     # See https://github.com/yt-dlp/yt-dlp/issues/2633
     websockets = None
 
+try:
+    import urllib3
+except ImportError:
+    urllib3 = None
+
+try:
+    import requests
+except ImportError:
+    requests = None
 
 try:
     import xattr  # xattr or pyxattr

yt_dlp/downloader/dash.py

@@ -15,12 +15,15 @@ class DashSegmentsFD(FragmentFD):
     FD_NAME = 'dashsegments'
 
     def real_download(self, filename, info_dict):
-        if info_dict.get('is_live') and set(info_dict['protocol'].split('+')) != {'http_dash_segments_generator'}:
+        if 'http_dash_segments_generator' in info_dict['protocol'].split('+'):
-            self.report_error('Live DASH videos are not supported')
+            real_downloader = None  # No external FD can support --live-from-start
+        else:
+            if info_dict.get('is_live'):
+                self.report_error('Live DASH videos are not supported')
+            real_downloader = get_suitable_downloader(
+                info_dict, self.params, None, protocol='dash_frag_urls', to_stdout=(filename == '-'))
 
         real_start = time.time()
-        real_downloader = get_suitable_downloader(
-            info_dict, self.params, None, protocol='dash_frag_urls', to_stdout=(filename == '-'))
 
         requested_formats = [{**info_dict, **fmt} for fmt in info_dict.get('requested_formats', [])]
         args = []

yt_dlp/downloader/external.py

@@ -335,7 +335,7 @@ class Aria2cFD(ExternalFD):
         cmd += ['--auto-file-renaming=false']
 
         if 'fragments' in info_dict:
-            cmd += ['--file-allocation=none', '--uri-selector=inorder']
+            cmd += ['--uri-selector=inorder']
             url_list_file = '%s.frag.urls' % tmpfilename
             url_list = []
             for frag_index, fragment in enumerate(info_dict['fragments']):

yt_dlp/downloader/fragment.py

@@ -14,6 +14,7 @@ from ..networking import Request
 from ..networking.exceptions import HTTPError, IncompleteRead
 from ..utils import DownloadError, RetryManager, encodeFilename, traverse_obj
 from ..utils.networking import HTTPHeaderDict
+from ..utils.progress import ProgressCalculator
 
 
 class HttpQuietDownloader(HttpFD):
@@ -226,8 +227,7 @@ class FragmentFD(FileDownloader):
         resume_len = ctx['complete_frags_downloaded_bytes']
         total_frags = ctx['total_frags']
         ctx_id = ctx.get('ctx_id')
-        # This dict stores the download progress, it's updated by the progress
+        # Stores the download progress, updated by the progress hook
-        # hook
         state = {
             'status': 'downloading',
             'downloaded_bytes': resume_len,
@@ -237,14 +237,8 @@ class FragmentFD(FileDownloader):
             'tmpfilename': ctx['tmpfilename'],
         }
 
-        start = time.time()
+        ctx['started'] = time.time()
-        ctx.update({
+        progress = ProgressCalculator(resume_len)
-            'started': start,
-            'fragment_started': start,
-            # Amount of fragment's bytes downloaded by the time of the previous
-            # frag progress hook invocation
-            'prev_frag_downloaded_bytes': 0,
-        })
 
         def frag_progress_hook(s):
             if s['status'] not in ('downloading', 'finished'):
@@ -259,38 +253,35 @@ class FragmentFD(FileDownloader):
             state['max_progress'] = ctx.get('max_progress')
             state['progress_idx'] = ctx.get('progress_idx')
 
-            time_now = time.time()
+            state['elapsed'] = progress.elapsed
-            state['elapsed'] = time_now - start
             frag_total_bytes = s.get('total_bytes') or 0
             s['fragment_info_dict'] = s.pop('info_dict', {})
+
+            # XXX: Fragment resume is not accounted for here
             if not ctx['live']:
                 estimated_size = (
                     (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes)
                     / (state['fragment_index'] + 1) * total_frags)
-                state['total_bytes_estimate'] = estimated_size
+                progress.total = estimated_size
+                progress.update(s.get('downloaded_bytes'))
+                state['total_bytes_estimate'] = progress.total
+            else:
+                progress.update(s.get('downloaded_bytes'))
 
             if s['status'] == 'finished':
                 state['fragment_index'] += 1
                 ctx['fragment_index'] = state['fragment_index']
-                state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
+                progress.thread_reset()
-                ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
+
-                ctx['speed'] = state['speed'] = self.calc_speed(
+            state['downloaded_bytes'] = ctx['complete_frags_downloaded_bytes'] = progress.downloaded
-                    ctx['fragment_started'], time_now, frag_total_bytes)
+            state['speed'] = ctx['speed'] = progress.speed.smooth
-                ctx['fragment_started'] = time.time()
+            state['eta'] = progress.eta.smooth
-                ctx['prev_frag_downloaded_bytes'] = 0
+
-            else:
-                frag_downloaded_bytes = s['downloaded_bytes']
-                state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
-                ctx['speed'] = state['speed'] = self.calc_speed(
-                    ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx.get('frag_resume_len', 0))
-                if not ctx['live']:
-                    state['eta'] = self.calc_eta(state['speed'], estimated_size - state['downloaded_bytes'])
-                ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state, info_dict)
 
         ctx['dl'].add_progress_hook(frag_progress_hook)
 
-        return start
+        return ctx['started']
 
     def _finish_frag_download(self, ctx, info_dict):
         ctx['dest_stream'].close()
@@ -500,7 +491,6 @@ class FragmentFD(FileDownloader):
                 download_fragment(fragment, ctx_copy)
                 return fragment, fragment['frag_index'], ctx_copy.get('fragment_filename_sanitized')
 
-            self.report_warning('The download speed shown is only of one thread. This is a known issue')
             with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool:
                 try:
                     for fragment, frag_index, frag_filename in pool.map(_download_fragment, fragments):

yt_dlp/extractor/_extractors.py

@@ -565,6 +565,7 @@ from .ellentube import (
 )
 from .elonet import ElonetIE
 from .elpais import ElPaisIE
+from .eltrecetv import ElTreceTVIE
 from .embedly import EmbedlyIE
 from .engadget import EngadgetIE
 from .epicon import (
@@ -574,6 +575,7 @@ from .epicon import (
 from .eplus import EplusIbIE
 from .epoch import EpochIE
 from .eporner import EpornerIE
+from .erocast import ErocastIE
 from .eroprofile import (
     EroProfileIE,
     EroProfileAlbumIE,
@@ -892,9 +894,17 @@ from .japandiet import (
     SangiinIE,
 )
 from .jeuxvideo import JeuxVideoIE
+from .jiosaavn import (
+    JioSaavnSongIE,
+    JioSaavnAlbumIE,
+)
 from .jove import JoveIE
 from .joj import JojIE
 from .jstream import JStreamIE
+from .jtbc import (
+    JTBCIE,
+    JTBCProgramIE,
+)
 from .jwplatform import JWPlatformIE
 from .kakao import KakaoIE
 from .kaltura import KalturaIE
@@ -948,9 +958,11 @@ from .lastfm import (
     LastFMPlaylistIE,
     LastFMUserIE,
 )
+from .laxarxames import LaXarxaMesIE
 from .lbry import (
     LBRYIE,
     LBRYChannelIE,
+    LBRYPlaylistIE,
 )
 from .lci import LCIIE
 from .lcp import (
@@ -1051,6 +1063,7 @@ from .markiza import (
 from .massengeschmacktv import MassengeschmackTVIE
 from .masters import MastersIE
 from .matchtv import MatchTVIE
+from .mbn import MBNIE
 from .mdr import MDRIE
 from .medaltv import MedalTVIE
 from .mediaite import MediaiteIE
@@ -1312,7 +1325,6 @@ from .ninegag import NineGagIE
 from .ninenow import NineNowIE
 from .nintendo import NintendoIE
 from .nitter import NitterIE
-from .njpwworld import NJPWWorldIE
 from .nobelprize import NobelPrizeIE
 from .noice import NoicePodcastIE
 from .nonktube import NonkTubeIE
@@ -1380,7 +1392,10 @@ from .oftv import (
 from .oktoberfesttv import OktoberfestTVIE
 from .olympics import OlympicsReplayIE
 from .on24 import On24IE
-from .ondemandkorea import OnDemandKoreaIE
+from .ondemandkorea import (
+    OnDemandKoreaIE,
+    OnDemandKoreaProgramIE,
+)
 from .onefootball import OneFootballIE
 from .onenewsnz import OneNewsNZIE
 from .oneplace import OnePlacePodcastIE
@@ -1409,6 +1424,7 @@ from .orf import (
     ORFTVthekIE,
     ORFFM4StoryIE,
     ORFRadioIE,
+    ORFPodcastIE,
     ORFIPTVIE,
 )
 from .outsidetv import OutsideTVIE
@@ -1571,6 +1587,10 @@ from .radiocanada import (
     RadioCanadaIE,
     RadioCanadaAudioVideoIE,
 )
+from .radiocomercial import (
+    RadioComercialIE,
+    RadioComercialPlaylistIE,
+)
 from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
@@ -1751,6 +1771,11 @@ from .samplefocus import SampleFocusIE
 from .sapo import SapoIE
 from .savefrom import SaveFromIE
 from .sbs import SBSIE
+from .sbscokr import (
+    SBSCoKrIE,
+    SBSCoKrAllvodProgramIE,
+    SBSCoKrProgramsVodIE,
+)
 from .screen9 import Screen9IE
 from .screencast import ScreencastIE
 from .screencastify import ScreencastifyIE
@@ -1895,6 +1920,8 @@ from .srmediathek import SRMediathekIE
 from .stacommu import (
     StacommuLiveIE,
     StacommuVODIE,
+    TheaterComplexTownVODIE,
+    TheaterComplexTownPPVIE,
 )
 from .stanfordoc import StanfordOpenClassroomIE
 from .startv import StarTVIE
@@ -1990,7 +2017,10 @@ from .tencent import (
     WeTvSeriesIE,
 )
 from .tennistv import TennisTVIE
-from .tenplay import TenPlayIE
+from .tenplay import (
+    TenPlayIE,
+    TenPlaySeasonIE,
+)
 from .testurl import TestURLIE
 from .tf1 import TF1IE
 from .tfo import TFOIE
@@ -2002,13 +2032,8 @@ from .theplatform import (
 )
 from .thestar import TheStarIE
 from .thesun import TheSunIE
-from .theta import (
-    ThetaVideoIE,
-    ThetaStreamIE,
-)
 from .theweatherchannel import TheWeatherChannelIE
 from .thisamericanlife import ThisAmericanLifeIE
-from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
 from .thisvid import (
     ThisVidIE,

yt_dlp/extractor/abc.py

@@ -16,6 +16,7 @@ from ..utils import (
     try_get,
     unescapeHTML,
     update_url_query,
+    url_or_none,
 )
 
 
@@ -181,18 +182,102 @@ class ABCIViewIE(InfoExtractor):
     _GEO_COUNTRIES = ['AU']
 
     _TESTS = [{
+        'url': 'https://iview.abc.net.au/show/utopia/series/1/video/CO1211V001S00',
+        'md5': '52a942bfd7a0b79a6bfe9b4ce6c9d0ed',
+        'info_dict': {
+            'id': 'CO1211V001S00',
+            'ext': 'mp4',
+            'title': 'Series 1 Ep 1 Wood For The Trees',
+            'series': 'Utopia',
+            'description': 'md5:0cfb2c183c1b952d1548fd65c8a95c00',
+            'upload_date': '20230726',
+            'uploader_id': 'abc1',
+            'series_id': 'CO1211V',
+            'episode_id': 'CO1211V001S00',
+            'season_number': 1,
+            'season': 'Season 1',
+            'episode_number': 1,
+            'episode': 'Wood For The Trees',
+            'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/co/CO1211V001S00_5ad8353f4df09_1280.jpg',
+            'timestamp': 1690403700,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'note': 'No episode name',
         'url': 'https://iview.abc.net.au/show/gruen/series/11/video/LE1927H001S00',
         'md5': '67715ce3c78426b11ba167d875ac6abf',
         'info_dict': {
             'id': 'LE1927H001S00',
             'ext': 'mp4',
-            'title': "Series 11 Ep 1",
+            'title': 'Series 11 Ep 1',
-            'series': "Gruen",
+            'series': 'Gruen',
             'description': 'md5:52cc744ad35045baf6aded2ce7287f67',
             'upload_date': '20190925',
             'uploader_id': 'abc1',
+            'series_id': 'LE1927H',
+            'episode_id': 'LE1927H001S00',
+            'season_number': 11,
+            'season': 'Season 11',
+            'episode_number': 1,
+            'episode': 'Episode 1',
+            'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/le/LE1927H001S00_5d954fbd79e25_1280.jpg',
             'timestamp': 1569445289,
         },
+        'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'note': 'No episode number',
+        'url': 'https://iview.abc.net.au/show/four-corners/series/2022/video/NC2203H039S00',
+        'md5': '77cb7d8434440e3b28fbebe331c2456a',
+        'info_dict': {
+            'id': 'NC2203H039S00',
+            'ext': 'mp4',
+            'title': 'Series 2022 Locking Up Kids',
+            'series': 'Four Corners',
+            'description': 'md5:54829ca108846d1a70e1fcce2853e720',
+            'upload_date': '20221114',
+            'uploader_id': 'abc1',
+            'series_id': 'NC2203H',
+            'episode_id': 'NC2203H039S00',
+            'season_number': 2022,
+            'season': 'Season 2022',
+            'episode_number': None,
+            'episode': 'Locking Up Kids',
+            'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/nc/NC2203H039S00_636d8a0944a22_1920.jpg',
+            'timestamp': 1668460497,
+
+        },
+        'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'note': 'No episode name or number',
+        'url': 'https://iview.abc.net.au/show/landline/series/2021/video/RF2004Q043S00',
+        'md5': '2e17dec06b13cc81dc119d2565289396',
+        'info_dict': {
+            'id': 'RF2004Q043S00',
+            'ext': 'mp4',
+            'title': 'Series 2021',
+            'series': 'Landline',
+            'description': 'md5:c9f30d9c0c914a7fd23842f6240be014',
+            'upload_date': '20211205',
+            'uploader_id': 'abc1',
+            'series_id': 'RF2004Q',
+            'episode_id': 'RF2004Q043S00',
+            'season_number': 2021,
+            'season': 'Season 2021',
+            'episode_number': None,
+            'episode': None,
+            'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/rf/RF2004Q043S00_61a950639dbc0_1920.jpg',
+            'timestamp': 1638710705,
+
+        },
+        'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
         'params': {
             'skip_download': True,
         },
@@ -254,6 +339,8 @@ class ABCIViewIE(InfoExtractor):
             'episode_number': int_or_none(self._search_regex(
                 r'\bEp\s+(\d+)\b', title, 'episode number', default=None)),
             'episode_id': house_number,
+            'episode': self._search_regex(
+                r'^(?:Series\s+\d+)?\s*(?:Ep\s+\d+)?\s*(.*)$', title, 'episode', default='') or None,
             'uploader_id': video_params.get('channel'),
             'formats': formats,
             'subtitles': subtitles,
@@ -293,6 +380,18 @@ class ABCIViewShowSeriesIE(InfoExtractor):
             'noplaylist': True,
             'skip_download': 'm3u8',
         },
+    }, {
+        # 'videoEpisodes' is a dict with `items` key
+        'url': 'https://iview.abc.net.au/show/7-30-mark-humphries-satire',
+        'info_dict': {
+            'id': '178458-0',
+            'title': 'Episodes',
+            'description': 'Satirist Mark Humphries brings his unique perspective on current political events for 7.30.',
+            'series': '7.30 Mark Humphries Satire',
+            'season': 'Episodes',
+            'thumbnail': r're:^https?://cdn\.iview\.abc\.net\.au/thumbs/.*\.jpg$'
+        },
+        'playlist_count': 15,
     }]
 
     def _real_extract(self, url):
@@ -312,12 +411,14 @@ class ABCIViewShowSeriesIE(InfoExtractor):
         series = video_data['selectedSeries']
         return {
             '_type': 'playlist',
-            'entries': [self.url_result(episode['shareUrl'])
+            'entries': [self.url_result(episode_url, ABCIViewIE)
-                        for episode in series['_embedded']['videoEpisodes']],
+                        for episode_url in traverse_obj(series, (
+                            '_embedded', 'videoEpisodes', (None, 'items'), ..., 'shareUrl', {url_or_none}))],
             'id': series.get('id'),
             'title': dict_get(series, ('title', 'displaySubtitle')),
             'description': series.get('description'),
             'series': dict_get(series, ('showTitle', 'displayTitle')),
             'season': dict_get(series, ('title', 'displaySubtitle')),
-            'thumbnail': series.get('thumbnail'),
+            'thumbnail': traverse_obj(
+                series, 'thumbnail', ('images', lambda _, v: v['name'] == 'seriesThumbnail', 'url'), get_all=False),
         }

yt_dlp/extractor/aenetworks.py

@@ -338,6 +338,7 @@ class BiographyIE(AENetworksBaseIE):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+        'skip': '404 Not Found',
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/amcnetworks.py

@@ -26,6 +26,7 @@ class AMCNetworksIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
             # m3u8 download
             'skip_download': True,
         },
+        'skip': '404 Not Found',
     }, {
         'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge',
         'only_matching': True,

yt_dlp/extractor/arte.py

@@ -48,17 +48,7 @@ class ArteTVIE(ArteTVBaseIE):
     }, {
         'note': 'No alt_title',
         'url': 'https://www.arte.tv/fr/videos/110371-000-A/la-chaleur-supplice-des-arbres-de-rue/',
-        'info_dict': {
+        'only_matching': True,
-            'id': '110371-000-A',
-            'ext': 'mp4',
-            'upload_date': '20220718',
-            'duration': 154,
-            'timestamp': 1658162460,
-            'description': 'md5:5890f36fe7dccfadb8b7c0891de54786',
-            'title': 'La chaleur, supplice des arbres de rue',
-            'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/CPE2sQDtD8GLQgt8DuYHLf/940x530',
-        },
-        'params': {'skip_download': 'm3u8'}
     }, {
         'url': 'https://api.arte.tv/api/player/v2/config/de/100605-013-A',
         'only_matching': True,
@@ -67,19 +57,20 @@ class ArteTVIE(ArteTVBaseIE):
         'only_matching': True,
     }, {
         'url': 'https://www.arte.tv/de/videos/110203-006-A/zaz/',
+        'only_matching': True,
+    }, {
+        'note': 'age-restricted',
+        'url': 'https://www.arte.tv/de/videos/006785-000-A/the-element-of-crime/',
         'info_dict': {
-            'id': '110203-006-A',
+            'id': '006785-000-A',
-            'chapters': 'count:16',
+            'description': 'md5:c2f94fdfefc8a280e4dab68ab96ab0ba',
-            'description': 'md5:cf592f1df52fe52007e3f8eac813c084',
+            'title': 'The Element of Crime',
-            'alt_title': 'Zaz',
+            'timestamp': 1696111200,
-            'title': 'Baloise Session 2022',
+            'duration': 5849,
-            'timestamp': 1668445200,
+            'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/q82dTTfyuCXupPsGxXsd7B/940x530',
-            'duration': 4054,
+            'upload_date': '20230930',
-            'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/ubQjmVCGyRx3hmBuZEK9QZ/940x530',
-            'upload_date': '20221114',
             'ext': 'mp4',
-        },
+        }
-        'expected_warnings': ['geo restricted']
     }]
 
     _GEO_BYPASS = True
@@ -136,7 +127,9 @@ class ArteTVIE(ArteTVBaseIE):
         lang = mobj.group('lang') or mobj.group('lang_2')
         langauge_code = self._LANG_MAP.get(lang)
 
-        config = self._download_json(f'{self._API_BASE}/config/{lang}/{video_id}', video_id)
+        config = self._download_json(f'{self._API_BASE}/config/{lang}/{video_id}', video_id, headers={
+            'x-validated-age': '18'
+        })
 
         geoblocking = traverse_obj(config, ('data', 'attributes', 'restriction', 'geoblocking')) or {}
         if geoblocking.get('restrictedArea'):

yt_dlp/extractor/banbye.py

@@ -31,7 +31,7 @@ class BanByeBaseIE(InfoExtractor):
 
 
 class BanByeIE(BanByeBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?banbye.com/(?:en/)?watch/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?banbye\.com/(?:en/)?watch/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://banbye.com/watch/v_ytfmvkVYLE8T',
         'md5': '2f4ea15c5ca259a73d909b2cfd558eb5',
@@ -120,7 +120,7 @@ class BanByeIE(BanByeBaseIE):
 
 
 class BanByeChannelIE(BanByeBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?banbye.com/(?:en/)?channel/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?banbye\.com/(?:en/)?channel/(?P<id>\w+)'
     _TESTS = [{
         'url': 'https://banbye.com/channel/ch_wrealu24',
         'info_dict': {

yt_dlp/extractor/beatbump.py

@@ -3,14 +3,13 @@ from .youtube import YoutubeIE, YoutubeTabIE
 
 
 class BeatBumpVideoIE(InfoExtractor):
-    _VALID_URL = r'https://beatbump\.ml/listen\?id=(?P<id>[\w-]+)'
+    _VALID_URL = r'https://beatbump\.(?:ml|io)/listen\?id=(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://beatbump.ml/listen?id=MgNrAu2pzNs',
         'md5': '5ff3fff41d3935b9810a9731e485fe66',
         'info_dict': {
             'id': 'MgNrAu2pzNs',
             'ext': 'mp4',
-            'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
             'artist': 'Stephen',
             'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
             'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
@@ -22,10 +21,9 @@ class BeatBumpVideoIE(InfoExtractor):
             'alt_title': 'Voyeur Girl',
             'view_count': int,
             'track': 'Voyeur Girl',
-            'uploader': 'Stephen - Topic',
+            'uploader': 'Stephen',
             'title': 'Voyeur Girl',
             'channel_follower_count': int,
-            'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
             'age_limit': 0,
             'availability': 'public',
             'live_status': 'not_live',
@@ -36,7 +34,12 @@ class BeatBumpVideoIE(InfoExtractor):
             'tags': 'count:11',
             'creator': 'Stephen',
             'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
-        }
+            'channel_is_verified': True,
+            'heatmap': 'count:100',
+        },
+    }, {
+        'url': 'https://beatbump.io/listen?id=LDGZAprNGWo',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -45,7 +48,7 @@ class BeatBumpVideoIE(InfoExtractor):
 
 
 class BeatBumpPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https://beatbump\.ml/(?:release\?id=|artist/|playlist/)(?P<id>[\w-]+)'
+    _VALID_URL = r'https://beatbump\.(?:ml|io)/(?:release\?id=|artist/|playlist/)(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://beatbump.ml/release?id=MPREb_gTAcphH99wE',
         'playlist_count': 50,
@@ -56,25 +59,28 @@ class BeatBumpPlaylistIE(InfoExtractor):
             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
             'description': '',
             'tags': [],
-            'modified_date': '20221223',
+            'modified_date': '20231110',
-        }
+        },
+        'expected_warnings': ['YouTube Music is not directly supported'],
     }, {
         'url': 'https://beatbump.ml/artist/UC_aEa8K-EOJ3D6gOs7HcyNg',
         'playlist_mincount': 1,
         'params': {'flatplaylist': True},
         'info_dict': {
             'id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
-            'uploader_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
+            'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
             'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
-            'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
+            'uploader_id': '@NoCopyrightSounds',
             'channel_follower_count': int,
-            'title': 'NoCopyrightSounds - Videos',
+            'title': 'NoCopyrightSounds',
             'uploader': 'NoCopyrightSounds',
             'description': 'md5:cd4fd53d81d363d05eee6c1b478b491a',
             'channel': 'NoCopyrightSounds',
-            'tags': 'count:12',
+            'tags': 'count:65',
             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
+            'channel_is_verified': True,
         },
+        'expected_warnings': ['YouTube Music is not directly supported'],
     }, {
         'url': 'https://beatbump.ml/playlist/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
         'playlist_mincount': 1,
@@ -84,16 +90,20 @@ class BeatBumpPlaylistIE(InfoExtractor):
             'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
             'view_count': int,
-            'channel_url': 'https://www.youtube.com/@NoCopyrightSounds',
+            'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
-            'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
+            'uploader_id': '@NoCopyrightSounds',
             'title': 'NCS : All Releases 💿',
             'uploader': 'NoCopyrightSounds',
             'availability': 'public',
             'channel': 'NoCopyrightSounds',
             'tags': [],
-            'modified_date': '20221225',
+            'modified_date': '20231112',
             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
-        }
+        },
+        'expected_warnings': ['YouTube Music is not directly supported'],
+    }, {
+        'url': 'https://beatbump.io/playlist/VLPLFCHGavqRG-q_2ZhmgU2XB2--ZY6irT1c',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/breitbart.py

@@ -2,7 +2,7 @@ from .common import InfoExtractor
 
 
 class BreitBartIE(InfoExtractor):
-    _VALID_URL = r'https?:\/\/(?:www\.)breitbart.com/videos/v/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?breitbart\.com/videos/v/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.breitbart.com/videos/v/5cOz1yup/?pl=Ij6NDOji',
         'md5': '0aa6d1d6e183ac5ca09207fe49f17ade',

yt_dlp/extractor/brilliantpala.py

@@ -21,10 +21,10 @@ class BrilliantpalaBaseIE(InfoExtractor):
 
     def _get_logged_in_username(self, url, video_id):
         webpage, urlh = self._download_webpage_handle(url, video_id)
-        if self._LOGIN_API == urlh.url:
+        if urlh.url.startswith(self._LOGIN_API):
             self.raise_login_required()
         return self._html_search_regex(
-            r'"username"\s*:\s*"(?P<username>[^"]+)"', webpage, 'stream page info', 'username')
+            r'"username"\s*:\s*"(?P<username>[^"]+)"', webpage, 'logged-in username')
 
     def _perform_login(self, username, password):
         login_form = self._hidden_inputs(self._download_webpage(

yt_dlp/extractor/cbc.py

@@ -1,8 +1,9 @@
-import re
-import json
 import base64
+import json
+import re
 import time
 import urllib.parse
+import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..compat import (
@@ -66,6 +67,7 @@ class CBCIE(InfoExtractor):
             'uploader': 'CBCC-NEW',
             'timestamp': 255977160,
         },
+        'skip': '404 Not Found',
     }, {
         # multiple iframes
         'url': 'http://www.cbc.ca/natureofthings/blog/birds-eye-view-from-vancouvers-burrard-street-bridge-how-we-got-the-shot',
@@ -97,7 +99,7 @@ class CBCIE(InfoExtractor):
         # multiple CBC.APP.Caffeine.initInstance(...)
         'url': 'http://www.cbc.ca/news/canada/calgary/dog-indoor-exercise-winter-1.3928238',
         'info_dict': {
-            'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',
+            'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',  # FIXME
             'id': 'dog-indoor-exercise-winter-1.3928238',
             'description': 'md5:c18552e41726ee95bd75210d1ca9194c',
         },
@@ -386,7 +388,7 @@ class CBCGemIE(InfoExtractor):
         url = re.sub(r'(Manifest\(.*?),format=[\w-]+(.*?\))', r'\1\2', base_url)
 
         secret_xml = self._download_xml(url, video_id, note='Downloading secret XML', fatal=False)
-        if not secret_xml:
+        if not isinstance(secret_xml, xml.etree.ElementTree.Element):
             return
 
         for child in secret_xml:
@@ -476,6 +478,10 @@ class CBCGemPlaylistIE(InfoExtractor):
             'id': 'schitts-creek/s06',
             'title': 'Season 6',
             'description': 'md5:6a92104a56cbeb5818cc47884d4326a2',
+            'series': 'Schitt\'s Creek',
+            'season_number': 6,
+            'season': 'Season 6',
+            'thumbnail': 'https://images.radio-canada.ca/v1/synps-cbc/season/perso/cbc_schitts_creek_season_06_carousel_v03.jpg?impolicy=ott&im=Resize=(_Size_)&quality=75',
         },
     }, {
         'url': 'https://gem.cbc.ca/schitts-creek/s06',

yt_dlp/extractor/cbs.py

@@ -101,6 +101,7 @@ class CBSIE(CBSBaseIE):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'Subscription required',
     }, {
         'url': 'https://www.cbs.com/shows/video/sZH1MGgomIosZgxGJ1l263MFq16oMtW1/',
         'info_dict': {
@@ -117,6 +118,7 @@ class CBSIE(CBSBaseIE):
         },
         'expected_warnings': [
             'This content expired on', 'No video formats found', 'Requested format is not available'],
+        'skip': '404 Not Found',
     }, {
         'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
         'only_matching': True,

yt_dlp/extractor/cnbc.py

@@ -19,6 +19,7 @@ class CNBCIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'Dead link',
     }
 
     def _real_extract(self, url):
@@ -49,6 +50,7 @@ class CNBCVideoIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'Dead link',
     }
 
     def _real_extract(self, url):

yt_dlp/extractor/common.py

@@ -1687,7 +1687,7 @@ class InfoExtractor:
     def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', *, fatal=True, traverse=('data', 0)):
         """Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
         rectx = re.escape(context_name)
-        FUNCTION_RE = r'\(function\((?P<arg_keys>.*?)\){(?:.*?)return\s+(?P<js>{.*?})\s*;?\s*}\((?P<arg_vals>.*?)\)'
+        FUNCTION_RE = r'\(function\((?P<arg_keys>.*?)\){.*?\breturn\s+(?P<js>{.*?})\s*;?\s*}\((?P<arg_vals>.*?)\)'
         js, arg_keys, arg_vals = self._search_regex(
             (rf'<script>\s*window\.{rectx}={FUNCTION_RE}\s*\)\s*;?\s*</script>', rf'{rectx}\(.*?{FUNCTION_RE}'),
             webpage, context_name, group=('js', 'arg_keys', 'arg_vals'),
@@ -2225,7 +2225,9 @@ class InfoExtractor:
             mpd_url, video_id,
             note='Downloading MPD VOD manifest' if note is None else note,
             errnote='Failed to download VOD manifest' if errnote is None else errnote,
-            fatal=False, data=data, headers=headers, query=query) or {}
+            fatal=False, data=data, headers=headers, query=query)
+        if not isinstance(mpd_doc, xml.etree.ElementTree.Element):
+            return None
         return int_or_none(parse_duration(mpd_doc.get('mediaPresentationDuration')))
 
     @staticmethod

yt_dlp/extractor/corus.py

@@ -41,7 +41,7 @@ class CorusIE(ThePlatformFeedIE):  # XXX: Do not subclass from concrete IE
                         )
                     '''
     _TESTS = [{
-        'url': 'http://www.hgtv.ca/shows/bryan-inc/videos/movie-night-popcorn-with-bryan-870923331648/',
+        'url': 'https://www.hgtv.ca/video/bryan-inc/movie-night-popcorn-with-bryan/870923331648/',
         'info_dict': {
             'id': '870923331648',
             'ext': 'mp4',
@@ -54,6 +54,7 @@ class CorusIE(ThePlatformFeedIE):  # XXX: Do not subclass from concrete IE
             'skip_download': True,
         },
         'expected_warnings': ['Failed to parse JSON'],
+        # FIXME: yt-dlp wrongly raises for geo restriction
     }, {
         'url': 'http://www.foodnetwork.ca/shows/chopped/video/episode/chocolate-obsession/video.html?v=872683587753',
         'only_matching': True,

yt_dlp/extractor/craftsy.py

@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class CraftsyIE(InfoExtractor):
-    _VALID_URL = r'https?://www.craftsy.com/class/(?P<id>[a-z0-9_-]+)/'
+    _VALID_URL = r'https?://www\.craftsy\.com/class/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://www.craftsy.com/class/the-midnight-quilt-show-season-5/',
         'info_dict': {

yt_dlp/extractor/cybrary.py

@@ -45,7 +45,7 @@ class CybraryBaseIE(InfoExtractor):
 
 
 class CybraryIE(CybraryBaseIE):
-    _VALID_URL = r'https?://app.cybrary.it/immersive/(?P<enrollment>[0-9]+)/activity/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://app\.cybrary\.it/immersive/(?P<enrollment>[0-9]+)/activity/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'https://app.cybrary.it/immersive/12487950/activity/63102',
         'md5': '9ae12d37e555cb2ed554223a71a701d0',
@@ -105,12 +105,12 @@ class CybraryIE(CybraryBaseIE):
             'chapter': module.get('title'),
             'chapter_id': str_or_none(module.get('id')),
             'title': activity.get('title'),
-            'url': smuggle_url(f'https://player.vimeo.com/video/{vimeo_id}', {'http_headers': {'Referer': 'https://api.cybrary.it'}})
+            'url': smuggle_url(f'https://player.vimeo.com/video/{vimeo_id}', {'referer': 'https://api.cybrary.it'})
         }
 
 
 class CybraryCourseIE(CybraryBaseIE):
-    _VALID_URL = r'https://app.cybrary.it/browse/course/(?P<id>[\w-]+)/?(?:$|[#?])'
+    _VALID_URL = r'https://app\.cybrary\.it/browse/course/(?P<id>[\w-]+)/?(?:$|[#?])'
     _TESTS = [{
         'url': 'https://app.cybrary.it/browse/course/az-500-microsoft-azure-security-technologies',
         'info_dict': {

yt_dlp/extractor/dailymotion.py

@@ -93,7 +93,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
     _VALID_URL = r'''(?ix)
                     https?://
                         (?:
-                            (?:(?:www|touch|geo)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:(?:embed|swf|\#)/)|player\.html\?)?video|swf)|
+                            (?:(?:www|touch|geo)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:(?:embed|swf|\#)/)|player(?:/\w+)?\.html\?)?video|swf)|
                             (?:www\.)?lequipe\.fr/video
                         )
                         [/=](?P<id>[^/?_&]+)(?:.+?\bplaylist=(?P<playlist_id>x[0-9a-z]+))?
@@ -107,13 +107,17 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'id': 'x5kesuj',
             'ext': 'mp4',
             'title': 'Office Christmas Party Review –  Jason Bateman, Olivia Munn, T.J. Miller',
-            'description': 'Office Christmas Party Review -  Jason Bateman, Olivia Munn, T.J. Miller',
+            'description': 'Office Christmas Party Review - Jason Bateman, Olivia Munn, T.J. Miller',
             'duration': 187,
             'timestamp': 1493651285,
             'upload_date': '20170501',
             'uploader': 'Deadline',
             'uploader_id': 'x1xm8ri',
             'age_limit': 0,
+            'view_count': int,
+            'like_count': int,
+            'tags': ['hollywood', 'celeb', 'celebrity', 'movies', 'red carpet'],
+            'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/K456B1aXqIx58LKWQ/x1080',
         },
     }, {
         'url': 'https://geo.dailymotion.com/player.html?video=x89eyek&mute=true',
@@ -132,7 +136,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'view_count': int,
             'like_count': int,
             'tags': ['en_quete_d_esprit'],
-            'thumbnail': 'https://s2.dmcdn.net/v/Tncwi1YGKdvFbDuDY/x1080',
+            'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/Tncwi1YNg_RUl7ueu/x1080',
         }
     }, {
         'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
@@ -201,6 +205,12 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
     }, {
         'url': 'https://www.dailymotion.com/video/x3z49k?playlist=xv4bw',
         'only_matching': True,
+    }, {
+        'url': 'https://geo.dailymotion.com/player/x86gw.html?video=k46oCapRs4iikoz9DWy',
+        'only_matching': True,
+    }, {
+        'url': 'https://geo.dailymotion.com/player/xakln.html?video=x8mjju4&customConfig%5BcustomParams%5D=%2Ffr-fr%2Ftennis%2Fwimbledon-mens-singles%2Farticles-video',
+        'only_matching': True,
     }]
     _GEO_BYPASS = False
     _COMMON_MEDIA_FIELDS = '''description

yt_dlp/extractor/drtv.py

@@ -1,21 +1,17 @@
-import binascii
+import json
-import hashlib
+import uuid
-import re
 
 from .common import InfoExtractor
-from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
-from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
-    float_or_none,
     int_or_none,
     mimetype2ext,
-    str_or_none,
+    parse_iso8601,
-    traverse_obj,
+    try_call,
-    unified_timestamp,
     update_url_query,
     url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 SERIES_API = 'https://production-cdn.dr-massive.com/api/page?device=web_browser&item_detail_expand=all&lang=da&max_list_prefetch=3&path=%s'
 
@@ -24,7 +20,7 @@ class DRTVIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:www\.)?dr\.dk/(?:tv/se|nyheder|(?P<radio>radio|lyd)(?:/ondemand)?)/(?:[^/]+/)*|
+                            (?:www\.)?dr\.dk/tv/se(?:/ondemand)?/(?:[^/?#]+/)*|
                             (?:www\.)?(?:dr\.dk|dr-massive\.com)/drtv/(?:se|episode|program)/
                         )
                         (?P<id>[\da-z_-]+)
@@ -53,22 +49,6 @@ class DRTVIE(InfoExtractor):
         },
         'expected_warnings': ['Unable to download f4m manifest'],
         'skip': 'this video has been removed',
-    }, {
-        # embed
-        'url': 'https://www.dr.dk/nyheder/indland/live-christianias-rydning-af-pusher-street-er-i-gang',
-        'info_dict': {
-            'id': 'urn:dr:mu:programcard:57c926176187a50a9c6e83c6',
-            'ext': 'mp4',
-            'title': 'christiania pusher street ryddes drdkrjpo',
-            'description': 'md5:2a71898b15057e9b97334f61d04e6eb5',
-            'timestamp': 1472800279,
-            'upload_date': '20160902',
-            'duration': 131.4,
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'expected_warnings': ['Unable to download f4m manifest'],
     }, {
         # with SignLanguage formats
         'url': 'https://www.dr.dk/tv/se/historien-om-danmark/-/historien-om-danmark-stenalder',
@@ -87,33 +67,54 @@ class DRTVIE(InfoExtractor):
             'season': 'Historien om Danmark',
             'series': 'Historien om Danmark',
         },
-        'params': {
+        'skip': 'this video has been removed',
-            'skip_download': True,
-        },
     }, {
-        'url': 'https://www.dr.dk/lyd/p4kbh/regionale-nyheder-kh4/p4-nyheder-2019-06-26-17-30-9',
+        'url': 'https://www.dr.dk/drtv/se/frank-and-kastaniegaarden_71769',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.dr.dk/drtv/se/bonderoeven_71769',
         'info_dict': {
             'id': '00951930010',
             'ext': 'mp4',
-            'title': 'Bonderøven 2019 (1:8)',
+            'title': 'Frank & Kastaniegaarden',
-            'description': 'md5:b6dcfe9b6f0bea6703e9a0092739a5bd',
+            'description': 'md5:974e1780934cf3275ef10280204bccb0',
-            'timestamp': 1654856100,
+            'release_timestamp': 1546545600,
-            'upload_date': '20220610',
+            'release_date': '20190103',
-            'duration': 2576.6,
+            'duration': 2576,
-            'season': 'Bonderøven 2019',
+            'season': 'Frank & Kastaniegaarden',
-            'season_id': 'urn:dr:mu:bundle:5c201667a11fa01ca4528ce5',
+            'season_id': '67125',
             'release_year': 2019,
             'season_number': 2019,
             'series': 'Frank & Kastaniegaarden',
             'episode_number': 1,
-            'episode': 'Episode 1',
+            'episode': 'Frank & Kastaniegaarden',
+            'thumbnail': r're:https?://.+',
         },
         'params': {
             'skip_download': True,
         },
+    }, {
+        # Foreign and Regular subtitle track
+        'url': 'https://www.dr.dk/drtv/se/spise-med-price_-pasta-selv_397445',
+        'info_dict': {
+            'id': '00212301010',
+            'ext': 'mp4',
+            'episode_number': 1,
+            'title': 'Spise med Price: Pasta Selv',
+            'alt_title': '1. Pasta Selv',
+            'release_date': '20230807',
+            'description': 'md5:2da9060524fed707810d71080b3d0cd8',
+            'duration': 1750,
+            'season': 'Spise med Price',
+            'release_timestamp': 1691438400,
+            'season_id': '397440',
+            'episode': 'Spise med Price: Pasta Selv',
+            'thumbnail': r're:https?://.+',
+            'season_number': 15,
+            'series': 'Spise med Price',
+            'release_year': 2022,
+            'subtitles': 'mincount:2',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
     }, {
         'url': 'https://www.dr.dk/drtv/episode/bonderoeven_71769',
         'only_matching': True,
@@ -123,226 +124,127 @@ class DRTVIE(InfoExtractor):
     }, {
         'url': 'https://www.dr.dk/drtv/program/jagten_220924',
         'only_matching': True,
-    }, {
-        'url': 'https://www.dr.dk/lyd/p4aarhus/regionale-nyheder-ar4/regionale-nyheder-2022-05-05-12-30-3',
-        'info_dict': {
-            'id': 'urn:dr:mu:programcard:6265cb2571401424d0360113',
-            'title': "Regionale nyheder",
-            'ext': 'mp4',
-            'duration': 120.043,
-            'series': 'P4 Østjylland regionale nyheder',
-            'timestamp': 1651746600,
-            'season': 'Regionale nyheder',
-            'release_year': 0,
-            'season_id': 'urn:dr:mu:bundle:61c26889539f0201586b73c5',
-            'description': '',
-            'upload_date': '20220505',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'skip': 'this video has been removed',
-    }, {
-        'url': 'https://www.dr.dk/lyd/p4kbh/regionale-nyheder-kh4/regionale-nyheder-2023-03-14-10-30-9',
-        'info_dict': {
-            'ext': 'mp4',
-            'id': '14802310112',
-            'timestamp': 1678786200,
-            'duration': 120.043,
-            'season_id': 'urn:dr:mu:bundle:63a4f7c87140143504b6710f',
-            'series': 'P4 København regionale nyheder',
-            'upload_date': '20230314',
-            'release_year': 0,
-            'description': 'Hør seneste regionale nyheder fra P4 København.',
-            'season': 'Regionale nyheder',
-            'title': 'Regionale nyheder',
-        },
     }]
 
+    SUBTITLE_LANGS = {
+        'DanishLanguageSubtitles': 'da',
+        'ForeignLanguageSubtitles': 'da_foreign',
+        'CombinedLanguageSubtitles': 'da_combined',
+    }
+
+    _TOKEN = None
+
+    def _real_initialize(self):
+        if self._TOKEN:
+            return
+
+        token_response = self._download_json(
+            'https://production.dr-massive.com/api/authorization/anonymous-sso', None,
+            note='Downloading anonymous token', headers={
+                'content-type': 'application/json',
+            }, query={
+                'device': 'web_browser',
+                'ff': 'idp,ldp,rpt',
+                'lang': 'da',
+                'supportFallbackToken': 'true',
+            }, data=json.dumps({
+                'deviceId': str(uuid.uuid4()),
+                'scopes': ['Catalog'],
+                'optout': True,
+            }).encode())
+
+        self._TOKEN = traverse_obj(
+            token_response, (lambda _, x: x['type'] == 'UserAccount', 'value', {str}), get_all=False)
+        if not self._TOKEN:
+            raise ExtractorError('Unable to get anonymous token')
+
     def _real_extract(self, url):
-        raw_video_id, is_radio_url = self._match_valid_url(url).group('id', 'radio')
+        url_slug = self._match_id(url)
+        webpage = self._download_webpage(url, url_slug)
 
-        webpage = self._download_webpage(url, raw_video_id)
+        json_data = self._search_json(
-
+            r'window\.__data\s*=', webpage, 'data', url_slug, fatal=False) or {}
-        if '>Programmet er ikke længere tilgængeligt' in webpage:
+        item = traverse_obj(
-            raise ExtractorError(
+            json_data, ('cache', 'page', ..., (None, ('entries', 0)), 'item', {dict}), get_all=False)
-                'Video %s is not available' % raw_video_id, expected=True)
+        if item:
-
+            item_id = item.get('id')
-        video_id = self._search_regex(
-            (r'data-(?:material-identifier|episode-slug)="([^"]+)"',
-             r'data-resource="[^>"]+mu/programcard/expanded/([^"]+)"'),
-            webpage, 'video id', default=None)
-
-        if not video_id:
-            video_id = self._search_regex(
-                r'(urn(?:%3A|:)dr(?:%3A|:)mu(?:%3A|:)programcard(?:%3A|:)[\da-f]+)',
-                webpage, 'urn', default=None)
-            if video_id:
-                video_id = compat_urllib_parse_unquote(video_id)
-
-        _PROGRAMCARD_BASE = 'https://www.dr.dk/mu-online/api/1.4/programcard'
-        query = {'expanded': 'true'}
-
-        if video_id:
-            programcard_url = '%s/%s' % (_PROGRAMCARD_BASE, video_id)
         else:
-            programcard_url = _PROGRAMCARD_BASE
+            item_id = url_slug.rsplit('_', 1)[-1]
-            if is_radio_url:
+            item = self._download_json(
-                video_id = self._search_nextjs_data(
+                f'https://production-cdn.dr-massive.com/api/items/{item_id}', item_id,
-                    webpage, raw_video_id)['props']['pageProps']['episode']['productionNumber']
+                note='Attempting to download backup item data', query={
-            else:
+                    'device': 'web_browser',
-                json_data = self._search_json(
+                    'expand': 'all',
-                    r'window\.__data\s*=', webpage, 'data', raw_video_id)
+                    'ff': 'idp,ldp,rpt',
-                video_id = traverse_obj(json_data, (
+                    'geoLocation': 'dk',
-                    'cache', 'page', ..., (None, ('entries', 0)), 'item', 'customId',
+                    'isDeviceAbroad': 'false',
-                    {lambda x: x.split(':')[-1]}), get_all=False)
+                    'lang': 'da',
-                if not video_id:
+                    'segments': 'drtv,optedout',
-                    raise ExtractorError('Unable to extract video id')
+                    'sub': 'Anonymous',
-            query['productionnumber'] = video_id
+                })
 
-        data = self._download_json(
+        video_id = try_call(lambda: item['customId'].rsplit(':', 1)[-1]) or item_id
-            programcard_url, video_id, 'Downloading video JSON', query=query)
+        stream_data = self._download_json(
-
+            f'https://production.dr-massive.com/api/account/items/{item_id}/videos', video_id,
-        supplementary_data = {}
+            note='Downloading stream data', query={
-        if re.search(r'_\d+$', raw_video_id):
+                'delivery': 'stream',
-            supplementary_data = self._download_json(
+                'device': 'web_browser',
-                SERIES_API % f'/episode/{raw_video_id}', raw_video_id, fatal=False) or {}
+                'ff': 'idp,ldp,rpt',
-
+                'lang': 'da',
-        title = str_or_none(data.get('Title')) or re.sub(
+                'resolution': 'HD-1080',
-            r'\s*\|\s*(?:TV\s*\|\s*DR|DRTV)$', '',
+                'sub': 'Anonymous',
-            self._og_search_title(webpage))
+            }, headers={'authorization': f'Bearer {self._TOKEN}'})
-        description = self._og_search_description(
-            webpage, default=None) or data.get('Description')
-
-        timestamp = unified_timestamp(
-            data.get('PrimaryBroadcastStartTime') or data.get('SortDateTime'))
-
-        thumbnail = None
-        duration = None
-
-        restricted_to_denmark = False
 
         formats = []
         subtitles = {}
+        for stream in traverse_obj(stream_data, (lambda _, x: x['url'])):
+            format_id = stream.get('format', 'na')
+            access_service = stream.get('accessService')
+            preference = None
+            subtitle_suffix = ''
+            if access_service in ('SpokenSubtitles', 'SignLanguage', 'VisuallyInterpreted'):
+                preference = -1
+                format_id += f'-{access_service}'
+                subtitle_suffix = f'-{access_service}'
+            elif access_service == 'StandardVideo':
+                preference = 1
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                stream.get('url'), video_id, preference=preference, m3u8_id=format_id, fatal=False)
+            formats.extend(fmts)
 
-        assets = []
+            api_subtitles = traverse_obj(stream, ('subtitles', lambda _, v: url_or_none(v['link']), {dict}))
-        primary_asset = data.get('PrimaryAsset')
+            if not api_subtitles:
-        if isinstance(primary_asset, dict):
+                self._merge_subtitles(subs, target=subtitles)
-            assets.append(primary_asset)
-        secondary_assets = data.get('SecondaryAssets')
-        if isinstance(secondary_assets, list):
-            for secondary_asset in secondary_assets:
-                if isinstance(secondary_asset, dict):
-                    assets.append(secondary_asset)
 
-        def hex_to_bytes(hex):
+            for sub_track in api_subtitles:
-            return binascii.a2b_hex(hex.encode('ascii'))
+                lang = sub_track.get('language') or 'da'
+                subtitles.setdefault(self.SUBTITLE_LANGS.get(lang, lang) + subtitle_suffix, []).append({
+                    'url': sub_track['link'],
+                    'ext': mimetype2ext(sub_track.get('format')) or 'vtt'
+                })
 
-        def decrypt_uri(e):
+        if not formats and traverse_obj(item, ('season', 'customFields', 'IsGeoRestricted')):
-            n = int(e[2:10], 16)
+            self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
-            a = e[10 + n:]
-            data = hex_to_bytes(e[10:10 + n])
-            key = hashlib.sha256(('%s:sRBzYNXBzkKgnjj8pGtkACch' % a).encode('utf-8')).digest()
-            iv = hex_to_bytes(a)
-            decrypted = unpad_pkcs7(aes_cbc_decrypt_bytes(data, key, iv))
-            return decrypted.decode('utf-8').split('?')[0]
-
-        for asset in assets:
-            kind = asset.get('Kind')
-            if kind == 'Image':
-                thumbnail = url_or_none(asset.get('Uri'))
-            elif kind in ('VideoResource', 'AudioResource'):
-                duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
-                restricted_to_denmark = asset.get('RestrictedToDenmark')
-                asset_target = asset.get('Target')
-                for link in asset.get('Links', []):
-                    uri = link.get('Uri')
-                    if not uri:
-                        encrypted_uri = link.get('EncryptedUri')
-                        if not encrypted_uri:
-                            continue
-                        try:
-                            uri = decrypt_uri(encrypted_uri)
-                        except Exception:
-                            self.report_warning(
-                                'Unable to decrypt EncryptedUri', video_id)
-                            continue
-                    uri = url_or_none(uri)
-                    if not uri:
-                        continue
-                    target = link.get('Target')
-                    format_id = target or ''
-                    if asset_target in ('SpokenSubtitles', 'SignLanguage', 'VisuallyInterpreted'):
-                        preference = -1
-                        format_id += '-%s' % asset_target
-                    elif asset_target == 'Default':
-                        preference = 1
-                    else:
-                        preference = None
-                    if target == 'HDS':
-                        f4m_formats = self._extract_f4m_formats(
-                            uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
-                            video_id, preference, f4m_id=format_id, fatal=False)
-                        if kind == 'AudioResource':
-                            for f in f4m_formats:
-                                f['vcodec'] = 'none'
-                        formats.extend(f4m_formats)
-                    elif target == 'HLS':
-                        fmts, subs = self._extract_m3u8_formats_and_subtitles(
-                            uri, video_id, 'mp4', entry_protocol='m3u8_native',
-                            quality=preference, m3u8_id=format_id, fatal=False)
-                        formats.extend(fmts)
-                        self._merge_subtitles(subs, target=subtitles)
-                    else:
-                        bitrate = link.get('Bitrate')
-                        if bitrate:
-                            format_id += '-%s' % bitrate
-                        formats.append({
-                            'url': uri,
-                            'format_id': format_id,
-                            'tbr': int_or_none(bitrate),
-                            'ext': link.get('FileFormat'),
-                            'vcodec': 'none' if kind == 'AudioResource' else None,
-                            'quality': preference,
-                        })
-            subtitles_list = asset.get('SubtitlesList') or asset.get('Subtitleslist')
-            if isinstance(subtitles_list, list):
-                LANGS = {
-                    'Danish': 'da',
-                }
-                for subs in subtitles_list:
-                    if not isinstance(subs, dict):
-                        continue
-                    sub_uri = url_or_none(subs.get('Uri'))
-                    if not sub_uri:
-                        continue
-                    lang = subs.get('Language') or 'da'
-                    subtitles.setdefault(LANGS.get(lang, lang), []).append({
-                        'url': sub_uri,
-                        'ext': mimetype2ext(subs.get('MimeType')) or 'vtt'
-                    })
-
-        if not formats and restricted_to_denmark:
-            self.raise_geo_restricted(
-                'Unfortunately, DR is not allowed to show this program outside Denmark.',
-                countries=self._GEO_COUNTRIES)
 
         return {
             'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
             'formats': formats,
             'subtitles': subtitles,
-            'series': str_or_none(data.get('SeriesTitle')),
+            **traverse_obj(item, {
-            'season': str_or_none(data.get('SeasonTitle')),
+                'title': 'title',
-            'season_number': int_or_none(data.get('SeasonNumber')),
+                'alt_title': 'contextualTitle',
-            'season_id': str_or_none(data.get('SeasonUrn')),
+                'description': 'description',
-            'episode': traverse_obj(supplementary_data, ('entries', 0, 'item', 'contextualTitle')) or str_or_none(data.get('EpisodeTitle')),
+                'thumbnail': ('images', 'wallpaper'),
-            'episode_number': traverse_obj(supplementary_data, ('entries', 0, 'item', 'episodeNumber')) or int_or_none(data.get('EpisodeNumber')),
+                'release_timestamp': ('customFields', 'BroadcastTimeDK', {parse_iso8601}),
-            'release_year': int_or_none(data.get('ProductionYear')),
+                'duration': ('duration', {int_or_none}),
+                'series': ('season', 'show', 'title'),
+                'season': ('season', 'title'),
+                'season_number': ('season', 'seasonNumber', {int_or_none}),
+                'season_id': 'seasonId',
+                'episode': 'episodeName',
+                'episode_number': ('episodeNumber', {int_or_none}),
+                'release_year': ('releaseYear', {int_or_none}),
+            }),
         }
 
 
@@ -412,6 +314,8 @@ class DRTVSeasonIE(InfoExtractor):
             'display_id': 'frank-and-kastaniegaarden',
             'title': 'Frank & Kastaniegaarden',
             'series': 'Frank & Kastaniegaarden',
+            'season_number': 2008,
+            'alt_title': 'Season 2008',
         },
         'playlist_mincount': 8
     }, {
@@ -421,6 +325,8 @@ class DRTVSeasonIE(InfoExtractor):
             'display_id': 'frank-and-kastaniegaarden',
             'title': 'Frank & Kastaniegaarden',
             'series': 'Frank & Kastaniegaarden',
+            'season_number': 2009,
+            'alt_title': 'Season 2009',
         },
         'playlist_mincount': 19
     }]
@@ -434,6 +340,7 @@ class DRTVSeasonIE(InfoExtractor):
             'url': f'https://www.dr.dk/drtv{episode["path"]}',
             'ie_key': DRTVIE.ie_key(),
             'title': episode.get('title'),
+            'alt_title': episode.get('contextualTitle'),
             'episode': episode.get('episodeName'),
             'description': episode.get('shortDescription'),
             'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
@@ -446,6 +353,7 @@ class DRTVSeasonIE(InfoExtractor):
             'id': season_id,
             'display_id': display_id,
             'title': traverse_obj(data, ('entries', 0, 'item', 'title')),
+            'alt_title': traverse_obj(data, ('entries', 0, 'item', 'contextualTitle')),
             'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
             'entries': entries,
             'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber'))
@@ -463,6 +371,7 @@ class DRTVSeriesIE(InfoExtractor):
             'display_id': 'frank-and-kastaniegaarden',
             'title': 'Frank & Kastaniegaarden',
             'series': 'Frank & Kastaniegaarden',
+            'alt_title': '',
         },
         'playlist_mincount': 15
     }]
@@ -476,6 +385,7 @@ class DRTVSeriesIE(InfoExtractor):
             'url': f'https://www.dr.dk/drtv{season.get("path")}',
             'ie_key': DRTVSeasonIE.ie_key(),
             'title': season.get('title'),
+            'alt_title': season.get('contextualTitle'),
             'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
             'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber'))
         } for season in traverse_obj(data, ('entries', 0, 'item', 'show', 'seasons', 'items'))]
@@ -485,6 +395,7 @@ class DRTVSeriesIE(InfoExtractor):
             'id': series_id,
             'display_id': display_id,
             'title': traverse_obj(data, ('entries', 0, 'item', 'title')),
+            'alt_title': traverse_obj(data, ('entries', 0, 'item', 'contextualTitle')),
             'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
             'entries': entries
         }

yt_dlp/extractor/duboku.py

@@ -138,7 +138,7 @@ class DubokuIE(InfoExtractor):
             # of the video.
             return {
                 '_type': 'url_transparent',
-                'url': smuggle_url(data_url, {'http_headers': headers}),
+                'url': smuggle_url(data_url, {'referer': webpage_url}),
                 'id': video_id,
                 'title': title,
                 'series': series_title,

yt_dlp/extractor/eltrecetv.py

@@ -0,0 +1,62 @@
+from .common import InfoExtractor
+
+
+class ElTreceTVIE(InfoExtractor):
+    IE_DESC = 'El Trece TV (Argentina)'
+    _VALID_URL = r'https?://(?:www\.)?eltrecetv\.com\.ar/[\w-]+/capitulos/temporada-\d+/(?P<id>[\w-]+)'
+    _TESTS = [
+        {
+            'url': 'https://www.eltrecetv.com.ar/ahora-caigo/capitulos/temporada-2023/programa-del-061023/',
+            'md5': '71a66673dc63f9a5939d97bfe4b311ba',
+            'info_dict': {
+                'id': 'AHCA05102023145553329621094',
+                'ext': 'mp4',
+                'title': 'AHORA CAIGO - Programa 06/10/23',
+                'thumbnail': 'https://thumbs.vodgc.net/AHCA05102023145553329621094.JPG?649339',
+            }
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/poco-correctos/capitulos/temporada-2023/programa-del-250923-invitada-dalia-gutmann/',
+            'only_matching': True,
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/argentina-tierra-de-amor-y-venganza/capitulos/temporada-2023/atav-2-capitulo-121-del-250923/',
+            'only_matching': True,
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/ahora-caigo/capitulos/temporada-2023/programa-del-250923/',
+            'only_matching': True,
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/pasaplatos/capitulos/temporada-2023/pasaplatos-el-restaurante-del-250923/',
+            'only_matching': True,
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/el-galpon/capitulos/temporada-2023/programa-del-160923-invitado-raul-lavie/',
+            'only_matching': True,
+        }
+    ]
+
+    def _real_extract(self, url):
+        slug = self._match_id(url)
+        webpage = self._download_webpage(url, slug)
+        config = self._search_json(
+            r'Fusion.globalContent\s*=', webpage, 'content', slug)['promo_items']['basic']['embed']['config']
+        video_url = config['m3u8']
+        video_id = self._search_regex(r'/(\w+)\.m3u8', video_url, 'video id', default=slug)
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', m3u8_id='hls')
+        formats.extend([{
+            'url': f['url'][:-23],
+            'format_id': f['format_id'].replace('hls', 'http'),
+            'width': f.get('width'),
+            'height': f.get('height'),
+        } for f in formats if f['url'].endswith('/tracks-v1a1/index.m3u8') and f.get('height') != 1080])
+
+        return {
+            'id': video_id,
+            'title': config.get('title'),
+            'thumbnail': config.get('thumbnail'),
+            'formats': formats,
+            'subtitles': subtitles,
+        }

yt_dlp/extractor/embedly.py

@@ -106,4 +106,4 @@ class EmbedlyIE(InfoExtractor):
             return self.url_result(src, YoutubeTabIE)
         return self.url_result(smuggle_url(
             urllib.parse.unquote(traverse_obj(qs, ('src', 0), ('url', 0))),
-            {'http_headers': {'Referer': url}}))
+            {'referer': url}))

yt_dlp/extractor/erocast.py

@@ -0,0 +1,63 @@
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    str_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class ErocastIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?erocast\.me/track/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://erocast.me/track/9787/f',
+        'md5': 'af63b91f5f231096aba54dd682abea3b',
+        'info_dict': {
+            'id': '9787',
+            'title': '[F4M] Your roommate, who is definitely not possessed by an alien, suddenly wants to fuck you',
+            'url': 'https://erocast.s3.us-east-2.wasabisys.com/1220419/track.m3u8',
+            'ext': 'm4a',
+            'age_limit': 18,
+            'release_timestamp': 1696178652,
+            'release_date': '20231001',
+            'modified_timestamp': int,
+            'modified_date': str,
+            'description': 'ExtraTerrestrial Tuesday!',
+            'uploader': 'clarissaisshy',
+            'uploader_id': '8113',
+            'uploader_url': 'https://erocast.me/clarissaisshy',
+            'thumbnail': 'https://erocast.s3.us-east-2.wasabisys.com/1220418/conversions/1696179247-lg.jpg',
+            'duration': 2307,
+            'view_count': int,
+            'comment_count': int,
+            'webpage_url': 'https://erocast.me/track/9787/f4m-your-roommate-who-is-definitely-not-possessed-by-an-alien-suddenly-wants-to-fuck-you',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        data = self._search_json(
+            rf'<script>\s*var song_data_{video_id}\s*=', webpage, 'data', video_id, end_pattern=r'</script>')
+
+        return {
+            'id': video_id,
+            'formats': self._extract_m3u8_formats(
+                data.get('file_url') or data['stream_url'], video_id, 'm4a', m3u8_id='hls'),
+            'age_limit': 18,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'release_timestamp': ('created_at', {parse_iso8601}),
+                'modified_timestamp': ('updated_at', {parse_iso8601}),
+                'uploader': ('user', 'name', {str}),
+                'uploader_id': ('user', 'id', {str_or_none}),
+                'uploader_url': ('user', 'permalink_url', {url_or_none}),
+                'thumbnail': ('artwork_url', {url_or_none}),
+                'duration': ('duration', {int_or_none}),
+                'view_count': ('plays', {int_or_none}),
+                'comment_count': ('comment_count', {int_or_none}),
+                'webpage_url': ('permalink_url', {url_or_none}),
+            }),
+        }

yt_dlp/extractor/fifa.py

@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class FifaIE(InfoExtractor):
-    _VALID_URL = r'https?://www.fifa.com/fifaplus/(?P<locale>\w{2})/watch/([^#?]+/)?(?P<id>\w+)'
+    _VALID_URL = r'https?://www\.fifa\.com/fifaplus/(?P<locale>\w{2})/watch/([^#?]+/)?(?P<id>\w+)'
     _TESTS = [{
         'url': 'https://www.fifa.com/fifaplus/en/watch/7on10qPcnyLajDDU3ntg6y',
         'info_dict': {

yt_dlp/extractor/filmmodu.py

@@ -3,7 +3,7 @@ from ..utils import int_or_none
 
 
 class FilmmoduIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www.)?filmmodu.org/(?P<id>[^/]+-(?:turkce-dublaj-izle|altyazili-izle))'
+    _VALID_URL = r'https?://(?:www\.)?filmmodu\.org/(?P<id>[^/]+-(?:turkce-dublaj-izle|altyazili-izle))'
     _TESTS = [{
         'url': 'https://www.filmmodu.org/f9-altyazili-izle',
         'md5': 'aeefd955c2a508a5bdaa3bcec8eeb0d4',

yt_dlp/extractor/generic.py

@@ -17,6 +17,7 @@ from ..utils import (
     determine_protocol,
     dict_get,
     extract_basic_auth,
+    filter_dict,
     format_field,
     int_or_none,
     is_html,
@@ -34,6 +35,7 @@ from ..utils import (
     unified_timestamp,
     unsmuggle_url,
     update_url_query,
+    urlhandle_detect_ext,
     url_or_none,
     urljoin,
     variadic,
@@ -58,6 +60,8 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'trailer',
                 'upload_date': '20100513',
+                'direct': True,
+                'timestamp': 1273772943.0,
             }
         },
         # Direct link to media delivered compressed (until Accept-Encoding is *)
@@ -101,6 +105,8 @@ class GenericIE(InfoExtractor):
                 'ext': 'webm',
                 'title': '5_Lennart_Poettering_-_Systemd',
                 'upload_date': '20141120',
+                'direct': True,
+                'timestamp': 1416498816.0,
             },
             'expected_warnings': [
                 'URL could be a direct video link, returning it as such.'
@@ -133,6 +139,7 @@ class GenericIE(InfoExtractor):
                     'upload_date': '20201204',
                 },
             }],
+            'skip': 'Dead link',
         },
         # RSS feed with item with description and thumbnails
         {
@@ -145,12 +152,12 @@ class GenericIE(InfoExtractor):
             'playlist': [{
                 'info_dict': {
                     'ext': 'm4a',
-                    'id': 'c1c879525ce2cb640b344507e682c36d',
+                    'id': '818a5d38-01cd-152f-2231-ee479677fa82',
                     'title': 're:Hydrogen!',
                     'description': 're:.*In this episode we are going.*',
                     'timestamp': 1567977776,
                     'upload_date': '20190908',
-                    'duration': 459,
+                    'duration': 423,
                     'thumbnail': r're:^https?://.*\.jpg$',
                     'episode_number': 1,
                     'season_number': 1,
@@ -267,6 +274,7 @@ class GenericIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': '404 Not Found',
         },
         # MPD from http://dash-mse-test.appspot.com/media.html
         {
@@ -278,6 +286,7 @@ class GenericIE(InfoExtractor):
                 'title': 'car-20120827-manifest',
                 'formats': 'mincount:9',
                 'upload_date': '20130904',
+                'timestamp': 1378272859.0,
             },
         },
         # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
@@ -318,7 +327,7 @@ class GenericIE(InfoExtractor):
                 'id': 'cmQHVoWB5FY',
                 'ext': 'mp4',
                 'upload_date': '20130224',
-                'uploader_id': 'TheVerge',
+                'uploader_id': '@TheVerge',
                 'description': r're:^Chris Ziegler takes a look at the\.*',
                 'uploader': 'The Verge',
                 'title': 'First Firefox OS phones side-by-side',
@@ -2427,10 +2436,10 @@ class GenericIE(InfoExtractor):
         # to accept raw bytes and being able to download only a chunk.
         # It may probably better to solve this by checking Content-Type for application/octet-stream
         # after a HEAD request, but not sure if we can rely on this.
-        full_response = self._request_webpage(url, video_id, headers={
+        full_response = self._request_webpage(url, video_id, headers=filter_dict({
             'Accept-Encoding': 'identity',
-            **smuggled_data.get('http_headers', {})
+            'Referer': smuggled_data.get('referer'),
-        })
+        }))
         new_url = full_response.url
         url = urllib.parse.urlparse(url)._replace(scheme=urllib.parse.urlparse(new_url).scheme).geturl()
         if new_url != extract_basic_auth(url)[0]:
@@ -2450,9 +2459,9 @@ class GenericIE(InfoExtractor):
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
             self.report_detected('direct video link')
-            headers = smuggled_data.get('http_headers', {})
+            headers = filter_dict({'Referer': smuggled_data.get('referer')})
             format_id = str(m.group('format_id'))
-            ext = determine_ext(url)
+            ext = determine_ext(url, default_ext=None) or urlhandle_detect_ext(full_response)
             subtitles = {}
             if format_id.endswith('mpegurl') or ext == 'm3u8':
                 formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4', headers=headers)
@@ -2464,6 +2473,7 @@ class GenericIE(InfoExtractor):
                 formats = [{
                     'format_id': format_id,
                     'url': url,
+                    'ext': ext,
                     'vcodec': 'none' if m.group('type') == 'audio' else None
                 }]
                 info_dict['direct'] = True
@@ -2701,7 +2711,7 @@ class GenericIE(InfoExtractor):
                 'url': smuggle_url(json_ld['url'], {
                     'force_videoid': video_id,
                     'to_generic': True,
-                    'http_headers': {'Referer': url},
+                    'referer': url,
                 }),
             }, json_ld)]
 

yt_dlp/extractor/gofile.py

@@ -60,7 +60,7 @@ class GofileIE(InfoExtractor):
         account_data = self._download_json(
             'https://api.gofile.io/createAccount', None, note='Getting a new guest account')
         self._TOKEN = account_data['data']['token']
-        self._set_cookie('gofile.io', 'accountToken', self._TOKEN)
+        self._set_cookie('.gofile.io', 'accountToken', self._TOKEN)
 
     def _entries(self, file_id):
         query_params = {

yt_dlp/extractor/iqiyi.py

@@ -499,9 +499,10 @@ class IqIE(InfoExtractor):
                     'tm': tm,
                     'qdy': 'a',
                     'qds': 0,
-                    'k_ft1': 141287244169348,
+                    'k_ft1': '143486267424900',
-                    'k_ft4': 34359746564,
+                    'k_ft4': '1572868',
-                    'k_ft5': 1,
+                    'k_ft7': '4',
+                    'k_ft5': '1',
                     'bop': JSON.stringify({
                         'version': '10.0',
                         'dfp': dfp
@@ -529,14 +530,22 @@ class IqIE(InfoExtractor):
         webpack_js_url = self._proto_relative_url(self._search_regex(
             r'<script src="((?:https?:)?//stc\.iqiyipic\.com/_next/static/chunks/webpack-\w+\.js)"', webpage, 'webpack URL'))
         webpack_js = self._download_webpage(webpack_js_url, video_id, note='Downloading webpack JS', errnote='Unable to download webpack JS')
+
         webpack_map = self._search_json(
             r'["\']\s*\+\s*', webpack_js, 'JS locations', video_id,
             contains_pattern=r'{\s*(?:\d+\s*:\s*["\'][\da-f]+["\']\s*,?\s*)+}',
             end_pattern=r'\[\w+\]\+["\']\.js', transform_source=js_to_json)
 
+        replacement_map = self._search_json(
+            r'["\']\s*\+\(\s*', webpack_js, 'replacement map', video_id,
+            contains_pattern=r'{\s*(?:\d+\s*:\s*["\'][\w.-]+["\']\s*,?\s*)+}',
+            end_pattern=r'\[\w+\]\|\|\w+\)\+["\']\.', transform_source=js_to_json,
+            fatal=False) or {}
+
         for module_index in reversed(webpack_map):
+            real_module = replacement_map.get(module_index) or module_index
             module_js = self._download_webpage(
-                f'https://stc.iqiyipic.com/_next/static/chunks/{module_index}.{webpack_map[module_index]}.js',
+                f'https://stc.iqiyipic.com/_next/static/chunks/{real_module}.{webpack_map[module_index]}.js',
                 video_id, note=f'Downloading #{module_index} module JS', errnote='Unable to download module JS', fatal=False) or ''
             if 'vms request' in module_js:
                 self.cache.store('iq', 'player_js', module_js)

yt_dlp/extractor/itprotv.py

@@ -31,7 +31,7 @@ class ITProTVBaseIE(InfoExtractor):
 
 
 class ITProTVIE(ITProTVBaseIE):
-    _VALID_URL = r'https://app.itpro.tv/course/(?P<course>[\w-]+)/(?P<id>[\w-]+)'
+    _VALID_URL = r'https://app\.itpro\.tv/course/(?P<course>[\w-]+)/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://app.itpro.tv/course/guided-tour/introductionitprotv',
         'md5': 'bca4a28c2667fd1a63052e71a94bb88c',
@@ -102,7 +102,7 @@ class ITProTVIE(ITProTVBaseIE):
 
 
 class ITProTVCourseIE(ITProTVBaseIE):
-    _VALID_URL = r'https?://app.itpro.tv/course/(?P<id>[\w-]+)/?(?:$|[#?])'
+    _VALID_URL = r'https?://app\.itpro\.tv/course/(?P<id>[\w-]+)/?(?:$|[#?])'
     _TESTS = [
         {
             'url': 'https://app.itpro.tv/course/guided-tour',

yt_dlp/extractor/jable.py

@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class JableIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?jable.tv/videos/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?jable\.tv/videos/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://jable.tv/videos/pppd-812/',
         'md5': 'f1537283a9bc073c31ff86ca35d9b2a6',
@@ -64,7 +64,7 @@ class JableIE(InfoExtractor):
 
 
 class JablePlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?jable.tv/(?:categories|models|tags)/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?jable\.tv/(?:categories|models|tags)/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://jable.tv/models/kaede-karen/',
         'info_dict': {

yt_dlp/extractor/jiosaavn.py

@@ -0,0 +1,79 @@
+from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    url_or_none,
+    urlencode_postdata,
+    urljoin,
+)
+from ..utils.traversal import traverse_obj
+
+
+class JioSaavnBaseIE(InfoExtractor):
+    def _extract_initial_data(self, url, audio_id):
+        webpage = self._download_webpage(url, audio_id)
+        return self._search_json(
+            r'window\.__INITIAL_DATA__\s*=', webpage,
+            'init json', audio_id, transform_source=js_to_json)
+
+
+class JioSaavnSongIE(JioSaavnBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:jiosaavn\.com/song/[^/?#]+/|saavn\.com/s/song/(?:[^/?#]+/){3})(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.jiosaavn.com/song/leja-re/OQsEfQFVUXk',
+        'md5': '7b1f70de088ede3a152ea34aece4df42',
+        'info_dict': {
+            'id': 'OQsEfQFVUXk',
+            'ext': 'mp3',
+            'title': 'Leja Re',
+            'album': 'Leja Re',
+            'thumbnail': 'https://c.saavncdn.com/258/Leja-Re-Hindi-2018-20181124024539-500x500.jpg',
+        },
+    }, {
+        'url': 'https://www.saavn.com/s/song/hindi/Saathiya/O-Humdum-Suniyo-Re/KAMiazoCblU',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        song_data = self._extract_initial_data(url, audio_id)['song']['song']
+        media_data = self._download_json(
+            'https://www.jiosaavn.com/api.php', audio_id, data=urlencode_postdata({
+                '__call': 'song.generateAuthToken',
+                '_format': 'json',
+                'bitrate': '128',
+                'url': song_data['encrypted_media_url'],
+            }))
+
+        return {
+            'id': audio_id,
+            'url': media_data['auth_url'],
+            'ext': media_data.get('type'),
+            'vcodec': 'none',
+            **traverse_obj(song_data, {
+                'title': ('title', 'text'),
+                'album': ('album', 'text'),
+                'thumbnail': ('image', 0, {url_or_none}),
+            }),
+        }
+
+
+class JioSaavnAlbumIE(JioSaavnBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:jio)?saavn\.com/album/[^/?#]+/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.jiosaavn.com/album/96/buIOjYZDrNA_',
+        'info_dict': {
+            'id': 'buIOjYZDrNA_',
+            'title': '96',
+        },
+        'playlist_count': 10,
+    }]
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+        album_view = self._extract_initial_data(url, album_id)['albumView']
+
+        return self.playlist_from_matches(
+            traverse_obj(album_view, (
+                'modules', lambda _, x: x['key'] == 'list', 'data', ..., 'title', 'action', {str})),
+            album_id, traverse_obj(album_view, ('album', 'title', 'text', {str})), ie=JioSaavnSongIE,
+            getter=lambda x: urljoin('https://www.jiosaavn.com/', x))

yt_dlp/extractor/jtbc.py

@@ -0,0 +1,156 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class JTBCIE(InfoExtractor):
+    IE_DESC = 'jtbc.co.kr'
+    _VALID_URL = r'''(?x)
+        https?://(?:
+            vod\.jtbc\.co\.kr/player/(?:program|clip)
+            |tv\.jtbc\.co\.kr/(?:replay|trailer|clip)/pr\d+/pm\d+
+        )/(?P<id>(?:ep|vo)\d+)'''
+    _GEO_COUNTRIES = ['KR']
+
+    _TESTS = [{
+        'url': 'https://tv.jtbc.co.kr/replay/pr10011629/pm10067930/ep20216321/view',
+        'md5': 'e6ade71d8c8685bbfd6e6ce4167c6a6c',
+        'info_dict': {
+            'id': 'VO10721192',
+            'display_id': 'ep20216321',
+            'ext': 'mp4',
+            'title': '힘쎈여자 강남순 2회 다시보기',
+            'description': 'md5:043c1d9019100ce271dba09995dbd1e2',
+            'duration': 3770.0,
+            'release_date': '20231008',
+            'age_limit': 15,
+            'thumbnail': 'https://fs.jtbc.co.kr//joydata/CP00000001/prog/drama/stronggirlnamsoon/img/20231008_163541_522_1.jpg',
+            'series': '힘쎈여자 강남순',
+        },
+    }, {
+        'url': 'https://vod.jtbc.co.kr/player/program/ep20216733',
+        'md5': '217a6d190f115a75e4bda0ceaa4cd7f4',
+        'info_dict': {
+            'id': 'VO10721429',
+            'display_id': 'ep20216733',
+            'ext': 'mp4',
+            'title': '헬로 마이 닥터 친절한 진료실 149회 다시보기',
+            'description': 'md5:1d70788a982dd5de26874a92fcffddb8',
+            'duration': 2720.0,
+            'release_date': '20231009',
+            'age_limit': 15,
+            'thumbnail': 'https://fs.jtbc.co.kr//joydata/CP00000001/prog/culture/hellomydoctor/img/20231009_095002_528_1.jpg',
+            'series': '헬로 마이 닥터 친절한 진료실',
+        },
+    }, {
+        'url': 'https://vod.jtbc.co.kr/player/clip/vo10721270',
+        'md5': '05782e2dc22a9c548aebefe62ae4328a',
+        'info_dict': {
+            'id': 'VO10721270',
+            'display_id': 'vo10721270',
+            'ext': 'mp4',
+            'title': '뭉쳐야 찬다3 2회 예고편 - A매치로 향하는 마지막 관문💥',
+            'description': 'md5:d48b51a8655c84843b4ed8d0c39aae68',
+            'duration': 46.0,
+            'release_date': '20231015',
+            'age_limit': 15,
+            'thumbnail': 'https://fs.jtbc.co.kr//joydata/CP00000001/prog/enter/soccer3/img/20231008_210957_775_1.jpg',
+            'series': '뭉쳐야 찬다3',
+        },
+    }, {
+        'url': 'https://tv.jtbc.co.kr/trailer/pr10010392/pm10032526/vo10720912/view',
+        'md5': '367d480eb3ef54a9cd7a4b4d69c4b32d',
+        'info_dict': {
+            'id': 'VO10720912',
+            'display_id': 'vo10720912',
+            'ext': 'mp4',
+            'title': '아는 형님 404회 예고편 | 10월 14일(토) 저녁 8시 50분 방송!',
+            'description': 'md5:2743bb1079ceb85bb00060f2ad8f0280',
+            'duration': 148.0,
+            'release_date': '20231014',
+            'age_limit': 15,
+            'thumbnail': 'https://fs.jtbc.co.kr//joydata/CP00000001/prog/enter/jtbcbros/img/20231006_230023_802_1.jpg',
+            'series': '아는 형님',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        if display_id.startswith('vo'):
+            video_id = display_id.upper()
+        else:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._search_regex(r'data-vod="(VO\d+)"', webpage, 'vod id')
+
+        playback_data = self._download_json(
+            f'https://api.jtbc.co.kr/vod/{video_id}', video_id, note='Downloading VOD playback data')
+
+        subtitles = {}
+        for sub in traverse_obj(playback_data, ('tracks', lambda _, v: v['file'])):
+            subtitles.setdefault(sub.get('label', 'und'), []).append({'url': sub['file']})
+
+        formats = []
+        for stream_url in traverse_obj(playback_data, ('sources', 'HLS', ..., 'file', {url_or_none})):
+            stream_url = re.sub(r'/playlist(?:_pd\d+)?\.m3u8', '/index.m3u8', stream_url)
+            formats.extend(self._extract_m3u8_formats(stream_url, video_id, fatal=False))
+
+        metadata = self._download_json(
+            'https://now-api.jtbc.co.kr/v1/vod/detail', video_id,
+            note='Downloading mobile details', fatal=False, query={'vodFileId': video_id})
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            **traverse_obj(metadata, ('vodDetail', {
+                'title': 'vodTitleView',
+                'series': 'programTitle',
+                'age_limit': ('watchAge', {int_or_none}),
+                'release_date': ('broadcastDate', {lambda x: re.match(r'\d{8}', x.replace('.', ''))}, 0),
+                'description': 'episodeContents',
+                'thumbnail': ('imgFileUrl', {url_or_none}),
+            })),
+            'duration': parse_duration(playback_data.get('playTime')),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+
+class JTBCProgramIE(InfoExtractor):
+    IE_NAME = 'JTBC:program'
+    _VALID_URL = r'https?://(?:vod\.jtbc\.co\.kr/program|tv\.jtbc\.co\.kr/replay)/(?P<id>pr\d+)/(?:replay|pm\d+)/?(?:$|[?#])'
+
+    _TESTS = [{
+        'url': 'https://tv.jtbc.co.kr/replay/pr10010392/pm10032710',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'pr10010392',
+        },
+        'playlist_count': 398,
+    }, {
+        'url': 'https://vod.jtbc.co.kr/program/pr10011491/replay',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'pr10011491',
+        },
+        'playlist_count': 59,
+    }]
+
+    def _real_extract(self, url):
+        program_id = self._match_id(url)
+
+        vod_list = self._download_json(
+            'https://now-api.jtbc.co.kr/v1/vodClip/programHome/programReplayVodList', program_id,
+            note='Downloading program replay list', query={
+                'programId': program_id,
+                'rowCount': '10000',
+            })
+
+        entries = [self.url_result(f'https://vod.jtbc.co.kr/player/program/{video_id}', JTBCIE, video_id)
+                   for video_id in traverse_obj(vod_list, ('programReplayVodList', ..., 'episodeId'))]
+        return self.playlist_result(entries, program_id)

yt_dlp/extractor/kommunetv.py

@@ -3,7 +3,7 @@ from ..utils import update_url
 
 
 class KommunetvIE(InfoExtractor):
-    _VALID_URL = r'https://(\w+).kommunetv.no/archive/(?P<id>\w+)'
+    _VALID_URL = r'https://\w+\.kommunetv\.no/archive/(?P<id>\w+)'
     _TEST = {
         'url': 'https://oslo.kommunetv.no/archive/921',
         'md5': '5f102be308ee759be1e12b63d5da4bbc',

yt_dlp/extractor/la7.py

@@ -208,9 +208,9 @@ class LA7PodcastIE(LA7PodcastEpisodeIE):  # XXX: Do not subclass from concrete I
         'url': 'https://www.la7.it/propagandalive/podcast',
         'info_dict': {
             'id': 'propagandalive',
-            'title': "Propaganda Live",
+            'title': 'Propaganda Live',
         },
-        'playlist_count_min': 10,
+        'playlist_mincount': 10,
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/laxarxames.py

@@ -0,0 +1,73 @@
+import json
+
+from .brightcove import BrightcoveNewIE
+from .common import InfoExtractor
+from ..utils import ExtractorError
+from ..utils.traversal import traverse_obj
+
+
+class LaXarxaMesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?laxarxames\.cat/(?:[^/?#]+/)*?(player|movie-details)/(?P<id>\d+)'
+    _NETRC_MACHINE = 'laxarxames'
+    _TOKEN = None
+    _TESTS = [{
+        'url': 'https://www.laxarxames.cat/player/3459421',
+        'md5': '0966f46c34275934c19af78f3df6e2bc',
+        'info_dict': {
+            'id': '6339612436112',
+            'ext': 'mp4',
+            'title': 'Resum | UA Horta — UD Viladecans',
+            'timestamp': 1697905186,
+            'thumbnail': r're:https?://.*\.jpg',
+            'description': '',
+            'upload_date': '20231021',
+            'duration': 129.44,
+            'tags': ['ott', 'esports', '23-24', ' futbol', ' futbol-partits', 'elit', 'resum'],
+            'uploader_id': '5779379807001',
+        },
+        'skip': 'Requires login',
+    }]
+
+    def _perform_login(self, username, password):
+        if self._TOKEN:
+            return
+
+        login = self._download_json(
+            'https://api.laxarxames.cat/Authorization/SignIn', None, note='Logging in', headers={
+                'X-Tenantorigin': 'https://laxarxames.cat',
+                'Content-Type': 'application/json',
+            }, data=json.dumps({
+                'Username': username,
+                'Password': password,
+                'Device': {
+                    'PlatformCode': 'WEB',
+                    'Name': 'Mac OS ()',
+                },
+            }).encode(), expected_status=401)
+
+        self._TOKEN = traverse_obj(login, ('AuthorizationToken', 'Token', {str}))
+        if not self._TOKEN:
+            raise ExtractorError('Login failed', expected=True)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        if not self._TOKEN:
+            self.raise_login_required()
+
+        media_play_info = self._download_json(
+            'https://api.laxarxames.cat/Media/GetMediaPlayInfo', video_id,
+            data=json.dumps({
+                'MediaId': int(video_id),
+                'StreamType': 'MAIN'
+            }).encode(), headers={
+                'Authorization': f'Bearer {self._TOKEN}',
+                'X-Tenantorigin': 'https://laxarxames.cat',
+                'Content-Type': 'application/json',
+            })
+
+        if not traverse_obj(media_play_info, ('ContentUrl', {str})):
+            self.raise_no_formats('No video found', expected=True)
+
+        return self.url_result(
+            f'https://players.brightcove.net/5779379807001/default_default/index.html?videoId={media_play_info["ContentUrl"]}',
+            BrightcoveNewIE, video_id, media_play_info.get('Title'))

yt_dlp/extractor/lbry.py

@@ -22,10 +22,11 @@ from ..utils import (
 
 
 class LBRYBaseIE(InfoExtractor):
-    _BASE_URL_REGEX = r'(?:https?://(?:www\.)?(?:lbry\.tv|odysee\.com)/|lbry://)'
+    _BASE_URL_REGEX = r'(?x)(?:https?://(?:www\.)?(?:lbry\.tv|odysee\.com)/|lbry://)'
     _CLAIM_ID_REGEX = r'[0-9a-f]{1,40}'
-    _OPT_CLAIM_ID = '[^:/?#&]+(?:[:#]%s)?' % _CLAIM_ID_REGEX
+    _OPT_CLAIM_ID = '[^$@:/?#&]+(?:[:#]%s)?' % _CLAIM_ID_REGEX
     _SUPPORTED_STREAM_TYPES = ['video', 'audio']
+    _PAGE_SIZE = 50
 
     def _call_api_proxy(self, method, display_id, params, resource):
         headers = {'Content-Type': 'application/json-rpc'}
@@ -69,18 +70,78 @@ class LBRYBaseIE(InfoExtractor):
             'duration': ('value', stream_type, 'duration', {int_or_none}),
             'channel': ('signing_channel', 'value', 'title', {str}),
             'channel_id': ('signing_channel', 'claim_id', {str}),
+            'uploader_id': ('signing_channel', 'name', {str}),
         })
 
-        channel_name = traverse_obj(stream, ('signing_channel', 'name', {str}))
+        if info.get('uploader_id') and info.get('channel_id'):
-        if channel_name and info.get('channel_id'):
+            info['channel_url'] = self._permanent_url(url, info['uploader_id'], info['channel_id'])
-            info['channel_url'] = self._permanent_url(url, channel_name, info['channel_id'])
 
         return info
 
+    def _fetch_page(self, display_id, url, params, page):
+        page += 1
+        page_params = {
+            'no_totals': True,
+            'page': page,
+            'page_size': self._PAGE_SIZE,
+            **params,
+        }
+        result = self._call_api_proxy(
+            'claim_search', display_id, page_params, f'page {page}')
+        for item in traverse_obj(result, ('items', lambda _, v: v['name'] and v['claim_id'])):
+            yield {
+                **self._parse_stream(item, url),
+                '_type': 'url',
+                'id': item['claim_id'],
+                'url': self._permanent_url(url, item['name'], item['claim_id']),
+            }
+
+    def _playlist_entries(self, url, display_id, claim_param, metadata):
+        qs = parse_qs(url)
+        content = qs.get('content', [None])[0]
+        params = {
+            'fee_amount': qs.get('fee_amount', ['>=0'])[0],
+            'order_by': {
+                'new': ['release_time'],
+                'top': ['effective_amount'],
+                'trending': ['trending_group', 'trending_mixed'],
+            }[qs.get('order', ['new'])[0]],
+            'claim_type': 'stream',
+            'stream_types': [content] if content in ['audio', 'video'] else self._SUPPORTED_STREAM_TYPES,
+            **claim_param,
+        }
+        duration = qs.get('duration', [None])[0]
+        if duration:
+            params['duration'] = {
+                'long': '>=1200',
+                'short': '<=240',
+            }[duration]
+        language = qs.get('language', ['all'])[0]
+        if language != 'all':
+            languages = [language]
+            if language == 'en':
+                languages.append('none')
+            params['any_languages'] = languages
+
+        entries = OnDemandPagedList(
+            functools.partial(self._fetch_page, display_id, url, params),
+            self._PAGE_SIZE)
+
+        return self.playlist_result(
+            entries, display_id, **traverse_obj(metadata, ('value', {
+                'title': 'title',
+                'description': 'description',
+            })))
+
 
 class LBRYIE(LBRYBaseIE):
     IE_NAME = 'lbry'
-    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + r'(?P<id>\$/[^/]+/[^/]+/{1}|@{0}/{0}|(?!@){0})'.format(LBRYBaseIE._OPT_CLAIM_ID, LBRYBaseIE._CLAIM_ID_REGEX)
+    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + rf'''
+        (?:\$/(?:download|embed)/)?
+        (?P<id>
+            [^$@:/?#]+/{LBRYBaseIE._CLAIM_ID_REGEX}
+            |(?:@{LBRYBaseIE._OPT_CLAIM_ID}/)?{LBRYBaseIE._OPT_CLAIM_ID}
+        )'''
     _TESTS = [{
         # Video
         'url': 'https://lbry.tv/@Mantega:1/First-day-LBRY:1',
@@ -98,6 +159,7 @@ class LBRYIE(LBRYBaseIE):
             'height': 720,
             'thumbnail': 'https://spee.ch/7/67f2d809c263288c.png',
             'license': 'None',
+            'uploader_id': '@Mantega',
             'duration': 346,
             'channel': 'LBRY/Odysee rats united!!!',
             'channel_id': '1c8ad6a2ab4e889a71146ae4deeb23bb92dab627',
@@ -131,6 +193,7 @@ class LBRYIE(LBRYBaseIE):
             'vcodec': 'none',
             'thumbnail': 'https://spee.ch/d/0bc63b0e6bf1492d.png',
             'license': 'None',
+            'uploader_id': '@LBRYFoundation',
         }
     }, {
         'url': 'https://odysee.com/@gardeningincanada:b/plants-i-will-never-grow-again.-the:e',
@@ -149,6 +212,7 @@ class LBRYIE(LBRYBaseIE):
             'channel': 'Gardening In Canada',
             'channel_id': 'b8be0e93b423dad221abe29545fbe8ec36e806bc',
             'channel_url': 'https://odysee.com/@gardeningincanada:b8be0e93b423dad221abe29545fbe8ec36e806bc',
+            'uploader_id': '@gardeningincanada',
             'formats': 'mincount:3',
             'thumbnail': 'https://thumbnails.lbry.com/AgHSc_HzrrE',
             'license': 'Copyrighted (contact publisher)',
@@ -174,6 +238,7 @@ class LBRYIE(LBRYBaseIE):
             'formats': 'mincount:1',
             'thumbnail': 'startswith:https://thumb',
             'license': 'None',
+            'uploader_id': '@RT',
         },
         'params': {'skip_download': True}
     }, {
@@ -184,12 +249,13 @@ class LBRYIE(LBRYBaseIE):
             'id': '41fbfe805eb73c8d3012c0c49faa0f563274f634',
             'ext': 'mp4',
             'title': 'Biotechnological Invasion of Skin (April 2023)',
-            'description': 'md5:709a2f4c07bd8891cda3a7cc2d6fcf5c',
+            'description': 'md5:fe28689db2cb7ba3436d819ac3ffc378',
             'channel': 'Wicked Truths',
             'channel_id': '23d2bbf856b0ceed5b1d7c5960bcc72da5a20cb0',
             'channel_url': 'https://odysee.com/@wickedtruths:23d2bbf856b0ceed5b1d7c5960bcc72da5a20cb0',
-            'timestamp': 1685790036,
+            'uploader_id': '@wickedtruths',
-            'upload_date': '20230603',
+            'timestamp': 1695114347,
+            'upload_date': '20230919',
             'release_timestamp': 1685617473,
             'release_date': '20230601',
             'duration': 1063,
@@ -229,10 +295,10 @@ class LBRYIE(LBRYBaseIE):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        if display_id.startswith('$/'):
+        if display_id.startswith('@'):
-            display_id = display_id.split('/', 2)[-1].replace('/', ':')
-        else:
             display_id = display_id.replace(':', '#')
+        else:
+            display_id = display_id.replace('/', ':')
         display_id = urllib.parse.unquote(display_id)
         uri = 'lbry://' + display_id
         result = self._resolve_url(uri, display_id, 'stream')
@@ -299,7 +365,7 @@ class LBRYIE(LBRYBaseIE):
 
 class LBRYChannelIE(LBRYBaseIE):
     IE_NAME = 'lbry:channel'
-    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + r'(?P<id>@%s)/?(?:[?&]|$)' % LBRYBaseIE._OPT_CLAIM_ID
+    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + rf'(?P<id>@{LBRYBaseIE._OPT_CLAIM_ID})/?(?:[?&]|$)'
     _TESTS = [{
         'url': 'https://lbry.tv/@LBRYFoundation:0',
         'info_dict': {
@@ -315,65 +381,50 @@ class LBRYChannelIE(LBRYBaseIE):
         'url': 'lbry://@lbry#3f',
         'only_matching': True,
     }]
-    _PAGE_SIZE = 50
-
-    def _fetch_page(self, claim_id, url, params, page):
-        page += 1
-        page_params = {
-            'channel_ids': [claim_id],
-            'claim_type': 'stream',
-            'no_totals': True,
-            'page': page,
-            'page_size': self._PAGE_SIZE,
-        }
-        page_params.update(params)
-        result = self._call_api_proxy(
-            'claim_search', claim_id, page_params, 'page %d' % page)
-        for item in (result.get('items') or []):
-            stream_claim_name = item.get('name')
-            stream_claim_id = item.get('claim_id')
-            if not (stream_claim_name and stream_claim_id):
-                continue
-
-            yield {
-                **self._parse_stream(item, url),
-                '_type': 'url',
-                'id': stream_claim_id,
-                'url': self._permanent_url(url, stream_claim_name, stream_claim_id),
-            }
 
     def _real_extract(self, url):
         display_id = self._match_id(url).replace(':', '#')
-        result = self._resolve_url(
+        result = self._resolve_url(f'lbry://{display_id}', display_id, 'channel')
-            'lbry://' + display_id, display_id, 'channel')
         claim_id = result['claim_id']
-        qs = parse_qs(url)
+
-        content = qs.get('content', [None])[0]
+        return self._playlist_entries(url, claim_id, {'channel_ids': [claim_id]}, result)
-        params = {
+
-            'fee_amount': qs.get('fee_amount', ['>=0'])[0],
+
-            'order_by': {
+class LBRYPlaylistIE(LBRYBaseIE):
-                'new': ['release_time'],
+    IE_NAME = 'lbry:playlist'
-                'top': ['effective_amount'],
+    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + r'\$/(?:play)?list/(?P<id>[0-9a-f-]+)'
-                'trending': ['trending_group', 'trending_mixed'],
+    _TESTS = [{
-            }[qs.get('order', ['new'])[0]],
+        'url': 'https://odysee.com/$/playlist/ffef782f27486f0ac138bde8777f72ebdd0548c2',
-            'stream_types': [content] if content in ['audio', 'video'] else self._SUPPORTED_STREAM_TYPES,
+        'info_dict': {
-        }
+            'id': 'ffef782f27486f0ac138bde8777f72ebdd0548c2',
-        duration = qs.get('duration', [None])[0]
+            'title': 'Théâtre Classique',
-        if duration:
+            'description': 'Théâtre Classique',
-            params['duration'] = {
+        },
-                'long': '>=1200',
+        'playlist_mincount': 4,
-                'short': '<=240',
+    }, {
-            }[duration]
+        'url': 'https://odysee.com/$/list/9c6658b3dd21e4f2a0602d523a13150e2b48b770',
-        language = qs.get('language', ['all'])[0]
+        'info_dict': {
-        if language != 'all':
+            'id': '9c6658b3dd21e4f2a0602d523a13150e2b48b770',
-            languages = [language]
+            'title': 'Social Media Exposed',
-            if language == 'en':
+            'description': 'md5:98af97317aacd5b85d595775ea37d80e',
-                languages.append('none')
+        },
-            params['any_languages'] = languages
+        'playlist_mincount': 34,
-        entries = OnDemandPagedList(
+    }, {
-            functools.partial(self._fetch_page, claim_id, url, params),
+        'url': 'https://odysee.com/$/playlist/938fb11d-215f-4d1c-ad64-723954df2184',
-            self._PAGE_SIZE)
+        'info_dict': {
-        result_value = result.get('value') or {}
+            'id': '938fb11d-215f-4d1c-ad64-723954df2184',
-        return self.playlist_result(
+        },
-            entries, claim_id, result_value.get('title'),
+        'playlist_mincount': 1000,
-            result_value.get('description'))
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        result = traverse_obj(self._call_api_proxy('claim_search', display_id, {
+            'claim_ids': [display_id],
+            'no_totals': True,
+            'page': 1,
+            'page_size': self._PAGE_SIZE,
+        }, 'playlist'), ('items', 0))
+        claim_param = {'claim_ids': traverse_obj(result, ('value', 'claims', ..., {str}))}
+
+        return self._playlist_entries(url, display_id, claim_param, result)

yt_dlp/extractor/litv.py

@@ -13,7 +13,7 @@ from ..utils import (
 class LiTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?litv\.tv/(?:vod|promo)/[^/]+/(?:content\.do)?\?.*?\b(?:content_)?id=(?P<id>[^&]+)'
 
-    _URL_TEMPLATE = 'https://www.litv.tv/vod/%s/content.do?id=%s'
+    _URL_TEMPLATE = 'https://www.litv.tv/vod/%s/content.do?content_id=%s'
 
     _TESTS = [{
         'url': 'https://www.litv.tv/vod/drama/content.do?brc_id=root&id=VOD00041610&isUHEnabled=true&autoPlay=1',
@@ -21,16 +21,18 @@ class LiTVIE(InfoExtractor):
             'id': 'VOD00041606',
             'title': '花千骨',
         },
-        'playlist_count': 50,
+        'playlist_count': 51,  # 50 episodes + 1 trailer
     }, {
         'url': 'https://www.litv.tv/vod/drama/content.do?brc_id=root&id=VOD00041610&isUHEnabled=true&autoPlay=1',
-        'md5': '969e343d9244778cb29acec608e53640',
+        'md5': 'b90ff1e9f1d8f5cfcd0a44c3e2b34c7a',
         'info_dict': {
             'id': 'VOD00041610',
             'ext': 'mp4',
             'title': '花千骨第1集',
             'thumbnail': r're:https?://.*\.jpg$',
-            'description': 'md5:c7017aa144c87467c4fb2909c4b05d6f',
+            'description': '《花千骨》陸劇線上看。十六年前，平靜的村莊內，一名女嬰隨異相出生，途徑此地的蜀山掌門清虛道長算出此女命運非同一般，她體內散發的異香易招惹妖魔。一念慈悲下，他在村莊周邊設下結界阻擋妖魔入侵，讓其年滿十六後去蜀山，並賜名花千骨。',
+            'categories': ['奇幻', '愛情', '中國', '仙俠'],
+            'episode': 'Episode 1',
             'episode_number': 1,
         },
         'params': {
@@ -46,20 +48,17 @@ class LiTVIE(InfoExtractor):
             'title': '芈月傳第1集　霸星芈月降世楚國',
             'description': '楚威王二年，太史令唐昧夜觀星象，發現霸星即將現世。王后得知霸星的預言後，想盡辦法不讓孩子順利出生，幸得莒姬相護化解危機。沒想到眾人期待下出生的霸星卻是位公主，楚威王對此失望至極。楚王后命人將女嬰丟棄河中，居然奇蹟似的被少司命像攔下，楚威王認為此女非同凡響，為她取名芈月。',
         },
-        'skip': 'Georestricted to Taiwan',
+        'skip': 'No longer exists',
     }]
 
-    def _extract_playlist(self, season_list, video_id, program_info, prompt=True):
+    def _extract_playlist(self, playlist_data, content_type):
-        episode_title = program_info['title']
-        content_id = season_list['contentId']
-
         all_episodes = [
             self.url_result(smuggle_url(
-                self._URL_TEMPLATE % (program_info['contentType'], episode['contentId']),
+                self._URL_TEMPLATE % (content_type, episode['contentId']),
                 {'force_noplaylist': True}))  # To prevent infinite recursion
-            for episode in season_list['episode']]
+            for episode in traverse_obj(playlist_data, ('seasons', ..., 'episode', lambda _, v: v['contentId']))]
 
-        return self.playlist_result(all_episodes, content_id, episode_title)
+        return self.playlist_result(all_episodes, playlist_data['contentId'], playlist_data.get('title'))
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
@@ -68,24 +67,31 @@ class LiTVIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        if self._search_regex(
+                r'(?i)<meta\s[^>]*http-equiv="refresh"\s[^>]*content="[0-9]+;\s*url=https://www\.litv\.tv/"',
+                webpage, 'meta refresh redirect', default=False, group=0):
+            raise ExtractorError('No such content found', expected=True)
+
         program_info = self._parse_json(self._search_regex(
             r'var\s+programInfo\s*=\s*([^;]+)', webpage, 'VOD data', default='{}'),
             video_id)
 
-        season_list = list(program_info.get('seasonList', {}).values())
+        # In browsers `getProgramInfo` request is always issued. Usually this
-        playlist_id = traverse_obj(season_list, 0, 'contentId')
-        if self._yes_playlist(playlist_id, video_id, smuggled_data):
-            return self._extract_playlist(season_list[0], video_id, program_info)
-
-        # In browsers `getMainUrl` request is always issued. Usually this
         # endpoint gives the same result as the data embedded in the webpage.
-        # If georestricted, there are no embedded data, so an extra request is
+        # If, for some reason, there are no embedded data, we do an extra request.
-        # necessary to get the error code
         if 'assetId' not in program_info:
             program_info = self._download_json(
                 'https://www.litv.tv/vod/ajax/getProgramInfo', video_id,
                 query={'contentId': video_id},
                 headers={'Accept': 'application/json'})
+
+        series_id = program_info['seriesId']
+        if self._yes_playlist(series_id, video_id, smuggled_data):
+            playlist_data = self._download_json(
+                'https://www.litv.tv/vod/ajax/getSeriesTree', video_id,
+                query={'seriesId': series_id}, headers={'Accept': 'application/json'})
+            return self._extract_playlist(playlist_data, program_info['contentType'])
+
         video_data = self._parse_json(self._search_regex(
             r'uiHlsUrl\s*=\s*testBackendData\(([^;]+)\);',
             webpage, 'video data', default='{}'), video_id)
@@ -96,7 +102,7 @@ class LiTVIE(InfoExtractor):
                 'contentType': program_info['contentType'],
             }
             video_data = self._download_json(
-                'https://www.litv.tv/vod/getMainUrl', video_id,
+                'https://www.litv.tv/vod/ajax/getMainUrlNoAuth', video_id,
                 data=json.dumps(payload).encode('utf-8'),
                 headers={'Content-Type': 'application/json'})
 

yt_dlp/extractor/mainstreaming.py

@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class MainStreamingIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:webtools-?)?(?P<host>[A-Za-z0-9-]*\.msvdn.net)/(?:embed|amp_embed|content)/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:webtools-?)?(?P<host>[A-Za-z0-9-]*\.msvdn\.net)/(?:embed|amp_embed|content)/(?P<id>\w+)'
     _EMBED_REGEX = [rf'<iframe[^>]+?src=["\']?(?P<url>{_VALID_URL})["\']?']
     IE_DESC = 'MainStreaming Player'
 

yt_dlp/extractor/mbn.py

@@ -0,0 +1,89 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class MBNIE(InfoExtractor):
+    IE_DESC = 'mbn.co.kr (매일방송)'
+    _VALID_URL = r'https?://(?:www\.)?mbn\.co\.kr/vod/programContents/preview(?:list)?/\d+/\d+/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://mbn.co.kr/vod/programContents/previewlist/861/5433/1276155',
+        'md5': '85e1694e5b247c04d1386b7e3c90fd76',
+        'info_dict': {
+            'id': '1276155',
+            'ext': 'mp4',
+            'title': '결국 사로잡힌 권유리, 그녀를 목숨 걸고 구하려는 정일우!',
+            'duration': 3891,
+            'release_date': '20210703',
+            'thumbnail': 'http://img.vod.mbn.co.kr/mbnvod2img/861/2021/07/03/20210703230811_20_861_1276155_360_7_0.jpg',
+            'series': '보쌈 - 운명을 훔치다',
+            'episode': 'Episode 19',
+            'episode_number': 19,
+        },
+    }, {
+        'url': 'https://www.mbn.co.kr/vod/programContents/previewlist/835/5294/1084744',
+        'md5': 'fc65d3aac85e85e0b5056f4ef99cde4a',
+        'info_dict': {
+            'id': '1084744',
+            'ext': 'mp4',
+            'title': '김정은♥최원영, 제자리를 찾은 위험한 부부! ＂결혼은 투쟁이면서, 어려운 방식이야..＂',
+            'duration': 93,
+            'release_date': '20201124',
+            'thumbnail': 'http://img.vod.mbn.co.kr/mbnvod2img/835/2020/11/25/20201125000221_21_835_1084744_360_7_0.jpg',
+            'series': '나의 위험한 아내',
+        },
+    }, {
+        'url': 'https://www.mbn.co.kr/vod/programContents/preview/952/6088/1054797?next=1',
+        'md5': 'c711103c72aeac8323a5cf1751f10097',
+        'info_dict': {
+            'id': '1054797',
+            'ext': 'mp4',
+            'title': '[2차 티저] MBN 주말 미니시리즈 <완벽한 결혼의 정석> l 그녀에게 주어진 두 번째 인생',
+            'duration': 65,
+            'release_date': '20231028',
+            'thumbnail': 'http://img.vod.mbn.co.kr/vod2/952/2023/09/11/20230911130223_22_952_1054797_1080_7.jpg',
+            'series': '완벽한 결혼의 정석',
+        },
+    }]
+
+    def _real_extract(self, url):
+        content_id = self._match_id(url)
+        webpage = self._download_webpage(url, content_id)
+
+        content_cls_cd = self._search_regex(
+            r'"\?content_cls_cd=(\d+)&', webpage, 'content cls cd', fatal=False) or '20'
+        media_info = self._download_json(
+            'https://www.mbn.co.kr/player/mbnVodPlayer_2020.mbn', content_id,
+            note='Fetching playback data', query={
+                'content_cls_cd': content_cls_cd,
+                'content_id': content_id,
+                'relay_type': '1',
+            })
+
+        formats = []
+        for stream_url in traverse_obj(media_info, ('movie_list', ..., 'url', {url_or_none})):
+            stream_url = re.sub(r'/(?:chunk|play)list(?:_pd\d+)?\.m3u8', '/manifest.m3u8', stream_url)
+            final_url = url_or_none(self._download_webpage(
+                f'https://www.mbn.co.kr/player/mbnStreamAuth_new_vod.mbn?vod_url={stream_url}',
+                content_id, note='Fetching authenticated m3u8 url'))
+
+            formats.extend(self._extract_m3u8_formats(final_url, content_id, fatal=False))
+
+        return {
+            'id': content_id,
+            **traverse_obj(media_info, {
+                'title': ('movie_title', {str}),
+                'duration': ('play_sec', {int_or_none}),
+                'release_date': ('bcast_date', {lambda x: x.replace('.', '')}, {unified_strdate}),
+                'thumbnail': ('movie_start_Img', {url_or_none}),
+                'series': ('prog_nm', {str}),
+                'episode_number': ('ad_contentnumber', {int_or_none}),
+            }),
+            'formats': formats,
+        }

yt_dlp/extractor/mediaite.py

@@ -2,7 +2,7 @@ from .common import InfoExtractor
 
 
 class MediaiteIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mediaite.com(?!/category)(?:/[\w-]+){2}'
+    _VALID_URL = r'https?://(?:www\.)?mediaite\.com(?!/category)(?:/[\w-]+){2}'
     _TESTS = [{
         'url': 'https://www.mediaite.com/sports/bill-burr-roasts-nfl-for-promoting-black-lives-matter-while-scheduling-more-games-after-all-the-sht-they-know-about-cte/',
         'info_dict': {

yt_dlp/extractor/mediaset.py

@@ -127,7 +127,8 @@ class MediasetIE(ThePlatformBaseIE):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Dead link',
     }, {
         # WittyTV embed
         'url': 'https://www.wittytv.it/mauriziocostanzoshow/ultima-puntata-venerdi-25-novembre/',

yt_dlp/extractor/mocha.py

@@ -3,7 +3,7 @@ from ..utils import int_or_none, traverse_obj
 
 
 class MochaVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://video.mocha.com.vn/(?P<video_slug>[\w-]+)'
+    _VALID_URL = r'https?://video\.mocha\.com\.vn/(?P<video_slug>[\w-]+)'
     _TESTS = [{
         'url': 'http://video.mocha.com.vn/chuyen-meo-gia-su-tu-thong-diep-cuoc-song-v18694039',
         'info_dict': {

yt_dlp/extractor/movieclips.py

@@ -23,6 +23,7 @@ class MovieClipsIE(InfoExtractor):
             'uploader': 'Movieclips',
         },
         'add_ie': ['ThePlatform'],
+        'skip': 'redirects to YouTube',
     }
 
     def _real_extract(self, url):

yt_dlp/extractor/mtv.py

@@ -1,4 +1,5 @@
 import re
+import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -137,7 +138,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         mediagen_doc = self._download_xml(
             mediagen_url, video_id, 'Downloading video urls', fatal=False)
 
-        if mediagen_doc is False:
+        if not isinstance(mediagen_doc, xml.etree.ElementTree.Element):
             return None
 
         item = mediagen_doc.find('./video/item')

yt_dlp/extractor/nationalgeographic.py

@@ -24,6 +24,7 @@ class NationalGeographicVideoIE(InfoExtractor):
                 'uploader': 'NAGS',
             },
             'add_ie': ['ThePlatform'],
+            'skip': 'Redirects to main page',
         },
         {
             'url': 'http://video.nationalgeographic.com/wild/when-sharks-attack/the-real-jaws',
@@ -38,6 +39,7 @@ class NationalGeographicVideoIE(InfoExtractor):
                 'uploader': 'NAGS',
             },
             'add_ie': ['ThePlatform'],
+            'skip': 'Redirects to main page',
         },
     ]
 
@@ -75,6 +77,7 @@ class NationalGeographicTVIE(FOXIE):  # XXX: Do not subclass from concrete IE
         'params': {
             'skip_download': True,
         },
+        'skip': 'Content not available',
     }]
     _HOME_PAGE_URL = 'https://www.nationalgeographic.com/tv/'
     _API_KEY = '238bb0a0c2aba67922c48709ce0c06fd'

yt_dlp/extractor/nbc.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import re
+import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from .theplatform import ThePlatformIE, default_ns
@@ -284,7 +285,7 @@ class NBCSportsIE(InfoExtractor):
 
     _TESTS = [{
         # iframe src
-        'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
+        'url': 'https://www.nbcsports.com/watch/nfl/profootballtalk/pft-pm/unpacking-addisons-reckless-driving-citation',
         'info_dict': {
             'id': 'PHJSaFWbrTY9',
             'ext': 'mp4',
@@ -379,7 +380,7 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
     _TESTS = [
         {
             'url': 'http://www.nbcnews.com/watch/nbcnews-com/how-twitter-reacted-to-the-snowden-interview-269389891880',
-            'md5': 'cf4bc9e6ce0130f00f545d80ecedd4bf',
+            'md5': 'fb3dcd2d7b1dd9804305fa2fc95ab610',  # md5 tends to fluctuate
             'info_dict': {
                 'id': '269389891880',
                 'ext': 'mp4',
@@ -387,6 +388,8 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
                 'timestamp': 1401363060,
                 'upload_date': '20140529',
+                'duration': 46.0,
+                'thumbnail': 'https://media-cldnry.s-nbcnews.com/image/upload/MSNBC/Components/Video/140529/p_tweet_snow_140529.jpg',
             },
         },
         {
@@ -402,7 +405,7 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
         },
         {
             'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
-            'md5': '8eb831eca25bfa7d25ddd83e85946548',
+            'md5': '40d0e48c68896359c80372306ece0fc3',
             'info_dict': {
                 'id': '394064451844',
                 'ext': 'mp4',
@@ -410,11 +413,13 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
                 'timestamp': 1423104900,
                 'upload_date': '20150205',
+                'duration': 1236.0,
+                'thumbnail': 'https://media-cldnry.s-nbcnews.com/image/upload/MSNBC/Components/Video/__NEW/nn_netcast_150204.jpg',
             },
         },
         {
             'url': 'http://www.nbcnews.com/business/autos/volkswagen-11-million-vehicles-could-have-suspect-software-emissions-scandal-n431456',
-            'md5': '4a8c4cec9e1ded51060bdda36ff0a5c0',
+            'md5': 'ffb59bcf0733dc3c7f0ace907f5e3939',
             'info_dict': {
                 'id': 'n431456',
                 'ext': 'mp4',
@@ -422,11 +427,13 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:d22d1281a24f22ea0880741bb4dd6301',
                 'upload_date': '20150922',
                 'timestamp': 1442917800,
+                'duration': 37.0,
+                'thumbnail': 'https://media-cldnry.s-nbcnews.com/image/upload/MSNBC/Components/Video/__NEW/x_lon_vwhorn_150922.jpg',
             },
         },
         {
             'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
-            'md5': '118d7ca3f0bea6534f119c68ef539f71',
+            'md5': '693d1fa21d23afcc9b04c66b227ed9ff',
             'info_dict': {
                 'id': '669831235788',
                 'ext': 'mp4',
@@ -434,6 +441,8 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
                 'upload_date': '20160420',
                 'timestamp': 1461152093,
+                'duration': 69.0,
+                'thumbnail': 'https://media-cldnry.s-nbcnews.com/image/upload/MSNBC/Components/Video/201604/2016-04-20T11-35-09-133Z--1280x720.jpg',
             },
         },
         {
@@ -447,6 +456,7 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'timestamp': 1406937606,
                 'upload_date': '20140802',
+                'duration': 940.0,
             },
         },
         {
@@ -535,6 +545,7 @@ class NBCOlympicsIE(InfoExtractor):
             'upload_date': '20160815',
             'uploader': 'NBCU-SPORTS',
         },
+        'skip': '404 Not Found',
     }
 
     def _real_extract(self, url):
@@ -578,6 +589,7 @@ class NBCOlympicsStreamIE(AdobePassIE):
             'params': {
                 'skip_download': 'm3u8',
             },
+            'skip': 'Livestream',
         }, {
             'note': 'Plain m3u8 source URL',
             'url': 'https://stream.nbcolympics.com/gymnastics-event-finals-mens-floor-pommel-horse-womens-vault-bars',
@@ -589,6 +601,7 @@ class NBCOlympicsStreamIE(AdobePassIE):
             'params': {
                 'skip_download': 'm3u8',
             },
+            'skip': 'Livestream',
         },
     ]
 
@@ -791,8 +804,10 @@ class NBCStationsIE(InfoExtractor):
             smil = self._download_xml(
                 f'https://link.theplatform.com/s/{pdk_acct}/{player_id}', video_id,
                 note='Downloading SMIL data', query=query, fatal=is_live)
-        subtitles = self._parse_smil_subtitles(smil, default_ns) if smil else {}
+            if not isinstance(smil, xml.etree.ElementTree.Element):
-        for video in smil.findall(self._xpath_ns('.//video', default_ns)) if smil else []:
+                smil = None
+        subtitles = self._parse_smil_subtitles(smil, default_ns) if smil is not None else {}
+        for video in smil.findall(self._xpath_ns('.//video', default_ns)) if smil is not None else []:
             info['duration'] = float_or_none(remove_end(video.get('dur'), 'ms'), 1000)
             video_src_url = video.get('src')
             ext = mimetype2ext(video.get('type'), default=determine_ext(video_src_url))

yt_dlp/extractor/neteasemusic.py

@@ -2,105 +2,74 @@ import itertools
 import json
 import re
 import time
-from base64 import b64encode
-from binascii import hexlify
-from datetime import datetime
 from hashlib import md5
 from random import randint
 
 from .common import InfoExtractor
 from ..aes import aes_ecb_encrypt, pkcs7_padding
-from ..compat import compat_urllib_parse_urlencode
-from ..networking import Request
 from ..utils import (
     ExtractorError,
-    bytes_to_intlist,
-    error_to_compat_str,
-    float_or_none,
     int_or_none,
-    intlist_to_bytes,
+    join_nonempty,
-    try_get,
+    str_or_none,
+    strftime_or_none,
+    traverse_obj,
+    unified_strdate,
+    url_or_none,
+    urljoin,
+    variadic,
 )
 
 
 class NetEaseMusicBaseIE(InfoExtractor):
     _FORMATS = ['bMusic', 'mMusic', 'hMusic']
-    _NETEASE_SALT = '3go8&$8*3*3h0k(2)2'
     _API_BASE = 'http://music.163.com/api/'
+    _GEO_BYPASS = False
 
-    @classmethod
+    @staticmethod
-    def _encrypt(cls, dfsid):
+    def kilo_or_none(value):
-        salt_bytes = bytearray(cls._NETEASE_SALT.encode('utf-8'))
+        return int_or_none(value, scale=1000)
-        string_bytes = bytearray(str(dfsid).encode('ascii'))
-        salt_len = len(salt_bytes)
-        for i in range(len(string_bytes)):
-            string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
-        m = md5()
-        m.update(bytes(string_bytes))
-        result = b64encode(m.digest()).decode('ascii')
-        return result.replace('/', '_').replace('+', '-')
 
-    def make_player_api_request_data_and_headers(self, song_id, bitrate):
+    def _create_eapi_cipher(self, api_path, query_body, cookies):
-        KEY = b'e82ckenh8dichen8'
+        request_text = json.dumps({**query_body, 'header': cookies}, separators=(',', ':'))
-        URL = '/api/song/enhance/player/url'
+
-        now = int(time.time() * 1000)
+        message = f'nobody{api_path}use{request_text}md5forencrypt'.encode('latin1')
-        rand = randint(0, 1000)
+        msg_digest = md5(message).hexdigest()
-        cookie = {
+
-            'osver': None,
+        data = pkcs7_padding(list(str.encode(
-            'deviceId': None,
+            f'{api_path}-36cd479b6b5-{request_text}-36cd479b6b5-{msg_digest}')))
+        encrypted = bytes(aes_ecb_encrypt(data, list(b'e82ckenh8dichen8')))
+        return f'params={encrypted.hex().upper()}'.encode()
+
+    def _download_eapi_json(self, path, video_id, query_body, headers={}, **kwargs):
+        cookies = {
+            'osver': 'undefined',
+            'deviceId': 'undefined',
             'appver': '8.0.0',
             'versioncode': '140',
-            'mobilename': None,
+            'mobilename': 'undefined',
             'buildver': '1623435496',
             'resolution': '1920x1080',
             '__csrf': '',
             'os': 'pc',
-            'channel': None,
+            'channel': 'undefined',
-            'requestId': '{0}_{1:04}'.format(now, rand),
+            'requestId': f'{int(time.time() * 1000)}_{randint(0, 1000):04}',
+            **traverse_obj(self._get_cookies(self._API_BASE), {
+                'MUSIC_U': ('MUSIC_U', {lambda i: i.value}),
+            })
         }
-        request_text = json.dumps(
+        return self._download_json(
-            {'ids': '[{0}]'.format(song_id), 'br': bitrate, 'header': cookie},
+            urljoin('https://interface3.music.163.com/', f'/eapi{path}'), video_id,
-            separators=(',', ':'))
+            data=self._create_eapi_cipher(f'/api{path}', query_body, cookies), headers={
-        message = 'nobody{0}use{1}md5forencrypt'.format(
+                'Referer': 'https://music.163.com',
-            URL, request_text).encode('latin1')
+                'Cookie': '; '.join([f'{k}={v}' for k, v in cookies.items()]),
-        msg_digest = md5(message).hexdigest()
+                **headers,
-
+            }, **kwargs)
-        data = '{0}-36cd479b6b5-{1}-36cd479b6b5-{2}'.format(
-            URL, request_text, msg_digest)
-        data = pkcs7_padding(bytes_to_intlist(data))
-        encrypted = intlist_to_bytes(aes_ecb_encrypt(data, bytes_to_intlist(KEY)))
-        encrypted_params = hexlify(encrypted).decode('ascii').upper()
-
-        cookie = '; '.join(
-            ['{0}={1}'.format(k, v if v is not None else 'undefined')
-             for [k, v] in cookie.items()])
-
-        headers = {
-            'User-Agent': self.extractor.get_param('http_headers')['User-Agent'],
-            'Content-Type': 'application/x-www-form-urlencoded',
-            'Referer': 'https://music.163.com',
-            'Cookie': cookie,
-        }
-        return ('params={0}'.format(encrypted_params), headers)
 
     def _call_player_api(self, song_id, bitrate):
-        url = 'https://interface3.music.163.com/eapi/song/enhance/player/url'
+        return self._download_eapi_json(
-        data, headers = self.make_player_api_request_data_and_headers(song_id, bitrate)
+            '/song/enhance/player/url', song_id, {'ids': f'[{song_id}]', 'br': bitrate},
-        try:
+            note=f'Downloading song URL info: bitrate {bitrate}')
-            msg = 'empty result'
-            result = self._download_json(
-                url, song_id, data=data.encode('ascii'), headers=headers)
-            if result:
-                return result
-        except ExtractorError as e:
-            if type(e.cause) in (ValueError, TypeError):
-                # JSON load failure
-                raise
-        except Exception as e:
-            msg = error_to_compat_str(e)
-            self.report_warning('%s API call (%s) failed: %s' % (
-                song_id, bitrate, msg))
-        return {}
 
     def extract_formats(self, info):
         err = 0
@@ -110,45 +79,50 @@ class NetEaseMusicBaseIE(InfoExtractor):
             details = info.get(song_format)
             if not details:
                 continue
-
             bitrate = int_or_none(details.get('bitrate')) or 999000
-            data = self._call_player_api(song_id, bitrate)
+            for song in traverse_obj(self._call_player_api(song_id, bitrate), ('data', lambda _, v: url_or_none(v['url']))):
-            for song in try_get(data, lambda x: x['data'], list) or []:
+                song_url = song['url']
-                song_url = try_get(song, lambda x: x['url'])
-                if not song_url:
-                    continue
                 if self._is_valid_url(song_url, info['id'], 'song'):
                     formats.append({
                         'url': song_url,
-                        'ext': details.get('extension'),
-                        'abr': float_or_none(song.get('br'), scale=1000),
                         'format_id': song_format,
-                        'filesize': int_or_none(song.get('size')),
+                        'asr': traverse_obj(details, ('sr', {int_or_none})),
-                        'asr': int_or_none(details.get('sr')),
+                        **traverse_obj(song, {
+                            'ext': ('type', {str}),
+                            'abr': ('br', {self.kilo_or_none}),
+                            'filesize': ('size', {int_or_none}),
+                        }),
                     })
                 elif err == 0:
-                    err = try_get(song, lambda x: x['code'], int)
+                    err = traverse_obj(song, ('code', {int})) or 0
 
         if not formats:
-            msg = 'No media links found'
             if err != 0 and (err < 200 or err >= 400):
-                raise ExtractorError(
+                raise ExtractorError(f'No media links found (site code {err})', expected=True)
-                    '%s (site code %d)' % (msg, err, ), expected=True)
             else:
                 self.raise_geo_restricted(
-                    msg + ': probably this video is not available from your location due to geo restriction.',
+                    'No media links found: probably due to geo restriction.', countries=['CN'])
-                    countries=['CN'])
-
         return formats
 
-    @classmethod
-    def convert_milliseconds(cls, ms):
-        return int(round(ms / 1000.0))
-
     def query_api(self, endpoint, video_id, note):
-        req = Request('%s%s' % (self._API_BASE, endpoint))
+        result = self._download_json(
-        req.headers['Referer'] = self._API_BASE
+            f'{self._API_BASE}{endpoint}', video_id, note, headers={'Referer': self._API_BASE})
-        return self._download_json(req, video_id, note)
+        code = traverse_obj(result, ('code', {int}))
+        message = traverse_obj(result, ('message', {str})) or ''
+        if code == -462:
+            self.raise_login_required(f'Login required to download: {message}')
+        elif code != 200:
+            raise ExtractorError(f'Failed to get meta info: {code} {message}')
+        return result
+
+    def _get_entries(self, songs_data, entry_keys=None, id_key='id', name_key='name'):
+        for song in traverse_obj(songs_data, (
+                *variadic(entry_keys, (str, bytes, dict, set)),
+                lambda _, v: int_or_none(v[id_key]) is not None)):
+            song_id = str(song[id_key])
+            yield self.url_result(
+                f'http://music.163.com/#/song?id={song_id}', NetEaseMusicIE,
+                song_id, traverse_obj(song, (name_key, {str})))
 
 
 class NetEaseMusicIE(NetEaseMusicBaseIE):
@@ -156,16 +130,21 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
     IE_DESC = '网易云音乐'
     _VALID_URL = r'https?://(y\.)?music\.163\.com/(?:[#m]/)?song\?.*?\bid=(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://music.163.com/#/song?id=32102397',
+        'url': 'https://music.163.com/#/song?id=548648087',
-        'md5': '3e909614ce09b1ccef4a3eb205441190',
         'info_dict': {
-            'id': '32102397',
+            'id': '548648087',
             'ext': 'mp3',
-            'title': 'Bad Blood',
+            'title': '戒烟 (Live)',
-            'creator': 'Taylor Swift / Kendrick Lamar',
+            'creator': '李荣浩 / 朱正廷 / 陈立农 / 尤长靖 / ONER灵超 / ONER木子洋 / 杨非同 / 陆定昊',
-            'upload_date': '20150516',
+            'timestamp': 1522944000,
-            'timestamp': 1431792000,
+            'upload_date': '20180405',
-            'description': 'md5:25fc5f27e47aad975aa6d36382c7833c',
+            'description': 'md5:3650af9ee22c87e8637cb2dde22a765c',
+            'subtitles': {'lyrics': [{'ext': 'lrc'}]},
+            "duration": 256,
+            'thumbnail': r're:^http.*\.jpg',
+            'album': '偶像练习生 表演曲目合集',
+            'average_rating': int,
+            'album_artist': '偶像练习生',
         },
     }, {
         'note': 'No lyrics.',
@@ -176,21 +155,12 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'title': 'Opus 28',
             'creator': 'Dustin O\'Halloran',
             'upload_date': '20080211',
-            'description': 'md5:f12945b0f6e0365e3b73c5032e1b0ff4',
             'timestamp': 1202745600,
-        },
+            'duration': 263,
-    }, {
+            'thumbnail': r're:^http.*\.jpg',
-        'note': 'Has translated name.',
+            'album': 'Piano Solos Vol. 2',
-        'url': 'http://music.163.com/#/song?id=22735043',
+            'album_artist': 'Dustin O\'Halloran',
-        'info_dict': {
+            'average_rating': int,
-            'id': '22735043',
-            'ext': 'mp3',
-            'title': '소원을 말해봐 (Genie)',
-            'creator': '少女时代',
-            'description': 'md5:79d99cc560e4ca97e0c4d86800ee4184',
-            'upload_date': '20100127',
-            'timestamp': 1264608000,
-            'alt_title': '说出愿望吧(Genie)',
         },
     }, {
         'url': 'https://y.music.163.com/m/song?app_version=8.8.45&id=95670&uct2=sKnvS4+0YStsWkqsPhFijw%3D%3D&dlt=0846',
@@ -203,59 +173,111 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'upload_date': '19911130',
             'timestamp': 691516800,
             'description': 'md5:1ba2f911a2b0aa398479f595224f2141',
+            'subtitles': {'lyrics': [{'ext': 'lrc'}]},
+            'duration': 268,
+            'alt_title': '伴唱:现代人乐队 合唱:总政歌舞团',
+            'thumbnail': r're:^http.*\.jpg',
+            'average_rating': int,
+            'album': '红色摇滚',
+            'album_artist': '侯牧人',
         },
+    }, {
+        'url': 'http://music.163.com/#/song?id=32102397',
+        'md5': '3e909614ce09b1ccef4a3eb205441190',
+        'info_dict': {
+            'id': '32102397',
+            'ext': 'mp3',
+            'title': 'Bad Blood',
+            'creator': 'Taylor Swift / Kendrick Lamar',
+            'upload_date': '20150516',
+            'timestamp': 1431792000,
+            'description': 'md5:21535156efb73d6d1c355f95616e285a',
+            'subtitles': {'lyrics': [{'ext': 'lrc'}]},
+            'duration': 199,
+            'thumbnail': r're:^http.*\.jpg',
+            'album': 'Bad Blood',
+            'average_rating': int,
+            'album_artist': 'Taylor Swift',
+        },
+        'skip': 'Blocked outside Mainland China',
+    }, {
+        'note': 'Has translated name.',
+        'url': 'http://music.163.com/#/song?id=22735043',
+        'info_dict': {
+            'id': '22735043',
+            'ext': 'mp3',
+            'title': '소원을 말해봐 (Genie)',
+            'creator': '少女时代',
+            'upload_date': '20100127',
+            'timestamp': 1264608000,
+            'description': 'md5:03d1ffebec3139aa4bafe302369269c5',
+            'subtitles': {'lyrics': [{'ext': 'lrc'}]},
+            'duration': 229,
+            'alt_title': '说出愿望吧(Genie)',
+            'thumbnail': r're:^http.*\.jpg',
+            'average_rating': int,
+            'album': 'Oh!',
+            'album_artist': '少女时代',
+        },
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _process_lyrics(self, lyrics_info):
-        original = lyrics_info.get('lrc', {}).get('lyric')
+        original = traverse_obj(lyrics_info, ('lrc', 'lyric', {str}))
-        translated = lyrics_info.get('tlyric', {}).get('lyric')
+        translated = traverse_obj(lyrics_info, ('tlyric', 'lyric', {str}))
+
+        if not original or original == '[99:00.00]纯音乐，请欣赏\n':
+            return None
 
         if not translated:
-            return original
+            return {
+                'lyrics': [{'data': original, 'ext': 'lrc'}],
+            }
 
         lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
         original_ts_texts = re.findall(lyrics_expr, original)
-        translation_ts_dict = dict(
+        translation_ts_dict = dict(re.findall(lyrics_expr, translated))
-            (time_stamp, text) for time_stamp, text in re.findall(lyrics_expr, translated)
+
-        )
+        merged = '\n'.join(
-        lyrics = '\n'.join([
+            join_nonempty(f'{timestamp}{text}', translation_ts_dict.get(timestamp, ''), delim=' / ')
-            '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
+            for timestamp, text in original_ts_texts)
-            for time_stamp, text in original_ts_texts
+
-        ])
+        return {
-        return lyrics
+            'lyrics_merged': [{'data': merged, 'ext': 'lrc'}],
+            'lyrics': [{'data': original, 'ext': 'lrc'}],
+            'lyrics_translated': [{'data': translated, 'ext': 'lrc'}],
+        }
 
     def _real_extract(self, url):
         song_id = self._match_id(url)
 
-        params = {
-            'id': song_id,
-            'ids': '[%s]' % song_id
-        }
         info = self.query_api(
-            'song/detail?' + compat_urllib_parse_urlencode(params),
+            f'song/detail?id={song_id}&ids=%5B{song_id}%5D', song_id, 'Downloading song info')['songs'][0]
-            song_id, 'Downloading song info')['songs'][0]
 
         formats = self.extract_formats(info)
 
-        lyrics_info = self.query_api(
+        lyrics = self._process_lyrics(self.query_api(
-            'song/lyric?id=%s&lv=-1&tv=-1' % song_id,
+            f'song/lyric?id={song_id}&lv=-1&tv=-1', song_id, 'Downloading lyrics data'))
-            song_id, 'Downloading lyrics data')
+        lyric_data = {
-        lyrics = self._process_lyrics(lyrics_info)
+            'description': traverse_obj(lyrics, (('lyrics_merged', 'lyrics'), 0, 'data'), get_all=False),
-
+            'subtitles': lyrics,
-        alt_title = None
+        } if lyrics else {}
-        if info.get('transNames'):
-            alt_title = '/'.join(info.get('transNames'))
 
         return {
             'id': song_id,
-            'title': info['name'],
-            'alt_title': alt_title,
-            'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
-            'timestamp': self.convert_milliseconds(info.get('album', {}).get('publishTime')),
-            'thumbnail': info.get('album', {}).get('picUrl'),
-            'duration': self.convert_milliseconds(info.get('duration', 0)),
-            'description': lyrics,
             'formats': formats,
+            'alt_title': '/'.join(traverse_obj(info, (('transNames', 'alias'), ...))) or None,
+            'creator': ' / '.join(traverse_obj(info, ('artists', ..., 'name'))) or None,
+            'album_artist': ' / '.join(traverse_obj(info, ('album', 'artists', ..., 'name'))) or None,
+            **lyric_data,
+            **traverse_obj(info, {
+                'title': ('name', {str}),
+                'timestamp': ('album', 'publishTime', {self.kilo_or_none}),
+                'thumbnail': ('album', 'picUrl', {url_or_none}),
+                'duration': ('duration', {self.kilo_or_none}),
+                'album': ('album', 'name', {str}),
+                'average_rating': ('score', {int_or_none}),
+            }),
         }
 
 
@@ -263,31 +285,44 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:album'
     IE_DESC = '网易云音乐 - 专辑'
     _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
+        'url': 'https://music.163.com/#/album?id=133153666',
+        'info_dict': {
+            'id': '133153666',
+            'title': '桃几的翻唱',
+            'upload_date': '20210913',
+            'description': '桃几2021年翻唱合集',
+            'thumbnail': r're:^http.*\.jpg',
+        },
+        'playlist_mincount': 13,
+    }, {
         'url': 'http://music.163.com/#/album?id=220780',
         'info_dict': {
             'id': '220780',
-            'title': 'B\'day',
+            'title': 'B\'Day',
+            'upload_date': '20060904',
+            'description': 'md5:71a74e1d8f392d88cf1bbe48879ad0b0',
+            'thumbnail': r're:^http.*\.jpg',
         },
         'playlist_count': 23,
-        'skip': 'Blocked outside Mainland China',
+    }]
-    }
 
     def _real_extract(self, url):
         album_id = self._match_id(url)
+        webpage = self._download_webpage(f'https://music.163.com/album?id={album_id}', album_id)
 
-        info = self.query_api(
+        songs = self._search_json(
-            'album/%s?id=%s' % (album_id, album_id),
+            r'<textarea[^>]+\bid="song-list-pre-data"[^>]*>', webpage, 'metainfo', album_id,
-            album_id, 'Downloading album data')['album']
+            end_pattern=r'</textarea>', contains_pattern=r'\[(?s:.+)\]')
-
+        metainfo = {
-        name = info['name']
+            'title': self._og_search_property('title', webpage, 'title', fatal=False),
-        desc = info.get('description')
+            'description': self._html_search_regex(
-        entries = [
+                (rf'<div[^>]+\bid="album-desc-{suffix}"[^>]*>(.*?)</div>' for suffix in ('more', 'dot')),
-            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                webpage, 'description', flags=re.S, fatal=False),
-                            'NetEaseMusic', song['id'])
+            'thumbnail': self._og_search_property('image', webpage, 'thumbnail', fatal=False),
-            for song in info['songs']
+            'upload_date': unified_strdate(self._html_search_meta('music:release_date', webpage, 'date', fatal=False)),
-        ]
+        }
-        return self.playlist_result(entries, album_id, name, desc)
+        return self.playlist_result(self._get_entries(songs), album_id, **metainfo)
 
 
 class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
@@ -299,10 +334,9 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
         'url': 'http://music.163.com/#/artist?id=10559',
         'info_dict': {
             'id': '10559',
-            'title': '张惠妹 - aMEI;阿密特',
+            'title': '张惠妹 - aMEI;阿妹;阿密特',
         },
         'playlist_count': 50,
-        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Singer has translated name.',
         'url': 'http://music.163.com/#/artist?id=124098',
@@ -311,28 +345,28 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
             'title': '李昇基 - 이승기',
         },
         'playlist_count': 50,
-        'skip': 'Blocked outside Mainland China',
+    }, {
+        'note': 'Singer with both translated and alias',
+        'url': 'https://music.163.com/#/artist?id=159692',
+        'info_dict': {
+            'id': '159692',
+            'title': '初音ミク - 初音未来;Hatsune Miku',
+        },
+        'playlist_count': 50,
     }]
 
     def _real_extract(self, url):
         singer_id = self._match_id(url)
 
         info = self.query_api(
-            'artist/%s?id=%s' % (singer_id, singer_id),
+            f'artist/{singer_id}?id={singer_id}', singer_id, note='Downloading singer data')
-            singer_id, 'Downloading singer data')
 
-        name = info['artist']['name']
+        name = join_nonempty(
-        if info['artist']['trans']:
+            traverse_obj(info, ('artist', 'name', {str})),
-            name = '%s - %s' % (name, info['artist']['trans'])
+            join_nonempty(*traverse_obj(info, ('artist', ('trans', ('alias', ...)), {str})), delim=';'),
-        if info['artist']['alias']:
+            delim=' - ')
-            name = '%s - %s' % (name, ';'.join(info['artist']['alias']))
 
-        entries = [
+        return self.playlist_result(self._get_entries(info, 'hotSongs'), singer_id, name)
-            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
-                            'NetEaseMusic', song['id'])
-            for song in info['hotSongs']
-        ]
-        return self.playlist_result(entries, singer_id, name)
 
 
 class NetEaseMusicListIE(NetEaseMusicBaseIE):
@@ -344,10 +378,28 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
         'info_dict': {
             'id': '79177352',
             'title': 'Billboard 2007 Top 100',
-            'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
+            'description': 'md5:12fd0819cab2965b9583ace0f8b7b022',
+            'tags': ['欧美'],
+            'uploader': '浑然破灭',
+            'uploader_id': '67549805',
+            'timestamp': int,
+            'upload_date': r're:\d{8}',
         },
-        'playlist_count': 99,
+        'playlist_mincount': 95,
-        'skip': 'Blocked outside Mainland China',
+    }, {
+        'note': 'Toplist/Charts sample',
+        'url': 'https://music.163.com/#/discover/toplist?id=60198',
+        'info_dict': {
+            'id': '60198',
+            'title': 're:美国Billboard榜 [0-9]{4}-[0-9]{2}-[0-9]{2}',
+            'description': '美国Billboard排行榜',
+            'tags': ['流行', '欧美', '榜单'],
+            'uploader': 'Billboard公告牌',
+            'uploader_id': '48171',
+            'timestamp': int,
+            'upload_date': r're:\d{8}',
+        },
+        'playlist_count': 100,
     }, {
         'note': 'Toplist/Charts sample',
         'url': 'http://music.163.com/#/discover/toplist?id=3733003',
@@ -363,64 +415,86 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
-        info = self.query_api(
+        info = self._download_eapi_json(
-            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id,
+            '/v3/playlist/detail', list_id,
-            list_id, 'Downloading playlist data')['result']
+            {'id': list_id, 't': '-1', 'n': '500', 's': '0'},
+            note="Downloading playlist info")
 
-        name = info['name']
+        metainfo = traverse_obj(info, ('playlist', {
-        desc = info.get('description')
+            'title': ('name', {str}),
+            'description': ('description', {str}),
+            'tags': ('tags', ..., {str}),
+            'uploader': ('creator', 'nickname', {str}),
+            'uploader_id': ('creator', 'userId', {str_or_none}),
+            'timestamp': ('updateTime', {self.kilo_or_none}),
+        }))
+        if traverse_obj(info, ('playlist', 'specialType')) == 10:
+            metainfo['title'] = f'{metainfo.get("title")} {strftime_or_none(metainfo.get("timestamp"), "%Y-%m-%d")}'
 
-        if info.get('specialType') == 10:  # is a chart/toplist
+        return self.playlist_result(self._get_entries(info, ('playlist', 'tracks')), list_id, **metainfo)
-            datestamp = datetime.fromtimestamp(
-                self.convert_milliseconds(info['updateTime'])).strftime('%Y-%m-%d')
-            name = '%s %s' % (name, datestamp)
-
-        entries = [
-            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
-                            'NetEaseMusic', song['id'])
-            for song in info['tracks']
-        ]
-        return self.playlist_result(entries, list_id, name, desc)
 
 
 class NetEaseMusicMvIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:mv'
     IE_DESC = '网易云音乐 - MV'
     _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
+        'url': 'https://music.163.com/#/mv?id=10958064',
+        'info_dict': {
+            'id': '10958064',
+            'ext': 'mp4',
+            'title': '交换余生',
+            'description': 'md5:e845872cff28820642a2b02eda428fea',
+            'creator': '林俊杰',
+            'upload_date': '20200916',
+            'thumbnail': r're:http.*\.jpg',
+            'duration': 364,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+    }, {
         'url': 'http://music.163.com/#/mv?id=415350',
         'info_dict': {
             'id': '415350',
             'ext': 'mp4',
             'title': '이럴거면 그러지말지',
             'description': '白雅言自作曲唱甜蜜爱情',
-            'creator': '白雅言',
+            'creator': '白娥娟',
             'upload_date': '20150520',
+            'thumbnail': r're:http.*\.jpg',
+            'duration': 216,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
         },
-        'skip': 'Blocked outside Mainland China',
+    }]
-    }
 
     def _real_extract(self, url):
         mv_id = self._match_id(url)
 
         info = self.query_api(
-            'mv/detail?id=%s&type=mp4' % mv_id,
+            f'mv/detail?id={mv_id}&type=mp4', mv_id, 'Downloading mv info')['data']
-            mv_id, 'Downloading mv info')['data']
 
         formats = [
-            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'height': int(brs)}
+            {'url': mv_url, 'ext': 'mp4', 'format_id': f'{brs}p', 'height': int_or_none(brs)}
             for brs, mv_url in info['brs'].items()
         ]
 
         return {
             'id': mv_id,
-            'title': info['name'],
-            'description': info.get('desc') or info.get('briefDesc'),
-            'creator': info['artistName'],
-            'upload_date': info['publishTime'].replace('-', ''),
             'formats': formats,
-            'thumbnail': info.get('cover'),
+            **traverse_obj(info, {
-            'duration': self.convert_milliseconds(info.get('duration', 0)),
+                'title': ('name', {str}),
+                'description': (('desc', 'briefDesc'), {str}, {lambda x: x or None}),
+                'creator': ('artistName', {str}),
+                'upload_date': ('publishTime', {unified_strdate}),
+                'thumbnail': ('cover', {url_or_none}),
+                'duration': ('duration', {self.kilo_or_none}),
+                'view_count': ('playCount', {int_or_none}),
+                'like_count': ('likeCount', {int_or_none}),
+                'comment_count': ('commentCount', {int_or_none}),
+            }, get_all=False),
         }
 
 
@@ -431,75 +505,74 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
     _TESTS = [{
         'url': 'http://music.163.com/#/program?id=10109055',
         'info_dict': {
-            'id': '10109055',
+            'id': '32593346',
             'ext': 'mp3',
             'title': '不丹足球背后的故事',
             'description': '喜马拉雅人的足球梦 ...',
             'creator': '大话西藏',
-            'timestamp': 1434179342,
+            'timestamp': 1434179287,
             'upload_date': '20150613',
+            'thumbnail': r're:http.*\.jpg',
             'duration': 900,
         },
-        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'This program has accompanying songs.',
         'url': 'http://music.163.com/#/program?id=10141022',
         'info_dict': {
             'id': '10141022',
-            'title': '25岁，你是自在如风的少年<27°C>',
+            'title': '滚滚电台的有声节目',
             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
+            'creator': '滚滚电台ORZ',
+            'timestamp': 1434450733,
+            'upload_date': '20150616',
+            'thumbnail': r're:http.*\.jpg',
         },
         'playlist_count': 4,
-        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'This program has accompanying songs.',
         'url': 'http://music.163.com/#/program?id=10141022',
         'info_dict': {
-            'id': '10141022',
+            'id': '32647209',
             'ext': 'mp3',
-            'title': '25岁，你是自在如风的少年<27°C>',
+            'title': '滚滚电台的有声节目',
             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
-            'timestamp': 1434450841,
+            'creator': '滚滚电台ORZ',
+            'timestamp': 1434450733,
             'upload_date': '20150616',
+            'thumbnail': r're:http.*\.jpg',
+            'duration': 1104,
         },
         'params': {
             'noplaylist': True
         },
-        'skip': 'Blocked outside Mainland China',
     }]
 
     def _real_extract(self, url):
         program_id = self._match_id(url)
 
         info = self.query_api(
-            'dj/program/detail?id=%s' % program_id,
+            f'dj/program/detail?id={program_id}', program_id, note='Downloading program info')['program']
-            program_id, 'Downloading program info')['program']
 
-        name = info['name']
+        metainfo = traverse_obj(info, {
-        description = info['description']
+            'title': ('name', {str}),
+            'description': ('description', {str}),
+            'creator': ('dj', 'brand', {str}),
+            'thumbnail': ('coverUrl', {url_or_none}),
+            'timestamp': ('createTime', {self.kilo_or_none}),
+        })
 
         if not self._yes_playlist(info['songs'] and program_id, info['mainSong']['id']):
             formats = self.extract_formats(info['mainSong'])
 
             return {
-                'id': info['mainSong']['id'],
+                'id': str(info['mainSong']['id']),
-                'title': name,
-                'description': description,
-                'creator': info['dj']['brand'],
-                'timestamp': self.convert_milliseconds(info['createTime']),
-                'thumbnail': info['coverUrl'],
-                'duration': self.convert_milliseconds(info.get('duration', 0)),
                 'formats': formats,
+                'duration': traverse_obj(info, ('mainSong', 'duration', {self.kilo_or_none})),
+                **metainfo,
             }
 
-        song_ids = [info['mainSong']['id']]
+        songs = traverse_obj(info, (('mainSong', ('songs', ...)),))
-        song_ids.extend([song['id'] for song in info['songs']])
+        return self.playlist_result(self._get_entries(songs), program_id, **metainfo)
-        entries = [
-            self.url_result('http://music.163.com/#/song?id=%s' % song_id,
-                            'NetEaseMusic', song_id)
-            for song_id in song_ids
-        ]
-        return self.playlist_result(entries, program_id, name, description)
 
 
 class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
@@ -511,38 +584,32 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
         'info_dict': {
             'id': '42',
             'title': '声音蔓延',
-            'description': 'md5:766220985cbd16fdd552f64c578a6b15'
+            'description': 'md5:c7381ebd7989f9f367668a5aee7d5f08'
         },
         'playlist_mincount': 40,
-        'skip': 'Blocked outside Mainland China',
     }
     _PAGE_SIZE = 1000
 
     def _real_extract(self, url):
         dj_id = self._match_id(url)
 
-        name = None
+        metainfo = {}
-        desc = None
         entries = []
         for offset in itertools.count(start=0, step=self._PAGE_SIZE):
             info = self.query_api(
-                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
+                f'dj/program/byradio?asc=false&limit={self._PAGE_SIZE}&radioId={dj_id}&offset={offset}',
-                % (self._PAGE_SIZE, dj_id, offset),
+                dj_id, note=f'Downloading dj programs - {offset}')
-                dj_id, 'Downloading dj programs - %d' % offset)
 
-            entries.extend([
+            entries.extend(self.url_result(
-                self.url_result(
+                f'http://music.163.com/#/program?id={program["id"]}', NetEaseMusicProgramIE,
-                    'http://music.163.com/#/program?id=%s' % program['id'],
+                program['id'], program.get('name')) for program in info['programs'])
-                    'NetEaseMusicProgram', program['id'])
+            if not metainfo:
-                for program in info['programs']
+                metainfo = traverse_obj(info, ('programs', 0, 'radio', {
-            ])
+                    'title': ('name', {str}),
-
+                    'description': ('desc', {str}),
-            if name is None:
+                }))
-                radio = info['programs'][0]['radio']
-                name = radio['name']
-                desc = radio['desc']
 
             if not info['more']:
                 break
 
-        return self.playlist_result(entries, dj_id, name, desc)
+        return self.playlist_result(entries, dj_id, **metainfo)

yt_dlp/extractor/nfl.py

@@ -247,7 +247,7 @@ class NFLArticleIE(NFLBaseIE):
 
 class NFLPlusReplayIE(NFLBaseIE):
     IE_NAME = 'nfl.com:plus:replay'
-    _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/games/(?P<slug>[\w-]+)(?:/(?P<id>\d+))?'
+    _VALID_URL = r'https?://(?:www\.)?nfl\.com/plus/games/(?P<slug>[\w-]+)(?:/(?P<id>\d+))?'
     _TESTS = [{
         'url': 'https://www.nfl.com/plus/games/giants-at-vikings-2022-post-1/1572108',
         'info_dict': {
@@ -342,7 +342,7 @@ class NFLPlusReplayIE(NFLBaseIE):
 
 class NFLPlusEpisodeIE(NFLBaseIE):
     IE_NAME = 'nfl.com:plus:episode'
-    _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/episodes/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?nfl\.com/plus/episodes/(?P<id>[\w-]+)'
     _TESTS = [{
         'note': 'Subscription required',
         'url': 'https://www.nfl.com/plus/episodes/kurt-s-qb-insider-conference-championships',

yt_dlp/extractor/nhk.py

@@ -3,6 +3,8 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    clean_html,
+    get_element_by_class,
     int_or_none,
     join_nonempty,
     parse_duration,
@@ -28,21 +30,71 @@ class NhkBaseIE(InfoExtractor):
                 m_id, lang, '/all' if is_video else ''),
             m_id, query={'apikey': 'EJfK8jdS57GqlupFgAfAAwr573q01y6k'})['data']['episodes'] or []
 
+    def _get_api_info(self, refresh=True):
+        if not refresh:
+            return self.cache.load('nhk', 'api_info')
+
+        self.cache.store('nhk', 'api_info', {})
+        movie_player_js = self._download_webpage(
+            'https://movie-a.nhk.or.jp/world/player/js/movie-player.js', None,
+            note='Downloading stream API information')
+        api_info = {
+            'url': self._search_regex(
+                r'prod:[^;]+\bapiUrl:\s*[\'"]([^\'"]+)[\'"]', movie_player_js, None, 'stream API url'),
+            'token': self._search_regex(
+                r'prod:[^;]+\btoken:\s*[\'"]([^\'"]+)[\'"]', movie_player_js, None, 'stream API token'),
+        }
+        self.cache.store('nhk', 'api_info', api_info)
+        return api_info
+
+    def _extract_stream_info(self, vod_id):
+        for refresh in (False, True):
+            api_info = self._get_api_info(refresh)
+            if not api_info:
+                continue
+
+            api_url = api_info.pop('url')
+            meta = traverse_obj(
+                self._download_json(
+                    api_url, vod_id, 'Downloading stream url info', fatal=False, query={
+                        **api_info,
+                        'type': 'json',
+                        'optional_id': vod_id,
+                        'active_flg': 1,
+                    }), ('meta', 0))
+            stream_url = traverse_obj(
+                meta, ('movie_url', ('mb_auto', 'auto_sp', 'auto_pc'), {url_or_none}), get_all=False)
+
+            if stream_url:
+                formats, subtitles = self._extract_m3u8_formats_and_subtitles(stream_url, vod_id)
+                return {
+                    **traverse_obj(meta, {
+                        'duration': ('duration', {int_or_none}),
+                        'timestamp': ('publication_date', {unified_timestamp}),
+                        'release_timestamp': ('insert_date', {unified_timestamp}),
+                        'modified_timestamp': ('update_date', {unified_timestamp}),
+                    }),
+                    'formats': formats,
+                    'subtitles': subtitles,
+                }
+        raise ExtractorError('Unable to extract stream url')
+
     def _extract_episode_info(self, url, episode=None):
         fetch_episode = episode is None
-        lang, m_type, episode_id = NhkVodIE._match_valid_url(url).groups()
+        lang, m_type, episode_id = NhkVodIE._match_valid_url(url).group('lang', 'type', 'id')
-        if len(episode_id) == 7:
+        is_video = m_type == 'video'
+
+        if is_video:
             episode_id = episode_id[:4] + '-' + episode_id[4:]
 
-        is_video = m_type == 'video'
         if fetch_episode:
             episode = self._call_api(
                 episode_id, lang, is_video, True, episode_id[:4] == '9999')[0]
-        title = episode.get('sub_title_clean') or episode['sub_title']
 
         def get_clean_field(key):
-            return episode.get(key + '_clean') or episode.get(key)
+            return clean_html(episode.get(key + '_clean') or episode.get(key))
 
+        title = get_clean_field('sub_title')
         series = get_clean_field('title')
 
         thumbnails = []
@@ -57,22 +109,32 @@ class NhkBaseIE(InfoExtractor):
                 'url': 'https://www3.nhk.or.jp' + img_path,
             })
 
+        episode_name = title
+        if series and title:
+            title = f'{series} - {title}'
+        elif series and not title:
+            title = series
+            series = None
+            episode_name = None
+        else:  # title, no series
+            episode_name = None
+
         info = {
             'id': episode_id + '-' + lang,
-            'title': '%s - %s' % (series, title) if series and title else title,
+            'title': title,
             'description': get_clean_field('description'),
             'thumbnails': thumbnails,
             'series': series,
-            'episode': title,
+            'episode': episode_name,
         }
+
         if is_video:
             vod_id = episode['vod_id']
             info.update({
-                '_type': 'url_transparent',
+                **self._extract_stream_info(vod_id),
-                'ie_key': 'Piksel',
-                'url': 'https://movie-s.nhk.or.jp/v/refid/nhkworld/prefid/' + vod_id,
                 'id': vod_id,
             })
+
         else:
             if fetch_episode:
                 audio_path = episode['audio']['audio']
@@ -93,47 +155,61 @@ class NhkBaseIE(InfoExtractor):
 
 class NhkVodIE(NhkBaseIE):
     # the 7-character IDs can have alphabetic chars too: assume [a-z] rather than just [a-f], eg
-    _VALID_URL = r'%s%s(?P<id>[0-9a-z]{7}|[^/]+?-\d{8}-[0-9a-z]+)' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
+    _VALID_URL = [rf'{NhkBaseIE._BASE_URL_REGEX}/(?P<type>video)/(?P<id>[0-9a-z]+)',
+                  rf'{NhkBaseIE._BASE_URL_REGEX}/(?P<type>audio)/(?P<id>[^/?#]+?-\d{{8}}-[0-9a-z]+)']
     # Content available only for a limited period of time. Visit
     # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
     _TESTS = [{
-        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2061601/',
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2049126/',
         'info_dict': {
-            'id': 'yd8322ch',
+            'id': 'nw_vod_v_en_2049_126_20230413233000_01_1681398302',
             'ext': 'mp4',
-            'description': 'md5:109c8b05d67a62d0592f2b445d2cd898',
+            'title': 'Japan Railway Journal - The Tohoku Shinkansen: Full Speed Ahead',
-            'title': 'GRAND SUMO Highlights - [Recap] May Tournament Day 1 (Opening Day)',
+            'description': 'md5:49f7c5b206e03868a2fdf0d0814b92f6',
-            'upload_date': '20230514',
+            'thumbnail': 'md5:51bcef4a21936e7fea1ff4e06353f463',
-            'timestamp': 1684083791,
+            'episode': 'The Tohoku Shinkansen: Full Speed Ahead',
-            'series': 'GRAND SUMO Highlights',
+            'series': 'Japan Railway Journal',
-            'episode': '[Recap] May Tournament Day 1 (Opening Day)',
+            'modified_timestamp': 1694243656,
-            'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1684084443/4028649.jpg?w=1920&h=1080',
+            'timestamp': 1681428600,
+            'release_timestamp': 1693883728,
+            'duration': 1679,
+            'upload_date': '20230413',
+            'modified_date': '20230909',
+            'release_date': '20230905',
+
         },
     }, {
         # video clip
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
-        'md5': '7a90abcfe610ec22a6bfe15bd46b30ca',
+        'md5': '153c3016dfd252ba09726588149cf0e7',
         'info_dict': {
-            'id': 'a95j5iza',
+            'id': 'lpZXIwaDE6_Z-976CPsFdxyICyWUzlT5',
             'ext': 'mp4',
-            'title': "Dining with the Chef - Chef Saito's Family recipe: MENCHI-KATSU",
+            'title': 'Dining with the Chef - Chef Saito\'s Family recipe: MENCHI-KATSU',
             'description': 'md5:5aee4a9f9d81c26281862382103b0ea5',
-            'timestamp': 1565965194,
+            'thumbnail': 'md5:d6a4d9b6e9be90aaadda0bcce89631ed',
-            'upload_date': '20190816',
-            'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1567086278/3715195.jpg?w=1920&h=1080',
             'series': 'Dining with the Chef',
             'episode': 'Chef Saito\'s Family recipe: MENCHI-KATSU',
+            'duration': 148,
+            'upload_date': '20190816',
+            'release_date': '20230902',
+            'release_timestamp': 1693619292,
+            'modified_timestamp': 1694168033,
+            'modified_date': '20230908',
+            'timestamp': 1565997540,
         },
     }, {
-        # audio clip
+        # radio
-        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/r_inventions-20201104-1/',
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/livinginjapan-20231001-1/',
         'info_dict': {
-            'id': 'r_inventions-20201104-1-en',
+            'id': 'livinginjapan-20231001-1-en',
             'ext': 'm4a',
-            'title': "Japan's Top Inventions - Miniature Video Cameras",
+            'title': 'Living in Japan - Tips for Travelers to Japan / Ramen Vending Machines',
-            'description': 'md5:07ea722bdbbb4936fdd360b6a480c25b',
+            'series': 'Living in Japan',
+            'description': 'md5:0a0e2077d8f07a03071e990a6f51bfab',
+            'thumbnail': 'md5:960622fb6e06054a4a1a0c97ea752545',
+            'episode': 'Tips for Travelers to Japan / Ramen Vending Machines'
         },
-        'skip': '404 Not Found',
     }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/',
         'only_matching': True,
@@ -159,6 +235,36 @@ class NhkVodIE(NhkBaseIE):
             'timestamp': 1623722008,
         },
         'skip': '404 Not Found',
+    }, {
+        # japanese-language, longer id than english
+        'url': 'https://www3.nhk.or.jp/nhkworld/ja/ondemand/video/0020271111/',
+        'info_dict': {
+            'id': 'nw_ja_v_jvod_ohayou_20231008',
+            'ext': 'mp4',
+            'title': 'おはよう日本（7時台） - 10月8日放送',
+            'series': 'おはよう日本（7時台）',
+            'episode': '10月8日放送',
+            'thumbnail': 'md5:d733b1c8e965ab68fb02b2d347d0e9b4',
+            'description': 'md5:9c1d6cbeadb827b955b20e99ab920ff0',
+        },
+        'skip': 'expires 2023-10-15',
+    }, {
+        # a one-off (single-episode series). title from the api is just '<p></p>'
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/3004952/',
+        'info_dict': {
+            'id': 'nw_vod_v_en_3004_952_20230723091000_01_1690074552',
+            'ext': 'mp4',
+            'title': 'Barakan Discovers AMAMI OSHIMA: Isson\'s Treasure Island',
+            'description': 'md5:5db620c46a0698451cc59add8816b797',
+            'thumbnail': 'md5:67d9ff28009ba379bfa85ad1aaa0e2bd',
+            'release_date': '20230905',
+            'timestamp': 1690103400,
+            'duration': 2939,
+            'release_timestamp': 1693898699,
+            'modified_timestamp': 1698057495,
+            'modified_date': '20231023',
+            'upload_date': '20230723',
+        },
     }]
 
     def _real_extract(self, url):
@@ -166,20 +272,22 @@ class NhkVodIE(NhkBaseIE):
 
 
 class NhkVodProgramIE(NhkBaseIE):
-    _VALID_URL = r'%s/program%s(?P<id>[0-9a-z]+)(?:.+?\btype=(?P<episode_type>clip|(?:radio|tv)Episode))?' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
+    _VALID_URL = rf'{NhkBaseIE._BASE_URL_REGEX}/program{NhkBaseIE._TYPE_REGEX}(?P<id>\w+)(?:.+?\btype=(?P<episode_type>clip|(?:radio|tv)Episode))?'
     _TESTS = [{
         # video program episodes
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/sumo',
         'info_dict': {
             'id': 'sumo',
             'title': 'GRAND SUMO Highlights',
+            'description': 'md5:fc20d02dc6ce85e4b72e0273aa52fdbf',
         },
-        'playlist_mincount': 12,
+        'playlist_mincount': 0,
     }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway',
         'info_dict': {
             'id': 'japanrailway',
             'title': 'Japan Railway Journal',
+            'description': 'md5:ea39d93af7d05835baadf10d1aae0e3f',
         },
         'playlist_mincount': 12,
     }, {
@@ -188,6 +296,7 @@ class NhkVodProgramIE(NhkBaseIE):
         'info_dict': {
             'id': 'japanrailway',
             'title': 'Japan Railway Journal',
+            'description': 'md5:ea39d93af7d05835baadf10d1aae0e3f',
         },
         'playlist_mincount': 5,
     }, {
@@ -200,8 +309,7 @@ class NhkVodProgramIE(NhkBaseIE):
     }]
 
     def _real_extract(self, url):
-        lang, m_type, program_id, episode_type = self._match_valid_url(url).groups()
+        lang, m_type, program_id, episode_type = self._match_valid_url(url).group('lang', 'type', 'id', 'episode_type')
-
         episodes = self._call_api(
             program_id, lang, m_type == 'video', False, episode_type == 'clip')
 
@@ -213,11 +321,11 @@ class NhkVodProgramIE(NhkBaseIE):
             entries.append(self._extract_episode_info(
                 urljoin(url, episode_path), episode))
 
-        program_title = None
+        html = self._download_webpage(url, program_id)
-        if entries:
+        program_title = clean_html(get_element_by_class('p-programDetail__title', html))
-            program_title = entries[0].get('series')
+        program_description = clean_html(get_element_by_class('p-programDetail__text', html))
 
-        return self.playlist_result(entries, program_id, program_title)
+        return self.playlist_result(entries, program_id, program_title, program_description)
 
 
 class NhkForSchoolBangumiIE(InfoExtractor):
@@ -369,6 +477,7 @@ class NhkRadiruIE(InfoExtractor):
         'skip': 'Episode expired on 2023-04-16',
         'info_dict': {
             'channel': 'NHK-FM',
+            'uploader': 'NHK-FM',
             'description': 'md5:94b08bdeadde81a97df4ec882acce3e9',
             'ext': 'm4a',
             'id': '0449_01_3853544',
@@ -389,6 +498,7 @@ class NhkRadiruIE(InfoExtractor):
             'title': 'ベストオブクラシック',
             'description': '世界中の上質な演奏会をじっくり堪能する本格派クラシック番組。',
             'channel': 'NHK-FM',
+            'uploader': 'NHK-FM',
             'thumbnail': 'https://www.nhk.or.jp/prog/img/458/g458.jpg',
         },
         'playlist_mincount': 3,
@@ -402,6 +512,7 @@ class NhkRadiruIE(InfoExtractor):
             'title': '有島武郎「一房のぶどう」',
             'description': '朗読：川野一宇（ラジオ深夜便アンカー）\r\n\r\n（2016年12月8日放送「ラジオ深夜便『アンカー朗読シリーズ』」より）',
             'channel': 'NHKラジオ第1、NHK-FM',
+            'uploader': 'NHKラジオ第1、NHK-FM',
             'timestamp': 1635757200,
             'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F300/img/corner/box_109_thumbnail.jpg',
             'release_date': '20161207',
@@ -417,6 +528,7 @@ class NhkRadiruIE(InfoExtractor):
             'id': 'F261_01_3855109',
             'ext': 'm4a',
             'channel': 'NHKラジオ第1',
+            'uploader': 'NHKラジオ第1',
             'timestamp': 1681635900,
             'release_date': '20230416',
             'series': 'NHKラジオニュース',
@@ -461,6 +573,7 @@ class NhkRadiruIE(InfoExtractor):
         series_meta = traverse_obj(meta, {
             'title': 'program_name',
             'channel': 'media_name',
+            'uploader': 'media_name',
             'thumbnail': (('thumbnail_c', 'thumbnail_p'), {url_or_none}),
         }, get_all=False)
 
@@ -489,6 +602,7 @@ class NhkRadioNewsPageIE(InfoExtractor):
             'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F261/img/RADIONEWS_640.jpg',
             'description': 'md5:bf2c5b397e44bc7eb26de98d8f15d79d',
             'channel': 'NHKラジオ第1',
+            'uploader': 'NHKラジオ第1',
             'title': 'NHKラジオニュース',
         }
     }]

yt_dlp/extractor/njpwworld.py

@@ -1,82 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    get_element_by_class,
-    urlencode_postdata,
-)
-
-
-class NJPWWorldIE(InfoExtractor):
-    _VALID_URL = r'https?://(front\.)?njpwworld\.com/p/(?P<id>[a-z0-9_]+)'
-    IE_DESC = '新日本プロレスワールド'
-    _NETRC_MACHINE = 'njpwworld'
-
-    _TESTS = [{
-        'url': 'http://njpwworld.com/p/s_series_00155_1_9/',
-        'info_dict': {
-            'id': 's_series_00155_1_9',
-            'ext': 'mp4',
-            'title': '闘強導夢2000 2000年1月4日 東京ドーム 第9試合 ランディ・サベージ VS リック・スタイナー',
-            'tags': list,
-        },
-        'params': {
-            'skip_download': True,  # AES-encrypted m3u8
-        },
-        'skip': 'Requires login',
-    }, {
-        'url': 'https://front.njpwworld.com/p/s_series_00563_16_bs',
-        'info_dict': {
-            'id': 's_series_00563_16_bs',
-            'ext': 'mp4',
-            'title': 'WORLD TAG LEAGUE 2020 & BEST OF THE SUPER Jr.27 2020年12月6日 福岡・福岡国際センター バックステージコメント（字幕あり）',
-            'tags': ["福岡・福岡国際センター", "バックステージコメント", "2020", "20年代"],
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    _LOGIN_URL = 'https://front.njpwworld.com/auth/login'
-
-    def _perform_login(self, username, password):
-        # Setup session (will set necessary cookies)
-        self._request_webpage(
-            'https://njpwworld.com/', None, note='Setting up session')
-
-        webpage, urlh = self._download_webpage_handle(
-            self._LOGIN_URL, None,
-            note='Logging in', errnote='Unable to login',
-            data=urlencode_postdata({'login_id': username, 'pw': password}),
-            headers={'Referer': 'https://front.njpwworld.com/auth'})
-        # /auth/login will return 302 for successful logins
-        if urlh.url == self._LOGIN_URL:
-            self.report_warning('unable to login')
-            return False
-
-        return True
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        formats = []
-        for kind, vid in re.findall(r'if\s+\(\s*imageQualityType\s*==\s*\'([^\']+)\'\s*\)\s*{\s*video_id\s*=\s*"(\d+)"', webpage):
-            player_path = '/intent?id=%s&type=url' % vid
-            player_url = compat_urlparse.urljoin(url, player_path)
-            formats += self._extract_m3u8_formats(
-                player_url, video_id, 'mp4', 'm3u8_native', m3u8_id=kind, fatal=False, quality=int(kind == 'high'))
-
-        tag_block = get_element_by_class('tag-block', webpage)
-        tags = re.findall(
-            r'<a[^>]+class="tag-[^"]+"[^>]*>([^<]+)</a>', tag_block
-        ) if tag_block else None
-
-        return {
-            'id': video_id,
-            'title': get_element_by_class('article-title', webpage) or self._og_search_title(webpage),
-            'formats': formats,
-            'tags': tags,
-        }

yt_dlp/extractor/nova.py

@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class NovaEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://media\.cms\.nova\.cz/embed/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://media(?:tn)?\.cms\.nova\.cz/embed/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://media.cms.nova.cz/embed/8o0n0r?autoplay=1',
         'info_dict': {
@@ -37,6 +37,16 @@ class NovaEmbedIE(InfoExtractor):
             'duration': 114,
         },
         'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://mediatn.cms.nova.cz/embed/EU5ELEsmOHt?autoplay=1',
+        'info_dict': {
+            'id': 'EU5ELEsmOHt',
+            'ext': 'mp4',
+            'title': 'Haptické křeslo, bionická ruka nebo roboti. Reportérka se podívala na Týden inovací',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 1780,
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/novaplay.py

@@ -3,7 +3,7 @@ from ..utils import int_or_none, parse_duration, parse_iso8601
 
 
 class NovaPlayIE(InfoExtractor):
-    _VALID_URL = r'https://play.nova\.bg/video/.*/(?P<id>\d+)'
+    _VALID_URL = r'https://play\.nova\.bg/video/[^?#]+/(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'https://play.nova.bg/video/ochakvaite/season-0/ochakvaite-2022-07-22-sybudi-se-sat/606627',

yt_dlp/extractor/npo.py

@@ -245,7 +245,7 @@ class NPOIE(InfoExtractor):
                     'quality': 'npoplus',
                     'tokenId': player_token,
                     'streamType': 'broadcast',
-                })
+                }, data=b'')  # endpoint requires POST
             if not streams:
                 continue
             stream = streams.get('stream')

yt_dlp/extractor/ntvde.py

@@ -1,21 +1,21 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     js_to_json,
-    parse_duration,
+    url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class NTVDeIE(InfoExtractor):
     IE_NAME = 'n-tv.de'
-    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/videos/[^/?#]+/[^/?#]+-article(?P<id>.+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/(?:videos|magazine)/[^/?#]+/[^/?#]+-article(?P<id>[^/?#]+)\.html'
 
     _TESTS = [{
         'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
-        'md5': '6ef2514d4b1e8e03ca24b49e2f167153',
+        'md5': '6bcf2a6638cb83f45d5561659a1cb498',
         'info_dict': {
             'id': '14438086',
             'ext': 'mp4',
@@ -23,51 +23,61 @@ class NTVDeIE(InfoExtractor):
             'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
             'alt_title': 'Winterchaos auf deutschen Straßen',
             'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
-            'duration': 4020,
+            'duration': 67,
             'timestamp': 1422892797,
             'upload_date': '20150202',
         },
+    }, {
+        'url': 'https://www.n-tv.de/mediathek/magazine/auslandsreport/Juedische-Siedler-wollten-Rache-die-wollten-nur-toeten-article24523089.html',
+        'md5': 'c5c6014c014ccc3359470e1d34472bfd',
+        'info_dict': {
+            'id': '24523089',
+            'ext': 'mp4',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'title': 'Jüdische Siedler "wollten Rache, die wollten nur töten"',
+            'alt_title': 'Israelische Gewalt fern von Gaza',
+            'description': 'Vier Tage nach dem Massaker der Hamas greifen jüdische Siedler das Haus einer palästinensischen Familie im Westjordanland an. Die Überlebenden berichten, sie waren unbewaffnet, die Angreifer seien nur auf "Rache und Töten" aus gewesen. Als die Toten beerdigt werden sollen, eröffnen die Siedler erneut das Feuer.',
+            'duration': 326,
+            'timestamp': 1699688294,
+            'upload_date': '20231111',
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        info = self._parse_json(self._search_regex(
+        info = self._search_json(
-            r'(?s)ntv\.pageInfo\.article\s*=\s*(\{.*?\});', webpage, 'info'),
+            r'article:', webpage, 'info', video_id, transform_source=js_to_json)
-            video_id, transform_source=js_to_json)
+
-        timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
+        vdata = self._search_json(
-        vdata = self._parse_json(self._search_regex(
+            r'\$\(\s*"#playerwrapper"\s*\)\s*\.data\(\s*"player",',
-            r'(?s)\$\(\s*"\#player"\s*\)\s*\.data\(\s*"player",\s*(\{.*?\})\);',
+            webpage, 'player data', video_id,
-            webpage, 'player data'), video_id,
+            transform_source=lambda s: js_to_json(re.sub(r'ivw:[^},]+', '', s)))['setup']['source']
-            transform_source=lambda s: js_to_json(re.sub(r'advertising:\s*{[^}]+},', '', s)))
-        duration = parse_duration(vdata.get('duration'))
 
         formats = []
-        if vdata.get('video'):
+        if vdata.get('progressive'):
             formats.append({
-                'format_id': 'flash',
+                'format_id': 'http',
-                'url': 'rtmp://fms.n-tv.de/%s' % vdata['video'],
+                'url': vdata['progressive'],
             })
-        if vdata.get('videoMp4'):
+        if vdata.get('hls'):
-            formats.append({
-                'format_id': 'mobile',
-                'url': compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoMp4']),
-                'tbr': 400,  # estimation
-            })
-        if vdata.get('videoM3u8'):
-            m3u8_url = compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoM3u8'])
             formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+                vdata['hls'], video_id, 'mp4', m3u8_id='hls', fatal=False))
-                quality=1, m3u8_id='hls', fatal=False))
+        if vdata.get('dash'):
+            formats.extend(self._extract_mpd_formats(vdata['dash'], video_id, fatal=False, mpd_id='dash'))
 
         return {
             'id': video_id,
-            'title': info['headline'],
+            **traverse_obj(info, {
-            'description': info.get('intro'),
+                'title': 'headline',
-            'alt_title': info.get('kicker'),
+                'description': 'intro',
-            'timestamp': timestamp,
+                'alt_title': 'kicker',
-            'thumbnail': vdata.get('html5VideoPoster'),
+                'timestamp': ('publishedDateAsUnixTimeStamp', {int_or_none}),
-            'duration': duration,
+            }),
+            **traverse_obj(vdata, {
+                'thumbnail': ('poster', {url_or_none}),
+                'duration': ('length', {int_or_none}),
+            }),
             'formats': formats,
         }

yt_dlp/extractor/nubilesporn.py

@@ -19,7 +19,7 @@ from ..utils import (
 class NubilesPornIE(InfoExtractor):
     _NETRC_MACHINE = 'nubiles-porn'
     _VALID_URL = r'''(?x)
-        https://members.nubiles-porn.com/video/watch/(?P<id>\d+)
+        https://members\.nubiles-porn\.com/video/watch/(?P<id>\d+)
         (?:/(?P<display_id>[\w\-]+-s(?P<season>\d+)e(?P<episode>\d+)))?
     '''