[version] update

Created by: pukkandan

:ci skip all :ci run dl

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -2,6 +2,13 @@ name: Broken site
 description: Report broken or misfunctioning site
 labels: [triage, site-bug]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:
@@ -11,7 +18,7 @@ body:
       options:
         - label: I'm reporting a broken site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.09.01** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -55,7 +62,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2022.09.01 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -63,8 +70,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2022.07.18, Current version: 2022.07.18
-        yt-dlp is up to date (2022.07.18)
+        Latest version: 2022.09.01, Current version: 2022.09.01
+        yt-dlp is up to date (2022.09.01)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -2,6 +2,13 @@ name: Site support request
 description: Request support for a new site
 labels: [triage, site-request]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:
@@ -11,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.09.01** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -67,7 +74,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2022.09.01 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -75,8 +82,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2022.07.18, Current version: 2022.07.18
-        yt-dlp is up to date (2022.07.18)
+        Latest version: 2022.09.01, Current version: 2022.09.01
+        yt-dlp is up to date (2022.09.01)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -2,6 +2,13 @@ name: Site feature request
 description: Request a new functionality for a supported site
 labels: [triage, site-enhancement]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:
@@ -11,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.09.01** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -63,7 +70,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2022.09.01 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -71,8 +78,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2022.07.18, Current version: 2022.07.18
-        yt-dlp is up to date (2022.07.18)
+        Latest version: 2022.09.01, Current version: 2022.09.01
+        yt-dlp is up to date (2022.09.01)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -2,6 +2,13 @@ name: Bug report
 description: Report a bug unrelated to any particular site or extractor
 labels: [triage, bug]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:
@@ -11,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.09.01** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -29,7 +36,7 @@ body:
       placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
-    - type: checkboxes
+  - type: checkboxes
     id: verbose
     attributes:
       label: Provide verbose output that clearly demonstrates the problem
@@ -48,7 +55,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2022.09.01 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -56,8 +63,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2022.07.18, Current version: 2022.07.18
-        yt-dlp is up to date (2022.07.18)
+        Latest version: 2022.09.01, Current version: 2022.09.01
+        yt-dlp is up to date (2022.09.01)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -2,6 +2,13 @@ name: Feature request
 description: Request a new functionality unrelated to any particular site or extractor
 labels: [triage, enhancement]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:
@@ -13,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.09.01** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -44,7 +51,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2022.09.01 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -52,7 +59,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2022.07.18, Current version: 2022.07.18
-        yt-dlp is up to date (2022.07.18)
+        Latest version: 2022.09.01, Current version: 2022.09.01
+        yt-dlp is up to date (2022.09.01)
         <more lines>
       render: shell

.github/ISSUE_TEMPLATE/6_question.yml

@@ -2,12 +2,19 @@ name: Ask question
 description: Ask yt-dlp related question
 labels: [question]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: markdown
     attributes:
       value: |
         ### Make sure you are **only** asking a question and not reporting a bug or requesting a feature.
         If your question contains "isn't working" or "can you add", this is most likely the wrong template.
-        If you are in doubt whether this is the right template, **use another template**!
+        If you are in doubt whether this is the right template, **USE ANOTHER TEMPLATE**!
   - type: checkboxes
     id: checklist
     attributes:
@@ -19,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.09.01** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -50,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2022.09.01 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -58,7 +65,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2022.07.18, Current version: 2022.07.18
-        yt-dlp is up to date (2022.07.18)
+        Latest version: 2022.09.01, Current version: 2022.09.01
+        yt-dlp is up to date (2022.09.01)
         <more lines>
       render: shell

.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml

@@ -2,6 +2,7 @@ name: Broken site
 description: Report broken or misfunctioning site
 labels: [triage, site-bug]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml

@@ -2,6 +2,7 @@ name: Site support request
 description: Request support for a new site
 labels: [triage, site-request]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml

@@ -2,6 +2,7 @@ name: Site feature request
 description: Request a new functionality for a supported site
 labels: [triage, site-enhancement]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml

@@ -2,6 +2,7 @@ name: Bug report
 description: Report a bug unrelated to any particular site or extractor
 labels: [triage, bug]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:
@@ -29,4 +30,4 @@ body:
       placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
-    %(verbose)s
+  %(verbose)s

.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml

@@ -2,6 +2,7 @@ name: Feature request
 description: Request a new functionality unrelated to any particular site or extractor
 labels: [triage, enhancement]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/6_question.yml

@@ -2,12 +2,13 @@ name: Ask question
 description: Ask yt-dlp related question
 labels: [question]
 body:
+  %(no_skip)s
   - type: markdown
     attributes:
       value: |
         ### Make sure you are **only** asking a question and not reporting a bug or requesting a feature.
         If your question contains "isn't working" or "can you add", this is most likely the wrong template.
-        If you are in doubt whether this is the right template, **use another template**!
+        If you are in doubt whether this is the right template, **USE ANOTHER TEMPLATE**!
   - type: checkboxes
     id: checklist
     attributes:

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,20 @@
+**IMPORTANT**: PRs without the template will be CLOSED
+
+### Description of your *pull request* and other information
+
+</details>
+
+<!--
+
+Explanation of your *pull request* in arbitrary form goes here. Please **make sure the description explains the purpose and effect** of your *pull request* and is worded well enough to be understood. Provide as much **context and examples** as possible
+
+-->
+
+ADD DESCRIPTION HERE
+
+Fixes #
+
+
 <details open><summary>Template</summary> <!-- OPEN is intentional -->
 
 <!--
@@ -24,17 +41,3 @@
 - [ ] New extractor ([Piracy websites will not be accepted](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy))
 - [ ] Core bug fix/improvement
 - [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes))
-
-### Description of your *pull request* and other information
-
-</details>
-
-<!--
-
-Explanation of your *pull request* in arbitrary form goes here. Please **make sure the description explains the purpose and effect** of your *pull request* and is worded well enough to be understood. Provide as much **context and examples** as possible
-
--->
-
-DESCRIPTION
-
-Fixes #

.github/workflows/build.yml

@@ -2,18 +2,17 @@ name: Build
 on: workflow_dispatch
 
 jobs:
-  create_release:
+  prepare:
     runs-on: ubuntu-latest
     outputs:
       version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
       ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
-      upload_url: ${{ steps.create_release.outputs.upload_url }}
-      release_id: ${{ steps.create_release.outputs.id }}
+      head_sha: ${{ steps.push_release.outputs.head_sha }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
           python-version: '3.10'
 
@@ -43,53 +42,15 @@ jobs:
         PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
       if: "env.PUSH_VERSION_COMMIT != ''"
       run: git push origin ${{ github.event.ref }}
-    - name: Get Changelog
-      run: |
-        changelog=$(grep -oPz '(?s)(?<=### ${{ steps.bump_version.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)' Changelog.md) || true
-        echo "changelog<<EOF" >> $GITHUB_ENV
-        echo "$changelog" >> $GITHUB_ENV
-        echo "EOF" >> $GITHUB_ENV
-
-    - name: Create Release
-      id: create_release
-      uses: actions/create-release@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        tag_name: ${{ steps.bump_version.outputs.ytdlp_version }}
-        release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }}
-        commitish: ${{ steps.push_release.outputs.head_sha }}
-        draft: true
-        prerelease: false
-        body: |
-          #### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
-
-          ---
-          <details open><summary><h3>Changelog</summary>
-          <p>
-
-          ${{ env.changelog }}
-
-          </p>
-          </details>
 
 
   build_unix:
-    needs: create_release
+    needs: prepare
     runs-on: ubuntu-18.04  # Standalone executable should be built on minimum supported OS
-    outputs:
-      sha256_bin: ${{ steps.get_sha.outputs.sha256_bin }}
-      sha512_bin: ${{ steps.get_sha.outputs.sha512_bin }}
-      sha256_tar: ${{ steps.get_sha.outputs.sha256_tar }}
-      sha512_tar: ${{ steps.get_sha.outputs.sha512_tar }}
-      sha256_linux: ${{ steps.get_sha.outputs.sha256_linux }}
-      sha512_linux: ${{ steps.get_sha.outputs.sha512_linux }}
-      sha256_linux_zip: ${{ steps.get_sha.outputs.sha256_linux_zip }}
-      sha512_linux_zip: ${{ steps.get_sha.outputs.sha512_linux_zip }}
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
       with:
           python-version: '3.10'
     - name: Install Requirements
@@ -100,7 +61,7 @@ jobs:
 
     - name: Prepare
       run: |
-          python devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }}
+          python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
           python devscripts/make_lazy_extractors.py
     - name: Build Unix executables
       run: |
@@ -111,51 +72,15 @@ jobs:
     - name: Get SHA2-SUMS
       id: get_sha
       run: |
-          echo "::set-output name=sha256_bin::$(sha256sum yt-dlp | awk '{print $1}')"
-          echo "::set-output name=sha512_bin::$(sha512sum yt-dlp | awk '{print $1}')"
-          echo "::set-output name=sha256_tar::$(sha256sum yt-dlp.tar.gz | awk '{print $1}')"
-          echo "::set-output name=sha512_tar::$(sha512sum yt-dlp.tar.gz | awk '{print $1}')"
-          echo "::set-output name=sha256_linux::$(sha256sum dist/yt-dlp_linux | awk '{print $1}')"
-          echo "::set-output name=sha512_linux::$(sha512sum dist/yt-dlp_linux | awk '{print $1}')"
-          echo "::set-output name=sha256_linux_zip::$(sha256sum dist/yt-dlp_linux.zip | awk '{print $1}')"
-          echo "::set-output name=sha512_linux_zip::$(sha512sum dist/yt-dlp_linux.zip | awk '{print $1}')"
 
-    - name: Upload zip binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
       with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./yt-dlp
-        asset_name: yt-dlp
-        asset_content_type: application/octet-stream
-    - name: Upload Source tar
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./yt-dlp.tar.gz
-        asset_name: yt-dlp.tar.gz
-        asset_content_type: application/gzip
-    - name: Upload standalone binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp_linux
-        asset_name: yt-dlp_linux
-        asset_content_type: application/octet-stream
-    - name: Upload onedir binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp_linux.zip
-        asset_name: yt-dlp_linux.zip
-        asset_content_type: application/zip
+        path: |
+          yt-dlp
+          yt-dlp.tar.gz
+          dist/yt-dlp_linux
+          dist/yt-dlp_linux.zip
 
     - name: Build and publish on PyPi
       env:
@@ -164,6 +89,7 @@ jobs:
       if: "env.TWINE_PASSWORD != ''"
       run: |
         rm -rf dist/*
+        python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
         python setup.py sdist bdist_wheel
         twine upload dist/*
 
@@ -180,24 +106,19 @@ jobs:
       if: "env.BREW_TOKEN != ''"
       run: |
         git clone git@github.com:yt-dlp/homebrew-taps taps/
-        python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.create_release.outputs.ytdlp_version }}"
+        python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.ytdlp_version }}"
         git -C taps/ config user.name github-actions
         git -C taps/ config user.email github-actions@example.com
-        git -C taps/ commit -am 'yt-dlp: ${{ needs.create_release.outputs.ytdlp_version }}'
+        git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.ytdlp_version }}'
         git -C taps/ push
 
 
   build_macos:
     runs-on: macos-11
-    needs: create_release
-    outputs:
-      sha256_macos: ${{ steps.get_sha.outputs.sha256_macos }}
-      sha512_macos: ${{ steps.get_sha.outputs.sha512_macos }}
-      sha256_macos_zip: ${{ steps.get_sha.outputs.sha256_macos_zip }}
-      sha512_macos_zip: ${{ steps.get_sha.outputs.sha512_macos_zip }}
+    needs: prepare
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     # NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used
     - name: Install Requirements
       run: |
@@ -206,50 +127,28 @@ jobs:
 
     - name: Prepare
       run: |
-          /usr/bin/python3 devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }}
+          /usr/bin/python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
           /usr/bin/python3 devscripts/make_lazy_extractors.py
     - name: Build
       run: |
           /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir
           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
           /usr/bin/python3 pyinst.py --target-architecture universal2
-    - name: Get SHA2-SUMS
-      id: get_sha
-      run: |
-          echo "::set-output name=sha256_macos::$(sha256sum dist/yt-dlp_macos | awk '{print $1}')"
-          echo "::set-output name=sha512_macos::$(sha512sum dist/yt-dlp_macos | awk '{print $1}')"
-          echo "::set-output name=sha256_macos_zip::$(sha256sum dist/yt-dlp_macos.zip | awk '{print $1}')"
-          echo "::set-output name=sha512_macos_zip::$(sha512sum dist/yt-dlp_macos.zip | awk '{print $1}')"
 
-    - name: Upload standalone binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
       with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp_macos
-        asset_name: yt-dlp_macos
-        asset_content_type: application/octet-stream
-    - name: Upload onedir binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp_macos.zip
-        asset_name: yt-dlp_macos.zip
-        asset_content_type: application/zip
+        path: |
+          dist/yt-dlp_macos
+          dist/yt-dlp_macos.zip
 
 
   build_macos_legacy:
     runs-on: macos-latest
-    needs: create_release
-    outputs:
-      sha256_macos_legacy: ${{ steps.get_sha.outputs.sha256_macos_legacy }}
-      sha512_macos_legacy: ${{ steps.get_sha.outputs.sha512_macos_legacy }}
+    needs: prepare
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Install Python
       # We need the official Python, because the GA ones only support newer macOS versions
       env:
@@ -269,52 +168,37 @@ jobs:
 
     - name: Prepare
       run: |
-          python3 devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }}
+          python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
           python3 devscripts/make_lazy_extractors.py
     - name: Build
       run: |
           python3 pyinst.py
-    - name: Get SHA2-SUMS
-      id: get_sha
-      run: |
-          echo "::set-output name=sha256_macos_legacy::$(sha256sum dist/yt-dlp_macos | awk '{print $1}')"
-          echo "::set-output name=sha512_macos_legacy::$(sha512sum dist/yt-dlp_macos | awk '{print $1}')"
+          mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
 
-    - name: Upload standalone binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
       with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp_macos
-        asset_name: yt-dlp_macos_legacy
-        asset_content_type: application/octet-stream
+        path: |
+          dist/yt-dlp_macos_legacy
 
 
   build_windows:
     runs-on: windows-latest
-    needs: create_release
-    outputs:
-      sha256_win: ${{ steps.get_sha.outputs.sha256_win }}
-      sha512_win: ${{ steps.get_sha.outputs.sha512_win }}
-      sha256_py2exe: ${{ steps.get_sha.outputs.sha256_py2exe }}
-      sha512_py2exe: ${{ steps.get_sha.outputs.sha512_py2exe }}
-      sha256_win_zip: ${{ steps.get_sha.outputs.sha256_win_zip }}
-      sha512_win_zip: ${{ steps.get_sha.outputs.sha512_win_zip }}
+    needs: prepare
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
       with:  # 3.8 is used for Win7 support
           python-version: '3.8'
     - name: Install Requirements
       run: |  # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
           python -m pip install --upgrade pip setuptools wheel py2exe
-          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.10-py3-none-any.whl" -r requirements.txt
+          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.3-py3-none-any.whl" -r requirements.txt
 
     - name: Prepare
       run: |
-          python devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }}
+          python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
           python devscripts/make_lazy_extractors.py
     - name: Build
       run: |
@@ -323,154 +207,118 @@ jobs:
           python pyinst.py
           python pyinst.py --onedir
           Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
-    - name: Get SHA2-SUMS
-      id: get_sha
-      run: |
-          echo "::set-output name=sha256_py2exe::$((Get-FileHash dist\yt-dlp_min.exe -Algorithm SHA256).Hash.ToLower())"
-          echo "::set-output name=sha512_py2exe::$((Get-FileHash dist\yt-dlp_min.exe -Algorithm SHA512).Hash.ToLower())"
-          echo "::set-output name=sha256_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())"
-          echo "::set-output name=sha512_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())"
-          echo "::set-output name=sha256_win_zip::$((Get-FileHash dist\yt-dlp_win.zip -Algorithm SHA256).Hash.ToLower())"
-          echo "::set-output name=sha512_win_zip::$((Get-FileHash dist\yt-dlp_win.zip -Algorithm SHA512).Hash.ToLower())"
 
-    - name: Upload py2exe binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
       with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp_min.exe
-        asset_name: yt-dlp_min.exe
-        asset_content_type: application/vnd.microsoft.portable-executable
-    - name: Upload standalone binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp.exe
-        asset_name: yt-dlp.exe
-        asset_content_type: application/vnd.microsoft.portable-executable
-    - name: Upload onedir binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp_win.zip
-        asset_name: yt-dlp_win.zip
-        asset_content_type: application/zip
+        path: |
+          dist/yt-dlp.exe
+          dist/yt-dlp_min.exe
+          dist/yt-dlp_win.zip
 
 
   build_windows32:
     runs-on: windows-latest
-    needs: create_release
-    outputs:
-      sha256_win32: ${{ steps.get_sha.outputs.sha256_win32 }}
-      sha512_win32: ${{ steps.get_sha.outputs.sha512_win32 }}
+    needs: prepare
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
       with:  # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390
           python-version: '3.7'
           architecture: 'x86'
     - name: Install Requirements
       run: |
           python -m pip install --upgrade pip setuptools wheel
-          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.10-py3-none-any.whl" -r requirements.txt
+          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.3-py3-none-any.whl" -r requirements.txt
 
     - name: Prepare
       run: |
-          python devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }}
+          python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
           python devscripts/make_lazy_extractors.py
     - name: Build
       run: |
           python pyinst.py
-    - name: Get SHA2-SUMS
-      id: get_sha
-      run: |
-          echo "::set-output name=sha256_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())"
-          echo "::set-output name=sha512_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())"
 
-    - name: Upload standalone binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
       with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp_x86.exe
-        asset_name: yt-dlp_x86.exe
-        asset_content_type: application/vnd.microsoft.portable-executable
+        path: |
+          dist/yt-dlp_x86.exe
 
 
-  finish:
+  publish_release:
     runs-on: ubuntu-latest
-    needs: [create_release, build_unix, build_windows, build_windows32, build_macos, build_macos_legacy]
+    needs: [prepare, build_unix, build_windows, build_windows32, build_macos, build_macos_legacy]
 
     steps:
-    - name: Make SHA2-SUMS files
+    - uses: actions/checkout@v3
+    - uses: actions/download-artifact@v3
+
+    - name: Get Changelog
       run: |
-        echo "${{ needs.build_unix.outputs.sha256_bin }}  yt-dlp" >> SHA2-256SUMS
-        echo "${{ needs.build_unix.outputs.sha256_tar }}  yt-dlp.tar.gz" >> SHA2-256SUMS
-        echo "${{ needs.build_unix.outputs.sha256_linux }}  yt-dlp_linux" >> SHA2-256SUMS
-        echo "${{ needs.build_unix.outputs.sha256_linux_zip }}  yt-dlp_linux.zip" >> SHA2-256SUMS
-        echo "${{ needs.build_windows.outputs.sha256_win }}  yt-dlp.exe" >> SHA2-256SUMS
-        echo "${{ needs.build_windows.outputs.sha256_py2exe }}  yt-dlp_min.exe" >> SHA2-256SUMS
-        echo "${{ needs.build_windows32.outputs.sha256_win32 }}  yt-dlp_x86.exe" >> SHA2-256SUMS
-        echo "${{ needs.build_windows.outputs.sha256_win_zip }}  yt-dlp_win.zip" >> SHA2-256SUMS
-        echo "${{ needs.build_macos.outputs.sha256_macos }}  yt-dlp_macos" >> SHA2-256SUMS
-        echo "${{ needs.build_macos.outputs.sha256_macos_zip }}  yt-dlp_macos.zip" >> SHA2-256SUMS
-        echo "${{ needs.build_macos_legacy.outputs.sha256_macos_legacy }}  yt-dlp_macos_legacy" >> SHA2-256SUMS
-        echo "${{ needs.build_unix.outputs.sha512_bin }}  yt-dlp" >> SHA2-512SUMS
-        echo "${{ needs.build_unix.outputs.sha512_tar }}  yt-dlp.tar.gz" >> SHA2-512SUMS
-        echo "${{ needs.build_unix.outputs.sha512_linux }}  yt-dlp_linux" >> SHA2-512SUMS
-        echo "${{ needs.build_unix.outputs.sha512_linux_zip }}  yt-dlp_linux.zip" >> SHA2-512SUMS
-        echo "${{ needs.build_windows.outputs.sha512_win }}  yt-dlp.exe" >> SHA2-512SUMS
-        echo "${{ needs.build_windows.outputs.sha512_py2exe }}  yt-dlp_min.exe" >> SHA2-512SUMS
-        echo "${{ needs.build_windows32.outputs.sha512_win32 }}  yt-dlp_x86.exe" >> SHA2-512SUMS
-        echo "${{ needs.build_windows.outputs.sha512_win_zip }}  yt-dlp_win.zip" >> SHA2-512SUMS
-        echo "${{ needs.build_macos.outputs.sha512_macos }}  yt-dlp_macos" >> SHA2-512SUMS
-        echo "${{ needs.build_macos.outputs.sha512_macos_zip }}  yt-dlp_macos.zip" >> SHA2-512SUMS
-        echo "${{ needs.build_macos_legacy.outputs.sha512_macos_legacy }}  yt-dlp_macos_legacy" >> SHA2-512SUMS
-
-    - name: Upload SHA2-256SUMS file
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./SHA2-256SUMS
-        asset_name: SHA2-256SUMS
-        asset_content_type: text/plain
-    - name: Upload SHA2-512SUMS file
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./SHA2-512SUMS
-        asset_name: SHA2-512SUMS
-        asset_content_type: text/plain
-
+        changelog=$(grep -oPz '(?s)(?<=### ${{ needs.prepare.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)' Changelog.md) || true
+        echo "changelog<<EOF" >> $GITHUB_ENV
+        echo "$changelog" >> $GITHUB_ENV
+        echo "EOF" >> $GITHUB_ENV
     - name: Make Update spec
       run: |
         echo "# This file is used for regulating self-update" >> _update_spec
         echo "lock 2022.07.18 .+ Python 3.6" >> _update_spec
-    - name: Upload update spec
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./_update_spec
-        asset_name: _update_spec
-        asset_content_type: text/plain
-
-    - name: Finalize release
-      env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Make SHA2-SUMS files
       run: |
-        gh api -X PATCH -H "Accept: application/vnd.github.v3+json" \
-          /repos/${{ github.repository }}/releases/${{ needs.create_release.outputs.release_id }} \
-          -F draft=false
+          sha256sum artifact/yt-dlp | awk '{print $1 "  yt-dlp"}' >> SHA2-256SUMS
+          sha256sum artifact/yt-dlp.tar.gz | awk '{print $1 "  yt-dlp.tar.gz"}' >> SHA2-256SUMS
+          sha256sum artifact/yt-dlp.exe | awk '{print $1 "  yt-dlp.exe"}' >> SHA2-256SUMS
+          sha256sum artifact/yt-dlp_win.zip | awk '{print $1 "  yt-dlp_win.zip"}' >> SHA2-256SUMS
+          sha256sum artifact/yt-dlp_min.exe | awk '{print $1 "  yt-dlp_min.exe"}' >> SHA2-256SUMS
+          sha256sum artifact/yt-dlp_x86.exe | awk '{print $1 "  yt-dlp_x86.exe"}' >> SHA2-256SUMS
+          sha256sum artifact/yt-dlp_macos | awk '{print $1 "  yt-dlp_macos"}' >> SHA2-256SUMS
+          sha256sum artifact/yt-dlp_macos.zip | awk '{print $1 "  yt-dlp_macos.zip"}' >> SHA2-256SUMS
+          sha256sum artifact/yt-dlp_macos_legacy | awk '{print $1 "  yt-dlp_macos_legacy"}' >> SHA2-256SUMS
+          sha256sum artifact/dist/yt-dlp_linux | awk '{print $1 "  yt-dlp_linux"}' >> SHA2-256SUMS
+          sha256sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 "  yt-dlp_linux.zip"}' >> SHA2-256SUMS
+          sha512sum artifact/yt-dlp | awk '{print $1 "  yt-dlp"}' >> SHA2-512SUMS
+          sha512sum artifact/yt-dlp.tar.gz | awk '{print $1 "  yt-dlp.tar.gz"}' >> SHA2-512SUMS
+          sha512sum artifact/yt-dlp.exe | awk '{print $1 "  yt-dlp.exe"}' >> SHA2-512SUMS
+          sha512sum artifact/yt-dlp_win.zip | awk '{print $1 "  yt-dlp_win.zip"}' >> SHA2-512SUMS
+          sha512sum artifact/yt-dlp_min.exe | awk '{print $1 "  yt-dlp_min.exe"}' >> SHA2-512SUMS
+          sha512sum artifact/yt-dlp_x86.exe | awk '{print $1 "  yt-dlp_x86.exe"}' >> SHA2-512SUMS
+          sha512sum artifact/yt-dlp_macos | awk '{print $1 "  yt-dlp_macos"}' >> SHA2-512SUMS
+          sha512sum artifact/yt-dlp_macos.zip | awk '{print $1 "  yt-dlp_macos.zip"}' >> SHA2-512SUMS
+          sha512sum artifact/yt-dlp_macos_legacy | awk '{print $1 "  yt-dlp_macos_legacy"}' >> SHA2-512SUMS
+          sha512sum artifact/dist/yt-dlp_linux | awk '{print $1 "  yt-dlp_linux"}' >> SHA2-512SUMS
+          sha512sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 "  yt-dlp_linux.zip"}' >> SHA2-512SUMS
+
+    - name: Publish Release
+      uses: yt-dlp/action-gh-release@v1
+      with:
+        tag_name: ${{ needs.prepare.outputs.ytdlp_version }}
+        name: yt-dlp ${{ needs.prepare.outputs.ytdlp_version }}
+        target_commitish: ${{ needs.prepare.outputs.head_sha }}
+        body: |
+          #### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
+
+          ---
+          <details open><summary><h3>Changelog</summary>
+          <p>
+
+          ${{ env.changelog }}
+
+          </p>
+          </details>
+        files: |
+          SHA2-256SUMS
+          SHA2-512SUMS
+          artifact/yt-dlp
+          artifact/yt-dlp.tar.gz
+          artifact/yt-dlp.exe
+          artifact/yt-dlp_win.zip
+          artifact/yt-dlp_min.exe
+          artifact/yt-dlp_x86.exe
+          artifact/yt-dlp_macos
+          artifact/yt-dlp_macos.zip
+          artifact/yt-dlp_macos_legacy
+          artifact/dist/yt-dlp_linux
+          artifact/dist/yt-dlp_linux.zip
+          _update_spec

.github/workflows/core.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         # CPython 3.9 is in quick-test
-        python-version: ['3.6', '3.7', '3.10', 3.11-dev, pypy-3.6, pypy-3.7, pypy-3.8]
+        python-version: ['3.7', '3.10', 3.11-dev, pypy-3.7, pypy-3.8]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
@@ -21,9 +21,9 @@ jobs:
           python-version: pypy-3.9
           run-tests-ext: bat
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pytest

.github/workflows/download.yml

@@ -6,9 +6,9 @@ jobs:
     if: "contains(github.event.head_commit.message, 'ci run dl')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install test requirements
@@ -25,7 +25,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.6', '3.7', '3.10', 3.11-dev, pypy-3.6, pypy-3.7, pypy-3.8]
+        python-version: ['3.7', '3.10', 3.11-dev, pypy-3.7, pypy-3.8]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
@@ -36,9 +36,9 @@ jobs:
           python-version: pypy-3.9
           run-tests-ext: bat
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pytest

.github/workflows/quick-test.yml

@@ -6,9 +6,9 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'ci skip all')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install test requirements
@@ -20,9 +20,9 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'ci skip all')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install flake8

CONTRIBUTING.md

@@ -195,7 +195,7 @@ After you have ensured this site is distributing its content legally, you can fo
                 # * A value
                 # * MD5 checksum; start the string with md5:
                 # * A regular expression; start the string with re:
-                # * Any Python type (for example int or float)
+                # * Any Python type, e.g. int or float
             }
         }]
 
@@ -222,7 +222,7 @@ After you have ensured this site is distributing its content legally, you can fo
 
         $ flake8 yt_dlp/extractor/yourextractor.py
 
-1. Make sure your code works under all [Python](https://www.python.org/) versions supported by yt-dlp, namely CPython and PyPy for Python 3.6 and above. Backward compatibility is not required for even older versions of Python.
+1. Make sure your code works under all [Python](https://www.python.org/) versions supported by yt-dlp, namely CPython and PyPy for Python 3.7 and above. Backward compatibility is not required for even older versions of Python.
 1. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files, [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
 
         $ git add yt_dlp/extractor/_extractors.py
@@ -261,7 +261,7 @@ The aforementioned metafields are the critical data that the extraction does not
 
 For pornographic sites, appropriate `age_limit` must also be returned.
 
-The extractor is allowed to return the info dict without url or formats in some special cases if it allows the user to extract usefull information with `--ignore-no-formats-error` - Eg: when the video is a live stream that has not started yet.
+The extractor is allowed to return the info dict without url or formats in some special cases if it allows the user to extract usefull information with `--ignore-no-formats-error` - e.g. when the video is a live stream that has not started yet.
 
 [Any field](yt_dlp/extractor/common.py#219-L426) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
 

CONTRIBUTORS

@@ -285,3 +285,26 @@ odo2063
 pritam20ps05
 scy
 sheerluck
+AxiosDeminence
+DjesonPV
+eren-kemer
+freezboltz
+Galiley
+haobinliang
+Mehavoid
+winterbird-code
+yashkc2025
+aldoridhoni
+bashonly
+jacobtruman
+masta79
+palewire
+cgrigis
+DavidH-2022
+dfaker
+jackyyf
+ohaiibuzzle
+SamantazFox
+shreyasminocha
+tejasa97
+xenov

Changelog.md

@@ -11,6 +11,188 @@
 -->
 
 
+### 2022.09.01
+
+* Add option `--use-extractors`
+* Merge youtube-dl: Upto [commit/ed5c44e](https://github.com/ytdl-org/youtube-dl/commit/ed5c44e7)
+* Add yt-dlp version to infojson
+* Fix `--break-per-url --max-downloads`
+* Fix bug in `--alias`
+* [cookies] Support firefox container in `--cookies-from-browser` by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+* [downloader/external] Smarter detection of executable
+* [extractor/generic] Don't return JW player without formats
+* [FormatSort] Fix `aext` for `--prefer-free-formats`
+* [jsinterp] Various improvements by [pukkandan](https://github.com/pukkandan), [dirkf](https://github.com/dirkf), [elyse0](https://github.com/elyse0)
+* [cache] Mechanism to invalidate old cache
+* [utils] Add `deprecation_warning`
+* [utils] Add `orderedSet_from_options`
+* [utils] `Popen`: Restore `LD_LIBRARY_PATH` when using PyInstaller by [Lesmiscore](https://github.com/Lesmiscore)
+* [build] `make tar` should not follow `DESTDIR` by [satan1st](https://github.com/satan1st)
+* [build] Update pyinstaller by [shirt-dev](https://github.com/shirt-dev)
+* [test] Fix `test_youtube_signature`
+* [cleanup] Misc fixes and cleanup by [DavidH-2022](https://github.com/DavidH-2022), [MrRawes](https://github.com/MrRawes), [pukkandan](https://github.com/pukkandan)
+* [extractor/epoch] Add extractor by [tejasa97](https://github.com/tejasa97)
+* [extractor/eurosport] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/IslamChannel] Add extractors by [Lesmiscore](https://github.com/Lesmiscore)
+* [extractor/newspicks] Add extractor by [Lesmiscore](https://github.com/Lesmiscore)
+* [extractor/triller] Add extractor by [bashonly](https://github.com/bashonly)
+* [extractor/VQQ] Add extractors by [elyse0](https://github.com/elyse0)
+* [extractor/youtube] Improvements to nsig extraction
+* [extractor/youtube] Fix bug in format sorting
+* [extractor/youtube] Update iOS Innertube clients by [SamantazFox](https://github.com/SamantazFox)
+* [extractor/youtube] Use device-specific user agent by [coletdjnz](https://github.com/coletdjnz)
+* [extractor/youtube] Add `--compat-option no-youtube-prefer-utc-upload-date` by [coletdjnz](https://github.com/coletdjnz)
+* [extractor/arte] Bug fix by [cgrigis](https://github.com/cgrigis)
+* [extractor/bilibili] Extract `flac` with premium account by [jackyyf](https://github.com/jackyyf)
+* [extractor/BiliBiliSearch] Don't sort by date
+* [extractor/BiliBiliSearch] Fix infinite loop
+* [extractor/bitchute] Mark errors as expected
+* [extractor/crunchyroll:beta] Use anonymous access by [tejing1](https://github.com/tejing1)
+* [extractor/huya] Fix stream extraction by [ohaiibuzzle](https://github.com/ohaiibuzzle)
+* [extractor/medaltv] Fix extraction by [xenova](https://github.com/xenova)
+* [extractor/mediaset] Fix embed extraction
+* [extractor/mixcloud] All formats are audio-only
+* [extractor/rtbf] Fix jwt extraction by [elyse0](https://github.com/elyse0)
+* [extractor/screencastomatic] Support `--video-password` by [shreyasminocha](https://github.com/shreyasminocha)
+* [extractor/stripchat] Don't modify input URL by [dfaker](https://github.com/dfaker)
+* [extractor/uktv] Improve `_VALID_URL` by [dirkf](https://github.com/dirkf)
+* [extractor/vimeo:user] Fix `_VALID_URL`
+
+
+### 2022.08.19
+
+* Fix bug in `--download-archive`
+* [jsinterp] **Fix for new youtube players** and related improvements by [dirkf](https://github.com/dirkf), [pukkandan](https://github.com/pukkandan)
+* [phantomjs] Add function to execute JS without a DOM by [MinePlayersPE](https://github.com/MinePlayersPE), [pukkandan](https://github.com/pukkandan)
+* [build] Exclude devscripts from installs by [Lesmiscore](https://github.com/Lesmiscore)
+* [cleanup] Misc fixes and cleanup
+* [extractor/youtube] **Add fallback to phantomjs** for nsig
+* [extractor/youtube] Fix error reporting of "Incomplete data"
+* [extractor/youtube] Improve format sorting for IOS formats
+* [extractor/youtube] Improve signature caching
+* [extractor/instagram] Fix extraction by [bashonly](https://github.com/bashonly), [pritam20ps05](https://github.com/pritam20ps05)
+* [extractor/rai] Minor fix by [nixxo](https://github.com/nixxo)
+* [extractor/rtbf] Fix stream extractor by [elyse0](https://github.com/elyse0)
+* [extractor/SovietsCloset] Fix extractor by [ChillingPepper](https://github.com/ChillingPepper)
+* [extractor/zattoo] Fix Zattoo resellers by [goggle](https://github.com/goggle)
+
+### 2022.08.14
+
+* Merge youtube-dl: Upto [commit/d231b56](https://github.com/ytdl-org/youtube-dl/commit/d231b56)
+* [jsinterp] Handle **new youtube signature functions**
+* [jsinterp] Truncate error messages
+* [extractor] Fix format sorting of `channels`
+* [ffmpeg] Disable avconv unless `--prefer-avconv`
+* [ffmpeg] Smarter detection of ffprobe filename
+* [embedthumbnail] Detect `libatomicparsley.so`
+* [ThumbnailsConvertor] Fix conversion after `fixup_webp`
+* [utils] Fix `get_compatible_ext`
+* [build] Fix changelog
+* [update] Set executable bit-mask by [pukkandan](https://github.com/pukkandan), [Lesmiscore](https://github.com/Lesmiscore)
+* [devscripts] Fix import
+* [docs] Consistent use of `e.g.` by [Lesmiscore](https://github.com/Lesmiscore)
+* [cleanup] Misc fixes and cleanup
+* [extractor/moview] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/parler] Add extractor by [palewire](https://github.com/palewire)
+* [extractor/patreon] Ignore erroneous media attachments by [coletdjnz](https://github.com/coletdjnz)
+* [extractor/truth] Add extractor by [palewire](https://github.com/palewire)
+* [extractor/aenetworks] Add formats parameter by [jacobtruman](https://github.com/jacobtruman)
+* [extractor/crunchyroll] Improve `_VALID_URL`s
+* [extractor/doodstream] Add `wf` domain by [aldoridhoni](https://github.com/aldoridhoni)
+* [extractor/facebook] Add reel support by [bashonly](https://github.com/bashonly)
+* [extractor/MLB] New extractor by [ischmidt20](https://github.com/ischmidt20)
+* [extractor/rai] Misc fixes by [nixxo](https://github.com/nixxo)
+* [extractor/toggo] Improve `_VALID_URL` by [masta79](https://github.com/masta79)
+* [extractor/tubitv] Extract additional formats by [shirt-dev](https://github.com/shirt-dev)
+* [extractor/zattoo] Potential fix for resellers
+
+
+### 2022.08.08
+
+* **Remove Python 3.6 support**
+* Determine merge container better by [pukkandan](https://github.com/pukkandan), [selfisekai](https://github.com/selfisekai)
+* Framework for embed detection by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+* Merge youtube-dl: Upto [commit/adb5294](https://github.com/ytdl-org/youtube-dl/commit/adb5294)
+* `--compat-option no-live-chat` should disable danmaku
+* Fix misleading DRM message
+* Import ctypes only when necessary
+* Minor bugfixes
+* Reject entire playlists faster with `--match-filter`
+* Remove filtered entries from `-J`
+* Standardize retry mechanism
+* Validate `--merge-output-format`
+* [downloader] Add average speed to final progress line
+* [extractor] Add field `audio_channels`
+* [extractor] Support multiple archive ids for one video
+* [ffmpeg] Set `ffmpeg_location` in a contextvar
+* [FFmpegThumbnailsConvertor] Fix conversion from GIF
+* [MetadataParser] Don't set `None` when the field didn't match
+* [outtmpl] Smarter replacing of unsupported characters
+* [outtmpl] Treat empty values as None in filenames
+* [utils] sanitize_open: Allow any IO stream as stdout
+* [build, devscripts] Add devscript to set a build variant
+* [build] Improve build process by [shirt-dev](https://github.com/shirt-dev)
+* [build] Update pyinstaller
+* [devscripts] Create `utils` and refactor
+* [docs] Clarify `best*`
+* [docs] Fix bug report issue template
+* [docs] Fix capitalization in references by [christoph-heinrich](https://github.com/christoph-heinrich)
+* [cleanup, mhtml] Use imghdr
+* [cleanup, utils] Consolidate known media extensions
+* [cleanup] Misc fixes and cleanup
+* [extractor/angel] Add extractor by [AxiosDeminence](https://github.com/AxiosDeminence)
+* [extractor/dplay] Add MotorTrend extractor by [Sipherdrakon](https://github.com/Sipherdrakon)
+* [extractor/harpodeon] Add extractor by [eren-kemer](https://github.com/eren-kemer)
+* [extractor/holodex] Add extractor by [pukkandan](https://github.com/pukkandan), [sqrtNOT](https://github.com/sqrtNOT)
+* [extractor/kompas] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/rai] Add raisudtirol extractor by [nixxo](https://github.com/nixxo)
+* [extractor/tempo] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/youtube] **Fixes for third party client detection** by [coletdjnz](https://github.com/coletdjnz)
+* [extractor/youtube] Add `live_status=post_live` by [lazypete365](https://github.com/lazypete365)
+* [extractor/youtube] Extract more format info
+* [extractor/youtube] Parse translated subtitles only when requested
+* [extractor/youtube, extractor/twitch] Allow waiting for channels to become live
+* [extractor/youtube, webvtt] Extract auto-subs from livestream VODs by [fstirlitz](https://github.com/fstirlitz), [pukkandan](https://github.com/pukkandan)
+* [extractor/AbemaTVTitle] Implement paging by [Lesmiscore](https://github.com/Lesmiscore)
+* [extractor/archiveorg] Improve handling of formats by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+* [extractor/arte] Fix title extraction
+* [extractor/arte] **Move to v2 API** by [fstirlitz](https://github.com/fstirlitz), [pukkandan](https://github.com/pukkandan)
+* [extractor/bbc] Fix news articles by [ajj8](https://github.com/ajj8)
+* [extractor/camtasia] Separate into own extractor by [coletdjnz](https://github.com/coletdjnz)
+* [extractor/cloudflarestream] Fix video_id padding by [haobinliang](https://github.com/haobinliang)
+* [extractor/crunchyroll] Fix conversion of thumbnail from GIF
+* [extractor/crunchyroll] Handle missing metadata correctly by [Burve](https://github.com/Burve), [pukkandan](https://github.com/pukkandan)
+* [extractor/crunchyroll:beta] Extract timestamp and fix tests by [tejing1](https://github.com/tejing1)
+* [extractor/crunchyroll:beta] Use streams API by [tejing1](https://github.com/tejing1)
+* [extractor/doodstream] Support more domains by [Galiley](https://github.com/Galiley)
+* [extractor/ESPN] Extract duration by [ischmidt20](https://github.com/ischmidt20)
+* [extractor/FIFA] Change API endpoint by [Bricio](https://github.com/Bricio), [yashkc2025](https://github.com/yashkc2025)
+* [extractor/globo:article] Remove false positives by [Bricio](https://github.com/Bricio)
+* [extractor/Go] Extract timestamp by [ischmidt20](https://github.com/ischmidt20)
+* [extractor/hidive] Fix cookie login when netrc is also given by [winterbird-code](https://github.com/winterbird-code)
+* [extractor/html5] Separate into own extractor by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+* [extractor/ina] Improve extractor by [elyse0](https://github.com/elyse0)
+* [extractor/NaverNow] Change endpoint by [ping](https://github.com/ping)
+* [extractor/ninegag] Extract uploader by [DjesonPV](https://github.com/DjesonPV)
+* [extractor/NovaPlay] Fix extractor by [Bojidarist](https://github.com/Bojidarist)
+* [extractor/orf:radio] Rewrite extractors
+* [extractor/patreon] Fix and improve extractors by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+* [extractor/rai] Fix RaiNews extraction by [nixxo](https://github.com/nixxo)
+* [extractor/redbee] Unify and update extractors by [elyse0](https://github.com/elyse0)
+* [extractor/stripchat] Fix _VALID_URL by [freezboltz](https://github.com/freezboltz)
+* [extractor/tubi] Exclude playlists from playlist entries by [sqrtNOT](https://github.com/sqrtNOT)
+* [extractor/tviplayer] Improve `_VALID_URL` by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/twitch] Extract chapters for single chapter VODs by [mpeter50](https://github.com/mpeter50)
+* [extractor/vgtv] Support tv.vg.no by [sqrtNOT](https://github.com/sqrtNOT)
+* [extractor/vidio] Support embed link by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/vk] Fix extractor by [Mehavoid](https://github.com/Mehavoid)
+* [extractor/WASDTV:record] Fix `_VALID_URL`
+* [extractor/xfileshare] Add Referer by [Galiley](https://github.com/Galiley)
+* [extractor/YahooJapanNews] Fix extractor by [Lesmiscore](https://github.com/Lesmiscore)
+* [extractor/yandexmusic] Extract higher quality format
+* [extractor/zee5] Update Device ID by [m4tu4g](https://github.com/m4tu4g)
+
+
 ### 2022.07.18
 
 * Allow users to specify encoding in each config files by [Lesmiscore](https://github.com/Lesmiscore)
@@ -125,7 +307,7 @@
 
 * [**Deprecate support for Python 3.6**](https://github.com/yt-dlp/yt-dlp/issues/3764#issuecomment-1154051119)
 * **Add option `--download-sections` to download video partially**
-    * Chapter regex and time ranges are accepted (Eg: `--download-sections *1:10-2:20`)
+    * Chapter regex and time ranges are accepted, e.g. `--download-sections *1:10-2:20`
 * Add option `--alias`
 * Add option `--lazy-playlist` to process entries as they are received
 * Add option `--retry-sleep`
@@ -1289,7 +1471,7 @@
 
 * Add new option `--netrc-location`
 * [outtmpl] Allow alternate fields using `,`
-* [outtmpl] Add format type `B` to treat the value as bytes (eg: to limit the filename to a certain number of bytes)
+* [outtmpl] Add format type `B` to treat the value as bytes, e.g. to limit the filename to a certain number of bytes
 * Separate the options `--ignore-errors` and `--no-abort-on-error`
 * Basic framework for simultaneous download of multiple formats by [nao20010128nao](https://github.com/nao20010128nao)
 * [17live] Add 17.live extractor by [nao20010128nao](https://github.com/nao20010128nao)
@@ -1679,7 +1861,7 @@
 
 * Merge youtube-dl: Upto [commit/a803582](https://github.com/ytdl-org/youtube-dl/commit/a8035827177d6b59aca03bd717acb6a9bdd75ada)
 * Add `--extractor-args` to pass some extractor-specific arguments. See [readme](https://github.com/yt-dlp/yt-dlp#extractor-arguments)
-    * Add extractor option `skip` for `youtube`. Eg: `--extractor-args youtube:skip=hls,dash`
+    * Add extractor option `skip` for `youtube`, e.g. `--extractor-args youtube:skip=hls,dash`
     * Deprecates `--youtube-skip-dash-manifest`, `--youtube-skip-hls-manifest`, `--youtube-include-dash-manifest`, `--youtube-include-hls-manifest`
 * Allow `--list...` options to work with `--print`, `--quiet` and other `--list...` options
 * [youtube] Use `player` API for additional video extraction requests by [coletdjnz](https://github.com/coletdjnz)

Collaborators.md

@@ -28,12 +28,12 @@ You can also find lists of all [contributors of yt-dlp](CONTRIBUTORS) and [autho
 [![gh-sponsor](https://img.shields.io/badge/_-Sponsor-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/coletdjnz)
 
 * YouTube improvements including: age-gate bypass, private playlists, multiple-clients (to avoid throttling) and a lot of under-the-hood improvements
-* Added support for downloading YoutubeWebArchive videos
-* Added support for new websites MainStreaming, PRX, nzherald, etc
+* Added support for new websites YoutubeWebArchive, MainStreaming, PRX, nzherald, Mediaklikk, StarTV etc
+* Improved/fixed support for Patreon, panopto, gfycat, itv, pbs, SouthParkDE etc
 
 
 
-## [Ashish0804](https://github.com/Ashish0804)
+## [Ashish0804](https://github.com/Ashish0804) <sub><sup>[Inactive]</sup></sub>
 
 [![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/ashish0804)
 
@@ -48,4 +48,5 @@ You can also find lists of all [contributors of yt-dlp](CONTRIBUTORS) and [autho
 **Monacoin**: mona1q3tf7dzvshrhfe3md379xtvt2n22duhglv5dskr
 
 * Download live from start to end for YouTube
-* Added support for new websites mildom, PixivSketch, skeb, radiko, voicy, mirrativ, openrec, whowatch, damtomo, 17.live, mixch etc
+* Added support for new websites AbemaTV, mildom, PixivSketch, skeb, radiko, voicy, mirrativ, openrec, whowatch, damtomo, 17.live, mixch etc
+* Improved/fixed support for fc2, YahooJapanNews, tver, iwara etc

Makefile

@@ -33,7 +33,6 @@ completion-zsh: completions/zsh/_yt-dlp
 lazy-extractors: yt_dlp/extractor/lazy_extractors.py
 
 PREFIX ?= /usr/local
-DESTDIR ?= .
 BINDIR ?= $(PREFIX)/bin
 MANDIR ?= $(PREFIX)/man
 SHAREDIR ?= $(PREFIX)/share
@@ -134,7 +133,7 @@ yt_dlp/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscrip
 	$(PYTHON) devscripts/make_lazy_extractors.py $@
 
 yt-dlp.tar.gz: all
-	@tar -czf $(DESTDIR)/yt-dlp.tar.gz --transform "s|^|yt-dlp/|" --owner 0 --group 0 \
+	@tar -czf yt-dlp.tar.gz --transform "s|^|yt-dlp/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
 		--exclude '*.pyc' \

README.md

@@ -71,7 +71,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 # NEW FEATURES
 
-* Merged with **youtube-dl v2021.12.17+ [commit/a03b977](https://github.com/ytdl-org/youtube-dl/commit/a03b9775d544b06a5b4f2aa630214c7c22fc2229)**<!--([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))--> and **youtube-dlc v2020.11.11-3+ [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
+* Merged with **youtube-dl v2021.12.17+ [commit/ed5c44e](https://github.com/ytdl-org/youtube-dl/commit/ed5c44e7b74ac77f87ca5ed6cb5e964a0c6a0678)**<!--([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))--> and **youtube-dlc v2020.11.11-3+ [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in youtube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 
@@ -105,7 +105,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 * **Multiple paths and output templates**: You can give different [output templates](#output-template) and download paths for different types of files. You can also set a temporary path where intermediary files are downloaded to using `--paths` (`-P`)
 
-* **Portable Configuration**: Configuration files are automatically loaded from the home and root directories. See [configuration](#configuration) for details
+* **Portable Configuration**: Configuration files are automatically loaded from the home and root directories. See [CONFIGURATION](#configuration) for details
 
 * **Output template improvements**: Output templates can now have date-time formatting, numeric offsets, object traversal etc. See [output template](#output-template) for details. Even more advanced operations can also be done with the help of `--parse-metadata` and `--replace-in-metadata`
 
@@ -127,7 +127,7 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 
 * The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
 * `avconv` is not supported as an alternative to `ffmpeg`
-* yt-dlp stores config files in slightly different locations to youtube-dl. See [configuration](#configuration) for a list of correct locations
+* yt-dlp stores config files in slightly different locations to youtube-dl. See [CONFIGURATION](#configuration) for a list of correct locations
 * The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s-%(id)s.%(ext)s`. Instead, you may use `--compat-options filename`
 * The default [format sorting](#sorting-formats) is different from youtube-dl and prefers higher resolution and better codecs rather than higher bitrates. You can use the `--format-sort` option to change this to any order you prefer, or use `--compat-options format-sort` to use youtube-dl's sorting order
 * The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be preferred. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this
@@ -138,16 +138,16 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 * Some metadata are embedded into different fields when using `--add-metadata` as compared to youtube-dl. Most notably, `comment` field contains the `webpage_url` and `synopsis` contains the `description`. You can [use `--parse-metadata`](#modifying-metadata) to modify this to your liking or use `--compat-options embed-metadata` to revert this
 * `playlist_index` behaves differently when used with options like `--playlist-reverse` and `--playlist-items`. See [#302](https://github.com/yt-dlp/yt-dlp/issues/302) for details. You can use `--compat-options playlist-index` if you want to keep the earlier behavior
 * The output of `-F` is listed in a new format. Use `--compat-options list-formats` to revert this
-* All *experiences* of a funimation episode are considered as a single video. This behavior breaks existing archives. Use `--compat-options seperate-video-versions` to extract information from only the default player
-* Youtube live chat (if available) is considered as a subtitle. Use `--sub-langs all,-live_chat` to download all subtitles except live chat. You can also use `--compat-options no-live-chat` to prevent live chat from downloading
+* Live chats (if available) are considered as subtitles. Use `--sub-langs all,-live_chat` to download all subtitles except live chat. You can also use `--compat-options no-live-chat` to prevent any live chat/danmaku from downloading
 * Youtube channel URLs are automatically redirected to `/video`. Append a `/featured` to the URL to download only the videos in the home page. If the channel does not have a videos tab, we try to download the equivalent `UU` playlist instead. For all other tabs, if the channel does not show the requested tab, an error will be raised. Also, `/live` URLs raise an error if there are no live videos instead of silently downloading the entire channel. You may use `--compat-options no-youtube-channel-redirect` to revert all these redirections
 * Unavailable videos are also listed for youtube playlists. Use `--compat-options no-youtube-unavailable-videos` to remove this
+* The upload dates extracted from YouTube are in UTC [when available](https://github.com/yt-dlp/yt-dlp/blob/89e4d86171c7b7c997c77d4714542e0383bf0db0/yt_dlp/extractor/youtube.py#L3898-L3900). Use `--compat-options no-youtube-prefer-utc-upload-date` to prefer the non-UTC upload date.
 * If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this
 * Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead
 * Some private fields such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this
 * When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the separate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this
 * `certifi` will be used for SSL root certificates, if installed. If you want to use system certificates (e.g. self-signed), use `--compat-options no-certifi`
-* youtube-dl tries to remove some superfluous punctuations from filenames. While this can sometimes be helpful, it is often undesirable. So yt-dlp tries to keep the fields in the filenames as close to their original values as possible. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
+* yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
 
 For ease of use, a few more compat options are available:
 
@@ -313,7 +313,7 @@ If you do not have the necessary dependencies for a task you are attempting, yt-
 ## COMPILE
 
 ### Standalone PyInstaller Builds
-To build the Windows/MacOS executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). Once you have all the necessary dependencies installed, simply run `pyinst.py`. The executable will be built for the same architecture (32/64 bit) as the Python used.
+To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). Once you have all the necessary dependencies installed, simply run `pyinst.py`. The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used.
 
     python3 -m pip install -U pyinstaller -r requirements.txt
     python3 devscripts/make_lazy_extractors.py
@@ -321,7 +321,7 @@ To build the Windows/MacOS executable, you must have Python and `pyinstaller` (p
 
 On some systems, you may need to use `py` or `python` instead of `python3`.
 
-Note that pyinstaller [does not support](https://github.com/pyinstaller/pyinstaller#requirements-and-tested-platforms) Python installed from the Windows store without using a virtual environment.
+Note that pyinstaller with versions below 4.4 [do not support](https://github.com/pyinstaller/pyinstaller#requirements-and-tested-platforms) Python installed from the Windows store without using a virtual environment.
 
 **Important**: Running `pyinstaller` directly **without** using `pyinst.py` is **not** officially supported. This may or may not work correctly.
 
@@ -330,7 +330,7 @@ You will need the build tools `python` (3.6+), `zip`, `make` (GNU), `pandoc`\* a
 
 After installing these, simply run `make`.
 
-You can also run `make yt-dlp` instead to compile only the binary without updating any of the additional files. (The dependencies marked with **\*** are not needed for this)
+You can also run `make yt-dlp` instead to compile only the binary without updating any of the additional files. (The build tools marked with **\*** are not needed for this)
 
 ### Standalone Py2Exe Builds (Windows)
 
@@ -344,7 +344,8 @@ If you wish to build it anyway, install Python and py2exe, and then simply run `
 
 ### Related scripts
 
-* **`devscripts/update-version.py`** - Update the version number based on current timestamp
+* **`devscripts/update-version.py [revision]`** - Update the version number based on current date
+* **`devscripts/set-variant.py variant [-M update_message]`** - Set the build variant of the executable
 * **`devscripts/make_lazy_extractors.py`** - Create lazy extractors. Running this before building the binaries (any variant) will improve their startup performance. Set the environment variable `YTDLP_NO_LAZY_EXTRACTORS=1` if you wish to forcefully disable lazy extractor loading.
 
 You can also fork the project on github and run your fork's [build workflow](.github/workflows/build.yml) to automatically build a full release
@@ -361,8 +362,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
 ## General Options:
     -h, --help                      Print this help text and exit
     --version                       Print program version and exit
-    -U, --update                    Update this program to latest version
-    --no-update                     Do not update (default)
+    -U, --update                    Update this program to the latest version
+    --no-update                     Do not check for updates (default)
     -i, --ignore-errors             Ignore download and postprocessing errors.
                                     The download will be considered successful
                                     even if the postprocessing fails
@@ -375,8 +376,14 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     --list-extractors               List all supported extractors and exit
     --extractor-descriptions        Output descriptions of all supported
                                     extractors and exit
-    --force-generic-extractor       Force extraction to use the generic extractor
-    --default-search PREFIX         Use this prefix for unqualified URLs. Eg:
+    --use-extractors NAMES          Extractor names to use separated by commas.
+                                    You can also use regexes, "all", "default"
+                                    and "end" (end URL matching); e.g. --ies
+                                    "holodex.*,end,youtube". Prefix the name
+                                    with a "-" to exclude it, e.g. --ies
+                                    default,-generic. Use --list-extractors for
+                                    a list of extractor names. (Alias: --ies)
+    --default-search PREFIX         Use this prefix for unqualified URLs. E.g.
                                     "gvsearch2:python" downloads two videos from
                                     google videos for the search term "python".
                                     Use the value "auto" to let yt-dlp guess
@@ -425,7 +432,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     an alias starts with a dash "-", it is
                                     prefixed with "--". Arguments are parsed
                                     according to the Python string formatting
-                                    mini-language. Eg: --alias get-audio,-X
+                                    mini-language. E.g. --alias get-audio,-X
                                     "-S=aext:{0},abr -x --audio-format {0}"
                                     creates options "--get-audio" and "-X" that
                                     takes an argument (ARG0) and expands to
@@ -439,10 +446,10 @@ You can also fork the project on github and run your fork's [build workflow](.gi
 
 ## Network Options:
     --proxy URL                     Use the specified HTTP/HTTPS/SOCKS proxy. To
-                                    enable SOCKS proxy, specify a proper scheme.
-                                    Eg: socks5://user:pass@127.0.0.1:1080/. Pass
-                                    in an empty string (--proxy "") for direct
-                                    connection
+                                    enable SOCKS proxy, specify a proper scheme,
+                                    e.g. socks5://user:pass@127.0.0.1:1080/.
+                                    Pass in an empty string (--proxy "") for
+                                    direct connection
     --socket-timeout SECONDS        Time to wait before giving up, in seconds
     --source-address IP             Client-side IP address to bind to
     -4, --force-ipv4                Make all connections via IPv4
@@ -471,17 +478,17 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     compatibility, START-STOP is also supported.
                                     Use negative indices to count from the right
                                     and negative STEP to download in reverse
-                                    order. Eg: "-I 1:3,7,-5::2" used on a
+                                    order. E.g. "-I 1:3,7,-5::2" used on a
                                     playlist of size 15 will download the videos
                                     at index 1,2,3,7,11,13,15
-    --min-filesize SIZE             Do not download any videos smaller than SIZE
-                                    (e.g. 50k or 44.6m)
-    --max-filesize SIZE             Do not download any videos larger than SIZE
-                                    (e.g. 50k or 44.6m)
+    --min-filesize SIZE             Do not download any videos smaller than
+                                    SIZE, e.g. 50k or 44.6M
+    --max-filesize SIZE             Do not download any videos larger than SIZE,
+                                    e.g. 50k or 44.6M
     --date DATE                     Download only videos uploaded on this date.
                                     The date can be "YYYYMMDD" or in the format 
                                     [now|today|yesterday][-N[day|week|month|year]].
-                                    Eg: --date today-2weeks
+                                    E.g. --date today-2weeks
     --datebefore DATE               Download only videos uploaded on or before
                                     this date. The date formats accepted is the
                                     same as --date
@@ -491,14 +498,14 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     --match-filters FILTER          Generic video filter. Any "OUTPUT TEMPLATE"
                                     field can be compared with a number or a
                                     string using the operators defined in
-                                    "Filtering formats". You can also simply
+                                    "Filtering Formats". You can also simply
                                     specify a field to match if the field is
                                     present, use "!field" to check if the field
                                     is not present, and "&" to check multiple
                                     conditions. Use a "\" to escape "&" or
                                     quotes if needed. If used multiple times,
                                     the filter matches if atleast one of the
-                                    conditions are met. Eg: --match-filter
+                                    conditions are met. E.g. --match-filter
                                     !is_live --match-filter "like_count>?100 &
                                     description~='(?i)\bcats \& dogs\b'" matches
                                     only videos that are not live OR those that
@@ -524,8 +531,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     a file that is in the archive
     --break-on-reject               Stop the download process when encountering
                                     a file that has been filtered out
-    --break-per-input               Make --break-on-existing, --break-on-reject
-                                    and --max-downloads act only on the current
+    --break-per-input               --break-on-existing, --break-on-reject,
+                                    --max-downloads, and autonumber resets per
                                     input URL
     --no-break-per-input            --break-on-existing and similar options
                                     terminates the entire download queue
@@ -536,25 +543,25 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     -N, --concurrent-fragments N    Number of fragments of a dash/hlsnative
                                     video that should be downloaded concurrently
                                     (default is 1)
-    -r, --limit-rate RATE           Maximum download rate in bytes per second
-                                    (e.g. 50K or 4.2M)
+    -r, --limit-rate RATE           Maximum download rate in bytes per second,
+                                    e.g. 50K or 4.2M
     --throttled-rate RATE           Minimum download rate in bytes per second
                                     below which throttling is assumed and the
-                                    video data is re-extracted (e.g. 100K)
+                                    video data is re-extracted, e.g. 100K
     -R, --retries RETRIES           Number of retries (default is 10), or
                                     "infinite"
     --file-access-retries RETRIES   Number of times to retry on file access
                                     error (default is 3), or "infinite"
     --fragment-retries RETRIES      Number of retries for a fragment (default is
                                     10), or "infinite" (DASH, hlsnative and ISM)
-    --retry-sleep [TYPE:]EXPR       An expression for the time to sleep between
-                                    retries in seconds (optionally) prefixed by
-                                    the type of retry (file_access, fragment,
-                                    http (default)) to apply the sleep to. EXPR
-                                    can be a number, linear=START[:END[:STEP=1]]
-                                    or exp=START[:END[:BASE=2]]. This option can
-                                    be used multiple times to set the sleep for
-                                    the different retry types. Eg: --retry-sleep
+    --retry-sleep [TYPE:]EXPR       Time to sleep between retries in seconds
+                                    (optionally) prefixed by the type of retry
+                                    (http (default), fragment, file_access,
+                                    extractor) to apply the sleep to. EXPR can
+                                    be a number, linear=START[:END[:STEP=1]] or
+                                    exp=START[:END[:BASE=2]]. This option can be
+                                    used multiple times to set the sleep for the
+                                    different retry types, e.g. --retry-sleep
                                     linear=1::2 --retry-sleep fragment:exp=1:20
     --skip-unavailable-fragments    Skip unavailable fragments for DASH,
                                     hlsnative and ISM downloads (default)
@@ -566,14 +573,14 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     downloading is finished
     --no-keep-fragments             Delete downloaded fragments after
                                     downloading is finished (default)
-    --buffer-size SIZE              Size of download buffer (e.g. 1024 or 16K)
+    --buffer-size SIZE              Size of download buffer, e.g. 1024 or 16K
                                     (default is 1024)
     --resize-buffer                 The buffer size is automatically resized
                                     from an initial value of --buffer-size
                                     (default)
     --no-resize-buffer              Do not automatically adjust the buffer size
     --http-chunk-size SIZE          Size of a chunk for chunk-based HTTP
-                                    downloading (e.g. 10485760 or 10M) (default
+                                    downloading, e.g. 10485760 or 10M (default
                                     is disabled). May be useful for bypassing
                                     bandwidth throttling imposed by a webserver
                                     (experimental)
@@ -598,10 +605,10 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     the given regular expression. Time ranges
                                     prefixed by a "*" can also be used in place
                                     of chapters to download the specified range.
-                                    Eg: --download-sections "*10:15-15:00"
-                                    --download-sections "intro". Needs ffmpeg.
-                                    This option can be used multiple times to
-                                    download multiple sections
+                                    Needs ffmpeg. This option can be used
+                                    multiple times to download multiple
+                                    sections, e.g. --download-sections
+                                    "*10:15-15:00" --download-sections "intro"
     --downloader [PROTO:]NAME       Name or path of the external downloader to
                                     use (optionally) prefixed by the protocols
                                     (http, ftp, m3u8, dash, rstp, rtmp, mms) to
@@ -609,7 +616,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     aria2c, avconv, axel, curl, ffmpeg, httpie,
                                     wget. You can use this option multiple times
                                     to set different downloaders for different
-                                    protocols. For example, --downloader aria2c
+                                    protocols. E.g. --downloader aria2c
                                     --downloader "dash,m3u8:native" will use
                                     aria2c for http/ftp downloads, and the
                                     native downloader for dash/m3u8 downloads
@@ -700,18 +707,20 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     and dump cookie jar in
     --no-cookies                    Do not read/dump cookies from/to file
                                     (default)
-    --cookies-from-browser BROWSER[+KEYRING][:PROFILE]
-                                    The name of the browser and (optionally) the
-                                    name/path of the profile to load cookies
-                                    from, separated by a ":". Currently
-                                    supported browsers are: brave, chrome,
-                                    chromium, edge, firefox, opera, safari,
-                                    vivaldi. By default, the most recently
-                                    accessed profile is used. The keyring used
-                                    for decrypting Chromium cookies on Linux can
-                                    be (optionally) specified after the browser
-                                    name separated by a "+". Currently supported
-                                    keyrings are: basictext, gnomekeyring, kwallet
+    --cookies-from-browser BROWSER[+KEYRING][:PROFILE][::CONTAINER]
+                                    The name of the browser to load cookies
+                                    from. Currently supported browsers are:
+                                    brave, chrome, chromium, edge, firefox,
+                                    opera, safari, vivaldi. Optionally, the
+                                    KEYRING used for decrypting Chromium cookies
+                                    on Linux, the name/path of the PROFILE to
+                                    load cookies from, and the CONTAINER name
+                                    (if Firefox) ("none" for no container) can
+                                    be given with their respective seperators.
+                                    By default, all containers of the most
+                                    recently accessed profile are used.
+                                    Currently supported keyrings are: basictext,
+                                    gnomekeyring, kwallet
     --no-cookies-from-browser       Do not load cookies from browser (default)
     --cache-dir DIR                 Location in the filesystem where youtube-dl
                                     can store some downloaded information (such
@@ -791,7 +800,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     "postprocess:",  or "postprocess-title:".
                                     The video's fields are accessible under the
                                     "info" key and the progress attributes are
-                                    accessible under "progress" key. E.g.:
+                                    accessible under "progress" key. E.g.
                                     --console-title --progress-template
                                     "download-title:%(info.id)s-%(progress.eta)s"
     -v, --verbose                   Print various debugging information
@@ -859,10 +868,10 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     downloadable
     -F, --list-formats              List available formats of each video.
                                     Simulate unless --no-simulate is used
-    --merge-output-format FORMAT    If a merge is required (e.g.
-                                    bestvideo+bestaudio), output to given
-                                    container format. One of mkv, mp4, ogg,
-                                    webm, flv. Ignored if no merge is required
+    --merge-output-format FORMAT    Containers that may be used when merging
+                                    formats, separated by "/", e.g. "mp4/mkv".
+                                    Ignored if no merge is required. (currently
+                                    supported: avi, flv, mkv, mov, mp4, webm)
 
 ## Subtitle Options:
     --write-subs                    Write subtitle file
@@ -874,13 +883,13 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     --list-subs                     List available subtitles of each video.
                                     Simulate unless --no-simulate is used
     --sub-format FORMAT             Subtitle format; accepts formats preference,
-                                    Eg: "srt" or "ass/srt/best"
+                                    e.g. "srt" or "ass/srt/best"
     --sub-langs LANGS               Languages of the subtitles to download (can
-                                    be regex) or "all" separated by commas. (Eg:
-                                    --sub-langs "en.*,ja") You can prefix the
+                                    be regex) or "all" separated by commas, e.g.
+                                    --sub-langs "en.*,ja". You can prefix the
                                     language code with a "-" to exclude it from
-                                    the requested languages. (Eg: --sub-langs
-                                    all,-live_chat) Use --list-subs for a list
+                                    the requested languages, e.g. --sub-langs
+                                    all,-live_chat. Use --list-subs for a list
                                     of available language tags
 
 ## Authentication Options:
@@ -916,7 +925,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     (requires ffmpeg and ffprobe)
     --audio-format FORMAT           Format to convert the audio to when -x is
                                     used. (currently supported: best (default),
-                                    mp3, aac, m4a, opus, vorbis, flac, alac,
+                                    aac, alac, flac, m4a, mp3, opus, vorbis,
                                     wav). You can specify multiple rules using
                                     similar syntax as --remux-video
     --audio-quality QUALITY         Specify ffmpeg audio quality to use when
@@ -924,12 +933,12 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     between 0 (best) and 10 (worst) for VBR or a
                                     specific bitrate like 128K (default 5)
     --remux-video FORMAT            Remux the video into another container if
-                                    necessary (currently supported: mp4, mkv,
-                                    flv, webm, mov, avi, mka, ogg, mp3, aac,
-                                    m4a, opus, vorbis, flac, alac, wav). If
+                                    necessary (currently supported: avi, flv,
+                                    mkv, mov, mp4, webm, aac, aiff, alac, flac,
+                                    m4a, mka, mp3, ogg, opus, vorbis, wav). If
                                     target container does not support the
                                     video/audio codec, remuxing will fail. You
-                                    can specify multiple rules; Eg.
+                                    can specify multiple rules; e.g.
                                     "aac>m4a/mov>mp4/mkv" will remux aac to m4a,
                                     mov to mp4 and anything else to mkv
     --recode-video FORMAT           Re-encode the video into another format if
@@ -954,7 +963,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     for ffmpeg/ffprobe, "_i"/"_o" can be
                                     appended to the prefix optionally followed
                                     by a number to pass the argument before the
-                                    specified input/output file. Eg: --ppa
+                                    specified input/output file, e.g. --ppa
                                     "Merger+ffmpeg_i1:-v quiet". You can use
                                     this option multiple times to give different
                                     arguments to different postprocessors.
@@ -1025,7 +1034,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     be used multiple times
     --no-exec                       Remove any previously defined --exec
     --convert-subs FORMAT           Convert the subtitles to another format
-                                    (currently supported: srt, vtt, ass, lrc)
+                                    (currently supported: ass, lrc, srt, vtt)
                                     (Alias: --convert-subtitles)
     --convert-thumbnails FORMAT     Convert the thumbnails to another format
                                     (currently supported: jpg, png, webp). You
@@ -1081,7 +1090,7 @@ Make chapter entries for, or remove various segments (sponsor,
                                     music_offtopic, poi_highlight, all and
                                     default (=all). You can prefix the category
                                     with a "-" to exclude it. See [1] for
-                                    description of the categories. Eg:
+                                    description of the categories. E.g.
                                     --sponsorblock-mark all,-preview
                                     [1] https://wiki.sponsor.ajay.app/w/Segment_Categories
     --sponsorblock-remove CATS      SponsorBlock categories to be removed from
@@ -1140,7 +1149,7 @@ You can configure yt-dlp by placing any supported command line option to a confi
 
 1. **System Configuration**: `/etc/yt-dlp.conf`
 
-For example, with the following configuration file yt-dlp will always extract the audio, not copy the mtime, use a proxy and save all videos under `YouTube` directory in your home directory:
+E.g. with the following configuration file yt-dlp will always extract the audio, not copy the mtime, use a proxy and save all videos under `YouTube` directory in your home directory:
 ```
 # Lines starting with # are comments
 
@@ -1178,7 +1187,7 @@ After that you can add credentials for an extractor in the following format, whe
 ```
 machine <extractor> login <username> password <password>
 ```
-For example:
+E.g.
 ```
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
@@ -1197,39 +1206,38 @@ The `-o` option is used to indicate a template for the output file names while `
 
 The simplest usage of `-o` is not to set any template arguments when downloading a single file, like in `yt-dlp -o funny_video.flv "https://some/video"` (hard-coding file extension like this is _not_ recommended and could break some post-processing).
 
-It may however also contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [Python string formatting operations](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations.
+It may however also contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [Python string formatting operations](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting), e.g. `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations.
 
 The field names themselves (the part inside the parenthesis) can also have some special formatting:
 
-1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a `.` (dot) separator. You can also do python slicing using `:`. Eg: `%(tags.0)s`, `%(subtitles.en.-1.ext)s`, `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. `%()s` refers to the entire infodict. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
+1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a `.` (dot) separator. You can also do python slicing using `:`. E.g. `%(tags.0)s`, `%(subtitles.en.-1.ext)s`, `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. `%()s` refers to the entire infodict. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
 
-1. **Addition**: Addition and subtraction of numeric fields can be done using `+` and `-` respectively. Eg: `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
+1. **Addition**: Addition and subtraction of numeric fields can be done using `+` and `-` respectively. E.g. `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
 
-1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. Eg: `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s`
+1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. E.g. `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s`
 
-1. **Alternatives**: Alternate fields can be specified separated with a `,`. Eg: `%(release_date>%Y,upload_date>%Y|Unknown)s`
+1. **Alternatives**: Alternate fields can be specified separated with a `,`. E.g. `%(release_date>%Y,upload_date>%Y|Unknown)s`
 
 1. **Replacement**: A replacement value can specified using a `&` separator. If the field is *not* empty, this replacement value will be used instead of the actual field content. This is done after alternate fields are considered; thus the replacement is used if *any* of the alternative fields is *not* empty.
 
-1. **Default**: A literal default value can be specified for when the field is empty using a `|` separator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s`
+1. **Default**: A literal default value can be specified for when the field is empty using a `|` separator. This overrides `--output-na-template`. E.g. `%(uploader|Unknown)s`
 
-1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, yt-dlp additionally supports converting to `B` = **B**ytes, `j` = **j**son (flag `#` for pretty-printing), `h` = HTML escaping, `l` = a comma separated **l**ist (flag `#` for `\n` newline-separated), `q` = a string **q**uoted for the terminal (flag `#` to split a list into different arguments), `D` = add **D**ecimal suffixes (Eg: 10M) (flag `#` to use 1024 as factor), and `S` = **S**anitize as filename (flag `#` for restricted)
+1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, yt-dlp additionally supports converting to `B` = **B**ytes, `j` = **j**son (flag `#` for pretty-printing), `h` = HTML escaping, `l` = a comma separated **l**ist (flag `#` for `\n` newline-separated), `q` = a string **q**uoted for the terminal (flag `#` to split a list into different arguments), `D` = add **D**ecimal suffixes (e.g. 10M) (flag `#` to use 1024 as factor), and `S` = **S**anitize as filename (flag `#` for restricted)
 
-1. **Unicode normalization**: The format type `U` can be used for NFC [unicode normalization](https://docs.python.org/3/library/unicodedata.html#unicodedata.normalize). The alternate form flag (`#`) changes the normalization to NFD and the conversion flag `+` can be used for NFKC/NFKD compatibility equivalence normalization. Eg: `%(title)+.100U` is NFKC
+1. **Unicode normalization**: The format type `U` can be used for NFC [unicode normalization](https://docs.python.org/3/library/unicodedata.html#unicodedata.normalize). The alternate form flag (`#`) changes the normalization to NFD and the conversion flag `+` can be used for NFKC/NFKD compatibility equivalence normalization. E.g. `%(title)+.100U` is NFKC
 
 To summarize, the general syntax for a field is:
 ```
 %(name[.keys][addition][>strf][,alternate][&replacement][|default])[flags][width][.precision][length]type
 ```
 
-Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`, `pl_video`. For example, `-o "%(title)s.%(ext)s" -o "thumbnail:%(title)s\%(title)s.%(ext)s"`  will put the thumbnails in a folder with the same name as the video. If any of the templates is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
+Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`, `pl_video`. E.g. `-o "%(title)s.%(ext)s" -o "thumbnail:%(title)s\%(title)s.%(ext)s"`  will put the thumbnails in a folder with the same name as the video. If any of the templates is empty, that type of file will not be written. E.g. `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
 
 The available fields are:
 
  - `id` (string): Video identifier
  - `title` (string): Video title
  - `fulltitle` (string): Video title ignoring live timestamp and generic title
- - `url` (string): Video URL
  - `ext` (string): Video filename extension
  - `alt_title` (string): A secondary title of the video
  - `description` (string): The description of the video
@@ -1257,32 +1265,13 @@ The available fields are:
  - `average_rating` (numeric): Average rating give by users, the scale used depends on the webpage
  - `comment_count` (numeric): Number of comments on the video (For some extractors, comments are only downloaded at the end, and so this field cannot be used)
  - `age_limit` (numeric): Age restriction for the video (years)
- - `live_status` (string): One of "is_live", "was_live", "is_upcoming", "not_live"
+ - `live_status` (string): One of "not_live", "is_live", "is_upcoming", "was_live", "post_live" (was live, but VOD is not yet processed)
  - `is_live` (boolean): Whether this video is a live stream or a fixed-length video
  - `was_live` (boolean): Whether this video was originally a live stream
  - `playable_in_embed` (string): Whether this video is allowed to play in embedded players on other sites
  - `availability` (string): Whether the video is "private", "premium_only", "subscriber_only", "needs_auth", "unlisted" or "public"
  - `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL
  - `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL
- - `format` (string): A human-readable description of the format
- - `format_id` (string): Format code specified by `--format`
- - `format_note` (string): Additional info about the format
- - `width` (numeric): Width of the video
- - `height` (numeric): Height of the video
- - `resolution` (string): Textual description of width and height
- - `tbr` (numeric): Average bitrate of audio and video in KBit/s
- - `abr` (numeric): Average audio bitrate in KBit/s
- - `acodec` (string): Name of the audio codec in use
- - `asr` (numeric): Audio sampling rate in Hertz
- - `vbr` (numeric): Average video bitrate in KBit/s
- - `fps` (numeric): Frame rate
- - `dynamic_range` (string): The dynamic range of the video
- - `stretched_ratio` (float): `width:height` of the video's pixels, if not square
- - `vcodec` (string): Name of the video codec in use
- - `container` (string): Name of the container format
- - `filesize` (numeric): The number of bytes, if known in advance
- - `filesize_approx` (numeric): An estimate for the number of bytes
- - `protocol` (string): The protocol that will be used for the actual download
  - `extractor` (string): Name of the extractor
  - `extractor_key` (string): Key name of the extractor
  - `epoch` (numeric): Unix epoch of when the information extraction was completed
@@ -1301,6 +1290,8 @@ The available fields are:
  - `webpage_url_basename` (string): The basename of the webpage URL
  - `webpage_url_domain` (string): The domain of the webpage URL
  - `original_url` (string): The URL given by the user (or same as `webpage_url` for playlist entries)
+ 
+All the fields in [Filtering Formats](#filtering-formats) can also be used
 
 Available for the video that belongs to some logical chapter or section:
 
@@ -1357,13 +1348,13 @@ Available only in `--sponsorblock-chapter-title`:
  - `category_names` (list): Friendly names of the categories
  - `name` (string): Friendly name of the smallest category
 
-Each aforementioned sequence when referenced in an output template will be replaced by the actual value corresponding to the sequence name. For example for `-o %(title)s-%(id)s.%(ext)s` and an mp4 video with title `yt-dlp test video` and id `BaW_jenozKc`, this will result in a `yt-dlp test video-BaW_jenozKc.mp4` file created in the current directory.
+Each aforementioned sequence when referenced in an output template will be replaced by the actual value corresponding to the sequence name. E.g. for `-o %(title)s-%(id)s.%(ext)s` and an mp4 video with title `yt-dlp test video` and id `BaW_jenozKc`, this will result in a `yt-dlp test video-BaW_jenozKc.mp4` file created in the current directory.
 
 Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with placeholder value provided with `--output-na-placeholder` (`NA` by default).
 
 **Tip**: Look at the `-j` output to identify which fields are available for the particular URL
 
-For numeric sequences you can use [numeric related formatting](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting), for example, `%(view_count)05d` will result in a string with view count padded with zeros up to 5 characters, like in `00042`.
+For numeric sequences you can use [numeric related formatting](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting); e.g. `%(view_count)05d` will result in a string with view count padded with zeros up to 5 characters, like in `00042`.
 
 Output templates can also contain arbitrary hierarchical path, e.g. `-o "%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s"` which will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
 
@@ -1382,13 +1373,13 @@ If you are using an output template inside a Windows batch file then you must es
 #### Output template examples
 
 ```bash
-$ yt-dlp --get-filename -o "test video.%(ext)s" BaW_jenozKc
+$ yt-dlp --print filename -o "test video.%(ext)s" BaW_jenozKc
 test video.webm    # Literal name with correct extension
 
-$ yt-dlp --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
+$ yt-dlp --print filename -o "%(title)s.%(ext)s" BaW_jenozKc
 youtube-dl test video ''_ä↭𝕐.webm    # All kinds of weird characters
 
-$ yt-dlp --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
+$ yt-dlp --print filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
 youtube-dl_test_video_.webm    # Restricted file name
 
 # Download YouTube playlist videos in separate directory indexed by video order in a playlist
@@ -1433,7 +1424,7 @@ The general syntax for format selection is `-f FORMAT` (or `--format FORMAT`) wh
 **tl;dr:** [navigate me to examples](#format-selection-examples).
 <!-- MANPAGE: END EXCLUDED SECTION -->
 
-The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific.
+The simplest case is requesting a specific format; e.g. with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific.
 
 You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`, `mp4`, `ogg`, `wav`, `webm` are supported) to download the best quality format of a particular file extension served as a single file, e.g. `-f webm` will download the best quality format with the `webm` extension served as a single file.
 
@@ -1443,7 +1434,7 @@ You can also use special names to select particular edge case formats:
 
  - `all`: Select **all formats** separately
  - `mergeall`: Select and **merge all formats** (Must be used with `--audio-multistreams`, `--video-multistreams` or both)
- - `b*`, `best*`: Select the best quality format that **contains either** a video or an audio
+ - `b*`, `best*`: Select the best quality format that **contains either** a video or an audio or both (ie; `vcodec!=none or acodec!=none`)
  - `b`, `best`: Select the best quality format that **contains both** video and audio. Equivalent to `best*[vcodec!=none][acodec!=none]`
  - `bv`, `bestvideo`: Select the best quality **video-only** format. Equivalent to `best*[acodec=none]`
  - `bv*`, `bestvideo*`: Select the best quality format that **contains video**. It may also contain audio. Equivalent to `best*[vcodec!=none]`
@@ -1456,19 +1447,19 @@ You can also use special names to select particular edge case formats:
  - `wa`, `worstaudio`: Select the worst quality audio-only format. Equivalent to `worst*[vcodec=none]`
  - `wa*`, `worstaudio*`: Select the worst quality format that contains audio. It may also contain video. Equivalent to `worst*[acodec!=none]`
 
-For example, to download the worst quality video-only format you can use `-f worstvideo`. It is however recommended not to use `worst` and related options. When your format selector is `worst`, the format which is worst in all respects is selected. Most of the time, what you actually want is the video with the smallest filesize instead. So it is generally better to use `-S +size` or more rigorously, `-S +size,+br,+res,+fps` instead of `-f worst`. See [sorting formats](#sorting-formats) for more details.
+For example, to download the worst quality video-only format you can use `-f worstvideo`. It is however recommended not to use `worst` and related options. When your format selector is `worst`, the format which is worst in all respects is selected. Most of the time, what you actually want is the video with the smallest filesize instead. So it is generally better to use `-S +size` or more rigorously, `-S +size,+br,+res,+fps` instead of `-f worst`. See [Sorting Formats](#sorting-formats) for more details.
 
 You can select the n'th best format of a type by using `best<type>.<n>`. For example, `best.2` will select the 2nd best combined format. Similarly, `bv*.3` will select the 3rd best format that contains a video stream.
 
-If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes. Note that formats on the left hand side are preferred, for example `-f 22/17/18` will download format 22 if it's available, otherwise it will download format 17 if it's available, otherwise it will download format 18 if it's available, otherwise it will complain that no suitable formats are available for download.
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes. Note that formats on the left hand side are preferred; e.g. `-f 22/17/18` will download format 22 if it's available, otherwise it will download format 17 if it's available, otherwise it will download format 18 if it's available, otherwise it will complain that no suitable formats are available for download.
 
 If you want to download several formats of the same video use a comma as a separator, e.g. `-f 22,17,18` will download all these three formats, of course if they are available. Or a more sophisticated example combined with the precedence feature: `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`.
 
-You can merge the video and audio of multiple formats into a single file using `-f <format1>+<format2>+...` (requires ffmpeg installed), for example `-f bestvideo+bestaudio` will download the best video-only format, the best audio-only format and mux them together with ffmpeg.
+You can merge the video and audio of multiple formats into a single file using `-f <format1>+<format2>+...` (requires ffmpeg installed); e.g. `-f bestvideo+bestaudio` will download the best video-only format, the best audio-only format and mux them together with ffmpeg.
 
 **Deprecation warning**: Since the *below* described behavior is complex and counter-intuitive, this will be removed and multistreams will be enabled by default in the future. A new operator will be instead added to limit formats to single audio/video
 
-Unless `--video-multistreams` is used, all formats with a video stream except the first one are ignored. Similarly, unless `--audio-multistreams` is used, all formats with an audio stream except the first one are ignored. For example, `-f bestvideo+best+bestaudio --video-multistreams --audio-multistreams` will download and merge all 3 given formats. The resulting file will have 2 video streams and 2 audio streams. But `-f bestvideo+best+bestaudio --no-video-multistreams` will download and merge only `bestvideo` and `bestaudio`. `best` is ignored since another format containing a video stream (`bestvideo`) has already been selected. The order of the formats is therefore important. `-f best+bestaudio --no-audio-multistreams` will download and merge both formats while `-f bestaudio+best --no-audio-multistreams` will ignore `best` and download only `bestaudio`.
+Unless `--video-multistreams` is used, all formats with a video stream except the first one are ignored. Similarly, unless `--audio-multistreams` is used, all formats with an audio stream except the first one are ignored. E.g. `-f bestvideo+best+bestaudio --video-multistreams --audio-multistreams` will download and merge all 3 given formats. The resulting file will have 2 video streams and 2 audio streams. But `-f bestvideo+best+bestaudio --no-video-multistreams` will download and merge only `bestvideo` and `bestaudio`. `best` is ignored since another format containing a video stream (`bestvideo`) has already been selected. The order of the formats is therefore important. `-f best+bestaudio --no-audio-multistreams` will download and merge both formats while `-f bestaudio+best --no-audio-multistreams` will ignore `best` and download only `bestaudio`.
 
 ## Filtering Formats
 
@@ -1477,6 +1468,7 @@ You can also filter the video formats by putting a condition in brackets, as in
 The following numeric meta fields can be used with comparisons `<`, `<=`, `>`, `>=`, `=` (equals), `!=` (not equals):
 
  - `filesize`: The number of bytes, if known in advance
+ - `filesize_approx`: An estimate for the number of bytes
  - `width`: Width of the video, if known
  - `height`: Height of the video, if known
  - `tbr`: Average bitrate of audio and video in KBit/s
@@ -1484,24 +1476,31 @@ The following numeric meta fields can be used with comparisons `<`, `<=`, `>`, `
  - `vbr`: Average video bitrate in KBit/s
  - `asr`: Audio sampling rate in Hertz
  - `fps`: Frame rate
+ - `audio_channels`: The number of audio channels
+ - `stretched_ratio`: `width:height` of the video's pixels, if not square
 
 Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends with), `*=` (contains), `~=` (matches regex) and following string meta fields:
 
+ - `url`: Video URL
  - `ext`: File extension
  - `acodec`: Name of the audio codec in use
  - `vcodec`: Name of the video codec in use
  - `container`: Name of the container format
  - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `http_dash_segments`, `m3u8`, or `m3u8_native`)
- - `format_id`: A short description of the format
  - `language`: Language code
+ - `dynamic_range`: The dynamic range of the video
+ - `format_id`: A short description of the format
+ - `format`: A human-readable description of the format
+ - `format_note`: Additional info about the format
+ - `resolution`: Textual description of width and height
 
 Any string comparison may be prefixed with negation `!` in order to produce an opposite comparison, e.g. `!*=` (does not contain). The comparand of a string comparison needs to be quoted with either double or single quotes if it contains spaces or special characters other than `._-`.
 
 Note that none of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by the website. Any other field made available by the extractor can also be used for filtering.
 
-Formats for which the value is not known are excluded unless you put a question mark (`?`) after the operator. You can combine format filters, so `-f "[height<=?720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. You can also use the filters with `all` to download all formats that satisfy the filter. For example, `-f "all[vcodec=none]"` selects all audio-only formats.
+Formats for which the value is not known are excluded unless you put a question mark (`?`) after the operator. You can combine format filters, so `-f "[height<=?720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. You can also use the filters with `all` to download all formats that satisfy the filter, e.g. `-f "all[vcodec=none]"` selects all audio-only formats.
 
-Format selectors can also be grouped using parentheses, for example if you want to download the best pre-merged mp4 and webm formats with a height lower than 480 you can use `-f "(mp4,webm)[height<480]"`.
+Format selectors can also be grouped using parentheses; e.g. `-f "(mp4,webm)[height<480]"` will download the best pre-merged mp4 and webm formats with a height lower than 480.
 
 ## Sorting Formats
 
@@ -1520,7 +1519,7 @@ The available fields are:
  - `acodec`: Audio Codec (`flac`/`alac` > `wav`/`aiff` > `opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `eac3` > `ac3` > `dts` > other)
  - `codec`: Equivalent to `vcodec,acodec`
  - `vext`: Video Extension (`mp4` > `webm` > `flv` > other). If `--prefer-free-formats` is used, `webm` is preferred.
- - `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other). If `--prefer-free-formats` is used, the order changes to `opus` > `ogg` > `webm` > `m4a` > `mp3` > `aac`.
+ - `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other). If `--prefer-free-formats` is used, the order changes to `ogg` > `opus` > `webm` > `mp3` > `m4a` > `aac`
  - `ext`: Equivalent to `vext,aext`
  - `filesize`: Exact filesize, if known in advance
  - `fs_approx`: Approximate filesize calculated from the manifests
@@ -1530,6 +1529,7 @@ The available fields are:
  - `res`: Video resolution, calculated as the smallest dimension.
  - `fps`: Framerate of video
  - `hdr`: The dynamic range of the video (`DV` > `HDR12` > `HDR10+` > `HDR10` > `HLG` > `SDR`)
+ - `channels`: The number of audio channels
  - `tbr`: Total average bitrate in KBit/s
  - `vbr`: Average video bitrate in KBit/s
  - `abr`: Average audio bitrate in KBit/s
@@ -1538,11 +1538,11 @@ The available fields are:
  
 **Deprecation warning**: Many of these fields have (currently undocumented) aliases, that may be removed in a future version. It is recommended to use only the documented field names.
 
-All fields, unless specified otherwise, are sorted in descending order. To reverse this, prefix the field with a `+`. Eg: `+res` prefers format with the smallest resolution. Additionally, you can suffix a preferred value for the fields, separated by a `:`. Eg: `res:720` prefers larger videos, but no larger than 720p and the smallest video if there are no videos less than 720p. For `codec` and `ext`, you can provide two preferred values, the first for video and the second for audio. Eg: `+codec:avc:m4a` (equivalent to `+vcodec:avc,+acodec:m4a`) sets the video codec preference to `h264` > `h265` > `vp9` > `vp9.2` > `av01` > `vp8` > `h263` > `theora` and audio codec preference to `mp4a` > `aac` > `vorbis` > `opus` > `mp3` > `ac3` > `dts`. You can also make the sorting prefer the nearest values to the provided by using `~` as the delimiter. Eg: `filesize~1G` prefers the format with filesize closest to 1 GiB.
+All fields, unless specified otherwise, are sorted in descending order. To reverse this, prefix the field with a `+`. E.g. `+res` prefers format with the smallest resolution. Additionally, you can suffix a preferred value for the fields, separated by a `:`. E.g. `res:720` prefers larger videos, but no larger than 720p and the smallest video if there are no videos less than 720p. For `codec` and `ext`, you can provide two preferred values, the first for video and the second for audio. E.g. `+codec:avc:m4a` (equivalent to `+vcodec:avc,+acodec:m4a`) sets the video codec preference to `h264` > `h265` > `vp9` > `vp9.2` > `av01` > `vp8` > `h263` > `theora` and audio codec preference to `mp4a` > `aac` > `vorbis` > `opus` > `mp3` > `ac3` > `dts`. You can also make the sorting prefer the nearest values to the provided by using `~` as the delimiter. E.g. `filesize~1G` prefers the format with filesize closest to 1 GiB.
 
-The fields `hasvid` and `ie_pref` are always given highest priority in sorting, irrespective of the user-defined order. This behaviour can be changed by using `--format-sort-force`. Apart from these, the default order used is: `lang,quality,res,fps,hdr:12,codec:vp9.2,size,br,asr,proto,ext,hasaud,source,id`. The extractors may override this default order, but they cannot override the user-provided order.
+The fields `hasvid` and `ie_pref` are always given highest priority in sorting, irrespective of the user-defined order. This behaviour can be changed by using `--format-sort-force`. Apart from these, the default order used is: `lang,quality,res,fps,hdr:12,vcodec:vp9.2,channels,acodec,size,br,asr,proto,ext,hasaud,source,id`. The extractors may override this default order, but they cannot override the user-provided order.
 
-Note that the default has `codec:vp9.2`; i.e. `av1` is not preferred. Similarly, the default for hdr is `hdr:12`; i.e. dolby vision is not preferred. These choices are made since DV and AV1 formats are not yet fully compatible with most devices. This may be changed in the future as more devices become capable of smoothly playing back these formats.
+Note that the default has `vcodec:vp9.2`; i.e. `av1` is not preferred. Similarly, the default for hdr is `hdr:12`; i.e. dolby vision is not preferred. These choices are made since DV and AV1 formats are not yet fully compatible with most devices. This may be changed in the future as more devices become capable of smoothly playing back these formats.
 
 If your format selector is `worst`, the last item is selected after sorting. This means it will select the format that is worst in all respects. Most of the time, what you actually want is the video with the smallest filesize instead. So it is generally better to use `-f best -S +size,+br,+res,+fps`.
 
@@ -1683,9 +1683,9 @@ Note that any field created by this can be used in the [output template](#output
 
 This option also has a few special uses:
 
-* You can download an additional URL based on the metadata of the currently downloaded video. To do this, set the field `additional_urls` to the URL that you want to download. Eg: `--parse-metadata "description:(?P<additional_urls>https?://www\.vimeo\.com/\d+)` will download the first vimeo video found in the description
+* You can download an additional URL based on the metadata of the currently downloaded video. To do this, set the field `additional_urls` to the URL that you want to download. E.g. `--parse-metadata "description:(?P<additional_urls>https?://www\.vimeo\.com/\d+)` will download the first vimeo video found in the description
 
-* You can use this to change the metadata that is embedded in the media file. To do this, set the value of the corresponding field with a `meta_` prefix. For example, any value you set to `meta_description` field will be added to the `description` field in the file. For example, you can use this to set a different "description" and "synopsis". To modify the metadata of individual streams, use the `meta<n>_` prefix (Eg: `meta1_language`). Any value set to the `meta_` field will overwrite all default values.
+* You can use this to change the metadata that is embedded in the media file. To do this, set the value of the corresponding field with a `meta_` prefix. For example, any value you set to `meta_description` field will be added to the `description` field in the file - you can use this to set a different "description" and "synopsis". To modify the metadata of individual streams, use the `meta<n>_` prefix (e.g. `meta1_language`). Any value set to the `meta_` field will overwrite all default values.
 
 **Note**: Metadata modification happens before format selection, post-extraction and other post-processing operations. Some fields may be added or changed during these steps, overriding your changes.
 
@@ -1744,21 +1744,19 @@ $ yt-dlp --replace-in-metadata "title,uploader" "[ _]" "-"
 
 # EXTRACTOR ARGUMENTS
 
-Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. Eg: `--extractor-args "youtube:player-client=android_embedded,web;include_live_dash" --extractor-args "funimation:version=uncut"`
+Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. E.g. `--extractor-args "youtube:player-client=android_embedded,web;include_live_dash" --extractor-args "funimation:version=uncut"`
 
 The following extractors use this feature:
 
 #### youtube
 * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
-* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (Eg: `web_embedded`); and `mweb` and `tv_embedded` (agegate bypass) with no variants. By default, `android,web` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
+* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb` and `tv_embedded` (agegate bypass) with no variants. By default, `android,web` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
 * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
 * `include_live_dash`: Include live dash formats even without `--live-from-start` (These formats don't download properly)
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
 * `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all`
     * E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total
-* `innertube_host`: Innertube API host to use for all API requests 
-  * e.g. `studio.youtube.com`, `youtubei.googleapis.com`
-  * Note: Cookies exported from `www.youtube.com` will not work with hosts other than `*.youtube.com`
+* `innertube_host`: Innertube API host to use for all API requests; e.g. `studio.youtube.com`, `youtubei.googleapis.com`. Note that cookies exported from one subdomain will not work on others
 * `innertube_key`: Innertube API key to use for all API requests
 
 #### youtubetab (YouTube playlists, channels, feeds, etc.)
@@ -1766,17 +1764,16 @@ The following extractors use this feature:
 * `approximate_date`: Extract approximate `upload_date` in flat-playlist. This may cause date-based filters to be slightly off
 
 #### funimation
-* `language`: Languages to extract. Eg: `funimation:language=english,japanese`
+* `language`: Languages to extract, e.g. `funimation:language=english,japanese`
 * `version`: The video version to extract - `uncut` or `simulcast`
 
 #### crunchyroll
-* `language`: Languages to extract. Eg: `crunchyroll:language=jaJp`
-* `hardsub`: Which hard-sub versions to extract. Eg: `crunchyroll:hardsub=None,enUS`
+* `language`: Languages to extract, e.g. `crunchyroll:language=jaJp`
+* `hardsub`: Which hard-sub versions to extract, e.g. `crunchyroll:hardsub=None,enUS`
 
 #### crunchyrollbeta
-* `format`: Which stream type(s) to extract. Default is `adaptive_hls` Eg: `crunchyrollbeta:format=vo_adaptive_hls`
-    * Potentially useful values include `adaptive_hls`, `adaptive_dash`, `vo_adaptive_hls`, `vo_adaptive_dash`, `download_hls`, `trailer_hls`, `trailer_dash`
-* `hardsub`: Preference order for which hardsub versions to extract. Default is `None` (no hardsubs). Eg: `crunchyrollbeta:hardsub=en-US,None`
+* `format`: Which stream type(s) to extract (default: `adaptive_hls`). Potentially useful values include `adaptive_hls`, `adaptive_dash`, `vo_adaptive_hls`, `vo_adaptive_dash`, `download_hls`, `download_dash`, `multitrack_adaptive_hls_v2`
+* `hardsub`: Preference order for which hardsub versions to extract (default: `None` = no hardsubs), e.g. `crunchyrollbeta:hardsub=en-US,None`
 
 #### vikichannel
 * `video_types`: Types of videos to download - one or more of `episodes`, `movies`, `clips`, `trailers`
@@ -1796,11 +1793,11 @@ The following extractors use this feature:
 * `dr`: dynamic range to ignore - one or more of `sdr`, `hdr10`, `dv`
 
 #### tiktok
-* `app_version`: App version to call mobile APIs with - should be set along with `manifest_app_version`. (e.g. `20.2.1`)
-* `manifest_app_version`: Numeric app version to call mobile APIs with. (e.g. `221`)
+* `app_version`: App version to call mobile APIs with - should be set along with `manifest_app_version`, e.g. `20.2.1`
+* `manifest_app_version`: Numeric app version to call mobile APIs with, e.g. `221`
 
 #### rokfinchannel
-* `tab`: Which tab to download. One of `new`, `top`, `videos`, `podcasts`, `streams`, `stacks`. (E.g. `rokfinchannel:tab=streams`)
+* `tab`: Which tab to download - one of `new`, `top`, `videos`, `podcasts`, `streams`, `stacks`
 
 
 NOTE: These options may be changed/removed in the future without concern for backward compatibility
@@ -2059,12 +2056,13 @@ While these options are redundant, they are still expected to be used due to the
 #### Not recommended
 While these options still work, their use is not recommended since there are other alternatives to achieve the same
 
+    --force-generic-extractor        --ies generic,default
     --exec-before-download CMD       --exec "before_dl:CMD"
     --no-exec-before-download        --no-exec
     --all-formats                    -f all
     --all-subs                       --sub-langs all --write-subs
     --print-json                     -j --no-simulate
-    --autonumber-size NUMBER         Use string formatting. Eg: %(autonumber)03d
+    --autonumber-size NUMBER         Use string formatting, e.g. %(autonumber)03d
     --autonumber-start NUMBER        Use internal field formatting like %(autonumber+NUMBER)s
     --id                             -o "%(id)s.%(ext)s"
     --metadata-from-title FORMAT     --parse-metadata "%(title)s:FORMAT"
@@ -2132,6 +2130,7 @@ These options may no longer work as intended
     --no-include-ads                 Default
     --write-annotations              No supported site has annotations now
     --no-write-annotations           Default
+    --compat-options seperate-video-versions  No longer needed
 
 #### Removed
 These options were deprecated since 2014 and have now been entirely removed

devscripts/__init__.py

@@ -0,0 +1 @@
+# Empty file needed to make devscripts.utils properly importable from outside

devscripts/lazy_load_template.py

@@ -9,14 +9,19 @@ from ..utils import (
     write_string,
 )
 
+# These bloat the lazy_extractors, so allow them to passthrough silently
+ALLOWED_CLASSMETHODS = {'get_testcases', 'extract_from_webpage'}
+_WARNED = False
+
 
 class LazyLoadMetaClass(type):
     def __getattr__(cls, name):
-        # "_TESTS" bloat the lazy_extractors
-        if '_real_class' not in cls.__dict__ and name != 'get_testcases':
-            write_string(
-                'WARNING: Falling back to normal extractor since lazy extractor '
-                f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n')
+        global _WARNED
+        if ('_real_class' not in cls.__dict__
+                and name not in ALLOWED_CLASSMETHODS and not _WARNED):
+            _WARNED = True
+            write_string('WARNING: Falling back to normal extractor since lazy extractor '
+                         f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n')
         return getattr(cls.real_class, name)
 
 

devscripts/make_issue_template.py

@@ -7,20 +7,14 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import optparse
 import re
 
-
-def read(fname):
-    with open(fname, encoding='utf-8') as f:
-        return f.read()
-
-
-# Get the version without importing the package
-def read_version(fname):
-    exec(compile(read(fname), fname, 'exec'))
-    return locals()['__version__']
-
+from devscripts.utils import (
+    get_filename_args,
+    read_file,
+    read_version,
+    write_file,
+)
 
 VERBOSE_TMPL = '''
   - type: checkboxes
@@ -58,20 +52,24 @@ VERBOSE_TMPL = '''
       required: true
 '''.strip()
 
+NO_SKIP = '''
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\\* field
+          required: true
+'''.strip()
+
 
 def main():
-    parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
-    _, args = parser.parse_args()
-    if len(args) != 2:
-        parser.error('Expected an input and an output filename')
-
-    fields = {'version': read_version('yt_dlp/version.py')}
+    fields = {'version': read_version(), 'no_skip': NO_SKIP}
     fields['verbose'] = VERBOSE_TMPL % fields
     fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose'])
 
-    infile, outfile = args
-    with open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(read(infile) % fields)
+    infile, outfile = get_filename_args(has_infile=True)
+    write_file(outfile, read_file(infile) % fields)
 
 
 if __name__ == '__main__':

devscripts/make_lazy_extractors.py

@@ -7,11 +7,14 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import optparse
 from inspect import getsource
 
+from devscripts.utils import get_filename_args, read_file, write_file
+
 NO_ATTR = object()
-STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE', 'age_limit']
+STATIC_CLASS_PROPERTIES = [
+    'IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_VALID_URL', '_WORKING', '_ENABLED', '_NETRC_MACHINE', 'age_limit'
+]
 CLASS_METHODS = [
     'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', 'is_suitable'
 ]
@@ -19,17 +22,11 @@ IE_TEMPLATE = '''
 class {name}({bases}):
     _module = {module!r}
 '''
-with open('devscripts/lazy_load_template.py', encoding='utf-8') as f:
-    MODULE_TEMPLATE = f.read()
+MODULE_TEMPLATE = read_file('devscripts/lazy_load_template.py')
 
 
 def main():
-    parser = optparse.OptionParser(usage='%prog [OUTFILE.py]')
-    args = parser.parse_args()[1] or ['yt_dlp/extractor/lazy_extractors.py']
-    if len(args) != 1:
-        parser.error('Expected only an output filename')
-
-    lazy_extractors_filename = args[0]
+    lazy_extractors_filename = get_filename_args(default_outfile='yt_dlp/extractor/lazy_extractors.py')
     if os.path.exists(lazy_extractors_filename):
         os.remove(lazy_extractors_filename)
 
@@ -46,8 +43,7 @@ def main():
         *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor),
     ))
 
-    with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
-        f.write(f'{module_src}\n')
+    write_file(lazy_extractors_filename, f'{module_src}\n')
 
 
 def get_all_ies():
@@ -116,11 +112,6 @@ def build_lazy_ie(ie, name, attr_base):
     }.get(base.__name__, base.__name__) for base in ie.__bases__)
 
     s = IE_TEMPLATE.format(name=name, module=ie.__module__, bases=bases)
-    valid_url = getattr(ie, '_VALID_URL', None)
-    if not valid_url and hasattr(ie, '_make_valid_url'):
-        valid_url = ie._make_valid_url()
-    if valid_url:
-        s += f'    _VALID_URL = {valid_url!r}\n'
     return s + '\n'.join(extra_ie_code(ie, attr_base))
 
 

devscripts/make_readme.py

@@ -5,10 +5,17 @@ yt-dlp --help | make_readme.py
 This must be run in a console of correct width
 """
 
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 
 import functools
 import re
-import sys
+
+from devscripts.utils import read_file, write_file
 
 README_FILE = 'README.md'
 
@@ -38,6 +45,10 @@ switch_col_width = len(re.search(r'(?m)^\s{5,}', options).group())
 delim = f'\n{" " * switch_col_width}'
 
 PATCHES = (
+    (   # Standardize update message
+        r'(?m)^(    -U, --update\s+).+(\n    \s.+)*$',
+        r'\1Update this program to the latest version',
+    ),
     (  # Headings
         r'(?m)^  (\w.+\n)(    (?=\w))?',
         r'## \1'
@@ -63,12 +74,10 @@ PATCHES = (
     ),
 )
 
-with open(README_FILE, encoding='utf-8') as f:
-    readme = f.read()
+readme = read_file(README_FILE)
 
-with open(README_FILE, 'w', encoding='utf-8') as f:
-    f.write(''.join((
-        take_section(readme, end=f'## {OPTIONS_START}'),
-        functools.reduce(apply_patch, PATCHES, options),
-        take_section(readme, f'# {OPTIONS_END}'),
-    )))
+write_file(README_FILE, ''.join((
+    take_section(readme, end=f'## {OPTIONS_START}'),
+    functools.reduce(apply_patch, PATCHES, options),
+    take_section(readme, f'# {OPTIONS_END}'),
+)))

devscripts/make_supportedsites.py

@@ -7,21 +7,13 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import optparse
-
+from devscripts.utils import get_filename_args, write_file
 from yt_dlp.extractor import list_extractor_classes
 
 
 def main():
-    parser = optparse.OptionParser(usage='%prog OUTFILE.md')
-    _, args = parser.parse_args()
-    if len(args) != 1:
-        parser.error('Expected an output filename')
-
     out = '\n'.join(ie.description() for ie in list_extractor_classes() if ie.IE_DESC is not False)
-
-    with open(args[0], 'w', encoding='utf-8') as outf:
-        outf.write(f'# Supported sites\n{out}\n')
+    write_file(get_filename_args(), f'# Supported sites\n{out}\n')
 
 
 if __name__ == '__main__':

devscripts/prepare_manpage.py

@@ -1,9 +1,22 @@
 #!/usr/bin/env python3
 
-import optparse
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
 import os.path
 import re
 
+from devscripts.utils import (
+    compose_functions,
+    get_filename_args,
+    read_file,
+    write_file,
+)
+
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 README_FILE = os.path.join(ROOT_DIR, 'README.md')
 
@@ -22,25 +35,6 @@ yt\-dlp \- A youtube-dl fork with additional features and patches
 '''
 
 
-def main():
-    parser = optparse.OptionParser(usage='%prog OUTFILE.md')
-    _, args = parser.parse_args()
-    if len(args) != 1:
-        parser.error('Expected an output filename')
-
-    outfile, = args
-
-    with open(README_FILE, encoding='utf-8') as f:
-        readme = f.read()
-
-    readme = filter_excluded_sections(readme)
-    readme = move_sections(readme)
-    readme = filter_options(readme)
-
-    with open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(PREFIX + readme)
-
-
 def filter_excluded_sections(readme):
     EXCLUDED_SECTION_BEGIN_STRING = re.escape('<!-- MANPAGE: BEGIN EXCLUDED SECTION -->')
     EXCLUDED_SECTION_END_STRING = re.escape('<!-- MANPAGE: END EXCLUDED SECTION -->')
@@ -92,5 +86,12 @@ def filter_options(readme):
     return readme.replace(section, options, 1)
 
 
+TRANSFORM = compose_functions(filter_excluded_sections, move_sections, filter_options)
+
+
+def main():
+    write_file(get_filename_args(), PREFIX + TRANSFORM(read_file(README_FILE)))
+
+
 if __name__ == '__main__':
     main()

devscripts/run_tests.sh

@@ -1,13 +1,13 @@
 #!/usr/bin/env sh
 
-if [ -z $1 ]; then
+if [ -z "$1" ]; then
     test_set='test'
-elif [ $1 = 'core' ]; then
+elif [ "$1" = 'core' ]; then
     test_set="-m not download"
-elif [ $1 = 'download' ]; then
+elif [ "$1" = 'download' ]; then
     test_set="-m download"
 else
-    echo 'Invalid test type "'$1'". Use "core" | "download"'
+    echo 'Invalid test type "'"$1"'". Use "core" | "download"'
     exit 1
 fi
 

devscripts/set-variant.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+import argparse
+import functools
+import re
+
+from devscripts.utils import compose_functions, read_file, write_file
+
+VERSION_FILE = 'yt_dlp/version.py'
+
+
+def parse_options():
+    parser = argparse.ArgumentParser(description='Set the build variant of the package')
+    parser.add_argument('variant', help='Name of the variant')
+    parser.add_argument('-M', '--update-message', default=None, help='Message to show in -U')
+    return parser.parse_args()
+
+
+def property_setter(name, value):
+    return functools.partial(re.sub, rf'(?m)^{name}\s*=\s*.+$', f'{name} = {value!r}')
+
+
+opts = parse_options()
+transform = compose_functions(
+    property_setter('VARIANT', opts.variant),
+    property_setter('UPDATE_HINT', opts.update_message)
+)
+
+write_file(VERSION_FILE, transform(read_file(VERSION_FILE)))

devscripts/update-formulae.py

@@ -1,5 +1,10 @@
 #!/usr/bin/env python3
 
+"""
+Usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
+version can be either 0-aligned (yt-dlp version) or normalized (PyPi version)
+"""
+
 # Allow direct execution
 import os
 import sys
@@ -11,8 +16,7 @@ import json
 import re
 import urllib.request
 
-# usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
-# version can be either 0-aligned (yt-dlp version) or normalized (PyPl version)
+from devscripts.utils import read_file, write_file
 
 filename, version = sys.argv[1:]
 
@@ -27,11 +31,9 @@ tarball_file = next(x for x in pypi_release['urls'] if x['filename'].endswith('.
 sha256sum = tarball_file['digests']['sha256']
 url = tarball_file['url']
 
-with open(filename) as r:
-    formulae_text = r.read()
+formulae_text = read_file(filename)
 
 formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text, count=1)
 formulae_text = re.sub(r'url "[^"]*?"', 'url "%s"' % url, formulae_text, count=1)
 
-with open(filename, 'w') as w:
-    w.write(formulae_text)
+write_file(filename, formulae_text)

devscripts/update-version.py

@@ -7,32 +7,35 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
+import contextlib
 import subprocess
 import sys
 from datetime import datetime
 
-with open('yt_dlp/version.py') as f:
-    exec(compile(f.read(), 'yt_dlp/version.py', 'exec'))
-old_version = locals()['__version__']
+from devscripts.utils import read_version, write_file
 
-old_version_list = old_version.split('.')
 
-old_ver = '.'.join(old_version_list[:3])
-old_rev = old_version_list[3] if len(old_version_list) > 3 else ''
+def get_new_version(revision):
+    version = datetime.utcnow().strftime('%Y.%m.%d')
 
-ver = datetime.utcnow().strftime("%Y.%m.%d")
+    if revision:
+        assert revision.isdigit(), 'Revision must be a number'
+    else:
+        old_version = read_version().split('.')
+        if version.split('.') == old_version[:3]:
+            revision = str(int((old_version + [0])[3]) + 1)
 
-rev = (sys.argv[1:] or [''])[0]  # Use first argument, if present as revision number
-if not rev:
-    rev = str(int(old_rev or 0) + 1) if old_ver == ver else ''
+    return f'{version}.{revision}' if revision else version
 
-VERSION = '.'.join((ver, rev)) if rev else ver
 
-try:
-    sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE)
-    GIT_HEAD = sp.communicate()[0].decode().strip() or None
-except Exception:
-    GIT_HEAD = None
+def get_git_head():
+    with contextlib.suppress(Exception):
+        sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE)
+        return sp.communicate()[0].decode().strip() or None
+
+
+VERSION = get_new_version((sys.argv + [''])[1])
+GIT_HEAD = get_git_head()
 
 VERSION_FILE = f'''\
 # Autogenerated by devscripts/update-version.py
@@ -40,10 +43,12 @@ VERSION_FILE = f'''\
 __version__ = {VERSION!r}
 
 RELEASE_GIT_HEAD = {GIT_HEAD!r}
+
+VARIANT = None
+
+UPDATE_HINT = None
 '''
 
-with open('yt_dlp/version.py', 'wt') as f:
-    f.write(VERSION_FILE)
-
-print('::set-output name=ytdlp_version::' + VERSION)
+write_file('yt_dlp/version.py', VERSION_FILE)
+print(f'::set-output name=ytdlp_version::{VERSION}')
 print(f'\nVersion = {VERSION}, Git HEAD = {GIT_HEAD}')

devscripts/utils.py

@@ -0,0 +1,35 @@
+import argparse
+import functools
+
+
+def read_file(fname):
+    with open(fname, encoding='utf-8') as f:
+        return f.read()
+
+
+def write_file(fname, content):
+    with open(fname, 'w', encoding='utf-8') as f:
+        return f.write(content)
+
+
+# Get the version without importing the package
+def read_version(fname='yt_dlp/version.py'):
+    exec(compile(read_file(fname), fname, 'exec'))
+    return locals()['__version__']
+
+
+def get_filename_args(has_infile=False, default_outfile=None):
+    parser = argparse.ArgumentParser()
+    if has_infile:
+        parser.add_argument('infile', help='Input file')
+    kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {}
+    parser.add_argument('outfile', **kwargs, help='Output file')
+
+    opts = parser.parse_args()
+    if has_infile:
+        return opts.infile, opts.outfile
+    return opts.outfile
+
+
+def compose_functions(*functions):
+    return lambda x: functools.reduce(lambda y, f: f(y), functions, x)

pyinst.py

@@ -1,11 +1,17 @@
 #!/usr/bin/env python3
 
+# Allow direct execution
 import os
-import platform
 import sys
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import platform
+
 from PyInstaller.__main__ import run as run_pyinstaller
 
+from devscripts.utils import read_version
+
 OS_NAME, MACHINE, ARCH = sys.platform, platform.machine(), platform.architecture()[0][:2]
 if MACHINE in ('x86_64', 'AMD64') or ('i' in MACHINE and '86' in MACHINE):
     # NB: Windows x86 has MACHINE = AMD64 irrespective of bitness
@@ -13,8 +19,7 @@ if MACHINE in ('x86_64', 'AMD64') or ('i' in MACHINE and '86' in MACHINE):
 
 
 def main():
-    opts = parse_options()
-    version = read_version('yt_dlp/version.py')
+    opts, version = parse_options(), read_version()
 
     onedir = '--onedir' in opts or '-D' in opts
     if not onedir and '-F' not in opts and '--onefile' not in opts:
@@ -53,13 +58,6 @@ def parse_options():
     return opts
 
 
-# Get the version from yt_dlp/version.py without importing the package
-def read_version(fname):
-    with open(fname, encoding='utf-8') as f:
-        exec(compile(f.read(), fname, 'exec'))
-        return locals()['__version__']
-
-
 def exe(onedir):
     """@returns (name, path)"""
     name = '_'.join(filter(None, (
@@ -83,7 +81,7 @@ def version_to_list(version):
 def dependency_options():
     # Due to the current implementation, these are auto-detected, but explicitly add them just in case
     dependencies = [pycryptodome_module(), 'mutagen', 'brotli', 'certifi', 'websockets']
-    excluded_modules = ['test', 'ytdlp_plugins', 'youtube_dl', 'youtube_dlc']
+    excluded_modules = ('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts')
 
     yield from (f'--hidden-import={module}' for module in dependencies)
     yield '--collect-submodules=websockets'

setup.cfg

@@ -31,7 +31,7 @@ setenv =
 
 
 [isort]
-py_version = 36
+py_version = 37
 multi_line_output = VERTICAL_HANGING_INDENT
 line_length = 80
 reverse_relative = true

setup.py

@@ -12,33 +12,23 @@ except ImportError:
     from distutils.core import Command, setup
     setuptools_available = False
 
+from devscripts.utils import read_file, read_version
 
-def read(fname):
-    with open(fname, encoding='utf-8') as f:
-        return f.read()
-
-
-# Get the version from yt_dlp/version.py without importing the package
-def read_version(fname):
-    exec(compile(read(fname), fname, 'exec'))
-    return locals()['__version__']
-
-
-VERSION = read_version('yt_dlp/version.py')
+VERSION = read_version()
 
 DESCRIPTION = 'A youtube-dl fork with additional features and patches'
 
 LONG_DESCRIPTION = '\n\n'.join((
     'Official repository: <https://github.com/yt-dlp/yt-dlp>',
     '**PS**: Some links in this document will not work since this is a copy of the README.md from Github',
-    read('README.md')))
+    read_file('README.md')))
 
-REQUIREMENTS = read('requirements.txt').splitlines()
+REQUIREMENTS = read_file('requirements.txt').splitlines()
 
 
 def packages():
     if setuptools_available:
-        return find_packages(exclude=('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins'))
+        return find_packages(exclude=('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts'))
 
     return [
         'yt_dlp', 'yt_dlp.extractor', 'yt_dlp.downloader', 'yt_dlp.postprocessor', 'yt_dlp.compat',
@@ -121,7 +111,7 @@ class build_lazy_extractors(Command):
         if self.dry_run:
             print('Skipping build of lazy extractors in dry run mode')
             return
-        subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py', 'yt_dlp/extractor/lazy_extractors.py'])
+        subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py'])
 
 
 params = py2exe_params() if sys.argv[1:2] == ['py2exe'] else build_params()
@@ -136,7 +126,7 @@ setup(
     url='https://github.com/yt-dlp/yt-dlp',
     packages=packages(),
     install_requires=REQUIREMENTS,
-    python_requires='>=3.6',
+    python_requires='>=3.7',
     project_urls={
         'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
         'Source': 'https://github.com/yt-dlp/yt-dlp',
@@ -148,7 +138,6 @@ setup(
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',

supportedsites.md

@@ -18,7 +18,7 @@
  - **8tracks**
  - **91porn**
  - **9c9media**
- - **9gag**
+ - **9gag**: 9GAG
  - **9now.com.au**
  - **abc.net.au**
  - **abc.net.au:iview**
@@ -64,6 +64,7 @@
  - **AmericasTestKitchenSeason**
  - **AmHistoryChannel**
  - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
+ - **Angel**
  - **AnimalPlanet**
  - **AnimeOnDemand**: [<abbr title="netrc machine"><em>animeondemand</em></abbr>]
  - **ant1newsgr:article**: ant1news.gr articles
@@ -127,6 +128,8 @@
  - **bbc.co.uk:iplayer:group**
  - **bbc.co.uk:playlist**
  - **BBVTV**: [<abbr title="netrc machine"><em>bbvtv</em></abbr>]
+ - **BBVTVLive**: [<abbr title="netrc machine"><em>bbvtv</em></abbr>]
+ - **BBVTVRecordings**: [<abbr title="netrc machine"><em>bbvtv</em></abbr>]
  - **Beatport**
  - **Beeg**
  - **BehindKink**
@@ -187,6 +190,7 @@
  - **Camdemy**
  - **CamdemyFolder**
  - **CamModels**
+ - **CamtasiaEmbed**
  - **CamWithHer**
  - **CanalAlpha**
  - **canalc2.tv**
@@ -232,6 +236,7 @@
  - **Clippit**
  - **ClipRs**
  - **Clipsyndicate**
+ - **ClipYouEmbed**
  - **CloserToTruth**
  - **CloudflareStream**
  - **Cloudy**
@@ -345,6 +350,8 @@
  - **ehftv**
  - **eHow**
  - **EinsUndEinsTV**: [<abbr title="netrc machine"><em>1und1tv</em></abbr>]
+ - **EinsUndEinsTVLive**: [<abbr title="netrc machine"><em>1und1tv</em></abbr>]
+ - **EinsUndEinsTVRecordings**: [<abbr title="netrc machine"><em>1und1tv</em></abbr>]
  - **Einthusan**
  - **eitb.tv**
  - **EllenTube**
@@ -357,6 +364,7 @@
  - **Engadget**
  - **Epicon**
  - **EpiconSeries**
+ - **Epoch**
  - **Eporner**
  - **EroProfile**: [<abbr title="netrc machine"><em>eroprofile</em></abbr>]
  - **EroProfile:album**
@@ -370,13 +378,17 @@
  - **EsriVideo**
  - **Europa**
  - **EuropeanTour**
+ - **Eurosport**
  - **EUScreen**
  - **EWETV**: [<abbr title="netrc machine"><em>ewetv</em></abbr>]
+ - **EWETVLive**: [<abbr title="netrc machine"><em>ewetv</em></abbr>]
+ - **EWETVRecordings**: [<abbr title="netrc machine"><em>ewetv</em></abbr>]
  - **ExpoTV**
  - **Expressen**
  - **ExtremeTube**
  - **EyedoTV**
  - **facebook**: [<abbr title="netrc machine"><em>facebook</em></abbr>]
+ - **facebook:reel**
  - **FacebookPluginsVideo**
  - **fancode:live**: [<abbr title="netrc machine"><em>fancode</em></abbr>]
  - **fancode:vod**: [<abbr title="netrc machine"><em>fancode</em></abbr>]
@@ -450,6 +462,8 @@
  - **GiantBomb**
  - **Giga**
  - **GlattvisionTV**: [<abbr title="netrc machine"><em>glattvisiontv</em></abbr>]
+ - **GlattvisionTVLive**: [<abbr title="netrc machine"><em>glattvisiontv</em></abbr>]
+ - **GlattvisionTVRecordings**: [<abbr title="netrc machine"><em>glattvisiontv</em></abbr>]
  - **Glide**: Glide mobile video messages (glide.me)
  - **Globo**: [<abbr title="netrc machine"><em>globo</em></abbr>]
  - **GloboArticle**
@@ -473,6 +487,7 @@
  - **gronkh:feed**
  - **gronkh:vods**
  - **Groupon**
+ - **Harpodeon**
  - **hbo**
  - **HearThisAt**
  - **Heise**
@@ -491,6 +506,7 @@
  - **hitbox:live**
  - **HitRecord**
  - **hketv**: 香港教育局教育電視 (HKETV) Educational Television, Hong Kong Educational Bureau
+ - **Holodex**
  - **HotNewHipHop**
  - **hotstar**
  - **hotstar:playlist**
@@ -502,6 +518,7 @@
  - **HRTiPlaylist**: [<abbr title="netrc machine"><em>hrti</em></abbr>]
  - **HSEProduct**
  - **HSEShow**
+ - **html5**
  - **Huajiao**: 花椒直播
  - **HuffPost**: Huffington Post
  - **Hungama**
@@ -538,6 +555,8 @@
  - **iq.com**: International version of iQiyi
  - **iq.com:album**
  - **iqiyi**: [<abbr title="netrc machine"><em>iqiyi</em></abbr>] 爱奇艺
+ - **IslamChannel**
+ - **IslamChannelSeries**
  - **ITProTV**
  - **ITProTVCourse**
  - **ITTF**
@@ -573,6 +592,7 @@
  - **KickStarter**
  - **KinjaEmbed**
  - **KinoPoisk**
+ - **KompasVideo**
  - **KonserthusetPlay**
  - **Koo**
  - **KrasView**: Красвью
@@ -702,10 +722,13 @@
  - **mixcloud:playlist**
  - **mixcloud:user**
  - **MLB**
+ - **MLBTV**: [<abbr title="netrc machine"><em>mlb</em></abbr>]
  - **MLBVideo**
  - **MLSSoccer**
  - **Mnet**
  - **MNetTV**: [<abbr title="netrc machine"><em>mnettv</em></abbr>]
+ - **MNetTVLive**: [<abbr title="netrc machine"><em>mnettv</em></abbr>]
+ - **MNetTVRecordings**: [<abbr title="netrc machine"><em>mnettv</em></abbr>]
  - **MochaVideo**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
  - **Mofosex**
@@ -715,9 +738,11 @@
  - **Motherless**
  - **MotherlessGroup**
  - **Motorsport**: motorsport.com
+ - **MotorTrend**
  - **MovieClips**
  - **MovieFap**
  - **Moviepilot**
+ - **MoviewPlay**
  - **Moviezine**
  - **MovingImage**
  - **MSN**
@@ -790,13 +815,16 @@
  - **netease:program**: 网易云音乐 - 电台节目
  - **netease:singer**: 网易云音乐 - 歌手
  - **netease:song**: 网易云音乐
- - **NetPlus**: [<abbr title="netrc machine"><em>netplus</em></abbr>]
+ - **NetPlusTV**: [<abbr title="netrc machine"><em>netplus</em></abbr>]
+ - **NetPlusTVLive**: [<abbr title="netrc machine"><em>netplus</em></abbr>]
+ - **NetPlusTVRecordings**: [<abbr title="netrc machine"><em>netplus</em></abbr>]
  - **Netverse**
  - **NetversePlaylist**
  - **Netzkino**
  - **Newgrounds**
  - **Newgrounds:playlist**
  - **Newgrounds:user**
+ - **NewsPicks**
  - **Newstube**
  - **Newsy**
  - **NextMedia**: 蘋果日報
@@ -890,22 +918,13 @@
  - **openrec:capture**
  - **openrec:movie**
  - **OraTV**
- - **orf:burgenland**: Radio Burgenland
- - **orf:fm4**: radio FM4
  - **orf:fm4:story**: fm4.orf.at stories
  - **orf:iptv**: iptv.ORF.at
- - **orf:kaernten**: Radio Kärnten
- - **orf:noe**: Radio Niederösterreich
- - **orf:oberoesterreich**: Radio Oberösterreich
- - **orf:oe1**: Radio Österreich 1
- - **orf:oe3**: Radio Österreich 3
- - **orf:salzburg**: Radio Salzburg
- - **orf:steiermark**: Radio Steiermark
- - **orf:tirol**: Radio Tirol
+ - **orf:radio**
  - **orf:tvthek**: ORF TVthek
- - **orf:vorarlberg**: Radio Vorarlberg
- - **orf:wien**: Radio Wien
  - **OsnatelTV**: [<abbr title="netrc machine"><em>osnateltv</em></abbr>]
+ - **OsnatelTVLive**: [<abbr title="netrc machine"><em>osnateltv</em></abbr>]
+ - **OsnatelTVRecordings**: [<abbr title="netrc machine"><em>osnateltv</em></abbr>]
  - **OutsideTV**
  - **PacktPub**: [<abbr title="netrc machine"><em>packtpub</em></abbr>]
  - **PacktPubCourse**
@@ -919,10 +938,11 @@
  - **ParamountNetwork**
  - **ParamountPlus**
  - **ParamountPlusSeries**
+ - **Parler**: Posts on parler.com
  - **parliamentlive.tv**: UK parliament videos
  - **Parlview**
  - **Patreon**
- - **PatreonUser**
+ - **PatreonCampaign**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
  - **PearVideo**
  - **PeekVids**
@@ -1012,6 +1032,8 @@
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
  - **QuantumTV**: [<abbr title="netrc machine"><em>quantumtv</em></abbr>]
+ - **QuantumTVLive**: [<abbr title="netrc machine"><em>quantumtv</em></abbr>]
+ - **QuantumTVRecordings**: [<abbr title="netrc machine"><em>quantumtv</em></abbr>]
  - **Qub**
  - **R7**
  - **R7Article**
@@ -1030,12 +1052,14 @@
  - **radlive:channel**
  - **radlive:season**
  - **Rai**
+ - **RaiNews**
  - **RaiPlay**
  - **RaiPlayLive**
  - **RaiPlayPlaylist**
  - **RaiPlaySound**
  - **RaiPlaySoundLive**
  - **RaiPlaySoundPlaylist**
+ - **RaiSudtirol**
  - **RayWenderlich**
  - **RayWenderlichCourse**
  - **RBMARadio**
@@ -1072,7 +1096,7 @@
  - **RoosterTeethSeries**: [<abbr title="netrc machine"><em>roosterteeth</em></abbr>]
  - **RottenTomatoes**
  - **Rozhlas**
- - **RTBF**
+ - **RTBF**: [<abbr title="netrc machine"><em>rtbf</em></abbr>]
  - **RTDocumentry**
  - **RTDocumentryPlaylist**
  - **rte**: Raidió Teilifís Éireann TV
@@ -1118,7 +1142,11 @@
  - **safari:course**: [<abbr title="netrc machine"><em>safari</em></abbr>] safaribooksonline.com online courses
  - **Saitosan**
  - **SAKTV**: [<abbr title="netrc machine"><em>saktv</em></abbr>]
+ - **SAKTVLive**: [<abbr title="netrc machine"><em>saktv</em></abbr>]
+ - **SAKTVRecordings**: [<abbr title="netrc machine"><em>saktv</em></abbr>]
  - **SaltTV**: [<abbr title="netrc machine"><em>salttv</em></abbr>]
+ - **SaltTVLive**: [<abbr title="netrc machine"><em>salttv</em></abbr>]
+ - **SaltTVRecordings**: [<abbr title="netrc machine"><em>salttv</em></abbr>]
  - **SampleFocus**
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
@@ -1144,6 +1172,7 @@
  - **Shahid**: [<abbr title="netrc machine"><em>shahid</em></abbr>]
  - **ShahidShow**
  - **Shared**: shared.sx
+ - **ShareVideosEmbed**
  - **ShemarooMe**
  - **ShowRoomLive**
  - **simplecast**
@@ -1268,6 +1297,7 @@
  - **TeleQuebecVideo**
  - **TeleTask**
  - **Telewebion**
+ - **Tempo**
  - **TennisTV**: [<abbr title="netrc machine"><em>tennistv</em></abbr>]
  - **TenPlay**: [<abbr title="netrc machine"><em>10play</em></abbr>]
  - **TF1**
@@ -1306,6 +1336,8 @@
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict**: (**Currently broken**)
  - **TravelChannel**
+ - **Triller**: [<abbr title="netrc machine"><em>triller</em></abbr>]
+ - **TrillerUser**: [<abbr title="netrc machine"><em>triller</em></abbr>]
  - **Trilulilu**
  - **Trovo**
  - **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix
@@ -1313,6 +1345,7 @@
  - **TrovoVod**
  - **TrueID**
  - **TruNews**
+ - **Truth**
  - **TruTV**
  - **Tube8**
  - **TubeTuGraz**: [<abbr title="netrc machine"><em>tubetugraz</em></abbr>] tube.tugraz.at
@@ -1480,6 +1513,8 @@
  - **VoxMedia**
  - **VoxMediaVolume**
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
+ - **vqq:series**
+ - **vqq:video**
  - **Vrak**
  - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza
  - **VrtNU**: [<abbr title="netrc machine"><em>vrtnu</em></abbr>] VrtNU.be
@@ -1488,6 +1523,8 @@
  - **VShare**
  - **VTM**
  - **VTXTV**: [<abbr title="netrc machine"><em>vtxtv</em></abbr>]
+ - **VTXTVLive**: [<abbr title="netrc machine"><em>vtxtv</em></abbr>]
+ - **VTXTVRecordings**: [<abbr title="netrc machine"><em>vtxtv</em></abbr>]
  - **VuClip**
  - **Vupload**
  - **VVVVID**
@@ -1497,6 +1534,8 @@
  - **Wakanim**
  - **Walla**
  - **WalyTV**: [<abbr title="netrc machine"><em>walytv</em></abbr>]
+ - **WalyTVLive**: [<abbr title="netrc machine"><em>walytv</em></abbr>]
+ - **WalyTVRecordings**: [<abbr title="netrc machine"><em>walytv</em></abbr>]
  - **wasdtv:clip**
  - **wasdtv:record**
  - **wasdtv:stream**
@@ -1583,7 +1622,7 @@
  - **youtube:clip**
  - **youtube:favorites**: YouTube liked videos; ":ytfav" keyword (requires cookies)
  - **youtube:history**: Youtube watch history; ":ythis" keyword (requires cookies)
- - **youtube:music:search_url**: YouTube music search URLs with selectable sections (Eg: #songs)
+ - **youtube:music:search_url**: YouTube music search URLs with selectable sections, e.g. #songs
  - **youtube:notif**: YouTube notifications; ":ytnotif" keyword (requires cookies)
  - **youtube:playlist**: YouTube playlists
  - **youtube:recommended**: YouTube recommended videos; ":ytrec" keyword

test/helper.py

@@ -92,6 +92,13 @@ def gettestcases(include_onlymatching=False):
         yield from ie.get_testcases(include_onlymatching)
 
 
+def getwebpagetestcases():
+    for ie in yt_dlp.extractor.gen_extractors():
+        for tc in ie.get_webpage_testcases():
+            tc.setdefault('add_ie', []).append('Generic')
+            yield tc
+
+
 md5 = lambda s: hashlib.md5(s.encode()).hexdigest()
 
 

test/test_YoutubeDL.py

@@ -668,7 +668,7 @@ class TestYoutubeDL(unittest.TestCase):
     def test_prepare_outtmpl_and_filename(self):
         def test(tmpl, expected, *, info=None, **params):
             params['outtmpl'] = tmpl
-            ydl = YoutubeDL(params)
+            ydl = FakeYDL(params)
             ydl._num_downloads = 1
             self.assertEqual(ydl.validate_outtmpl(tmpl), None)
 
@@ -722,7 +722,7 @@ class TestYoutubeDL(unittest.TestCase):
         test('%(id)s', '-abcd', info={'id': '-abcd'})
         test('%(id)s', '.abcd', info={'id': '.abcd'})
         test('%(id)s', 'ab__cd', info={'id': 'ab__cd'})
-        test('%(id)s', ('ab:cd', 'ab -cd'), info={'id': 'ab:cd'})
+        test('%(id)s', ('ab:cd', 'ab：cd'), info={'id': 'ab:cd'})
         test('%(id.0)s', '-', info={'id': '--'})
 
         # Invalid templates
@@ -770,7 +770,7 @@ class TestYoutubeDL(unittest.TestCase):
         test('a%(width|)d', 'a', outtmpl_na_placeholder='none')
 
         FORMATS = self.outtmpl_info['formats']
-        sanitize = lambda x: x.replace(':', ' -').replace('"', "'").replace('\n', ' ')
+        sanitize = lambda x: x.replace(':', '：').replace('"', "＂").replace('\n', ' ')
 
         # Custom type casting
         test('%(formats.:.id)l', 'id 1, id 2, id 3')
@@ -788,13 +788,13 @@ class TestYoutubeDL(unittest.TestCase):
         test('%(filesize)#D', '1Ki')
         test('%(height)5.2D', ' 1.08k')
         test('%(title4)#S', 'foo_bar_test')
-        test('%(title4).10S', ('foo \'bar\' ', 'foo \'bar\'' + ('#' if compat_os_name == 'nt' else ' ')))
+        test('%(title4).10S', ('foo ＂bar＂ ', 'foo ＂bar＂' + ('#' if compat_os_name == 'nt' else ' ')))
         if compat_os_name == 'nt':
-            test('%(title4)q', ('"foo \\"bar\\" test"', "'foo _'bar_' test'"))
-            test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', "'id 1' 'id 2' 'id 3'"))
-            test('%(formats.0.id)#q', ('"id 1"', "'id 1'"))
+            test('%(title4)q', ('"foo \\"bar\\" test"', "＂foo ⧹＂bar⧹＂ test＂"))
+            test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', '＂id 1＂ ＂id 2＂ ＂id 3＂'))
+            test('%(formats.0.id)#q', ('"id 1"', '＂id 1＂'))
         else:
-            test('%(title4)q', ('\'foo "bar" test\'', "'foo 'bar' test'"))
+            test('%(title4)q', ('\'foo "bar" test\'', '\'foo ＂bar＂ test\''))
             test('%(formats.:.id)#q', "'id 1' 'id 2' 'id 3'")
             test('%(formats.0.id)#q', "'id 1'")
 
@@ -852,8 +852,8 @@ class TestYoutubeDL(unittest.TestCase):
         # Path expansion and escaping
         test('Hello %(title1)s', 'Hello $PATH')
         test('Hello %(title2)s', 'Hello %PATH%')
-        test('%(title3)s', ('foo/bar\\test', 'foo_bar_test'))
-        test('folder/%(title3)s', ('folder/foo/bar\\test', 'folder%sfoo_bar_test' % os.path.sep))
+        test('%(title3)s', ('foo/bar\\test', 'foo⧸bar⧹test'))
+        test('folder/%(title3)s', ('folder/foo/bar\\test', 'folder%sfoo⧸bar⧹test' % os.path.sep))
 
     def test_format_note(self):
         ydl = YoutubeDL()

test/test_compat.py

@@ -28,7 +28,8 @@ class TestCompat(unittest.TestCase):
         with self.assertWarns(DeprecationWarning):
             compat.WINDOWS_VT_MODE
 
-        compat.asyncio.events  # Must not raise error
+        # TODO: Test submodule
+        # compat.asyncio.events  # Must not raise error
 
     def test_compat_expanduser(self):
         old_home = os.environ.get('HOME')

test/test_download.py

@@ -8,6 +8,7 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
+import collections
 import hashlib
 import http.client
 import json
@@ -20,6 +21,7 @@ from test.helper import (
     expect_warnings,
     get_params,
     gettestcases,
+    getwebpagetestcases,
     is_download_test,
     report_warning,
     try_rm,
@@ -32,6 +34,7 @@ from yt_dlp.utils import (
     ExtractorError,
     UnavailableVideoError,
     format_bytes,
+    join_nonempty,
 )
 
 RETRIES = 3
@@ -57,7 +60,9 @@ def _file_md5(fn):
         return hashlib.md5(f.read()).hexdigest()
 
 
-defs = gettestcases()
+normal_test_cases = gettestcases()
+webpage_test_cases = getwebpagetestcases()
+tests_counter = collections.defaultdict(collections.Counter)
 
 
 @is_download_test
@@ -72,24 +77,13 @@ class TestDownload(unittest.TestCase):
 
     def __str__(self):
         """Identify each test with the `add_ie` attribute, if available."""
+        cls, add_ie = type(self), getattr(self, self._testMethodName).add_ie
+        return f'{self._testMethodName} ({cls.__module__}.{cls.__name__}){f" [{add_ie}]" if add_ie else ""}:'
 
-        def strclass(cls):
-            """From 2.7's unittest; 2.6 had _strclass so we can't import it."""
-            return f'{cls.__module__}.{cls.__name__}'
-
-        add_ie = getattr(self, self._testMethodName).add_ie
-        return '%s (%s)%s:' % (self._testMethodName,
-                               strclass(self.__class__),
-                               ' [%s]' % add_ie if add_ie else '')
-
-    def setUp(self):
-        self.defs = defs
 
 # Dynamically generate tests
 
-
 def generator(test_case, tname):
-
     def test_template(self):
         if self.COMPLETED_TESTS.get(tname):
             return
@@ -111,11 +105,11 @@ def generator(test_case, tname):
             info_dict = tc.get('info_dict', {})
             params = tc.get('params', {})
             if not info_dict.get('id'):
-                raise Exception('Test definition incorrect. \'id\' key is not present')
+                raise Exception(f'Test {tname} definition incorrect - "id" key is not present')
             elif not info_dict.get('ext'):
                 if params.get('skip_download') and params.get('ignore_no_formats_error'):
                     continue
-                raise Exception('Test definition incorrect. The output file cannot be known. \'ext\' key is not present')
+                raise Exception(f'Test {tname} definition incorrect - "ext" key must be present to define the output file')
 
         if 'skip' in test_case:
             print_skipping(test_case['skip'])
@@ -167,7 +161,9 @@ def generator(test_case, tname):
                         force_generic_extractor=params.get('force_generic_extractor', False))
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
-                    if not err.exc_info[0] in (urllib.error.URLError, socket.timeout, UnavailableVideoError, http.client.BadStatusLine) or (err.exc_info[0] == urllib.error.HTTPError and err.exc_info[1].code == 503):
+                    if (err.exc_info[0] not in (urllib.error.URLError, socket.timeout, UnavailableVideoError, http.client.BadStatusLine)
+                            or (err.exc_info[0] == urllib.error.HTTPError and err.exc_info[1].code == 503)):
+                        err.msg = f'{getattr(err, "msg", err)} ({tname})'
                         raise
 
                     if try_num == RETRIES:
@@ -255,39 +251,43 @@ def generator(test_case, tname):
 
 
 # And add them to TestDownload
-tests_counter = {}
-for test_case in defs:
-    name = test_case['name']
-    i = tests_counter.get(name, 0)
-    tests_counter[name] = i + 1
-    tname = f'test_{name}_{i}' if i else f'test_{name}'
-    test_method = generator(test_case, tname)
-    test_method.__name__ = str(tname)
-    ie_list = test_case.get('add_ie')
-    test_method.add_ie = ie_list and ','.join(ie_list)
-    setattr(TestDownload, test_method.__name__, test_method)
-    del test_method
+def inject_tests(test_cases, label=''):
+    for test_case in test_cases:
+        name = test_case['name']
+        tname = join_nonempty('test', name, label, tests_counter[name][label], delim='_')
+        tests_counter[name][label] += 1
+
+        test_method = generator(test_case, tname)
+        test_method.__name__ = tname
+        test_method.add_ie = ','.join(test_case.get('add_ie', []))
+        setattr(TestDownload, test_method.__name__, test_method)
 
 
-def batch_generator(name, num_tests):
+inject_tests(normal_test_cases)
 
+# TODO: disable redirection to the IE to ensure we are actually testing the webpage extraction
+inject_tests(webpage_test_cases, 'webpage')
+
+
+def batch_generator(name):
     def test_template(self):
-        for i in range(num_tests):
-            test_name = f'test_{name}_{i}' if i else f'test_{name}'
-            try:
-                getattr(self, test_name)()
-            except unittest.SkipTest:
-                print(f'Skipped {test_name}')
+        for label, num_tests in tests_counter[name].items():
+            for i in range(num_tests):
+                test_name = join_nonempty('test', name, label, i, delim='_')
+                try:
+                    getattr(self, test_name)()
+                except unittest.SkipTest:
+                    print(f'Skipped {test_name}')
 
     return test_template
 
 
-for name, num_tests in tests_counter.items():
-    test_method = batch_generator(name, num_tests)
+for name in tests_counter:
+    test_method = batch_generator(name)
     test_method.__name__ = f'test_{name}_all'
     test_method.add_ie = ''
     setattr(TestDownload, test_method.__name__, test_method)
-    del test_method
+del test_method
 
 
 if __name__ == '__main__':

test/test_downloader_http.py

@@ -95,8 +95,8 @@ class TestHttpFD(unittest.TestCase):
         try_rm(encodeFilename(filename))
         self.assertTrue(downloader.real_download(filename, {
             'url': 'http://127.0.0.1:%d/%s' % (self.port, ep),
-        }))
-        self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE)
+        }), ep)
+        self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE, ep)
         try_rm(encodeFilename(filename))
 
     def download_all(self, params):

test/test_execution.py

@@ -11,41 +11,46 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import contextlib
 import subprocess
 
-from yt_dlp.utils import encodeArgument
+from yt_dlp.utils import Popen
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-
-try:
-    _DEV_NULL = subprocess.DEVNULL
-except AttributeError:
-    _DEV_NULL = open(os.devnull, 'wb')
+LAZY_EXTRACTORS = 'yt_dlp/extractor/lazy_extractors.py'
 
 
 class TestExecution(unittest.TestCase):
-    def test_import(self):
-        subprocess.check_call([sys.executable, '-c', 'import yt_dlp'], cwd=rootDir)
-
-    def test_module_exec(self):
-        subprocess.check_call([sys.executable, '-m', 'yt_dlp', '--ignore-config', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+    def run_yt_dlp(self, exe=(sys.executable, 'yt_dlp/__main__.py'), opts=('--version', )):
+        stdout, stderr, returncode = Popen.run(
+            [*exe, '--ignore-config', *opts], cwd=rootDir, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        print(stderr, file=sys.stderr)
+        self.assertEqual(returncode, 0)
+        return stdout.strip(), stderr.strip()
 
     def test_main_exec(self):
-        subprocess.check_call([sys.executable, 'yt_dlp/__main__.py', '--ignore-config', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+        self.run_yt_dlp()
+
+    def test_import(self):
+        self.run_yt_dlp(exe=(sys.executable, '-c', 'import yt_dlp'))
+
+    def test_module_exec(self):
+        self.run_yt_dlp(exe=(sys.executable, '-m', 'yt_dlp'))
 
     def test_cmdline_umlauts(self):
-        p = subprocess.Popen(
-            [sys.executable, 'yt_dlp/__main__.py', '--ignore-config', encodeArgument('ä'), '--version'],
-            cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
-        _, stderr = p.communicate()
+        _, stderr = self.run_yt_dlp(opts=('ä', '--version'))
         self.assertFalse(stderr)
 
     def test_lazy_extractors(self):
         try:
-            subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', 'yt_dlp/extractor/lazy_extractors.py'], cwd=rootDir, stdout=_DEV_NULL)
-            subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=_DEV_NULL)
+            subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', LAZY_EXTRACTORS],
+                                  cwd=rootDir, stdout=subprocess.DEVNULL)
+            self.assertTrue(os.path.exists(LAZY_EXTRACTORS))
+
+            _, stderr = self.run_yt_dlp(opts=('-s', 'test:'))
+            self.assertFalse(stderr)
+
+            subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=subprocess.DEVNULL)
         finally:
             with contextlib.suppress(OSError):
-                os.remove('yt_dlp/extractor/lazy_extractors.py')
+                os.remove(LAZY_EXTRACTORS)
 
 
 if __name__ == '__main__':

test/test_http.py

@@ -85,7 +85,7 @@ class TestHTTPS(unittest.TestCase):
 
         ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
         r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
-        self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
+        self.assertEqual(r['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
 
 
 class TestClientCert(unittest.TestCase):
@@ -113,7 +113,7 @@ class TestClientCert(unittest.TestCase):
             **params,
         })
         r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
-        self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
+        self.assertEqual(r['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
 
     def test_certificate_combined_nopass(self):
         self._run_test(client_certificate=os.path.join(self.certdir, 'clientwithkey.crt'))

test/test_jsinterp.py

@@ -7,8 +7,10 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import math
+import re
 
-from yt_dlp.jsinterp import JSInterpreter
+from yt_dlp.jsinterp import JS_Undefined, JSInterpreter
 
 
 class TestJSInterpreter(unittest.TestCase):
@@ -19,6 +21,9 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function x3(){return 42;}')
         self.assertEqual(jsi.call_function('x3'), 42)
 
+        jsi = JSInterpreter('function x3(){42}')
+        self.assertEqual(jsi.call_function('x3'), None)
+
         jsi = JSInterpreter('var x5 = function(){return 42;}')
         self.assertEqual(jsi.call_function('x5'), 42)
 
@@ -45,14 +50,32 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function f(){return 1 << 5;}')
         self.assertEqual(jsi.call_function('f'), 32)
 
+        jsi = JSInterpreter('function f(){return 2 ** 5}')
+        self.assertEqual(jsi.call_function('f'), 32)
+
         jsi = JSInterpreter('function f(){return 19 & 21;}')
         self.assertEqual(jsi.call_function('f'), 17)
 
         jsi = JSInterpreter('function f(){return 11 >> 2;}')
         self.assertEqual(jsi.call_function('f'), 2)
 
+        jsi = JSInterpreter('function f(){return []? 2+3: 4;}')
+        self.assertEqual(jsi.call_function('f'), 5)
+
+        jsi = JSInterpreter('function f(){return 1 == 2}')
+        self.assertEqual(jsi.call_function('f'), False)
+
+        jsi = JSInterpreter('function f(){return 0 && 1 || 2;}')
+        self.assertEqual(jsi.call_function('f'), 2)
+
+        jsi = JSInterpreter('function f(){return 0 ?? 42;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+
+        jsi = JSInterpreter('function f(){return "life, the universe and everything" < 42;}')
+        self.assertFalse(jsi.call_function('f'))
+
     def test_array_access(self):
-        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2] = 7; return x;}')
+        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
         self.assertEqual(jsi.call_function('f'), [5, 2, 7])
 
     def test_parens(self):
@@ -62,6 +85,10 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
         self.assertEqual(jsi.call_function('f'), 9)
 
+    def test_quotes(self):
+        jsi = JSInterpreter(R'function f(){return "a\"\\("}')
+        self.assertEqual(jsi.call_function('f'), R'a"\(')
+
     def test_assignments(self):
         jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
         self.assertEqual(jsi.call_function('f'), 31)
@@ -104,17 +131,33 @@ class TestJSInterpreter(unittest.TestCase):
         }''')
         self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
 
+    def test_builtins(self):
+        jsi = JSInterpreter('''
+        function x() { return NaN }
+        ''')
+        self.assertTrue(math.isnan(jsi.call_function('x')))
+
+        jsi = JSInterpreter('''
+        function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 86000)
+        jsi = JSInterpreter('''
+        function x(dt) { return new Date(dt) - 0; }
+        ''')
+        self.assertEqual(jsi.call_function('x', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
+
     def test_call(self):
         jsi = JSInterpreter('''
         function x() { return 2; }
-        function y(a) { return x() + a; }
+        function y(a) { return x() + (a?a:0); }
         function z() { return y(3); }
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
+        self.assertEqual(jsi.call_function('y'), 2)
 
     def test_for_loop(self):
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) {a++} a }
+        function x() { a=0; for (i=0; i-10; i++) {a++} return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 10)
 
@@ -153,21 +196,53 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 10)
 
+    def test_catch(self):
+        jsi = JSInterpreter('''
+        function x() { try{throw 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+
+    def test_finally(self):
+        jsi = JSInterpreter('''
+        function x() { try{throw 10} finally {return 42} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 42)
+        jsi = JSInterpreter('''
+        function x() { try{throw 10} catch(e){return 5} finally {return 42} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 42)
+
+    def test_nested_try(self):
+        jsi = JSInterpreter('''
+        function x() {try {
+            try{throw 10} finally {throw 42}
+            } catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+
     def test_for_loop_continue(self):
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
     def test_for_loop_break(self):
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
+    def test_for_loop_try(self):
+        jsi = JSInterpreter('''
+        function x() {
+            for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} };
+            return 42 }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 42)
+
     def test_literal_list(self):
         jsi = JSInterpreter('''
-        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        function x() { return [1, 2, "asdf", [5, 6, 7]][3] }
         ''')
         self.assertEqual(jsi.call_function('x'), [5, 6, 7])
 
@@ -177,6 +252,162 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 7)
 
+        jsi = JSInterpreter('''
+        function x() { a=5; return (a -= 1, a+=3, a); }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
+        jsi = JSInterpreter('''
+        function x() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+
+    def test_void(self):
+        jsi = JSInterpreter('''
+        function x() { return void 42; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), None)
+
+    def test_return_function(self):
+        jsi = JSInterpreter('''
+        function x() { return [1, function(){return 1}][1] }
+        ''')
+        self.assertEqual(jsi.call_function('x')([]), 1)
+
+    def test_null(self):
+        jsi = JSInterpreter('''
+        function x() { return null; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), None)
+
+        jsi = JSInterpreter('''
+        function x() { return [null > 0, null < 0, null == 0, null === 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [null >= 0, null <= 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [True, True])
+
+    def test_undefined(self):
+        jsi = JSInterpreter('''
+        function x() { return undefined === undefined; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), True)
+
+        jsi = JSInterpreter('''
+        function x() { return undefined; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), JS_Undefined)
+
+        jsi = JSInterpreter('''
+        function x() { let v; return v; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), JS_Undefined)
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined === undefined, undefined == undefined, undefined < undefined, undefined > undefined]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [True, True, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined === 0, undefined == 0, undefined < 0, undefined > 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined >= 0, undefined <= 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined > null, undefined < null, undefined == null, undefined === null]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, True, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined === null, undefined == null, undefined < null, undefined > null]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, True, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
+        ''')
+        for y in jsi.call_function('x'):
+            self.assertTrue(math.isnan(y))
+
+        jsi = JSInterpreter('''
+        function x() { let v; return v**0; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 1)
+
+        jsi = JSInterpreter('''
+        function x() { let v; return [v>42, v<=42, v&&42, 42&&v]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, JS_Undefined, JS_Undefined])
+
+        jsi = JSInterpreter('function x(){return undefined ?? 42; }')
+        self.assertEqual(jsi.call_function('x'), 42)
+
+    def test_object(self):
+        jsi = JSInterpreter('''
+        function x() { return {}; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), {})
+
+        jsi = JSInterpreter('''
+        function x() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [42, 0])
+
+        jsi = JSInterpreter('''
+        function x() { let a; return a?.qq; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), JS_Undefined)
+
+        jsi = JSInterpreter('''
+        function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), JS_Undefined)
+
+    def test_regex(self):
+        jsi = JSInterpreter('''
+        function x() { let a=/,,[/,913,/](,)}/; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), None)
+
+        jsi = JSInterpreter('''
+        function x() { let a=/,,[/,913,/](,)}/; return a; }
+        ''')
+        self.assertIsInstance(jsi.call_function('x'), re.Pattern)
+
+        jsi = JSInterpreter('''
+        function x() { let a=/,,[/,913,/](,)}/i; return a; }
+        ''')
+        self.assertEqual(jsi.call_function('x').flags & re.I, re.I)
+
+        jsi = JSInterpreter(R'''
+        function x() { let a=/,][}",],()}(\[)/; return a; }
+        ''')
+        self.assertEqual(jsi.call_function('x').pattern, r',][}",],()}(\[)')
+
+    def test_char_code_at(self):
+        jsi = JSInterpreter('function x(i){return "test".charCodeAt(i)}')
+        self.assertEqual(jsi.call_function('x', 0), 116)
+        self.assertEqual(jsi.call_function('x', 1), 101)
+        self.assertEqual(jsi.call_function('x', 2), 115)
+        self.assertEqual(jsi.call_function('x', 3), 116)
+        self.assertEqual(jsi.call_function('x', 4), None)
+        self.assertEqual(jsi.call_function('x', 'not_a_number'), 116)
+
+    def test_bitwise_operators_overflow(self):
+        jsi = JSInterpreter('function x(){return -524999584 << 5}')
+        self.assertEqual(jsi.call_function('x'), 379882496)
+
+        jsi = JSInterpreter('function x(){return 1236566549 << 5}')
+        self.assertEqual(jsi.call_function('x'), 915423904)
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_utils.py

@@ -53,6 +53,7 @@ from yt_dlp.utils import (
     fix_xml_ampersands,
     float_or_none,
     format_bytes,
+    get_compatible_ext,
     get_element_by_attribute,
     get_element_by_class,
     get_element_html_by_attribute,
@@ -139,13 +140,13 @@ class TestUtil(unittest.TestCase):
 
         self.assertEqual(sanitize_filename('123'), '123')
 
-        self.assertEqual('abc_de', sanitize_filename('abc/de'))
+        self.assertEqual('abc⧸de', sanitize_filename('abc/de'))
         self.assertFalse('/' in sanitize_filename('abc/de///'))
 
-        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de'))
-        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|'))
-        self.assertEqual('yes no', sanitize_filename('yes? no'))
-        self.assertEqual('this - that', sanitize_filename('this: that'))
+        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', is_id=False))
+        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', is_id=False))
+        self.assertEqual('yes no', sanitize_filename('yes? no', is_id=False))
+        self.assertEqual('this - that', sanitize_filename('this: that', is_id=False))
 
         self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
         aumlaut = 'ä'
@@ -368,6 +369,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
         self.assertEqual(unified_strdate('1968 12 10'), '19681210')
         self.assertEqual(unified_strdate('1968-12-10'), '19681210')
+        self.assertEqual(unified_strdate('31-07-2022 20:00'), '20220731')
         self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128')
         self.assertEqual(
             unified_strdate('11/26/2014 11:30:00 AM PST', day_first=False),
@@ -411,6 +413,10 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140)
         self.assertEqual(unified_timestamp('2018-03-14T08:32:43.1493874+00:00'), 1521016363)
 
+        self.assertEqual(unified_timestamp('December 31 1969 20:00:01 EDT'), 1)
+        self.assertEqual(unified_timestamp('Wednesday 31 December 1969 18:01:26 MDT'), 86)
+        self.assertEqual(unified_timestamp('12/31/1969 20:01:18 EDT', False), 78)
+
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
         self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
@@ -1842,6 +1848,31 @@ Line 1
         self.assertEqual(determine_file_encoding('# coding: utf-32-be'.encode('utf-32-be')), ('utf-32-be', 0))
         self.assertEqual(determine_file_encoding('# coding: utf-16-le'.encode('utf-16-le')), ('utf-16-le', 0))
 
+    def test_get_compatible_ext(self):
+        self.assertEqual(get_compatible_ext(
+            vcodecs=[None], acodecs=[None, None], vexts=['mp4'], aexts=['m4a', 'm4a']), 'mkv')
+        self.assertEqual(get_compatible_ext(
+            vcodecs=[None], acodecs=[None], vexts=['flv'], aexts=['flv']), 'flv')
+
+        self.assertEqual(get_compatible_ext(
+            vcodecs=[None], acodecs=[None], vexts=['mp4'], aexts=['m4a']), 'mp4')
+        self.assertEqual(get_compatible_ext(
+            vcodecs=[None], acodecs=[None], vexts=['mp4'], aexts=['webm']), 'mkv')
+        self.assertEqual(get_compatible_ext(
+            vcodecs=[None], acodecs=[None], vexts=['webm'], aexts=['m4a']), 'mkv')
+        self.assertEqual(get_compatible_ext(
+            vcodecs=[None], acodecs=[None], vexts=['webm'], aexts=['webm']), 'webm')
+
+        self.assertEqual(get_compatible_ext(
+            vcodecs=['h264'], acodecs=['mp4a'], vexts=['mov'], aexts=['m4a']), 'mp4')
+        self.assertEqual(get_compatible_ext(
+            vcodecs=['av01.0.12M.08'], acodecs=['opus'], vexts=['mp4'], aexts=['webm']), 'webm')
+
+        self.assertEqual(get_compatible_ext(
+            vcodecs=['vp9'], acodecs=['opus'], vexts=['webm'], aexts=['webm'], preferences=['flv', 'mp4']), 'mp4')
+        self.assertEqual(get_compatible_ext(
+            vcodecs=['av1'], acodecs=['mp4a'], vexts=['webm'], aexts=['m4a'], preferences=('webm', 'mkv')), 'mkv')
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_youtube_signature.py

@@ -94,6 +94,38 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/5dd88d1d/player-plasma-ias-phone-en_US.vflset/base.js',
         'kSxKFLeqzv_ZyHSAt', 'n8gS8oRlHOxPFA',
     ),
+    (
+        'https://www.youtube.com/s/player/324f67b9/player_ias.vflset/en_US/base.js',
+        'xdftNy7dh9QGnhW', '22qLGxrmX8F1rA',
+    ),
+    (
+        'https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js',
+        'TDCstCG66tEAO5pR9o', 'dbxNtZ14c-yWyw',
+    ),
+    (
+        'https://www.youtube.com/s/player/c81bbb4a/player_ias.vflset/en_US/base.js',
+        'gre3EcLurNY2vqp94', 'Z9DfGxWP115WTg',
+    ),
+    (
+        'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
+        'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
+    ),
+    (
+        'https://www.youtube.com/s/player/009f1d77/player_ias.vflset/en_US/base.js',
+        '5dwFHw8aFWQUQtffRq', 'audescmLUzI3jw',
+    ),
+    (
+        'https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/en_US/base.js',
+        '5EHDMgYLV6HPGk_Mu-kk', 'n9lUJLHbxUI0GQ',
+    ),
+    (
+        'https://www.youtube.com/s/player/113ca41c/player_ias.vflset/en_US/base.js',
+        'cgYl-tlYkhjT7A', 'hI7BBr2zUgcmMg',
+    ),
+    (
+        'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js',
+        'M92UUMHa8PdvPd3wyM', '3hPqLJsiNZx7yA',
+    ),
 ]
 
 
@@ -101,6 +133,7 @@ _NSIG_TESTS = [
 class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
         PLAYER_URLS = (
+            ('https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js', '4c3f79c5'),
             ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/en_US/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/fr_FR/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'),

yt_dlp/YoutubeDL.py

@@ -29,6 +29,7 @@ from .cookies import load_cookies
 from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name
 from .downloader.rtmp import rtmpdump_version
 from .extractor import gen_extractor_classes, get_info_extractor
+from .extractor.common import UnsupportedURLIE
 from .extractor.openload import PhantomJSwrapper
 from .minicurses import format_text
 from .postprocessor import _PLUGIN_CLASSES as plugin_postprocessors
@@ -47,11 +48,12 @@ from .postprocessor import (
     get_postprocessor,
 )
 from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
-from .update import detect_variant
+from .update import REPOSITORY, current_git_head, detect_variant
 from .utils import (
     DEFAULT_OUTTMPL,
     IDENTITY,
     LINK_TEMPLATES,
+    MEDIA_EXTENSIONS,
     NO_DEFAULT,
     NUMBER_RE,
     OUTTMPL_TYPES,
@@ -80,6 +82,7 @@ from .utils import (
     RejectedVideoReached,
     SameFileError,
     UnavailableVideoError,
+    UserNotLive,
     YoutubeDLCookieProcessor,
     YoutubeDLHandler,
     YoutubeDLRedirectHandler,
@@ -87,6 +90,7 @@ from .utils import (
     args_to_str,
     bug_reports_message,
     date_from_str,
+    deprecation_warning,
     determine_ext,
     determine_protocol,
     encode_compat_str,
@@ -100,17 +104,20 @@ from .utils import (
     format_decimal_suffix,
     format_field,
     formatSeconds,
+    get_compatible_ext,
     get_domain,
     int_or_none,
     iri_to_uri,
     join_nonempty,
     locked_file,
+    make_archive_id,
     make_dir,
     make_HTTPS_handler,
     merge_headers,
     network_exceptions,
     number_of_digits,
     orderedSet,
+    orderedSet_from_options,
     parse_filesize,
     preferredencoding,
     prepend_extension,
@@ -131,6 +138,7 @@ from .utils import (
     timetuple_from_msec,
     to_high_limit_path,
     traverse_obj,
+    try_call,
     try_get,
     url_basename,
     variadic,
@@ -139,7 +147,7 @@ from .utils import (
     write_json_file,
     write_string,
 )
-from .version import RELEASE_GIT_HEAD, __version__
+from .version import RELEASE_GIT_HEAD, VARIANT, __version__
 
 if compat_os_name == 'nt':
     import ctypes
@@ -231,7 +239,7 @@ class YoutubeDL:
                        Default is 'only_download' for CLI, but False for API
     skip_playlist_after_errors: Number of allowed failures until the rest of
                        the playlist is skipped
-    force_generic_extractor: Force downloader to use the generic extractor
+    allowed_extractors:  List of regexes to match against extractor names that are allowed
     overwrites:        Overwrite all video and metadata files if True,
                        overwrite only non-video files if None
                        and don't overwrite any file if False
@@ -267,7 +275,7 @@ class YoutubeDL:
     subtitleslangs:    List of languages of the subtitles to download (can be regex).
                        The list may contain "all" to refer to all the available
                        subtitles. The language can be prefixed with a "-" to
-                       exclude it from the requested languages. Eg: ['all', '-live_chat']
+                       exclude it from the requested languages, e.g. ['all', '-live_chat']
     keepvideo:         Keep the video file after post-processing
     daterange:         A DateRange object, download only if the upload_date is in the range.
     skip_download:     Skip the actual download of the video file
@@ -296,8 +304,9 @@ class YoutubeDL:
                        should act on each input URL as opposed to for the entire queue
     cookiefile:        File name or text stream from where cookies should be read and dumped to
     cookiesfrombrowser:  A tuple containing the name of the browser, the profile
-                       name/pathfrom where cookies are loaded, and the name of the
-                       keyring. Eg: ('chrome', ) or ('vivaldi', 'default', 'BASICTEXT')
+                       name/path from where cookies are loaded, the name of the keyring,
+                       and the container name, e.g. ('chrome', ) or
+                       ('vivaldi', 'default', 'BASICTEXT') or ('firefox', 'default', None, 'Meta')
     legacyserverconnect: Explicitly allow HTTPS connection to servers that do not
                        support RFC 5746 secure renegotiation
     nocheckcertificate:  Do not verify SSL certificates
@@ -369,7 +378,7 @@ class YoutubeDL:
 
                        Progress hooks are guaranteed to be called at least twice
                        (with status "started" and "finished") if the processing is successful.
-    merge_output_format: Extension to use when merging formats.
+    merge_output_format: "/" separated list of extensions to use when merging formats.
     final_ext:         Expected final extension; used to detect when the file was
                        already downloaded and converted
     fixup:             Automatically correct known faults of the file.
@@ -439,6 +448,7 @@ class YoutubeDL:
                        * index: Section number (Optional)
     force_keyframes_at_cuts: Re-encode the video when downloading ranges to get precise cuts
     noprogress:        Do not print the progress bar
+    live_from_start:   Whether to download livestreams videos from the start
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the downloader (see yt_dlp/downloader/common.py):
@@ -465,11 +475,13 @@ class YoutubeDL:
                        discontinuities such as ad breaks (default: False)
     extractor_args:    A dictionary of arguments to be passed to the extractors.
                        See "EXTRACTOR ARGUMENTS" for details.
-                       Eg: {'youtube': {'skip': ['dash', 'hls']}}
+                       E.g. {'youtube': {'skip': ['dash', 'hls']}}
     mark_watched:      Mark videos watched (even with --simulate). Only for YouTube
 
     The following options are deprecated and may be removed in the future:
 
+    force_generic_extractor: Force downloader to use the generic extractor
+                       - Use allowed_extractors = ['generic', 'default']
     playliststart:     - Use playlist_items
                        Playlist item to start at.
     playlistend:       - Use playlist_items
@@ -522,7 +534,8 @@ class YoutubeDL:
     """
 
     _NUMERIC_FIELDS = {
-        'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
+        'width', 'height', 'asr', 'audio_channels', 'fps',
+        'tbr', 'abr', 'vbr', 'filesize', 'filesize_approx',
         'timestamp', 'release_timestamp',
         'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
         'average_rating', 'comment_count', 'age_limit',
@@ -534,7 +547,7 @@ class YoutubeDL:
     _format_fields = {
         # NB: Keep in sync with the docstring of extractor/common.py
         'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note',
-        'width', 'height', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr',
+        'width', 'height', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels',
         'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx',
         'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start',
         'preference', 'language', 'language_preference', 'quality', 'source_preference',
@@ -542,9 +555,9 @@ class YoutubeDL:
         'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time'
     }
     _format_selection_exts = {
-        'audio': {'m4a', 'mp3', 'ogg', 'aac'},
-        'video': {'mp4', 'flv', 'webm', '3gp'},
-        'storyboards': {'mhtml'},
+        'audio': set(MEDIA_EXTENSIONS.common_audio),
+        'video': set(MEDIA_EXTENSIONS.common_video + ('3gp', )),
+        'storyboards': set(MEDIA_EXTENSIONS.storyboards),
     }
 
     def __init__(self, params=None, auto_init=True):
@@ -584,7 +597,8 @@ class YoutubeDL:
             for type_, stream in self._out_files.items_ if type_ != 'console'
         })
 
-        MIN_SUPPORTED, MIN_RECOMMENDED = (3, 6), (3, 7)
+        # The code is left like this to be reused for future deprecations
+        MIN_SUPPORTED, MIN_RECOMMENDED = (3, 7), (3, 7)
         current_version = sys.version_info[:2]
         if current_version < MIN_RECOMMENDED:
             msg = ('Support for Python version %d.%d has been deprecated. '
@@ -619,7 +633,7 @@ class YoutubeDL:
         for msg in self.params.get('_warnings', []):
             self.report_warning(msg)
         for msg in self.params.get('_deprecation_warnings', []):
-            self.deprecation_warning(msg)
+            self.deprecated_feature(msg)
 
         self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
         if 'list-formats' in self.params['compat_opts']:
@@ -749,13 +763,6 @@ class YoutubeDL:
             self._ies_instances[ie_key] = ie
             ie.set_downloader(self)
 
-    def _get_info_extractor_class(self, ie_key):
-        ie = self._ies.get(ie_key)
-        if ie is None:
-            ie = get_info_extractor(ie_key)
-            self.add_info_extractor(ie)
-        return ie
-
     def get_info_extractor(self, ie_key):
         """
         Get an instance of an IE with name ie_key, it will try to get one from
@@ -772,8 +779,19 @@ class YoutubeDL:
         """
         Add the InfoExtractors returned by gen_extractors to the end of the list
         """
-        for ie in gen_extractor_classes():
-            self.add_info_extractor(ie)
+        all_ies = {ie.IE_NAME.lower(): ie for ie in gen_extractor_classes()}
+        all_ies['end'] = UnsupportedURLIE()
+        try:
+            ie_names = orderedSet_from_options(
+                self.params.get('allowed_extractors', ['default']), {
+                    'all': list(all_ies),
+                    'default': [name for name, ie in all_ies.items() if ie._ENABLED],
+                }, use_regex=True)
+        except re.error as e:
+            raise ValueError(f'Wrong regex for allowed_extractors: {e.pattern}')
+        for name in ie_names:
+            self.add_info_extractor(all_ies[name])
+        self.write_debug(f'Loaded {len(ie_names)} extractors')
 
     def add_post_processor(self, pp, when='post_process'):
         """Add a PostProcessor object to the end of the chain."""
@@ -819,9 +837,11 @@ class YoutubeDL:
     def to_stdout(self, message, skip_eol=False, quiet=None):
         """Print message to stdout"""
         if quiet is not None:
-            self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument quiet. Use "YoutubeDL.to_screen" instead')
+            self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument quiet. '
+                                     'Use "YoutubeDL.to_screen" instead')
         if skip_eol is not False:
-            self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument skip_eol. Use "YoutubeDL.to_screen" instead')
+            self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument skip_eol. '
+                                     'Use "YoutubeDL.to_screen" instead')
         self._write_string(f'{self._bidi_workaround(message)}\n', self._out_files.out)
 
     def to_screen(self, message, skip_eol=False, quiet=None):
@@ -957,11 +977,14 @@ class YoutubeDL:
                 return
             self.to_stderr(f'{self._format_err("WARNING:", self.Styles.WARNING)} {message}', only_once)
 
-    def deprecation_warning(self, message):
+    def deprecation_warning(self, message, *, stacklevel=0):
+        deprecation_warning(
+            message, stacklevel=stacklevel + 1, printer=self.report_error, is_error=False)
+
+    def deprecated_feature(self, message):
         if self.params.get('logger') is not None:
-            self.params['logger'].warning(f'DeprecationWarning: {message}')
-        else:
-            self.to_stderr(f'{self._format_err("DeprecationWarning:", self.Styles.ERROR)} {message}', True)
+            self.params['logger'].warning(f'Deprecated Feature: {message}')
+        self.to_stderr(f'{self._format_err("Deprecated Feature:", self.Styles.ERROR)} {message}', True)
 
     def report_error(self, message, *args, **kwargs):
         '''
@@ -1021,7 +1044,7 @@ class YoutubeDL:
 
     def get_output_path(self, dir_type='', filename=None):
         paths = self.params.get('paths', {})
-        assert isinstance(paths, dict)
+        assert isinstance(paths, dict), '"paths" parameter must be a dictionary'
         path = os.path.join(
             expand_path(paths.get('home', '').strip()),
             expand_path(paths.get(dir_type, '').strip()) if dir_type else '',
@@ -1039,7 +1062,7 @@ class YoutubeDL:
 
         # outtmpl should be expand_path'ed before template dict substitution
         # because meta fields may contain env variables we don't want to
-        # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
+        # be expanded. E.g. for outtmpl "%(title)s.%(ext)s" and
         # title "Hello $PATH", we don't want `$PATH` to be expanded.
         return expand_path(outtmpl).replace(sep, '')
 
@@ -1159,6 +1182,9 @@ class YoutubeDL:
             if mdict['strf_format']:
                 value = strftime_or_none(value, mdict['strf_format'].replace('\\,', ','))
 
+            # XXX: Workaround for https://github.com/yt-dlp/yt-dlp/issues/4485
+            if sanitize and value == '':
+                value = None
             return value
 
         na = self.params.get('outtmpl_na_placeholder', 'NA')
@@ -1308,7 +1334,7 @@ class YoutubeDL:
     def _match_entry(self, info_dict, incomplete=False, silent=False):
         """ Returns None if the file should be downloaded """
 
-        video_title = info_dict.get('title', info_dict.get('id', 'video'))
+        video_title = info_dict.get('title', info_dict.get('id', 'entry'))
 
         def check_filter():
             if 'title' in info_dict:
@@ -1401,11 +1427,11 @@ class YoutubeDL:
             ie_key = 'Generic'
 
         if ie_key:
-            ies = {ie_key: self._get_info_extractor_class(ie_key)}
+            ies = {ie_key: self._ies[ie_key]} if ie_key in self._ies else {}
         else:
             ies = self._ies
 
-        for ie_key, ie in ies.items():
+        for key, ie in ies.items():
             if not ie.suitable(url):
                 continue
 
@@ -1414,14 +1440,16 @@ class YoutubeDL:
                                     'and will probably not work.')
 
             temp_id = ie.get_temp_id(url)
-            if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': ie_key}):
-                self.to_screen(f'[{ie_key}] {temp_id}: has already been recorded in the archive')
+            if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': key}):
+                self.to_screen(f'[{key}] {temp_id}: has already been recorded in the archive')
                 if self.params.get('break_on_existing', False):
                     raise ExistingVideoReached()
                 break
-            return self.__extract_info(url, self.get_info_extractor(ie_key), download, extra_info, process)
+            return self.__extract_info(url, self.get_info_extractor(key), download, extra_info, process)
         else:
-            self.report_error('no suitable InfoExtractor for URL %s' % url)
+            extractors_restricted = self.params.get('allowed_extractors') not in (None, ['default'])
+            self.report_error(f'No suitable extractor{format_field(ie_key, None, " (%s)")} found for URL {url}',
+                              tb=False if extractors_restricted else None)
 
     def _handle_extraction_exceptions(func):
         @functools.wraps(func)
@@ -1455,7 +1483,7 @@ class YoutubeDL:
                 break
         return wrapper
 
-    def _wait_for_video(self, ie_result):
+    def _wait_for_video(self, ie_result={}):
         if (not self.params.get('wait_for_video')
                 or ie_result.get('_type', 'video') != 'video'
                 or ie_result.get('formats') or ie_result.get('url')):
@@ -1479,7 +1507,7 @@ class YoutubeDL:
         if diff is None and ie_result.get('live_status') == 'is_upcoming':
             diff = round(random.uniform(min_wait, max_wait) if (max_wait and min_wait) else (max_wait or min_wait), 0)
             self.report_warning('Release time of video is not known')
-        elif (diff or 0) <= 0:
+        elif ie_result and (diff or 0) <= 0:
             self.report_warning('Video should already be available according to extracted info')
         diff = min(max(diff or 0, min_wait or 0), max_wait or float('inf'))
         self.to_screen(f'[wait] Waiting for {format_dur(diff)} - Press Ctrl+C to try now')
@@ -1503,7 +1531,14 @@ class YoutubeDL:
 
     @_handle_extraction_exceptions
     def __extract_info(self, url, ie, download, extra_info, process):
-        ie_result = ie.extract(url)
+        try:
+            ie_result = ie.extract(url)
+        except UserNotLive as e:
+            if process:
+                if self.params.get('wait_for_video'):
+                    self.report_warning(e)
+                self._wait_for_video()
+            raise
         if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
             self.report_warning(f'Extractor {ie.IE_NAME} returned nothing{bug_reports_message()}')
             return
@@ -1553,7 +1588,8 @@ class YoutubeDL:
         result_type = ie_result.get('_type', 'video')
 
         if result_type in ('url', 'url_transparent'):
-            ie_result['url'] = sanitize_url(ie_result['url'])
+            ie_result['url'] = sanitize_url(
+                ie_result['url'], scheme='http' if self.params.get('prefer_insecure') else 'https')
             if ie_result.get('original_url'):
                 extra_info.setdefault('original_url', ie_result['original_url'])
 
@@ -1676,23 +1712,37 @@ class YoutubeDL:
         return make_dir(path, self.report_error)
 
     @staticmethod
-    def _playlist_infodict(ie_result, **kwargs):
-        return {
-            **ie_result,
+    def _playlist_infodict(ie_result, strict=False, **kwargs):
+        info = {
+            'playlist_count': ie_result.get('playlist_count'),
             'playlist': ie_result.get('title') or ie_result.get('id'),
             'playlist_id': ie_result.get('id'),
             'playlist_title': ie_result.get('title'),
             'playlist_uploader': ie_result.get('uploader'),
             'playlist_uploader_id': ie_result.get('uploader_id'),
-            'playlist_index': 0,
             **kwargs,
         }
+        if strict:
+            return info
+        return {
+            **info,
+            'playlist_index': 0,
+            '__last_playlist_index': max(ie_result['requested_entries'] or (0, 0)),
+            'extractor': ie_result['extractor'],
+            'webpage_url': ie_result['webpage_url'],
+            'webpage_url_basename': url_basename(ie_result['webpage_url']),
+            'webpage_url_domain': get_domain(ie_result['webpage_url']),
+            'extractor_key': ie_result['extractor_key'],
+        }
 
     def __process_playlist(self, ie_result, download):
         """Process each entry in the playlist"""
         assert ie_result['_type'] in ('playlist', 'multi_video')
 
-        title = ie_result.get('title') or ie_result.get('id') or '<Untitled>'
+        common_info = self._playlist_infodict(ie_result, strict=True)
+        title = common_info.get('playlist') or '<Untitled>'
+        if self._match_entry(common_info, incomplete=True) is not None:
+            return
         self.to_screen(f'[download] Downloading {ie_result["_type"]}: {title}')
 
         all_entries = PlaylistEntries(self, ie_result)
@@ -1710,12 +1760,14 @@ class YoutubeDL:
             # Better to do this after potentially exhausting entries
             ie_result['playlist_count'] = all_entries.get_full_count()
 
+        extra = self._playlist_infodict(ie_result, n_entries=int_or_none(n_entries))
+        ie_copy = collections.ChainMap(ie_result, extra)
+
         _infojson_written = False
         write_playlist_files = self.params.get('allow_playlist_files', True)
         if write_playlist_files and self.params.get('list_thumbnails'):
             self.list_thumbnails(ie_result)
         if write_playlist_files and not self.params.get('simulate'):
-            ie_copy = self._playlist_infodict(ie_result, n_entries=int_or_none(n_entries))
             _infojson_written = self._write_info_json(
                 'playlist', ie_result, self.prepare_filename(ie_copy, 'pl_infojson'))
             if _infojson_written is None:
@@ -1724,7 +1776,7 @@ class YoutubeDL:
                                        self.prepare_filename(ie_copy, 'pl_description')) is None:
                 return
             # TODO: This should be passed to ThumbnailsConvertor if necessary
-            self._write_thumbnails('playlist', ie_copy, self.prepare_filename(ie_copy, 'pl_thumbnail'))
+            self._write_thumbnails('playlist', ie_result, self.prepare_filename(ie_copy, 'pl_thumbnail'))
 
         if lazy:
             if self.params.get('playlistreverse') or self.params.get('playlistrandom'):
@@ -1748,35 +1800,33 @@ class YoutubeDL:
         for i, (playlist_index, entry) in enumerate(entries):
             if lazy:
                 resolved_entries.append((playlist_index, entry))
-
-            # TODO: Add auto-generated fields
-            if not entry or self._match_entry(entry, incomplete=True) is not None:
+            if not entry:
                 continue
 
-            self.to_screen('[download] Downloading video %s of %s' % (
-                self._format_screen(i + 1, self.Styles.ID), self._format_screen(n_entries, self.Styles.EMPHASIS)))
-
             entry['__x_forwarded_for_ip'] = ie_result.get('__x_forwarded_for_ip')
             if not lazy and 'playlist-index' in self.params.get('compat_opts', []):
                 playlist_index = ie_result['requested_entries'][i]
 
-            entry_result = self.__process_iterable_entry(entry, download, {
+            entry_copy = collections.ChainMap(entry, {
+                **common_info,
                 'n_entries': int_or_none(n_entries),
-                '__last_playlist_index': max(ie_result['requested_entries'] or (0, 0)),
-                'playlist_count': ie_result.get('playlist_count'),
                 'playlist_index': playlist_index,
                 'playlist_autonumber': i + 1,
-                'playlist': title,
-                'playlist_id': ie_result.get('id'),
-                'playlist_title': ie_result.get('title'),
-                'playlist_uploader': ie_result.get('uploader'),
-                'playlist_uploader_id': ie_result.get('uploader_id'),
-                'extractor': ie_result['extractor'],
-                'webpage_url': ie_result['webpage_url'],
-                'webpage_url_basename': url_basename(ie_result['webpage_url']),
-                'webpage_url_domain': get_domain(ie_result['webpage_url']),
-                'extractor_key': ie_result['extractor_key'],
             })
+
+            if self._match_entry(entry_copy, incomplete=True) is not None:
+                # For compatabilty with youtube-dl. See https://github.com/yt-dlp/yt-dlp/issues/4369
+                resolved_entries[i] = (playlist_index, NO_DEFAULT)
+                continue
+
+            self.to_screen('[download] Downloading video %s of %s' % (
+                self._format_screen(i + 1, self.Styles.ID), self._format_screen(n_entries, self.Styles.EMPHASIS)))
+
+            extra.update({
+                'playlist_index': playlist_index,
+                'playlist_autonumber': i + 1,
+            })
+            entry_result = self.__process_iterable_entry(entry, download, extra)
             if not entry_result:
                 failures += 1
             if failures >= max_failures:
@@ -1787,7 +1837,8 @@ class YoutubeDL:
                 resolved_entries[i] = (playlist_index, entry_result)
 
         # Update with processed data
-        ie_result['requested_entries'], ie_result['entries'] = tuple(zip(*resolved_entries)) or ([], [])
+        ie_result['requested_entries'] = [i for i, e in resolved_entries if e is not NO_DEFAULT]
+        ie_result['entries'] = [e for _, e in resolved_entries if e is not NO_DEFAULT]
 
         # Write the updated info to json
         if _infojson_written is True and self._write_info_json(
@@ -1944,8 +1995,8 @@ class YoutubeDL:
                     filter_parts.append(string)
 
         def _remove_unused_ops(tokens):
-            # Remove operators that we don't use and join them with the surrounding strings
-            # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
+            # Remove operators that we don't use and join them with the surrounding strings.
+            # E.g. 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
             ALLOWED_OPS = ('/', '+', ',', '(', ')')
             last_string, last_start, last_end, last_line = None, None, None, None
             for type, string, start, end, line in tokens:
@@ -2061,14 +2112,13 @@ class YoutubeDL:
             the_only_video = video_fmts[0] if len(video_fmts) == 1 else None
             the_only_audio = audio_fmts[0] if len(audio_fmts) == 1 else None
 
-            output_ext = self.params.get('merge_output_format')
-            if not output_ext:
-                if the_only_video:
-                    output_ext = the_only_video['ext']
-                elif the_only_audio and not video_fmts:
-                    output_ext = the_only_audio['ext']
-                else:
-                    output_ext = 'mkv'
+            output_ext = get_compatible_ext(
+                vcodecs=[f.get('vcodec') for f in video_fmts],
+                acodecs=[f.get('acodec') for f in audio_fmts],
+                vexts=[f['ext'] for f in video_fmts],
+                aexts=[f['ext'] for f in audio_fmts],
+                preferences=(try_call(lambda: self.params['merge_output_format'].split('/'))
+                             or self.params.get('prefer_free_formats') and ('webm', 'mkv')))
 
             filtered = lambda *keys: filter(None, (traverse_obj(fmt, *keys) for fmt in formats_info))
 
@@ -2101,6 +2151,7 @@ class YoutubeDL:
                     'acodec': the_only_audio.get('acodec'),
                     'abr': the_only_audio.get('abr'),
                     'asr': the_only_audio.get('asr'),
+                    'audio_channels': the_only_audio.get('audio_channels')
                 })
 
             return new_dict
@@ -2461,7 +2512,7 @@ class YoutubeDL:
         info_dict['_has_drm'] = any(f.get('has_drm') for f in formats) or None
         if not self.params.get('allow_unplayable_formats'):
             formats = [f for f in formats if not f.get('has_drm')]
-            if info_dict['_has_drm'] and all(
+            if info_dict['_has_drm'] and formats and all(
                     f.get('acodec') == f.get('vcodec') == 'none' for f in formats):
                 self.report_warning(
                     'This video is DRM protected and only images are available for download. '
@@ -2477,9 +2528,6 @@ class YoutubeDL:
                     '--live-from-start is passed, but there are no formats that can be downloaded from the start. '
                     'If you want to download from the current time, use --no-live-from-start'))
 
-        if not formats:
-            self.raise_no_formats(info_dict)
-
         def is_wellformed(f):
             url = f.get('url')
             if not url:
@@ -2492,7 +2540,10 @@ class YoutubeDL:
             return True
 
         # Filter out malformed formats for better extraction robustness
-        formats = list(filter(is_wellformed, formats))
+        formats = list(filter(is_wellformed, formats or []))
+
+        if not formats:
+            self.raise_no_formats(info_dict)
 
         formats_dict = {}
 
@@ -2694,42 +2745,26 @@ class YoutubeDL:
                 if lang not in available_subs:
                     available_subs[lang] = cap_info
 
-        if (not self.params.get('writesubtitles') and not
-                self.params.get('writeautomaticsub') or not
-                available_subs):
+        if not available_subs or (
+                not self.params.get('writesubtitles')
+                and not self.params.get('writeautomaticsub')):
             return None
 
         all_sub_langs = tuple(available_subs.keys())
         if self.params.get('allsubtitles', False):
             requested_langs = all_sub_langs
         elif self.params.get('subtitleslangs', False):
-            # A list is used so that the order of languages will be the same as
-            # given in subtitleslangs. See https://github.com/yt-dlp/yt-dlp/issues/1041
-            requested_langs = []
-            for lang_re in self.params.get('subtitleslangs'):
-                discard = lang_re[0] == '-'
-                if discard:
-                    lang_re = lang_re[1:]
-                if lang_re == 'all':
-                    if discard:
-                        requested_langs = []
-                    else:
-                        requested_langs.extend(all_sub_langs)
-                    continue
-                current_langs = filter(re.compile(lang_re + '$').match, all_sub_langs)
-                if discard:
-                    for lang in current_langs:
-                        while lang in requested_langs:
-                            requested_langs.remove(lang)
-                else:
-                    requested_langs.extend(current_langs)
-            requested_langs = orderedSet(requested_langs)
+            try:
+                requested_langs = orderedSet_from_options(
+                    self.params.get('subtitleslangs'), {'all': all_sub_langs}, use_regex=True)
+            except re.error as e:
+                raise ValueError(f'Wrong regex for subtitlelangs: {e.pattern}')
         elif normal_sub_langs:
             requested_langs = ['en'] if 'en' in normal_sub_langs else normal_sub_langs[:1]
         else:
             requested_langs = ['en'] if 'en' in all_sub_langs else all_sub_langs[:1]
         if requested_langs:
-            self.write_debug('Downloading subtitles: %s' % ', '.join(requested_langs))
+            self.to_screen(f'[info] {video_id}: Downloading subtitles: {", ".join(requested_langs)}')
 
         formats_query = self.params.get('subtitlesformat', 'best')
         formats_preference = formats_query.split('/') if formats_query else []
@@ -3040,33 +3075,9 @@ class YoutubeDL:
                         return
 
                 if info_dict.get('requested_formats') is not None:
-
-                    def compatible_formats(formats):
-                        # TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
-                        video_formats = [format for format in formats if format.get('vcodec') != 'none']
-                        audio_formats = [format for format in formats if format.get('acodec') != 'none']
-                        if len(video_formats) > 2 or len(audio_formats) > 2:
-                            return False
-
-                        # Check extension
-                        exts = {format.get('ext') for format in formats}
-                        COMPATIBLE_EXTS = (
-                            {'mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'},
-                            {'webm'},
-                        )
-                        for ext_sets in COMPATIBLE_EXTS:
-                            if ext_sets.issuperset(exts):
-                                return True
-                        # TODO: Check acodec/vcodec
-                        return False
-
                     requested_formats = info_dict['requested_formats']
                     old_ext = info_dict['ext']
                     if self.params.get('merge_output_format') is None:
-                        if not compatible_formats(requested_formats):
-                            info_dict['ext'] = 'mkv'
-                            self.report_warning(
-                                'Requested formats are incompatible for merge and will be merged into mkv')
                         if (info_dict['ext'] == 'webm'
                                 and info_dict.get('thumbnails')
                                 # check with type instead of pp_key, __name__, or isinstance
@@ -3261,6 +3272,7 @@ class YoutubeDL:
                 self.to_screen(f'[info] {e}')
                 if not self.params.get('break_per_url'):
                     raise
+                self._num_downloads = 0
             else:
                 if self.params.get('dump_single_json', False):
                     self.post_extract(res)
@@ -3309,6 +3321,12 @@ class YoutubeDL:
             return info_dict
         info_dict.setdefault('epoch', int(time.time()))
         info_dict.setdefault('_type', 'video')
+        info_dict.setdefault('_version', {
+            'version': __version__,
+            'current_git_head': current_git_head(),
+            'release_git_head': RELEASE_GIT_HEAD,
+            'repository': REPOSITORY,
+        })
 
         if remove_private_keys:
             reject = lambda k, v: v is None or k.startswith('__') or k in {
@@ -3426,18 +3444,16 @@ class YoutubeDL:
                     break
             else:
                 return
-        return f'{extractor.lower()} {video_id}'
+        return make_archive_id(extractor, video_id)
 
     def in_download_archive(self, info_dict):
         fn = self.params.get('download_archive')
         if fn is None:
             return False
 
-        vid_id = self._make_archive_id(info_dict)
-        if not vid_id:
-            return False  # Incomplete video information
-
-        return vid_id in self.archive
+        vid_ids = [self._make_archive_id(info_dict)]
+        vid_ids.extend(info_dict.get('_old_archive_ids') or [])
+        return any(id_ in self.archive for id_ in vid_ids)
 
     def record_download_archive(self, info_dict):
         fn = self.params.get('download_archive')
@@ -3567,6 +3583,7 @@ class YoutubeDL:
                 format_field(f, func=self.format_resolution, ignore=('audio only', 'images')),
                 format_field(f, 'fps', '\t%d', func=round),
                 format_field(f, 'dynamic_range', '%s', ignore=(None, 'SDR')).replace('HDR', ''),
+                format_field(f, 'audio_channels', '\t%s'),
                 delim,
                 format_field(f, 'filesize', ' \t%s', func=format_bytes) + format_field(f, 'filesize_approx', '~\t%s', func=format_bytes),
                 format_field(f, 'tbr', '\t%dk', func=round),
@@ -3586,7 +3603,7 @@ class YoutubeDL:
                     delim=' '),
             ] for f in formats if f.get('preference') is None or f['preference'] >= -1000]
         header_line = self._list_format_headers(
-            'ID', 'EXT', 'RESOLUTION', '\tFPS', 'HDR', delim, '\tFILESIZE', '\tTBR', 'PROTO',
+            'ID', 'EXT', 'RESOLUTION', '\tFPS', 'HDR', 'CH', delim, '\tFILESIZE', '\tTBR', 'PROTO',
             delim, 'VCODEC', '\tVBR', 'ACODEC', '\tABR', '\tASR', 'MORE INFO')
 
         return render_table(
@@ -3671,8 +3688,11 @@ class YoutubeDL:
             write_debug = lambda msg: self._write_string(f'[debug] {msg}\n')
 
         source = detect_variant()
+        if VARIANT not in (None, 'pip'):
+            source += '*'
         write_debug(join_nonempty(
-            'yt-dlp version', __version__,
+            f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
+            __version__,
             f'[{RELEASE_GIT_HEAD}]' if RELEASE_GIT_HEAD else '',
             '' if source == 'unknown' else f'({source})',
             delim=' '))
@@ -3688,18 +3708,8 @@ class YoutubeDL:
         if self.params['compat_opts']:
             write_debug('Compatibility options: %s' % ', '.join(self.params['compat_opts']))
 
-        if source == 'source':
-            try:
-                stdout, _, _ = Popen.run(
-                    ['git', 'rev-parse', '--short', 'HEAD'],
-                    text=True, cwd=os.path.dirname(os.path.abspath(__file__)),
-                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-                if re.fullmatch('[0-9a-f]+', stdout.strip()):
-                    write_debug(f'Git HEAD: {stdout.strip()}')
-            except Exception:
-                with contextlib.suppress(Exception):
-                    sys.exc_clear()
-
+        if current_git_head():
+            write_debug(f'Git HEAD: {current_git_head()}')
         write_debug(system_identifier())
 
         exe_versions, ffmpeg_features = FFmpegPostProcessor.get_versions_and_features(self)

yt_dlp/__init__.py

@@ -1,4 +1,8 @@
-f'You are using an unsupported version of Python. Only Python versions 3.6 and above are supported by yt-dlp'  # noqa: F541
+try:
+    import contextvars  # noqa: F401
+except Exception:
+    raise Exception(
+        f'You are using an unsupported version of Python. Only Python versions 3.7 and above are supported by yt-dlp')  # noqa: F541
 
 __license__ = 'Public Domain'
 
@@ -20,6 +24,8 @@ from .extractor.common import InfoExtractor
 from .options import parseOpts
 from .postprocessor import (
     FFmpegExtractAudioPP,
+    FFmpegMergerPP,
+    FFmpegPostProcessor,
     FFmpegSubtitlesConvertorPP,
     FFmpegThumbnailsConvertorPP,
     FFmpegVideoConvertorPP,
@@ -57,6 +63,8 @@ from .utils import (
 )
 from .YoutubeDL import YoutubeDL
 
+_IN_CLI = False
+
 
 def _exit(status=0, *args):
     for msg in args:
@@ -222,6 +230,8 @@ def validate_options(opts):
         validate_regex('format sorting', f, InfoExtractor.FormatSort.regex)
 
     # Postprocessor formats
+    validate_regex('merge output format', opts.merge_output_format,
+                   r'({0})(/({0}))*'.format('|'.join(map(re.escape, FFmpegMergerPP.SUPPORTED_EXTS))))
     validate_regex('audio format', opts.audioformat, FFmpegExtractAudioPP.FORMAT_RE)
     validate_in('subtitle format', opts.convertsubtitles, FFmpegSubtitlesConvertorPP.SUPPORTED_EXTS)
     validate_regex('thumbnail format', opts.convertthumbnails, FFmpegThumbnailsConvertorPP.FORMAT_RE)
@@ -336,10 +346,16 @@ def validate_options(opts):
 
     # Cookies from browser
     if opts.cookiesfrombrowser:
-        mobj = re.match(r'(?P<name>[^+:]+)(\s*\+\s*(?P<keyring>[^:]+))?(\s*:(?P<profile>.+))?', opts.cookiesfrombrowser)
+        container = None
+        mobj = re.fullmatch(r'''(?x)
+            (?P<name>[^+:]+)
+            (?:\s*\+\s*(?P<keyring>[^:]+))?
+            (?:\s*:\s*(?P<profile>.+?))?
+            (?:\s*::\s*(?P<container>.+))?
+        ''', opts.cookiesfrombrowser)
         if mobj is None:
             raise ValueError(f'invalid cookies from browser arguments: {opts.cookiesfrombrowser}')
-        browser_name, keyring, profile = mobj.group('name', 'keyring', 'profile')
+        browser_name, keyring, profile, container = mobj.group('name', 'keyring', 'profile', 'container')
         browser_name = browser_name.lower()
         if browser_name not in SUPPORTED_BROWSERS:
             raise ValueError(f'unsupported browser specified for cookies: "{browser_name}". '
@@ -349,7 +365,7 @@ def validate_options(opts):
             if keyring not in SUPPORTED_KEYRINGS:
                 raise ValueError(f'unsupported keyring specified for cookies: "{keyring}". '
                                  f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
-        opts.cookiesfrombrowser = (browser_name, profile, keyring)
+        opts.cookiesfrombrowser = (browser_name, profile, keyring, container)
 
     # MetadataParser
     def metadataparser_actions(f):
@@ -758,6 +774,7 @@ def parse_options(argv=None):
         'windowsfilenames': opts.windowsfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'force_generic_extractor': opts.force_generic_extractor,
+        'allowed_extractors': opts.allowed_extractors or ['default'],
         'ratelimit': opts.ratelimit,
         'throttledratelimit': opts.throttledratelimit,
         'overwrites': opts.overwrites,
@@ -899,6 +916,11 @@ def _real_main(argv=None):
     if print_extractor_information(opts, all_urls):
         return
 
+    # We may need ffmpeg_location without having access to the YoutubeDL instance
+    # See https://github.com/yt-dlp/yt-dlp/issues/2191
+    if opts.ffmpeg_location:
+        FFmpegPostProcessor._ffmpeg_location.set(opts.ffmpeg_location)
+
     with YoutubeDL(ydl_opts) as ydl:
         pre_process = opts.update_self or opts.rm_cachedir
         actual_use = all_urls or opts.load_info_filename

yt_dlp/__main__.py

@@ -14,4 +14,5 @@ if __package__ is None and not hasattr(sys, 'frozen'):
 import yt_dlp
 
 if __name__ == '__main__':
+    yt_dlp._IN_CLI = True
     yt_dlp.main()

yt_dlp/cache.py

@@ -6,7 +6,8 @@ import re
 import shutil
 import traceback
 
-from .utils import expand_path, write_json_file
+from .utils import expand_path, traverse_obj, version_tuple, write_json_file
+from .version import __version__
 
 
 class Cache:
@@ -45,12 +46,20 @@ class Cache:
                 if ose.errno != errno.EEXIST:
                     raise
             self._ydl.write_debug(f'Saving {section}.{key} to cache')
-            write_json_file(data, fn)
+            write_json_file({'yt-dlp_version': __version__, 'data': data}, fn)
         except Exception:
             tb = traceback.format_exc()
             self._ydl.report_warning(f'Writing cache to {fn!r} failed: {tb}')
 
-    def load(self, section, key, dtype='json', default=None):
+    def _validate(self, data, min_ver):
+        version = traverse_obj(data, 'yt-dlp_version')
+        if not version:  # Backward compatibility
+            data, version = {'data': data}, '2022.08.19'
+        if not min_ver or version_tuple(version) >= version_tuple(min_ver):
+            return data['data']
+        self._ydl.write_debug(f'Discarding old cache from version {version} (needs {min_ver})')
+
+    def load(self, section, key, dtype='json', default=None, *, min_ver=None):
         assert dtype in ('json',)
 
         if not self.enabled:
@@ -61,8 +70,8 @@ class Cache:
             try:
                 with open(cache_fn, encoding='utf-8') as cachef:
                     self._ydl.write_debug(f'Loading {section}.{key} from cache')
-                    return json.load(cachef)
-            except ValueError:
+                    return self._validate(json.load(cachef), min_ver)
+            except (ValueError, KeyError):
                 try:
                     file_size = os.path.getsize(cache_fn)
                 except OSError as oe:

yt_dlp/compat/__init__.py

@@ -3,13 +3,12 @@ import sys
 import warnings
 import xml.etree.ElementTree as etree
 
-from . import re
 from ._deprecated import *  # noqa: F401, F403
 from .compat_utils import passthrough_module
 
 # XXX: Implement this the same way as other DeprecationWarnings without circular import
 passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
-    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=2))
+    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=3))
 
 
 # HTMLParseError has been deprecated in Python 3.3 and removed in
@@ -33,6 +32,7 @@ compat_os_name = os._name if os.name == 'java' else os.name
 
 if compat_os_name == 'nt':
     def compat_shlex_quote(s):
+        import re
         return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
 else:
     from shlex import quote as compat_shlex_quote  # noqa: F401

yt_dlp/compat/_legacy.py

@@ -22,10 +22,14 @@ import urllib.request
 import xml.etree.ElementTree as etree
 from subprocess import DEVNULL
 
-from .compat_utils import passthrough_module  # isort: split
-from .asyncio import run as compat_asyncio_run  # noqa: F401
-from .re import Pattern as compat_Pattern  # noqa: F401
-from .re import match as compat_Match  # noqa: F401
+# isort: split
+import asyncio  # noqa: F401
+import re  # noqa: F401
+from asyncio import run as compat_asyncio_run  # noqa: F401
+from re import Pattern as compat_Pattern  # noqa: F401
+from re import match as compat_Match  # noqa: F401
+
+from .compat_utils import passthrough_module
 from ..dependencies import Cryptodome_AES as compat_pycrypto_AES  # noqa: F401
 from ..dependencies import brotli as compat_brotli  # noqa: F401
 from ..dependencies import websockets as compat_websockets  # noqa: F401

yt_dlp/compat/asyncio.py

@@ -1,23 +0,0 @@
-# flake8: noqa: F405
-from asyncio import *  # noqa: F403
-
-from .compat_utils import passthrough_module
-
-passthrough_module(__name__, 'asyncio')
-del passthrough_module
-
-try:
-    run  # >= 3.7
-except NameError:
-    def run(coro):
-        try:
-            loop = get_event_loop()
-        except RuntimeError:
-            loop = new_event_loop()
-            set_event_loop(loop)
-        loop.run_until_complete(coro)
-
-try:
-    all_tasks  # >= 3.7
-except NameError:
-    all_tasks = Task.all_tasks

yt_dlp/compat/imghdr.py

@@ -2,13 +2,15 @@ tests = {
     'webp': lambda h: h[0:4] == b'RIFF' and h[8:] == b'WEBP',
     'png': lambda h: h[:8] == b'\211PNG\r\n\032\n',
     'jpeg': lambda h: h[6:10] in (b'JFIF', b'Exif'),
+    'gif': lambda h: h[:6] in (b'GIF87a', b'GIF89a'),
 }
 
 
-def what(path):
-    """Detect format of image (Currently supports jpeg, png, webp only)
+def what(file=None, h=None):
+    """Detect format of image (Currently supports jpeg, png, webp, gif only)
     Ref: https://github.com/python/cpython/blob/3.10/Lib/imghdr.py
     """
-    with open(path, 'rb') as f:
-        head = f.read(12)
-    return next((type_ for type_, test in tests.items() if test(head)), None)
+    if h is None:
+        with open(file, 'rb') as f:
+            h = f.read(12)
+    return next((type_ for type_, test in tests.items() if test(h)), None)

yt_dlp/compat/re.py

@@ -1,18 +0,0 @@
-# flake8: noqa: F405
-from re import *  # F403
-
-from .compat_utils import passthrough_module
-
-passthrough_module(__name__, 're')
-del passthrough_module
-
-try:
-    Pattern  # >= 3.7
-except NameError:
-    Pattern = type(compile(''))
-
-
-try:
-    Match  # >= 3.7
-except NameError:
-    Match = type(compile('').match(''))

yt_dlp/cookies.py

@@ -1,9 +1,9 @@
 import base64
 import contextlib
-import ctypes
 import http.cookiejar
 import json
 import os
+import re
 import shutil
 import struct
 import subprocess
@@ -25,7 +25,13 @@ from .dependencies import (
     sqlite3,
 )
 from .minicurses import MultilinePrinter, QuietMultilinePrinter
-from .utils import Popen, YoutubeDLCookieJar, error_to_str, expand_path
+from .utils import (
+    Popen,
+    YoutubeDLCookieJar,
+    error_to_str,
+    expand_path,
+    try_call,
+)
 
 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
 SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
@@ -86,8 +92,9 @@ def _create_progress_bar(logger):
 def load_cookies(cookie_file, browser_specification, ydl):
     cookie_jars = []
     if browser_specification is not None:
-        browser_name, profile, keyring = _parse_browser_specification(*browser_specification)
-        cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring))
+        browser_name, profile, keyring, container = _parse_browser_specification(*browser_specification)
+        cookie_jars.append(
+            extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring, container=container))
 
     if cookie_file is not None:
         is_filename = YoutubeDLCookieJar.is_path(cookie_file)
@@ -102,9 +109,9 @@ def load_cookies(cookie_file, browser_specification, ydl):
     return _merge_cookie_jars(cookie_jars)
 
 
-def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None):
+def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None, container=None):
     if browser_name == 'firefox':
-        return _extract_firefox_cookies(profile, logger)
+        return _extract_firefox_cookies(profile, container, logger)
     elif browser_name == 'safari':
         return _extract_safari_cookies(profile, logger)
     elif browser_name in CHROMIUM_BASED_BROWSERS:
@@ -113,7 +120,7 @@ def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(),
         raise ValueError(f'unknown browser: {browser_name}')
 
 
-def _extract_firefox_cookies(profile, logger):
+def _extract_firefox_cookies(profile, container, logger):
     logger.info('Extracting cookies from firefox')
     if not sqlite3:
         logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
@@ -132,11 +139,36 @@ def _extract_firefox_cookies(profile, logger):
         raise FileNotFoundError(f'could not find firefox cookies database in {search_root}')
     logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
 
+    container_id = None
+    if container not in (None, 'none'):
+        containers_path = os.path.join(os.path.dirname(cookie_database_path), 'containers.json')
+        if not os.path.isfile(containers_path) or not os.access(containers_path, os.R_OK):
+            raise FileNotFoundError(f'could not read containers.json in {search_root}')
+        with open(containers_path) as containers:
+            identities = json.load(containers).get('identities', [])
+        container_id = next((context.get('userContextId') for context in identities if container in (
+            context.get('name'),
+            try_call(lambda: re.fullmatch(r'userContext([^\.]+)\.label', context['l10nID']).group())
+        )), None)
+        if not isinstance(container_id, int):
+            raise ValueError(f'could not find firefox container "{container}" in containers.json')
+
     with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
         cursor = None
         try:
             cursor = _open_database_copy(cookie_database_path, tmpdir)
-            cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
+            if isinstance(container_id, int):
+                logger.debug(
+                    f'Only loading cookies from firefox container "{container}", ID {container_id}')
+                cursor.execute(
+                    'SELECT host, name, value, path, expiry, isSecure FROM moz_cookies WHERE originAttributes LIKE ? OR originAttributes LIKE ?',
+                    (f'%userContextId={container_id}', f'%userContextId={container_id}&%'))
+            elif container == 'none':
+                logger.debug('Only loading cookies not belonging to any container')
+                cursor.execute(
+                    'SELECT host, name, value, path, expiry, isSecure FROM moz_cookies WHERE NOT INSTR(originAttributes,"userContextId=")')
+            else:
+                cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
             jar = YoutubeDLCookieJar()
             with _create_progress_bar(logger) as progress_bar:
                 table = cursor.fetchall()
@@ -876,10 +908,12 @@ def _decrypt_windows_dpapi(ciphertext, logger):
     References:
         - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata
     """
-    from ctypes.wintypes import DWORD
+
+    import ctypes
+    import ctypes.wintypes
 
     class DATA_BLOB(ctypes.Structure):
-        _fields_ = [('cbData', DWORD),
+        _fields_ = [('cbData', ctypes.wintypes.DWORD),
                     ('pbData', ctypes.POINTER(ctypes.c_char))]
 
     buffer = ctypes.create_string_buffer(ciphertext)
@@ -947,11 +981,11 @@ def _is_path(value):
     return os.path.sep in value
 
 
-def _parse_browser_specification(browser_name, profile=None, keyring=None):
+def _parse_browser_specification(browser_name, profile=None, keyring=None, container=None):
     if browser_name not in SUPPORTED_BROWSERS:
         raise ValueError(f'unsupported browser: "{browser_name}"')
     if keyring not in (None, *SUPPORTED_KEYRINGS):
         raise ValueError(f'unsupported keyring: "{keyring}"')
     if profile is not None and _is_path(profile):
         profile = os.path.expanduser(profile)
-    return browser_name, profile, keyring
+    return browser_name, profile, keyring, container

yt_dlp/dependencies.py

@@ -28,7 +28,7 @@ try:
 except ImportError:
     try:
         from Crypto.Cipher import AES as Cryptodome_AES
-    except ImportError:
+    except (ImportError, SyntaxError):  # Old Crypto gives SyntaxError in newer Python
         Cryptodome_AES = None
     else:
         try:

yt_dlp/downloader/common.py

@@ -1,5 +1,6 @@
 import contextlib
 import errno
+import functools
 import os
 import random
 import re
@@ -12,14 +13,15 @@ from ..minicurses import (
     QuietMultilinePrinter,
 )
 from ..utils import (
+    IDENTITY,
+    NO_DEFAULT,
     NUMBER_RE,
     LockingUnsupportedError,
     Namespace,
+    RetryManager,
     classproperty,
     decodeArgument,
     encodeFilename,
-    error_to_compat_str,
-    float_or_none,
     format_bytes,
     join_nonempty,
     sanitize_open,
@@ -90,6 +92,7 @@ class FileDownloader:
 
         for func in (
             'deprecation_warning',
+            'deprecated_feature',
             'report_error',
             'report_file_already_downloaded',
             'report_warning',
@@ -215,27 +218,24 @@ class FileDownloader:
         return filename + '.ytdl'
 
     def wrap_file_access(action, *, fatal=False):
-        def outer(func):
-            def inner(self, *args, **kwargs):
-                file_access_retries = self.params.get('file_access_retries', 0)
-                retry = 0
-                while True:
-                    try:
-                        return func(self, *args, **kwargs)
-                    except OSError as err:
-                        retry = retry + 1
-                        if retry > file_access_retries or err.errno not in (errno.EACCES, errno.EINVAL):
-                            if not fatal:
-                                self.report_error(f'unable to {action} file: {err}')
-                                return
-                            raise
-                        self.to_screen(
-                            f'[download] Unable to {action} file due to file access error. '
-                            f'Retrying (attempt {retry} of {self.format_retries(file_access_retries)}) ...')
-                        if not self.sleep_retry('file_access', retry):
-                            time.sleep(0.01)
-            return inner
-        return outer
+        def error_callback(err, count, retries, *, fd):
+            return RetryManager.report_retry(
+                err, count, retries, info=fd.__to_screen,
+                warn=lambda e: (time.sleep(0.01), fd.to_screen(f'[download] Unable to {action} file: {e}')),
+                error=None if fatal else lambda e: fd.report_error(f'Unable to {action} file: {e}'),
+                sleep_func=fd.params.get('retry_sleep_functions', {}).get('file_access'))
+
+        def wrapper(self, func, *args, **kwargs):
+            for retry in RetryManager(self.params.get('file_access_retries'), error_callback, fd=self):
+                try:
+                    return func(self, *args, **kwargs)
+                except OSError as err:
+                    if err.errno in (errno.EACCES, errno.EINVAL):
+                        retry.error = err
+                        continue
+                    retry.error_callback(err, 1, 0)
+
+        return functools.partial(functools.partialmethod, wrapper)
 
     @wrap_file_access('open', fatal=True)
     def sanitize_open(self, filename, open_mode):
@@ -335,7 +335,10 @@ class FileDownloader:
         if s['status'] == 'finished':
             if self.params.get('noprogress'):
                 self.to_screen('[download] Download completed')
+            speed = try_call(lambda: s['total_bytes'] / s['elapsed'])
             s.update({
+                'speed': speed,
+                '_speed_str': self.format_speed(speed).strip(),
                 '_total_bytes_str': format_bytes(s.get('total_bytes')),
                 '_elapsed_str': self.format_seconds(s.get('elapsed')),
                 '_percent_str': self.format_percent(100),
@@ -344,6 +347,7 @@ class FileDownloader:
                 '100%%',
                 with_fields(('total_bytes', 'of %(_total_bytes_str)s')),
                 with_fields(('elapsed', 'in %(_elapsed_str)s')),
+                with_fields(('speed', 'at %(_speed_str)s')),
                 delim=' '))
 
         if s['status'] != 'downloading':
@@ -378,25 +382,20 @@ class FileDownloader:
         """Report attempt to resume at given byte."""
         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 
-    def report_retry(self, err, count, retries):
-        """Report retry in case of HTTP error 5xx"""
-        self.__to_screen(
-            '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
-            % (error_to_compat_str(err), count, self.format_retries(retries)))
-        self.sleep_retry('http', count)
+    def report_retry(self, err, count, retries, frag_index=NO_DEFAULT, fatal=True):
+        """Report retry"""
+        is_frag = False if frag_index is NO_DEFAULT else 'fragment'
+        RetryManager.report_retry(
+            err, count, retries, info=self.__to_screen,
+            warn=lambda msg: self.__to_screen(f'[download] Got error: {msg}'),
+            error=IDENTITY if not fatal else lambda e: self.report_error(f'\r[download] Got error: {e}'),
+            sleep_func=self.params.get('retry_sleep_functions', {}).get(is_frag or 'http'),
+            suffix=f'fragment{"s" if frag_index is None else f" {frag_index}"}' if is_frag else None)
 
     def report_unable_to_resume(self):
         """Report it was impossible to resume download."""
         self.to_screen('[download] Unable to resume')
 
-    def sleep_retry(self, retry_type, count):
-        sleep_func = self.params.get('retry_sleep_functions', {}).get(retry_type)
-        delay = float_or_none(sleep_func(n=count - 1)) if sleep_func else None
-        if delay:
-            self.__to_screen(f'Sleeping {delay:.2f} seconds ...')
-            time.sleep(delay)
-        return sleep_func is not None
-
     @staticmethod
     def supports_manifest(manifest):
         """ Whether the downloader can download the fragments from the manifest.

yt_dlp/downloader/external.py

@@ -10,6 +10,7 @@ from ..compat import functools
 from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor
 from ..utils import (
     Popen,
+    RetryManager,
     _configuration_args,
     check_executable,
     classproperty,
@@ -134,29 +135,22 @@ class ExternalFD(FragmentFD):
                 self.to_stderr(stderr)
             return returncode
 
-        fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
-        count = 0
-        while count <= fragment_retries:
+        retry_manager = RetryManager(self.params.get('fragment_retries'), self.report_retry,
+                                     frag_index=None, fatal=not skip_unavailable_fragments)
+        for retry in retry_manager:
             _, stderr, returncode = Popen.run(cmd, text=True, stderr=subprocess.PIPE)
             if not returncode:
                 break
-
             # TODO: Decide whether to retry based on error code
             # https://aria2.github.io/manual/en/html/aria2c.html#exit-status
             if stderr:
                 self.to_stderr(stderr)
-            count += 1
-            if count <= fragment_retries:
-                self.to_screen(
-                    '[%s] Got error. Retrying fragments (attempt %d of %s)...'
-                    % (self.get_basename(), count, self.format_retries(fragment_retries)))
-                self.sleep_retry('fragment', count)
-        if count > fragment_retries:
-            if not skip_unavailable_fragments:
-                self.report_error('Giving up after %s fragment retries' % fragment_retries)
-                return -1
+            retry.error = Exception()
+            continue
+        if not skip_unavailable_fragments and retry_manager.error:
+            return -1
 
         decrypt_fragment = self.decrypter(info_dict)
         dest, _ = self.sanitize_open(tmpfilename, 'wb')
@@ -521,16 +515,14 @@ _BY_NAME = {
     if name.endswith('FD') and name not in ('ExternalFD', 'FragmentFD')
 }
 
-_BY_EXE = {klass.EXE_NAME: klass for klass in _BY_NAME.values()}
-
 
 def list_external_downloaders():
     return sorted(_BY_NAME.keys())
 
 
 def get_external_downloader(external_downloader):
-    """ Given the name of the executable, see whether we support the given
-        downloader . """
-    # Drop .exe extension on Windows
+    """ Given the name of the executable, see whether we support the given downloader """
     bn = os.path.splitext(os.path.basename(external_downloader))[0]
-    return _BY_NAME.get(bn, _BY_EXE.get(bn))
+    return _BY_NAME.get(bn) or next((
+        klass for klass in _BY_NAME.values() if klass.EXE_NAME in bn
+    ), None)

yt_dlp/downloader/f4m.py

@@ -184,7 +184,7 @@ def build_fragments_list(boot_info):
     first_frag_number = fragment_run_entry_table[0]['first']
     fragments_counter = itertools.count(first_frag_number)
     for segment, fragments_count in segment_run_table['segment_run']:
-        # In some live HDS streams (for example Rai), `fragments_count` is
+        # In some live HDS streams (e.g. Rai), `fragments_count` is
         # abnormal and causing out-of-memory errors. It's OK to change the
         # number of fragments for live streams as they are updated periodically
         if fragments_count == 4294967295 and boot_info['live']:

yt_dlp/downloader/fragment.py

@@ -14,8 +14,8 @@ from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
 from ..compat import compat_os_name
 from ..utils import (
     DownloadError,
+    RetryManager,
     encodeFilename,
-    error_to_compat_str,
     sanitized_Request,
     traverse_obj,
 )
@@ -65,10 +65,9 @@ class FragmentFD(FileDownloader):
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):
-        self.to_screen(
-            '\r[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s) ...'
-            % (error_to_compat_str(err), frag_index, count, self.format_retries(retries)))
-        self.sleep_retry('fragment', count)
+        self.deprecation_warning('yt_dlp.downloader.FragmentFD.report_retry_fragment is deprecated. '
+                                 'Use yt_dlp.downloader.FileDownloader.report_retry instead')
+        return self.report_retry(err, count, retries, frag_index)
 
     def report_skip_fragment(self, frag_index, err=None):
         err = f' {err};' if err else ''
@@ -347,6 +346,8 @@ class FragmentFD(FileDownloader):
             return _key_cache[url]
 
         def decrypt_fragment(fragment, frag_content):
+            if frag_content is None:
+                return
             decrypt_info = fragment.get('decrypt_info')
             if not decrypt_info or decrypt_info['METHOD'] != 'AES-128':
                 return frag_content
@@ -432,7 +433,6 @@ class FragmentFD(FileDownloader):
         if not interrupt_trigger:
             interrupt_trigger = (True, )
 
-        fragment_retries = self.params.get('fragment_retries', 0)
         is_fatal = (
             ((lambda _: False) if info_dict.get('is_live') else (lambda idx: idx == 0))
             if self.params.get('skip_unavailable_fragments', True) else (lambda _: True))
@@ -452,32 +452,25 @@ class FragmentFD(FileDownloader):
                 headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
 
             # Never skip the first fragment
-            fatal, count = is_fatal(fragment.get('index') or (frag_index - 1)), 0
-            while count <= fragment_retries:
+            fatal = is_fatal(fragment.get('index') or (frag_index - 1))
+
+            def error_callback(err, count, retries):
+                if fatal and count > retries:
+                    ctx['dest_stream'].close()
+                self.report_retry(err, count, retries, frag_index, fatal)
+                ctx['last_error'] = err
+
+            for retry in RetryManager(self.params.get('fragment_retries'), error_callback):
                 try:
                     ctx['fragment_count'] = fragment.get('fragment_count')
-                    if self._download_fragment(ctx, fragment['url'], info_dict, headers):
-                        break
-                    return
+                    if not self._download_fragment(ctx, fragment['url'], info_dict, headers):
+                        return
                 except (urllib.error.HTTPError, http.client.IncompleteRead) as err:
-                    # Unavailable (possibly temporary) fragments may be served.
-                    # First we try to retry then either skip or abort.
-                    # See https://github.com/ytdl-org/youtube-dl/issues/10165,
-                    # https://github.com/ytdl-org/youtube-dl/issues/10448).
-                    count += 1
-                    ctx['last_error'] = err
-                    if count <= fragment_retries:
-                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
-                except DownloadError:
-                    # Don't retry fragment if error occurred during HTTP downloading
-                    # itself since it has own retry settings
-                    if not fatal:
-                        break
-                    raise
-
-            if count > fragment_retries and fatal:
-                ctx['dest_stream'].close()
-                self.report_error('Giving up after %s fragment retries' % fragment_retries)
+                    retry.error = err
+                    continue
+                except DownloadError:  # has own retry settings
+                    if fatal:
+                        raise
 
         def append_fragment(frag_content, frag_index, ctx):
             if frag_content:

yt_dlp/downloader/http.py

@@ -9,6 +9,7 @@ import urllib.error
 from .common import FileDownloader
 from ..utils import (
     ContentTooShortError,
+    RetryManager,
     ThrottledDownload,
     XAttrMetadataError,
     XAttrUnavailableError,
@@ -72,9 +73,6 @@ class HttpFD(FileDownloader):
 
         ctx.is_resume = ctx.resume_len > 0
 
-        count = 0
-        retries = self.params.get('retries', 0)
-
         class SucceedDownload(Exception):
             pass
 
@@ -349,9 +347,7 @@ class HttpFD(FileDownloader):
 
             if data_len is not None and byte_counter != data_len:
                 err = ContentTooShortError(byte_counter, int(data_len))
-                if count <= retries:
-                    retry(err)
-                raise err
+                retry(err)
 
             self.try_rename(ctx.tmpfilename, ctx.filename)
 
@@ -370,24 +366,20 @@ class HttpFD(FileDownloader):
 
             return True
 
-        while count <= retries:
+        for retry in RetryManager(self.params.get('retries'), self.report_retry):
             try:
                 establish_connection()
                 return download()
-            except RetryDownload as e:
-                count += 1
-                if count <= retries:
-                    self.report_retry(e.source_error, count, retries)
-                else:
-                    self.to_screen(f'[download] Got server HTTP error: {e.source_error}')
+            except RetryDownload as err:
+                retry.error = err.source_error
                 continue
             except NextFragment:
+                retry.error = None
+                retry.attempt -= 1
                 continue
             except SucceedDownload:
                 return True
             except:  # noqa: E722
                 close_stream()
                 raise
-
-        self.report_error('giving up after %s retries' % retries)
         return False

yt_dlp/downloader/ism.py

@@ -5,6 +5,7 @@ import time
 import urllib.error
 
 from .fragment import FragmentFD
+from ..utils import RetryManager
 
 u8 = struct.Struct('>B')
 u88 = struct.Struct('>Bx')
@@ -245,7 +246,6 @@ class IsmFD(FragmentFD):
             'ism_track_written': False,
         })
 
-        fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
         frag_index = 0
@@ -253,8 +253,10 @@ class IsmFD(FragmentFD):
             frag_index += 1
             if frag_index <= ctx['fragment_index']:
                 continue
-            count = 0
-            while count <= fragment_retries:
+
+            retry_manager = RetryManager(self.params.get('fragment_retries'), self.report_retry,
+                                         frag_index=frag_index, fatal=not skip_unavailable_fragments)
+            for retry in retry_manager:
                 try:
                     success = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
@@ -267,18 +269,14 @@ class IsmFD(FragmentFD):
                         write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
                         extra_state['ism_track_written'] = True
                     self._append_fragment(ctx, frag_content)
-                    break
                 except urllib.error.HTTPError as err:
-                    count += 1
-                    if count <= fragment_retries:
-                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
-            if count > fragment_retries:
-                if skip_unavailable_fragments:
-                    self.report_skip_fragment(frag_index)
+                    retry.error = err
                     continue
-                self.report_error('giving up after %s fragment retries' % fragment_retries)
-                return False
+
+            if retry_manager.error:
+                if not skip_unavailable_fragments:
+                    return False
+                self.report_skip_fragment(frag_index)
 
         self._finish_frag_download(ctx, info_dict)
-
         return True

yt_dlp/downloader/mhtml.py

@@ -4,6 +4,7 @@ import re
 import uuid
 
 from .fragment import FragmentFD
+from ..compat import imghdr
 from ..utils import escapeHTML, formatSeconds, srt_subtitles_timecode, urljoin
 from ..version import __version__ as YT_DLP_VERSION
 
@@ -166,21 +167,13 @@ body > figure > img {
                 continue
             frag_content = self._read_fragment(ctx)
 
-            mime_type = b'image/jpeg'
-            if frag_content.startswith(b'\x89PNG\r\n\x1a\n'):
-                mime_type = b'image/png'
-            if frag_content.startswith((b'GIF87a', b'GIF89a')):
-                mime_type = b'image/gif'
-            if frag_content.startswith(b'RIFF') and frag_content[8:12] == b'WEBP':
-                mime_type = b'image/webp'
-
             frag_header = io.BytesIO()
             frag_header.write(
                 b'--%b\r\n' % frag_boundary.encode('us-ascii'))
             frag_header.write(
                 b'Content-ID: <%b>\r\n' % self._gen_cid(i, fragment, frag_boundary).encode('us-ascii'))
             frag_header.write(
-                b'Content-type: %b\r\n' % mime_type)
+                b'Content-type: %b\r\n' % f'image/{imghdr.what(h=frag_content) or "jpeg"}'.encode())
             frag_header.write(
                 b'Content-length: %u\r\n' % len(frag_content))
             frag_header.write(

yt_dlp/downloader/websocket.py

@@ -1,3 +1,4 @@
+import asyncio
 import contextlib
 import os
 import signal
@@ -5,7 +6,6 @@ import threading
 
 from .common import FileDownloader
 from .external import FFmpegFD
-from ..compat import asyncio
 from ..dependencies import websockets
 
 

yt_dlp/downloader/youtube_live_chat.py

@@ -3,7 +3,13 @@ import time
 import urllib.error
 
 from .fragment import FragmentFD
-from ..utils import RegexNotFoundError, dict_get, int_or_none, try_get
+from ..utils import (
+    RegexNotFoundError,
+    RetryManager,
+    dict_get,
+    int_or_none,
+    try_get,
+)
 
 
 class YoutubeLiveChatFD(FragmentFD):
@@ -16,7 +22,6 @@ class YoutubeLiveChatFD(FragmentFD):
             self.report_warning('Live chat download runs until the livestream ends. '
                                 'If you wish to download the video simultaneously, run a separate yt-dlp instance')
 
-        fragment_retries = self.params.get('fragment_retries', 0)
         test = self.params.get('test', False)
 
         ctx = {
@@ -104,8 +109,7 @@ class YoutubeLiveChatFD(FragmentFD):
             return continuation_id, live_offset, click_tracking_params
 
         def download_and_parse_fragment(url, frag_index, request_data=None, headers=None):
-            count = 0
-            while count <= fragment_retries:
+            for retry in RetryManager(self.params.get('fragment_retries'), self.report_retry, frag_index=frag_index):
                 try:
                     success = dl_fragment(url, request_data, headers)
                     if not success:
@@ -120,21 +124,15 @@ class YoutubeLiveChatFD(FragmentFD):
                     live_chat_continuation = try_get(
                         data,
                         lambda x: x['continuationContents']['liveChatContinuation'], dict) or {}
-                    if info_dict['protocol'] == 'youtube_live_chat_replay':
-                        if frag_index == 1:
-                            continuation_id, offset, click_tracking_params = try_refresh_replay_beginning(live_chat_continuation)
-                        else:
-                            continuation_id, offset, click_tracking_params = parse_actions_replay(live_chat_continuation)
-                    elif info_dict['protocol'] == 'youtube_live_chat':
-                        continuation_id, offset, click_tracking_params = parse_actions_live(live_chat_continuation)
-                    return True, continuation_id, offset, click_tracking_params
+
+                    func = (info_dict['protocol'] == 'youtube_live_chat' and parse_actions_live
+                            or frag_index == 1 and try_refresh_replay_beginning
+                            or parse_actions_replay)
+                    return (True, *func(live_chat_continuation))
                 except urllib.error.HTTPError as err:
-                    count += 1
-                    if count <= fragment_retries:
-                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
-            if count > fragment_retries:
-                self.report_error('giving up after %s fragment retries' % fragment_retries)
-                return False, None, None, None
+                    retry.error = err
+                    continue
+            return False, None, None, None
 
         self._prepare_and_start_frag_download(ctx, info_dict)
 

yt_dlp/extractor/_extractors.py

@@ -1,5 +1,28 @@
 # flake8: noqa: F401
 
+from .youtube import (  # Youtube is moved to the top to improve performance
+    YoutubeIE,
+    YoutubeClipIE,
+    YoutubeFavouritesIE,
+    YoutubeNotificationsIE,
+    YoutubeHistoryIE,
+    YoutubeTabIE,
+    YoutubeLivestreamEmbedIE,
+    YoutubePlaylistIE,
+    YoutubeRecommendedIE,
+    YoutubeSearchDateIE,
+    YoutubeSearchIE,
+    YoutubeSearchURLIE,
+    YoutubeMusicSearchURLIE,
+    YoutubeSubscriptionsIE,
+    YoutubeStoriesIE,
+    YoutubeTruncatedIDIE,
+    YoutubeTruncatedURLIE,
+    YoutubeYtBeIE,
+    YoutubeYtUserIE,
+    YoutubeWatchLaterIE,
+)
+
 from .abc import (
     ABCIE,
     ABCIViewIE,
@@ -60,6 +83,7 @@ from .americastestkitchen import (
     AmericasTestKitchenIE,
     AmericasTestKitchenSeasonIE,
 )
+from .angel import AngelIE
 from .animeondemand import AnimeOnDemandIE
 from .anvato import AnvatoIE
 from .aol import AolIE
@@ -218,6 +242,7 @@ from .camdemy import (
     CamdemyFolderIE
 )
 from .cammodels import CamModelsIE
+from .camtasia import CamtasiaEmbedIE
 from .camwithher import CamWithHerIE
 from .canalalpha import CanalAlphaIE
 from .canalplus import CanalplusIE
@@ -408,6 +433,7 @@ from .dplay import (
     DiscoveryLifeIE,
     AnimalPlanetIE,
     TLCIE,
+    MotorTrendIE,
     DiscoveryPlusIndiaIE,
     DiscoveryNetworksDeIE,
     DiscoveryPlusItalyIE,
@@ -443,7 +469,7 @@ from .dw import (
     DWIE,
     DWArticleIE,
 )
-from .eagleplatform import EaglePlatformIE
+from .eagleplatform import EaglePlatformIE, ClipYouEmbedIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
 from .egghead import (
@@ -467,6 +493,7 @@ from .epicon import (
     EpiconIE,
     EpiconSeriesIE,
 )
+from .epoch import EpochIE
 from .eporner import EpornerIE
 from .eroprofile import (
     EroProfileIE,
@@ -488,6 +515,7 @@ from .espn import (
 from .esri import EsriVideoIE
 from .europa import EuropaIE
 from .europeantour import EuropeanTourIE
+from .eurosport import EurosportIE
 from .euscreen import EUScreenIE
 from .expotv import ExpoTVIE
 from .expressen import ExpressenIE
@@ -497,6 +525,7 @@ from .facebook import (
     FacebookIE,
     FacebookPluginsVideoIE,
     FacebookRedirectURLIE,
+    FacebookReelIE,
 )
 from .fancode import (
     FancodeVodIE,
@@ -628,6 +657,7 @@ from .gronkh import (
     GronkhVodsIE
 )
 from .groupon import GrouponIE
+from .harpodeon import HarpodeonIE
 from .hbo import HBOIE
 from .hearthisat import HearThisAtIE
 from .heise import HeiseIE
@@ -640,6 +670,7 @@ from .hidive import HiDiveIE
 from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hitrecord import HitRecordIE
+from .holodex import HolodexIE
 from .hotnewhiphop import HotNewHipHopIE
 from .hotstar import (
     HotStarIE,
@@ -658,6 +689,7 @@ from .hse import (
     HSEShowIE,
     HSEProductIE,
 )
+from .genericembeds import HTML5MediaEmbedIE
 from .huajiao import HuajiaoIE
 from .huya import HuyaLiveIE
 from .huffpost import HuffPostIE
@@ -713,6 +745,10 @@ from .iqiyi import (
     IqIE,
     IqAlbumIE
 )
+from .islamchannel import (
+    IslamChannelIE,
+    IslamChannelSeriesIE,
+)
 from .itprotv import (
     ITProTVIE,
     ITProTVCourseIE
@@ -760,6 +796,7 @@ from .kicker import KickerIE
 from .kickstarter import KickStarterIE
 from .kinja import KinjaEmbedIE
 from .kinopoisk import KinoPoiskIE
+from .kompas import KompasVideoIE
 from .konserthusetplay import KonserthusetPlayIE
 from .koo import KooIE
 from .kth import KTHIE
@@ -949,6 +986,7 @@ from .mixcloud import (
 from .mlb import (
     MLBIE,
     MLBVideoIE,
+    MLBTVIE,
 )
 from .mlssoccer import MLSSoccerIE
 from .mnet import MnetIE
@@ -967,6 +1005,7 @@ from .motherless import (
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
 from .moviepilot import MoviepilotIE
+from .moview import MoviewPlayIE
 from .moviezine import MoviezineIE
 from .movingimage import MovingImageIE
 from .msn import MSNIE
@@ -1069,6 +1108,7 @@ from .newgrounds import (
     NewgroundsPlaylistIE,
     NewgroundsUserIE,
 )
+from .newspicks import NewsPicksIE
 from .newstube import NewstubeIE
 from .newsy import NewsyIE
 from .nextmedia import (
@@ -1205,19 +1245,8 @@ from .openrec import (
 from .ora import OraTVIE
 from .orf import (
     ORFTVthekIE,
-    ORFFM4IE,
     ORFFM4StoryIE,
-    ORFOE1IE,
-    ORFOE3IE,
-    ORFNOEIE,
-    ORFWIEIE,
-    ORFBGLIE,
-    ORFOOEIE,
-    ORFSTMIE,
-    ORFKTNIE,
-    ORFSBGIE,
-    ORFTIRIE,
-    ORFVBGIE,
+    ORFRadioIE,
     ORFIPTVIE,
 )
 from .outsidetv import OutsideTVIE
@@ -1240,11 +1269,11 @@ from .paramountplus import (
     ParamountPlusIE,
     ParamountPlusSeriesIE,
 )
-from .parliamentliveuk import ParliamentLiveUKIE
+from .parler import ParlerIE
 from .parlview import ParlviewIE
 from .patreon import (
     PatreonIE,
-    PatreonUserIE
+    PatreonCampaignIE
 )
 from .pbs import PBSIE
 from .pearvideo import PearVideoIE
@@ -1391,6 +1420,8 @@ from .rai import (
     RaiPlaySoundIE,
     RaiPlaySoundLiveIE,
     RaiPlaySoundPlaylistIE,
+    RaiNewsIE,
+    RaiSudtirolIE,
     RaiIE,
 )
 from .raywenderlich import (
@@ -1409,6 +1440,7 @@ from .rcti import (
     RCTIPlusTVIE,
 )
 from .rds import RDSIE
+from .redbee import ParliamentLiveUKIE, RTBFIE
 from .redbulltv import (
     RedBullTVIE,
     RedBullEmbedIE,
@@ -1442,7 +1474,6 @@ from .rokfin import (
 from .roosterteeth import RoosterTeethIE, RoosterTeethSeriesIE
 from .rottentomatoes import RottenTomatoesIE
 from .rozhlas import RozhlasIE
-from .rtbf import RTBFIE
 from .rte import RteIE, RteRadioIE
 from .rtlnl import (
     RtlNlIE,
@@ -1548,6 +1579,7 @@ from .shared import (
     SharedIE,
     VivoIE,
 )
+from .sharevideos import ShareVideosEmbedIE
 from .shemaroome import ShemarooMeIE
 from .showroomlive import ShowRoomLiveIE
 from .simplecast import (
@@ -1725,6 +1757,13 @@ from .telequebec import (
 )
 from .teletask import TeleTaskIE
 from .telewebion import TelewebionIE
+from .tempo import TempoIE
+from .tencent import (
+    VQQSeriesIE,
+    VQQVideoIE,
+    WeTvEpisodeIE,
+    WeTvSeriesIE,
+)
 from .tennistv import TennisTVIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
@@ -1784,6 +1823,10 @@ from .toongoggles import ToonGogglesIE
 from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
+from .triller import (
+    TrillerIE,
+    TrillerUserIE,
+)
 from .trilulilu import TriluliluIE
 from .trovo import (
     TrovoIE,
@@ -1793,6 +1836,7 @@ from .trovo import (
 )
 from .trueid import TrueIDIE
 from .trunews import TruNewsIE
+from .truth import TruthIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
 from .tubetugraz import TubeTuGrazIE, TubeTuGrazSeriesIE
@@ -2088,7 +2132,6 @@ from .weibo import (
     WeiboMobileIE
 )
 from .weiqitv import WeiqiTVIE
-from .wetv import WeTvEpisodeIE, WeTvSeriesIE
 from .wikimedia import WikimediaIE
 from .willow import WillowIE
 from .wimtv import WimTVIE
@@ -2171,42 +2214,44 @@ from .younow import (
 from .youporn import YouPornIE
 from .yourporn import YourPornIE
 from .yourupload import YourUploadIE
-from .youtube import (
-    YoutubeIE,
-    YoutubeClipIE,
-    YoutubeFavouritesIE,
-    YoutubeNotificationsIE,
-    YoutubeHistoryIE,
-    YoutubeTabIE,
-    YoutubeLivestreamEmbedIE,
-    YoutubePlaylistIE,
-    YoutubeRecommendedIE,
-    YoutubeSearchDateIE,
-    YoutubeSearchIE,
-    YoutubeSearchURLIE,
-    YoutubeMusicSearchURLIE,
-    YoutubeSubscriptionsIE,
-    YoutubeStoriesIE,
-    YoutubeTruncatedIDIE,
-    YoutubeTruncatedURLIE,
-    YoutubeYtBeIE,
-    YoutubeYtUserIE,
-    YoutubeWatchLaterIE,
-)
 from .zapiks import ZapiksIE
 from .zattoo import (
     BBVTVIE,
+    BBVTVLiveIE,
+    BBVTVRecordingsIE,
     EinsUndEinsTVIE,
+    EinsUndEinsTVLiveIE,
+    EinsUndEinsTVRecordingsIE,
     EWETVIE,
+    EWETVLiveIE,
+    EWETVRecordingsIE,
     GlattvisionTVIE,
+    GlattvisionTVLiveIE,
+    GlattvisionTVRecordingsIE,
     MNetTVIE,
-    NetPlusIE,
+    MNetTVLiveIE,
+    MNetTVRecordingsIE,
+    NetPlusTVIE,
+    NetPlusTVLiveIE,
+    NetPlusTVRecordingsIE,
     OsnatelTVIE,
+    OsnatelTVLiveIE,
+    OsnatelTVRecordingsIE,
     QuantumTVIE,
+    QuantumTVLiveIE,
+    QuantumTVRecordingsIE,
     SaltTVIE,
+    SaltTVLiveIE,
+    SaltTVRecordingsIE,
     SAKTVIE,
+    SAKTVLiveIE,
+    SAKTVRecordingsIE,
     VTXTVIE,
+    VTXTVLiveIE,
+    VTXTVRecordingsIE,
     WalyTVIE,
+    WalyTVLiveIE,
+    WalyTVRecordingsIE,
     ZattooIE,
     ZattooLiveIE,
     ZattooMoviesIE,

yt_dlp/extractor/abematv.py

@@ -1,5 +1,6 @@
 import base64
 import binascii
+import functools
 import hashlib
 import hmac
 import io
@@ -20,11 +21,11 @@ from ..utils import (
     decode_base_n,
     int_or_none,
     intlist_to_bytes,
+    OnDemandPagedList,
     request_to_url,
     time_seconds,
     traverse_obj,
     update_url_query,
-    urljoin,
 )
 
 # NOTE: network handler related code is temporary thing until network stack overhaul PRs are merged (#2861/#2862)
@@ -145,17 +146,106 @@ class AbemaLicenseHandler(urllib.request.BaseHandler):
 
 
 class AbemaTVBaseIE(InfoExtractor):
+    _USERTOKEN = None
+    _DEVICE_ID = None
+    _MEDIATOKEN = None
+
+    _SECRETKEY = b'v+Gjs=25Aw5erR!J8ZuvRrCx*rGswhB&qdHd_SYerEWdU&a?3DzN9BRbp5KwY4hEmcj5#fykMjJ=AuWz5GSMY-d@H7DMEh3M@9n2G552Us$$k9cD=3TxwWe86!x#Zyhe'
+
+    @classmethod
+    def _generate_aks(cls, deviceid):
+        deviceid = deviceid.encode('utf-8')
+        # add 1 hour and then drop minute and secs
+        ts_1hour = int((time_seconds(hours=9) // 3600 + 1) * 3600)
+        time_struct = time.gmtime(ts_1hour)
+        ts_1hour_str = str(ts_1hour).encode('utf-8')
+
+        tmp = None
+
+        def mix_once(nonce):
+            nonlocal tmp
+            h = hmac.new(cls._SECRETKEY, digestmod=hashlib.sha256)
+            h.update(nonce)
+            tmp = h.digest()
+
+        def mix_tmp(count):
+            nonlocal tmp
+            for i in range(count):
+                mix_once(tmp)
+
+        def mix_twist(nonce):
+            nonlocal tmp
+            mix_once(base64.urlsafe_b64encode(tmp).rstrip(b'=') + nonce)
+
+        mix_once(cls._SECRETKEY)
+        mix_tmp(time_struct.tm_mon)
+        mix_twist(deviceid)
+        mix_tmp(time_struct.tm_mday % 5)
+        mix_twist(ts_1hour_str)
+        mix_tmp(time_struct.tm_hour % 5)
+
+        return base64.urlsafe_b64encode(tmp).rstrip(b'=').decode('utf-8')
+
+    def _get_device_token(self):
+        if self._USERTOKEN:
+            return self._USERTOKEN
+
+        AbemaTVBaseIE._DEVICE_ID = str(uuid.uuid4())
+        aks = self._generate_aks(self._DEVICE_ID)
+        user_data = self._download_json(
+            'https://api.abema.io/v1/users', None, note='Authorizing',
+            data=json.dumps({
+                'deviceId': self._DEVICE_ID,
+                'applicationKeySecret': aks,
+            }).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json',
+            })
+        AbemaTVBaseIE._USERTOKEN = user_data['token']
+
+        # don't allow adding it 2 times or more, though it's guarded
+        remove_opener(self._downloader, AbemaLicenseHandler)
+        add_opener(self._downloader, AbemaLicenseHandler(self))
+
+        return self._USERTOKEN
+
+    def _get_media_token(self, invalidate=False, to_show=True):
+        if not invalidate and self._MEDIATOKEN:
+            return self._MEDIATOKEN
+
+        AbemaTVBaseIE._MEDIATOKEN = self._download_json(
+            'https://api.abema.io/v1/media/token', None, note='Fetching media token' if to_show else False,
+            query={
+                'osName': 'android',
+                'osVersion': '6.0.1',
+                'osLang': 'ja_JP',
+                'osTimezone': 'Asia/Tokyo',
+                'appId': 'tv.abema',
+                'appVersion': '3.27.1'
+            }, headers={
+                'Authorization': f'bearer {self._get_device_token()}',
+            })['token']
+
+        return self._MEDIATOKEN
+
+    def _call_api(self, endpoint, video_id, query=None, note='Downloading JSON metadata'):
+        return self._download_json(
+            f'https://api.abema.io/{endpoint}', video_id, query=query or {},
+            note=note,
+            headers={
+                'Authorization': f'bearer {self._get_device_token()}',
+            })
+
     def _extract_breadcrumb_list(self, webpage, video_id):
         for jld in re.finditer(
                 r'(?is)</span></li></ul><script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
                 webpage):
             jsonld = self._parse_json(jld.group('json_ld'), video_id, fatal=False)
-            if jsonld:
-                if jsonld.get('@type') != 'BreadcrumbList':
-                    continue
-                trav = traverse_obj(jsonld, ('itemListElement', ..., 'name'))
-                if trav:
-                    return trav
+            if traverse_obj(jsonld, '@type') != 'BreadcrumbList':
+                continue
+            items = traverse_obj(jsonld, ('itemListElement', ..., 'name'))
+            if items:
+                return items
         return []
 
 
@@ -207,87 +297,7 @@ class AbemaTVIE(AbemaTVBaseIE):
         },
         'skip': 'Not supported until yt-dlp implements native live downloader OR AbemaTV can start a local HTTP server',
     }]
-    _USERTOKEN = None
-    _DEVICE_ID = None
     _TIMETABLE = None
-    _MEDIATOKEN = None
-
-    _SECRETKEY = b'v+Gjs=25Aw5erR!J8ZuvRrCx*rGswhB&qdHd_SYerEWdU&a?3DzN9BRbp5KwY4hEmcj5#fykMjJ=AuWz5GSMY-d@H7DMEh3M@9n2G552Us$$k9cD=3TxwWe86!x#Zyhe'
-
-    def _generate_aks(self, deviceid):
-        deviceid = deviceid.encode('utf-8')
-        # add 1 hour and then drop minute and secs
-        ts_1hour = int((time_seconds(hours=9) // 3600 + 1) * 3600)
-        time_struct = time.gmtime(ts_1hour)
-        ts_1hour_str = str(ts_1hour).encode('utf-8')
-
-        tmp = None
-
-        def mix_once(nonce):
-            nonlocal tmp
-            h = hmac.new(self._SECRETKEY, digestmod=hashlib.sha256)
-            h.update(nonce)
-            tmp = h.digest()
-
-        def mix_tmp(count):
-            nonlocal tmp
-            for i in range(count):
-                mix_once(tmp)
-
-        def mix_twist(nonce):
-            nonlocal tmp
-            mix_once(base64.urlsafe_b64encode(tmp).rstrip(b'=') + nonce)
-
-        mix_once(self._SECRETKEY)
-        mix_tmp(time_struct.tm_mon)
-        mix_twist(deviceid)
-        mix_tmp(time_struct.tm_mday % 5)
-        mix_twist(ts_1hour_str)
-        mix_tmp(time_struct.tm_hour % 5)
-
-        return base64.urlsafe_b64encode(tmp).rstrip(b'=').decode('utf-8')
-
-    def _get_device_token(self):
-        if self._USERTOKEN:
-            return self._USERTOKEN
-
-        self._DEVICE_ID = str(uuid.uuid4())
-        aks = self._generate_aks(self._DEVICE_ID)
-        user_data = self._download_json(
-            'https://api.abema.io/v1/users', None, note='Authorizing',
-            data=json.dumps({
-                'deviceId': self._DEVICE_ID,
-                'applicationKeySecret': aks,
-            }).encode('utf-8'),
-            headers={
-                'Content-Type': 'application/json',
-            })
-        self._USERTOKEN = user_data['token']
-
-        # don't allow adding it 2 times or more, though it's guarded
-        remove_opener(self._downloader, AbemaLicenseHandler)
-        add_opener(self._downloader, AbemaLicenseHandler(self))
-
-        return self._USERTOKEN
-
-    def _get_media_token(self, invalidate=False, to_show=True):
-        if not invalidate and self._MEDIATOKEN:
-            return self._MEDIATOKEN
-
-        self._MEDIATOKEN = self._download_json(
-            'https://api.abema.io/v1/media/token', None, note='Fetching media token' if to_show else False,
-            query={
-                'osName': 'android',
-                'osVersion': '6.0.1',
-                'osLang': 'ja_JP',
-                'osTimezone': 'Asia/Tokyo',
-                'appId': 'tv.abema',
-                'appVersion': '3.27.1'
-            }, headers={
-                'Authorization': 'bearer ' + self._get_device_token()
-            })['token']
-
-        return self._MEDIATOKEN
 
     def _perform_login(self, username, password):
         if '@' in username:  # don't strictly check if it's email address or not
@@ -301,13 +311,13 @@ class AbemaTVIE(AbemaTVBaseIE):
                 method: username,
                 'password': password
             }).encode('utf-8'), headers={
-                'Authorization': 'bearer ' + self._get_device_token(),
+                'Authorization': f'bearer {self._get_device_token()}',
                 'Origin': 'https://abema.tv',
                 'Referer': 'https://abema.tv/',
                 'Content-Type': 'application/json',
             })
 
-        self._USERTOKEN = login_response['token']
+        AbemaTVBaseIE._USERTOKEN = login_response['token']
         self._get_media_token(True)
 
     def _real_extract(self, url):
@@ -355,7 +365,7 @@ class AbemaTVIE(AbemaTVBaseIE):
         # read breadcrumb on top of page
         breadcrumb = self._extract_breadcrumb_list(webpage, video_id)
         if breadcrumb:
-            # breadcrumb list translates to: (example is 1st test for this IE)
+            # breadcrumb list translates to: (e.g. 1st test for this IE)
             # Home > Anime (genre) > Isekai Shokudo 2 (series name) > Episode 1 "Cheese cakes" "Morning again" (episode title)
             # hence this works
             info['series'] = breadcrumb[-2]
@@ -442,6 +452,7 @@ class AbemaTVIE(AbemaTVBaseIE):
 
 class AbemaTVTitleIE(AbemaTVBaseIE):
     _VALID_URL = r'https?://abema\.tv/video/title/(?P<id>[^?/]+)'
+    _PAGE_SIZE = 25
 
     _TESTS = [{
         'url': 'https://abema.tv/video/title/90-1597',
@@ -457,18 +468,39 @@ class AbemaTVTitleIE(AbemaTVBaseIE):
             'title': '真心が届く~僕とスターのオフィス・ラブ!?~',
         },
         'playlist_mincount': 16,
+    }, {
+        'url': 'https://abema.tv/video/title/25-102',
+        'info_dict': {
+            'id': '25-102',
+            'title': 'ソードアート・オンライン アリシゼーション',
+        },
+        'playlist_mincount': 24,
     }]
 
+    def _fetch_page(self, playlist_id, series_version, page):
+        programs = self._call_api(
+            f'v1/video/series/{playlist_id}/programs', playlist_id,
+            note=f'Downloading page {page + 1}',
+            query={
+                'seriesVersion': series_version,
+                'offset': str(page * self._PAGE_SIZE),
+                'order': 'seq',
+                'limit': str(self._PAGE_SIZE),
+            })
+        yield from (
+            self.url_result(f'https://abema.tv/video/episode/{x}')
+            for x in traverse_obj(programs, ('programs', ..., 'id'), default=[]))
+
+    def _entries(self, playlist_id, series_version):
+        return OnDemandPagedList(
+            functools.partial(self._fetch_page, playlist_id, series_version),
+            self._PAGE_SIZE)
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        playlist_id = self._match_id(url)
+        series_info = self._call_api(f'v1/video/series/{playlist_id}', playlist_id)
 
-        playlist_title, breadcrumb = None, self._extract_breadcrumb_list(webpage, video_id)
-        if breadcrumb:
-            playlist_title = breadcrumb[-1]
-
-        playlist = [
-            self.url_result(urljoin('https://abema.tv/', mobj.group(1)))
-            for mobj in re.finditer(r'<li\s*class=".+?EpisodeList.+?"><a\s*href="(/[^"]+?)"', webpage)]
-
-        return self.playlist_result(playlist, playlist_title=playlist_title, playlist_id=video_id)
+        return self.playlist_result(
+            self._entries(playlist_id, series_info['version']), playlist_id=playlist_id,
+            playlist_title=series_info.get('title'),
+            playlist_description=series_info.get('content'))

yt_dlp/extractor/adobetv.py

@@ -232,6 +232,7 @@ class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
 class AdobeTVVideoIE(AdobeTVBaseIE):
     IE_NAME = 'adobetv:video'
     _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
+    _EMBED_REGEX = [r'<iframe[^>]+src=[\'"](?P<url>(?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]']
 
     _TEST = {
         # From https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners

yt_dlp/extractor/aenetworks.py

@@ -28,14 +28,17 @@ class AENetworksBaseIE(ThePlatformIE):
     }
 
     def _extract_aen_smil(self, smil_url, video_id, auth=None):
-        query = {'mbr': 'true'}
+        query = {
+            'mbr': 'true',
+            'formats': 'M3U+none,MPEG-DASH+none,MPEG4,MP3',
+        }
         if auth:
             query['auth'] = auth
         TP_SMIL_QUERY = [{
             'assetTypes': 'high_video_ak',
-            'switch': 'hls_high_ak'
+            'switch': 'hls_high_ak',
         }, {
-            'assetTypes': 'high_video_s3'
+            'assetTypes': 'high_video_s3',
         }, {
             'assetTypes': 'high_video_s3',
             'switch': 'hls_high_fastly',

yt_dlp/extractor/angel.py

@@ -0,0 +1,56 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import url_or_none, merge_dicts
+
+
+class AngelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?angel\.com/watch/(?P<series>[^/?#]+)/episode/(?P<id>[\w-]+)/season-(?P<season_number>\d+)/episode-(?P<episode_number>\d+)/(?P<title>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.angel.com/watch/tuttle-twins/episode/2f3d0382-ea82-4cdc-958e-84fbadadc710/season-1/episode-1/when-laws-give-you-lemons',
+        'md5': '4734e5cfdd64a568e837246aa3eaa524',
+        'info_dict': {
+            'id': '2f3d0382-ea82-4cdc-958e-84fbadadc710',
+            'ext': 'mp4',
+            'title': 'Tuttle Twins Season 1, Episode 1: When Laws Give You Lemons',
+            'description': 'md5:73b704897c20ab59c433a9c0a8202d5e',
+            'thumbnail': r're:^https?://images.angelstudios.com/image/upload/angel-app/.*$',
+            'duration': 1359.0
+        }
+    }, {
+        'url': 'https://www.angel.com/watch/the-chosen/episode/8dfb714d-bca5-4812-8125-24fb9514cd10/season-1/episode-1/i-have-called-you-by-name',
+        'md5': 'e4774bad0a5f0ad2e90d175cafdb797d',
+        'info_dict': {
+            'id': '8dfb714d-bca5-4812-8125-24fb9514cd10',
+            'ext': 'mp4',
+            'title': 'The Chosen Season 1, Episode 1: I Have Called You By Name',
+            'description': 'md5:aadfb4827a94415de5ff6426e6dee3be',
+            'thumbnail': r're:^https?://images.angelstudios.com/image/upload/angel-app/.*$',
+            'duration': 3276.0
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        json_ld = self._search_json_ld(webpage, video_id)
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            json_ld.pop('url'), video_id, note='Downloading HD m3u8 information')
+
+        info_dict = {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'formats': formats,
+            'subtitles': subtitles
+        }
+
+        # Angel uses cloudinary in the background and supports image transformations.
+        # We remove these transformations and return the source file
+        base_thumbnail_url = url_or_none(self._og_search_thumbnail(webpage)) or json_ld.pop('thumbnails')
+        if base_thumbnail_url:
+            info_dict['thumbnail'] = re.sub(r'(/upload)/.+(/angel-app/.+)$', r'\1\2', base_thumbnail_url)
+
+        return merge_dicts(info_dict, json_ld)

yt_dlp/extractor/ant1newsgr.py

@@ -1,4 +1,3 @@
-import re
 import urllib.parse
 
 from .common import InfoExtractor
@@ -7,7 +6,6 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     scale_thumbnails_to_max_format_width,
-    unescapeHTML,
 )
 
 
@@ -91,7 +89,7 @@ class Ant1NewsGrArticleIE(Ant1NewsGrBaseIE):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         info = self._search_json_ld(webpage, video_id, expected_type='NewsArticle')
-        embed_urls = list(Ant1NewsGrEmbedIE._extract_urls(webpage))
+        embed_urls = list(Ant1NewsGrEmbedIE._extract_embed_urls(url, webpage))
         if not embed_urls:
             raise ExtractorError('no videos found for %s' % video_id, expected=True)
         return self.playlist_from_matches(
@@ -104,6 +102,7 @@ class Ant1NewsGrEmbedIE(Ant1NewsGrBaseIE):
     IE_DESC = 'ant1news.gr embedded videos'
     _BASE_PLAYER_URL_RE = r'(?:https?:)?//(?:[a-zA-Z0-9\-]+\.)?(?:antenna|ant1news)\.gr/templates/pages/player'
     _VALID_URL = rf'{_BASE_PLAYER_URL_RE}\?([^#]+&)?cid=(?P<id>[^#&]+)'
+    _EMBED_REGEX = [rf'<iframe[^>]+?src=(?P<_q1>["\'])(?P<url>{_BASE_PLAYER_URL_RE}\?(?:(?!(?P=_q1)).)+)(?P=_q1)']
     _API_PATH = '/news/templates/data/jsonPlayer'
 
     _TESTS = [{
@@ -117,16 +116,6 @@ class Ant1NewsGrEmbedIE(Ant1NewsGrBaseIE):
         },
     }]
 
-    @classmethod
-    def _extract_urls(cls, webpage):
-        _EMBED_URL_RE = rf'{cls._BASE_PLAYER_URL_RE}\?(?:(?!(?P=_q1)).)+'
-        _EMBED_RE = rf'<iframe[^>]+?src=(?P<_q1>["\'])(?P<url>{_EMBED_URL_RE})(?P=_q1)'
-        for mobj in re.finditer(_EMBED_RE, webpage):
-            url = unescapeHTML(mobj.group('url'))
-            if not cls.suitable(url):
-                continue
-            yield url
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

yt_dlp/extractor/anvato.py

@@ -340,30 +340,16 @@ class AnvatoIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
-    @staticmethod
-    def _extract_urls(ie, webpage, video_id):
-        entries = []
-        for mobj in re.finditer(AnvatoIE._ANVP_RE, webpage):
-            anvplayer_data = ie._parse_json(
-                mobj.group('anvp'), video_id, transform_source=unescapeHTML,
-                fatal=False)
-            if not anvplayer_data:
-                continue
-            video = anvplayer_data.get('video')
-            if not isinstance(video, compat_str) or not video.isdigit():
-                continue
-            access_key = anvplayer_data.get('accessKey')
-            if not access_key:
-                mcp = anvplayer_data.get('mcp')
-                if mcp:
-                    access_key = AnvatoIE._MCP_TO_ACCESS_KEY_TABLE.get(
-                        mcp.lower())
+    @classmethod
+    def _extract_from_webpage(cls, url, webpage):
+        for mobj in re.finditer(cls._ANVP_RE, webpage):
+            anvplayer_data = unescapeHTML(json.loads(mobj.group('anvp'))) or {}
+            video_id, access_key = anvplayer_data.get('video'), anvplayer_data.get('accessKey')
             if not access_key:
+                access_key = cls._MCP_TO_ACCESS_KEY_TABLE.get((anvplayer_data.get('mcp') or '').lower())
+            if not (video_id or '').isdigit() or not access_key:
                 continue
-            entries.append(ie.url_result(
-                'anvato:%s:%s' % (access_key, video), ie=AnvatoIE.ie_key(),
-                video_id=video))
-        return entries
+            yield cls.url_result(f'anvato:{access_key}:{video_id}', AnvatoIE, video_id)
 
     def _extract_anvato_videos(self, webpage, video_id):
         anvplayer_data = self._parse_json(

yt_dlp/extractor/apa.py

@@ -1,5 +1,3 @@
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
@@ -10,6 +8,7 @@ from ..utils import (
 
 class APAIE(InfoExtractor):
     _VALID_URL = r'(?P<base_url>https?://[^/]+\.apa\.at)/embed/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+    _EMBED_REGEX = [r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//[^/]+\.apa\.at/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}.*?)\1']
     _TESTS = [{
         'url': 'http://uvp.apa.at/embed/293f6d17-692a-44e3-9fd5-7b178f3a1029',
         'md5': '2b12292faeb0a7d930c778c7a5b4759b',
@@ -30,14 +29,6 @@ class APAIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return [
-            mobj.group('url')
-            for mobj in re.finditer(
-                r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//[^/]+\.apa\.at/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}.*?)\1',
-                webpage)]
-
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         video_id, base_url = mobj.group('id', 'base_url')

yt_dlp/extractor/aparat.py

@@ -10,6 +10,7 @@ from ..utils import (
 
 class AparatIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P<id>[a-zA-Z0-9]+)'
+    _EMBED_REGEX = [r'<iframe .*?src="(?P<url>http://www\.aparat\.com/video/[^"]+)"']
 
     _TESTS = [{
         'url': 'http://www.aparat.com/v/wP8On',

yt_dlp/extractor/archiveorg.py

@@ -49,6 +49,11 @@ class ArchiveOrgIE(InfoExtractor):
             'upload_date': '20100315',
             'creator': 'SRI International',
             'uploader': 'laura@archive.org',
+            'thumbnail': r're:https://archive\.org/download/.*\.jpg',
+            'release_year': 1968,
+            'display_id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.cdr',
+            'track': 'XD300-23 68HighlightsAResearchCntAugHumanIntellect',
+
         },
     }, {
         'url': 'https://archive.org/details/Cops1922',
@@ -57,33 +62,43 @@ class ArchiveOrgIE(InfoExtractor):
             'id': 'Cops1922',
             'ext': 'mp4',
             'title': 'Buster Keaton\'s "Cops" (1922)',
-            'description': 'md5:43a603fd6c5b4b90d12a96b921212b9c',
+            'description': 'md5:cd6f9910c35aedd5fc237dbc3957e2ca',
             'uploader': 'yorkmba99@hotmail.com',
             'timestamp': 1387699629,
             'upload_date': '20131222',
+            'display_id': 'Cops-v2.mp4',
+            'thumbnail': r're:https://archive\.org/download/.*\.jpg',
+            'duration': 1091.96,
         },
     }, {
         'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
         'only_matching': True,
     }, {
         'url': 'https://archive.org/details/Election_Ads',
-        'md5': '284180e857160cf866358700bab668a3',
+        'md5': 'eec5cddebd4793c6a653b69c3b11f2e6',
         'info_dict': {
             'id': 'Election_Ads/Commercial-JFK1960ElectionAdCampaignJingle.mpg',
             'title': 'Commercial-JFK1960ElectionAdCampaignJingle.mpg',
-            'ext': 'mp4',
+            'ext': 'mpg',
+            'thumbnail': r're:https://archive\.org/download/.*\.jpg',
+            'duration': 59.77,
+            'display_id': 'Commercial-JFK1960ElectionAdCampaignJingle.mpg',
         },
     }, {
         'url': 'https://archive.org/details/Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpg',
-        'md5': '7915213ef02559b5501fe630e1a53f59',
+        'md5': 'ea1eed8234e7d4165f38c8c769edef38',
         'info_dict': {
             'id': 'Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpg',
             'title': 'Commercial-Nixon1960ElectionAdToughonDefense.mpg',
-            'ext': 'mp4',
+            'ext': 'mpg',
             'timestamp': 1205588045,
             'uploader': 'mikedavisstripmaster@yahoo.com',
             'description': '1960 Presidential Campaign Election Commercials John F Kennedy, Richard M Nixon',
             'upload_date': '20080315',
+            'display_id': 'Commercial-Nixon1960ElectionAdToughonDefense.mpg',
+            'duration': 59.51,
+            'license': 'http://creativecommons.org/licenses/publicdomain/',
+            'thumbnail': r're:https://archive\.org/download/.*\.jpg',
         },
     }, {
         'url': 'https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16',
@@ -92,6 +107,12 @@ class ArchiveOrgIE(InfoExtractor):
             'id': 'gd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t01.flac',
             'title': 'Turning',
             'ext': 'flac',
+            'track': 'Turning',
+            'creator': 'Grateful Dead',
+            'display_id': 'gd1977-05-08d01t01.flac',
+            'track_number': 1,
+            'album': '1977-05-08 - Barton Hall - Cornell University',
+            'duration': 39.8,
         },
     }, {
         'url': 'https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t07.flac',
@@ -102,11 +123,20 @@ class ArchiveOrgIE(InfoExtractor):
             'ext': 'flac',
             'timestamp': 1205895624,
             'uploader': 'mvernon54@yahoo.com',
-            'description': 'md5:6a31f1996db0aa0fc9da6d6e708a1bb0',
+            'description': 'md5:6c921464414814720c6593810a5c7e3d',
             'upload_date': '20080319',
             'location': 'Barton Hall - Cornell University',
+            'duration': 438.68,
+            'track': 'Deal',
+            'creator': 'Grateful Dead',
+            'album': '1977-05-08 - Barton Hall - Cornell University',
+            'release_date': '19770508',
+            'display_id': 'gd1977-05-08d01t07.flac',
+            'release_year': 1977,
+            'track_number': 7,
         },
     }, {
+        # FIXME: give a better error message than just IndexError when all available formats are restricted
         'url': 'https://archive.org/details/lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik',
         'md5': '7cb019baa9b332e82ea7c10403acd180',
         'info_dict': {
@@ -114,6 +144,7 @@ class ArchiveOrgIE(InfoExtractor):
             'title': 'Bells Of Rostov',
             'ext': 'mp3',
         },
+        'skip': 'restricted'
     }, {
         'url': 'https://archive.org/details/lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/02.02.+Song+And+Chorus+In+The+Polovetsian+Camp+From+%22Prince+Igor%22+(Act+2%2C+Scene+1).mp3',
         'md5': '1d0aabe03edca83ca58d9ed3b493a3c3',
@@ -126,6 +157,52 @@ class ArchiveOrgIE(InfoExtractor):
             'description': 'md5:012b2d668ae753be36896f343d12a236',
             'upload_date': '20190928',
         },
+        'skip': 'restricted'
+    }, {
+        # Original formats are private
+        'url': 'https://archive.org/details/irelandthemakingofarepublic',
+        'info_dict': {
+            'id': 'irelandthemakingofarepublic',
+            'title': 'Ireland: The Making of a Republic',
+            'upload_date': '20160610',
+            'description': 'md5:f70956a156645a658a0dc9513d9e78b7',
+            'uploader': 'dimitrios@archive.org',
+            'creator': ['British Broadcasting Corporation', 'Time-Life Films'],
+            'timestamp': 1465594947,
+        },
+        'playlist': [
+            {
+                'md5': '0b211261b26590d49df968f71b90690d',
+                'info_dict': {
+                    'id': 'irelandthemakingofarepublic/irelandthemakingofarepublicreel1_01.mov',
+                    'ext': 'mp4',
+                    'title': 'irelandthemakingofarepublicreel1_01.mov',
+                    'duration': 130.46,
+                    'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_01_000117.jpg',
+                    'display_id': 'irelandthemakingofarepublicreel1_01.mov',
+                },
+            }, {
+                'md5': '67335ee3b23a0da930841981c1e79b02',
+                'info_dict': {
+                    'id': 'irelandthemakingofarepublic/irelandthemakingofarepublicreel1_02.mov',
+                    'ext': 'mp4',
+                    'duration': 1395.13,
+                    'title': 'irelandthemakingofarepublicreel1_02.mov',
+                    'display_id': 'irelandthemakingofarepublicreel1_02.mov',
+                    'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_02_001374.jpg',
+                },
+            }, {
+                'md5': 'e470e86787893603f4a341a16c281eb5',
+                'info_dict': {
+                    'id': 'irelandthemakingofarepublic/irelandthemakingofarepublicreel2.mov',
+                    'ext': 'mp4',
+                    'duration': 1602.67,
+                    'title': 'irelandthemakingofarepublicreel2.mov',
+                    'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel2_001554.jpg',
+                    'display_id': 'irelandthemakingofarepublicreel2.mov',
+                },
+            }
+        ]
     }]
 
     @staticmethod
@@ -216,17 +293,25 @@ class ArchiveOrgIE(InfoExtractor):
                     'filesize': int_or_none(f.get('size'))})
 
             extension = (f['name'].rsplit('.', 1) + [None])[1]
-            if extension in KNOWN_EXTENSIONS:
+
+            # We don't want to skip private formats if the user has access to them,
+            # however without access to an account with such privileges we can't implement/test this.
+            # For now to be safe, we will only skip them if there is no user logged in.
+            is_logged_in = bool(self._get_cookies('https://archive.org').get('logged-in-sig'))
+            if extension in KNOWN_EXTENSIONS and (not f.get('private') or is_logged_in):
                 entry['formats'].append({
                     'url': 'https://archive.org/download/' + identifier + '/' + f['name'],
                     'format': f.get('format'),
                     'width': int_or_none(f.get('width')),
                     'height': int_or_none(f.get('height')),
                     'filesize': int_or_none(f.get('size')),
-                    'protocol': 'https'})
+                    'protocol': 'https',
+                    'source_preference': 0 if f.get('source') == 'original' else -1,
+                    'format_note': f.get('source')
+                })
 
         for entry in entries.values():
-            self._sort_formats(entry['formats'])
+            self._sort_formats(entry['formats'], ('source', ))
 
         if len(entries) == 1:
             # If there's only one item, use it as the main info dict

yt_dlp/extractor/arcpublishing.py

@@ -70,8 +70,8 @@ class ArcPublishingIE(InfoExtractor):
         ], 'video-api-cdn.%s.arcpublishing.com/api'),
     ]
 
-    @staticmethod
-    def _extract_urls(webpage):
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
         entries = []
         # https://arcpublishing.atlassian.net/wiki/spaces/POWA/overview
         for powa_el in re.findall(r'(<div[^>]+class="[^"]*\bpowa\b[^"]*"[^>]+data-uuid="%s"[^>]*>)' % ArcPublishingIE._UUID_REGEX, webpage):

yt_dlp/extractor/arkena.py

@@ -1,5 +1,3 @@
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -19,6 +17,8 @@ class ArkenaIE(InfoExtractor):
                                 play\.arkena\.com/(?:config|embed)/avp/v\d/player/media/(?P<id>[^/]+)/[^/]+/(?P<account_id>\d+)
                             )
                         '''
+    # See https://support.arkena.com/display/PLAY/Ways+to+embed+your+video
+    _EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//play\.arkena\.com/embed/avp/.+?)\1']
     _TESTS = [{
         'url': 'https://video.qbrick.com/play2/embed/player?accountId=1034090&mediaId=d8ab4607-00090107-aab86310',
         'md5': '97f117754e5f3c020f5f26da4a44ebaf',
@@ -50,15 +50,6 @@ class ArkenaIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def _extract_url(webpage):
-        # See https://support.arkena.com/display/PLAY/Ways+to+embed+your+video
-        mobj = re.search(
-            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//play\.arkena\.com/embed/avp/.+?)\1',
-            webpage)
-        if mobj:
-            return mobj.group('url')
-
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         video_id = mobj.group('id')

yt_dlp/extractor/arte.py

@@ -1,190 +1,210 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-)
 from ..utils import (
     ExtractorError,
+    GeoRestrictedError,
     int_or_none,
+    parse_iso8601,
     parse_qs,
-    qualities,
     strip_or_none,
-    try_get,
-    unified_strdate,
+    traverse_obj,
     url_or_none,
 )
 
 
 class ArteTVBaseIE(InfoExtractor):
     _ARTE_LANGUAGES = 'fr|de|en|es|it|pl'
-    _API_BASE = 'https://api.arte.tv/api/player/v1'
+    _API_BASE = 'https://api.arte.tv/api/player/v2'
 
 
 class ArteTVIE(ArteTVBaseIE):
     _VALID_URL = r'''(?x)
-                    https?://
+                    (?:https?://
                         (?:
                             (?:www\.)?arte\.tv/(?P<lang>%(langs)s)/videos|
                             api\.arte\.tv/api/player/v\d+/config/(?P<lang_2>%(langs)s)
                         )
-                        /(?P<id>\d{6}-\d{3}-[AF])
+                    |arte://program)
+                        /(?P<id>\d{6}-\d{3}-[AF]|LIVE)
                     ''' % {'langs': ArteTVBaseIE._ARTE_LANGUAGES}
     _TESTS = [{
         'url': 'https://www.arte.tv/en/videos/088501-000-A/mexico-stealing-petrol-to-survive/',
-        'info_dict': {
-            'id': '088501-000-A',
-            'ext': 'mp4',
-            'title': 'Mexico: Stealing Petrol to Survive',
-            'upload_date': '20190628',
-        },
+        'only_matching': True,
     }, {
         'url': 'https://www.arte.tv/pl/videos/100103-000-A/usa-dyskryminacja-na-porodowce/',
-        'only_matching': True,
+        'info_dict': {
+            'id': '100103-000-A',
+            'title': 'USA: Dyskryminacja na porodówce',
+            'description': 'md5:242017b7cce59ffae340a54baefcafb1',
+            'alt_title': 'ARTE Reportage',
+            'upload_date': '20201103',
+            'duration': 554,
+            'thumbnail': r're:https://api-cdn\.arte\.tv/.+940x530',
+            'timestamp': 1604417980,
+            'ext': 'mp4',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }, {
+        'note': 'No alt_title',
+        'url': 'https://www.arte.tv/fr/videos/110371-000-A/la-chaleur-supplice-des-arbres-de-rue/',
+        'info_dict': {
+            'id': '110371-000-A',
+            'ext': 'mp4',
+            'upload_date': '20220718',
+            'duration': 154,
+            'timestamp': 1658162460,
+            'description': 'md5:5890f36fe7dccfadb8b7c0891de54786',
+            'title': 'La chaleur, supplice des arbres de rue',
+            'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/CPE2sQDtD8GLQgt8DuYHLf/940x530',
+        },
+        'params': {'skip_download': 'm3u8'}
     }, {
         'url': 'https://api.arte.tv/api/player/v2/config/de/100605-013-A',
         'only_matching': True,
+    }, {
+        'url': 'https://api.arte.tv/api/player/v2/config/de/LIVE',
+        'only_matching': True,
     }]
 
+    _GEO_BYPASS = True
+
+    _LANG_MAP = {  # ISO639 -> French abbreviations
+        'fr': 'F',
+        'de': 'A',
+        'en': 'E[ANG]',
+        'es': 'E[ESP]',
+        'it': 'E[ITA]',
+        'pl': 'E[POL]',
+        # XXX: probably means mixed; <https://www.arte.tv/en/videos/107710-029-A/dispatches-from-ukraine-local-journalists-report/>
+        # uses this code for audio that happens to be in Ukrainian, but the manifest uses the ISO code 'mul' (mixed)
+        'mul': 'EU',
+    }
+
+    _VERSION_CODE_RE = re.compile(r'''(?x)
+        V
+        (?P<original_voice>O?)
+        (?P<vlang>[FA]|E\[[A-Z]+\]|EU)?
+        (?P<audio_desc>AUD|)
+        (?:
+            (?P<has_sub>-ST)
+            (?P<sdh_sub>M?)
+            (?P<sub_lang>[FA]|E\[[A-Z]+\]|EU)
+        )?
+    ''')
+
+    # all obtained by exhaustive testing
+    _COUNTRIES_MAP = {
+        'DE_FR': (
+            'BL', 'DE', 'FR', 'GF', 'GP', 'MF', 'MQ', 'NC',
+            'PF', 'PM', 'RE', 'WF', 'YT',
+        ),
+        # with both of the below 'BE' sometimes works, sometimes doesn't
+        'EUR_DE_FR': (
+            'AT', 'BL', 'CH', 'DE', 'FR', 'GF', 'GP', 'LI',
+            'MC', 'MF', 'MQ', 'NC', 'PF', 'PM', 'RE', 'WF',
+            'YT',
+        ),
+        'SAT': (
+            'AD', 'AT', 'AX', 'BG', 'BL', 'CH', 'CY', 'CZ',
+            'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GF',
+            'GR', 'HR', 'HU', 'IE', 'IS', 'IT', 'KN', 'LI',
+            'LT', 'LU', 'LV', 'MC', 'MF', 'MQ', 'MT', 'NC',
+            'NL', 'NO', 'PF', 'PL', 'PM', 'PT', 'RE', 'RO',
+            'SE', 'SI', 'SK', 'SM', 'VA', 'WF', 'YT',
+        ),
+    }
+
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         video_id = mobj.group('id')
         lang = mobj.group('lang') or mobj.group('lang_2')
+        langauge_code = self._LANG_MAP.get(lang)
 
-        info = self._download_json(
-            '%s/config/%s/%s' % (self._API_BASE, lang, video_id), video_id)
-        player_info = info['videoJsonPlayer']
+        config = self._download_json(f'{self._API_BASE}/config/{lang}/{video_id}', video_id)
 
-        vsr = try_get(player_info, lambda x: x['VSR'], dict)
-        if not vsr:
-            error = None
-            if try_get(player_info, lambda x: x['custom_msg']['type']) == 'error':
-                error = try_get(
-                    player_info, lambda x: x['custom_msg']['msg'], compat_str)
-            if not error:
-                error = 'Video %s is not available' % player_info.get('VID') or video_id
-            raise ExtractorError(error, expected=True)
+        geoblocking = traverse_obj(config, ('data', 'attributes', 'restriction', 'geoblocking')) or {}
+        if geoblocking.get('restrictedArea'):
+            raise GeoRestrictedError(f'Video restricted to {geoblocking["code"]!r}',
+                                     countries=self._COUNTRIES_MAP.get(geoblocking['code'], ('DE', 'FR')))
 
-        upload_date_str = player_info.get('shootingDate')
-        if not upload_date_str:
-            upload_date_str = (player_info.get('VRA') or player_info.get('VDA') or '').split(' ')[0]
+        if not traverse_obj(config, ('data', 'attributes', 'rights')):
+            # Eg: https://www.arte.tv/de/videos/097407-215-A/28-minuten
+            # Eg: https://www.arte.tv/es/videos/104351-002-A/serviteur-du-peuple-1-23
+            raise ExtractorError(
+                'Video is not available in this language edition of Arte or broadcast rights expired', expected=True)
 
-        title = (player_info.get('VTI') or player_info['VID']).strip()
-        subtitle = player_info.get('VSU', '').strip()
-        if subtitle:
-            title += ' - %s' % subtitle
+        formats, subtitles = [], {}
+        for stream in config['data']['attributes']['streams']:
+            # official player contains code like `e.get("versions")[0].eStat.ml5`
+            stream_version = stream['versions'][0]
+            stream_version_code = stream_version['eStat']['ml5']
 
-        qfunc = qualities(['MQ', 'HQ', 'EQ', 'SQ'])
+            lang_pref = -1
+            m = self._VERSION_CODE_RE.match(stream_version_code)
+            if m:
+                lang_pref = int(''.join('01'[x] for x in (
+                    m.group('vlang') == langauge_code,      # we prefer voice in the requested language
+                    not m.group('audio_desc'),              # and not the audio description version
+                    bool(m.group('original_voice')),        # but if voice is not in the requested language, at least choose the original voice
+                    m.group('sub_lang') == langauge_code,   # if subtitles are present, we prefer them in the requested language
+                    not m.group('has_sub'),                 # but we prefer no subtitles otherwise
+                    not m.group('sdh_sub'),                 # and we prefer not the hard-of-hearing subtitles if there are subtitles
+                )))
 
-        LANGS = {
-            'fr': 'F',
-            'de': 'A',
-            'en': 'E[ANG]',
-            'es': 'E[ESP]',
-            'it': 'E[ITA]',
-            'pl': 'E[POL]',
-        }
-
-        langcode = LANGS.get(lang, lang)
-
-        formats = []
-        for format_id, format_dict in vsr.items():
-            f = dict(format_dict)
-            format_url = url_or_none(f.get('url'))
-            streamer = f.get('streamer')
-            if not format_url and not streamer:
-                continue
-            versionCode = f.get('versionCode')
-            l = re.escape(langcode)
-
-            # Language preference from most to least priority
-            # Reference: section 6.8 of
-            # https://www.arte.tv/sites/en/corporate/files/complete-technical-guidelines-arte-geie-v1-07-1.pdf
-            PREFERENCES = (
-                # original version in requested language, without subtitles
-                r'VO{0}$'.format(l),
-                # original version in requested language, with partial subtitles in requested language
-                r'VO{0}-ST{0}$'.format(l),
-                # original version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
-                r'VO{0}-STM{0}$'.format(l),
-                # non-original (dubbed) version in requested language, without subtitles
-                r'V{0}$'.format(l),
-                # non-original (dubbed) version in requested language, with subtitles partial subtitles in requested language
-                r'V{0}-ST{0}$'.format(l),
-                # non-original (dubbed) version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
-                r'V{0}-STM{0}$'.format(l),
-                # original version in requested language, with partial subtitles in different language
-                r'VO{0}-ST(?!{0}).+?$'.format(l),
-                # original version in requested language, with subtitles for the deaf and hard-of-hearing in different language
-                r'VO{0}-STM(?!{0}).+?$'.format(l),
-                # original version in different language, with partial subtitles in requested language
-                r'VO(?:(?!{0}).+?)?-ST{0}$'.format(l),
-                # original version in different language, with subtitles for the deaf and hard-of-hearing in requested language
-                r'VO(?:(?!{0}).+?)?-STM{0}$'.format(l),
-                # original version in different language, without subtitles
-                r'VO(?:(?!{0}))?$'.format(l),
-                # original version in different language, with partial subtitles in different language
-                r'VO(?:(?!{0}).+?)?-ST(?!{0}).+?$'.format(l),
-                # original version in different language, with subtitles for the deaf and hard-of-hearing in different language
-                r'VO(?:(?!{0}).+?)?-STM(?!{0}).+?$'.format(l),
-            )
-
-            for pref, p in enumerate(PREFERENCES):
-                if re.match(p, versionCode):
-                    lang_pref = len(PREFERENCES) - pref
-                    break
-            else:
-                lang_pref = -1
-            format_note = '%s, %s' % (f.get('versionCode'), f.get('versionLibelle'))
-
-            media_type = f.get('mediaType')
-            if media_type == 'hls':
-                m3u8_formats = self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id=format_id, fatal=False)
-                for m3u8_format in m3u8_formats:
-                    m3u8_format.update({
+            if stream['protocol'].startswith('HLS'):
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    stream['url'], video_id=video_id, ext='mp4', m3u8_id=stream_version_code, fatal=False)
+                for fmt in fmts:
+                    fmt.update({
+                        'format_note': f'{stream_version.get("label", "unknown")} [{stream_version.get("shortLabel", "?")}]',
                         'language_preference': lang_pref,
-                        'format_note': format_note,
                     })
-                formats.extend(m3u8_formats)
-                continue
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
 
-            format = {
-                'format_id': format_id,
-                'language_preference': lang_pref,
-                'format_note': format_note,
-                'width': int_or_none(f.get('width')),
-                'height': int_or_none(f.get('height')),
-                'tbr': int_or_none(f.get('bitrate')),
-                'quality': qfunc(f.get('quality')),
-            }
+            elif stream['protocol'] in ('HTTPS', 'RTMP'):
+                formats.append({
+                    'format_id': f'{stream["protocol"]}-{stream_version_code}',
+                    'url': stream['url'],
+                    'format_note': f'{stream_version.get("label", "unknown")} [{stream_version.get("shortLabel", "?")}]',
+                    'language_preference': lang_pref,
+                    # 'ext': 'mp4',  # XXX: may or may not be necessary, at least for HTTPS
+                })
 
-            if media_type == 'rtmp':
-                format['url'] = f['streamer']
-                format['play_path'] = 'mp4:' + f['url']
-                format['ext'] = 'flv'
             else:
-                format['url'] = f['url']
+                self.report_warning(f'Skipping stream with unknown protocol {stream["protocol"]}')
 
-            formats.append(format)
+            # TODO: chapters from stream['segments']?
+            # The JS also looks for chapters in config['data']['attributes']['chapters'],
+            # but I am yet to find a video having those
 
-        # For this extractor, quality only represents the relative quality
-        # with respect to other formats with the same resolution
-        self._sort_formats(formats, ('res', 'quality'))
+        self._sort_formats(formats)
+
+        metadata = config['data']['attributes']['metadata']
 
         return {
-            'id': player_info.get('VID') or video_id,
-            'title': title,
-            'description': player_info.get('VDE') or player_info.get('V7T'),
-            'upload_date': unified_strdate(upload_date_str),
-            'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
+            'id': metadata['providerId'],
+            'webpage_url': traverse_obj(metadata, ('link', 'url')),
+            'title': traverse_obj(metadata, 'subtitle', 'title'),
+            'alt_title': metadata.get('subtitle') and metadata.get('title'),
+            'description': metadata.get('description'),
+            'duration': traverse_obj(metadata, ('duration', 'seconds')),
+            'language': metadata.get('language'),
+            'timestamp': traverse_obj(config, ('data', 'attributes', 'rights', 'begin'), expected_type=parse_iso8601),
+            'is_live': config['data']['attributes'].get('live', False),
             'formats': formats,
+            'subtitles': subtitles,
+            'thumbnails': [
+                {'url': image['url'], 'id': image.get('caption')}
+                for image in metadata.get('images') or [] if url_or_none(image.get('url'))
+            ],
         }
 
 
 class ArteTVEmbedIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?arte\.tv/player/v\d+/index\.php\?.*?\bjson_url=.+'
+    _EMBED_REGEX = [r'<(?:iframe|script)[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?arte\.tv/player/v\d+/index\.php\?.*?\bjson_url=.+?)\1']
     _TESTS = [{
         'url': 'https://www.arte.tv/player/v5/index.php?json_url=https%3A%2F%2Fapi.arte.tv%2Fapi%2Fplayer%2Fv2%2Fconfig%2Fde%2F100605-013-A&lang=de&autoplay=true&mute=0100605-013-A',
         'info_dict': {
@@ -194,17 +214,12 @@ class ArteTVEmbedIE(InfoExtractor):
             'description': 'md5:be40b667f45189632b78c1425c7c2ce1',
             'upload_date': '20201116',
         },
+        'skip': 'No video available'
     }, {
         'url': 'https://www.arte.tv/player/v3/index.php?json_url=https://api.arte.tv/api/player/v2/config/de/100605-013-A',
         'only_matching': True,
     }]
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return [url for _, url in re.findall(
-            r'<(?:iframe|script)[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?arte\.tv/player/v\d+/index\.php\?.*?\bjson_url=.+?)\1',
-            webpage)]
-
     def _real_extract(self, url):
         qs = parse_qs(url)
         json_url = qs['json_url'][0]
@@ -217,44 +232,36 @@ class ArteTVPlaylistIE(ArteTVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?arte\.tv/(?P<lang>%s)/videos/(?P<id>RC-\d{6})' % ArteTVBaseIE._ARTE_LANGUAGES
     _TESTS = [{
         'url': 'https://www.arte.tv/en/videos/RC-016954/earn-a-living/',
-        'info_dict': {
-            'id': 'RC-016954',
-            'title': 'Earn a Living',
-            'description': 'md5:d322c55011514b3a7241f7fb80d494c2',
-        },
-        'playlist_mincount': 6,
+        'only_matching': True,
     }, {
         'url': 'https://www.arte.tv/pl/videos/RC-014123/arte-reportage/',
-        'only_matching': True,
+        'playlist_mincount': 100,
+        'info_dict': {
+            'description': 'md5:84e7bf1feda248bc325ebfac818c476e',
+            'id': 'RC-014123',
+            'title': 'ARTE Reportage - najlepsze reportaże',
+        },
     }]
 
     def _real_extract(self, url):
-        lang, playlist_id = self._match_valid_url(url).groups()
-        collection = self._download_json(
-            '%s/collectionData/%s/%s?source=videos'
-            % (self._API_BASE, lang, playlist_id), playlist_id)
-        entries = []
-        for video in collection['videos']:
-            if not isinstance(video, dict):
-                continue
-            video_url = url_or_none(video.get('url')) or url_or_none(video.get('jsonUrl'))
-            if not video_url:
-                continue
-            video_id = video.get('programId')
-            entries.append({
-                '_type': 'url_transparent',
-                'url': video_url,
-                'id': video_id,
-                'title': video.get('title'),
-                'alt_title': video.get('subtitle'),
-                'thumbnail': url_or_none(try_get(video, lambda x: x['mainImage']['url'], compat_str)),
-                'duration': int_or_none(video.get('durationSeconds')),
-                'view_count': int_or_none(video.get('views')),
-                'ie_key': ArteTVIE.ie_key(),
-            })
-        title = collection.get('title')
-        description = collection.get('shortDescription') or collection.get('teaserText')
-        return self.playlist_result(entries, playlist_id, title, description)
+        lang, playlist_id = self._match_valid_url(url).group('lang', 'id')
+        playlist = self._download_json(
+            f'{self._API_BASE}/playlist/{lang}/{playlist_id}', playlist_id)['data']['attributes']
+
+        entries = [{
+            '_type': 'url_transparent',
+            'url': video['config']['url'],
+            'ie_key': ArteTVIE.ie_key(),
+            'id': video.get('providerId'),
+            'title': video.get('title'),
+            'alt_title': video.get('subtitle'),
+            'thumbnail': url_or_none(traverse_obj(video, ('mainImage', 'url'))),
+            'duration': int_or_none(traverse_obj(video, ('duration', 'seconds'))),
+        } for video in traverse_obj(playlist, ('items', lambda _, v: v['config']['url']))]
+
+        return self.playlist_result(entries, playlist_id,
+                                    traverse_obj(playlist, ('metadata', 'title')),
+                                    traverse_obj(playlist, ('metadata', 'description')))
 
 
 class ArteTVCategoryIE(ArteTVBaseIE):
@@ -267,14 +274,13 @@ class ArteTVCategoryIE(ArteTVBaseIE):
             'description': 'Investigative documentary series, geopolitical analysis, and international commentary',
         },
         'playlist_mincount': 13,
-    },
-    ]
+    }]
 
     @classmethod
     def suitable(cls, url):
         return (
             not any(ie.suitable(url) for ie in (ArteTVIE, ArteTVPlaylistIE, ))
-            and super(ArteTVCategoryIE, cls).suitable(url))
+            and super().suitable(url))
 
     def _real_extract(self, url):
         lang, playlist_id = self._match_valid_url(url).groups()

yt_dlp/extractor/bandcamp.py

@@ -22,6 +22,7 @@ from ..utils import (
 
 class BandcampIE(InfoExtractor):
     _VALID_URL = r'https?://[^/]+\.bandcamp\.com/track/(?P<id>[^/?#&]+)'
+    _EMBED_REGEX = [r'<meta property="og:url"[^>]*?content="(?P<url>.*?bandcamp\.com.*?)"']
     _TESTS = [{
         'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
         'md5': 'c557841d5e50261777a6585648adf439',

yt_dlp/extractor/bbc.py

@@ -46,6 +46,7 @@ class BBCCoUkIE(InfoExtractor):
                         )
                         (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
                     ''' % _ID_REGEX
+    _EMBED_REGEX = [r'setPlaylist\("(?P<url>https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)']
 
     _LOGIN_URL = 'https://account.bbc.com/signin'
     _NETRC_MACHINE = 'bbc'
@@ -1231,7 +1232,7 @@ class BBCIE(BBCCoUkIE):
                                           (lambda x: x['data']['blocks'],
                                            lambda x: x['data']['content']['model']['blocks'],),
                                           list) or []):
-                        if block.get('type') != 'media':
+                        if block.get('type') not in ['media', 'video']:
                             continue
                         parse_media(block.get('model'))
             return self.playlist_result(

yt_dlp/extractor/bilibili.py

@@ -218,6 +218,9 @@ class BiliBiliIE(InfoExtractor):
 
         durl = traverse_obj(video_info, ('dash', 'video'))
         audios = traverse_obj(video_info, ('dash', 'audio')) or []
+        flac_audio = traverse_obj(video_info, ('dash', 'flac', 'audio'))
+        if flac_audio:
+            audios.append(flac_audio)
         entries = []
 
         RENDITIONS = ('qn=80&quality=80&type=', 'quality=2&type=mp4')
@@ -620,14 +623,15 @@ class BiliBiliSearchIE(SearchInfoExtractor):
                     'keyword': query,
                     'page': page_num,
                     'context': '',
-                    'order': 'pubdate',
                     'duration': 0,
                     'tids_2': '',
                     '__refresh__': 'true',
                     'search_type': 'video',
                     'tids': 0,
                     'highlight': 1,
-                })['data'].get('result') or []
+                })['data'].get('result')
+            if not videos:
+                break
             for video in videos:
                 yield self.url_result(video['arcurl'], 'BiliBili', str(video['aid']))
 

yt_dlp/extractor/bitchute.py

@@ -13,6 +13,7 @@ from ..utils import (
 
 class BitChuteIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bitchute\.com/(?:video|embed|torrent/[^/]+)/(?P<id>[^/?#&]+)'
+    _EMBED_REGEX = [rf'<(?:script|iframe)[^>]+\bsrc=(["\'])(?P<url>{_VALID_URL})']
     _TESTS = [{
         'url': 'https://www.bitchute.com/video/UGlrF9o9b-Q/',
         'md5': '7e427d7ed7af5a75b5855705ec750e2b',
@@ -33,14 +34,6 @@ class BitChuteIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return [
-            mobj.group('url')
-            for mobj in re.finditer(
-                r'<(?:script|iframe)[^>]+\bsrc=(["\'])(?P<url>%s)' % BitChuteIE._VALID_URL,
-                webpage)]
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -72,10 +65,12 @@ class BitChuteIE(InfoExtractor):
                 error = self._html_search_regex(r'<h1 class="page-title">([^<]+)</h1>', webpage, 'error', default='Cannot find video')
                 if error == 'Video Unavailable':
                     raise GeoRestrictedError(error)
-                raise ExtractorError(error)
+                raise ExtractorError(error, expected=True)
             formats = entries[0]['formats']
 
         self._check_formats(formats, video_id)
+        if not formats:
+            raise self.raise_no_formats('Video is unavailable', expected=True, video_id=video_id)
         self._sort_formats(formats)
 
         description = self._html_search_regex(

yt_dlp/extractor/blogger.py

@@ -1,5 +1,3 @@
-import re
-
 from ..utils import (
     mimetype2ext,
     parse_duration,
@@ -13,7 +11,7 @@ from .common import InfoExtractor
 class BloggerIE(InfoExtractor):
     IE_NAME = 'blogger.com'
     _VALID_URL = r'https?://(?:www\.)?blogger\.com/video\.g\?token=(?P<id>.+)'
-    _VALID_EMBED = r'''<iframe[^>]+src=["']((?:https?:)?//(?:www\.)?blogger\.com/video\.g\?token=[^"']+)["']'''
+    _EMBED_REGEX = [r'''<iframe[^>]+src=["'](?P<url>(?:https?:)?//(?:www\.)?blogger\.com/video\.g\?token=[^"']+)["']''']
     _TESTS = [{
         'url': 'https://www.blogger.com/video.g?token=AD6v5dzEe9hfcARr5Hlq1WTkYy6t-fXH3BBahVhGvVHe5szdEUBEloSEDSTA8-b111089KbfWuBvTN7fnbxMtymsHhXAXwVvyzHH4Qch2cfLQdGxKQrrEuFpC1amSl_9GuLWODjPgw',
         'md5': 'f1bc19b6ea1b0fd1d81e84ca9ec467ac',
@@ -26,10 +24,6 @@ class BloggerIE(InfoExtractor):
         }
     }]
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return re.findall(BloggerIE._VALID_EMBED, webpage)
-
     def _real_extract(self, url):
         token_id = self._match_id(url)
         webpage = self._download_webpage(url, token_id)

yt_dlp/extractor/brightcove.py

@@ -402,11 +402,11 @@ class BrightcoveNewIE(AdobePassIE):
 
     @staticmethod
     def _extract_url(ie, webpage):
-        urls = BrightcoveNewIE._extract_urls(ie, webpage)
+        urls = BrightcoveNewIE._extract_brightcove_urls(ie, webpage)
         return urls[0] if urls else None
 
     @staticmethod
-    def _extract_urls(ie, webpage):
+    def _extract_brightcove_urls(ie, webpage):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag

yt_dlp/extractor/buzzfeed.py

@@ -81,7 +81,7 @@ class BuzzFeedIE(InfoExtractor):
                 continue
             entries.append(self.url_result(video['url']))
 
-        facebook_urls = FacebookIE._extract_urls(webpage)
+        facebook_urls = FacebookIE._extract_embed_urls(url, webpage)
         entries.extend([
             self.url_result(facebook_url)
             for facebook_url in facebook_urls])

yt_dlp/extractor/camtasia.py

@@ -0,0 +1,71 @@
+import os
+import urllib.parse
+
+from .common import InfoExtractor
+from ..utils import float_or_none
+
+
+class CamtasiaEmbedIE(InfoExtractor):
+    _VALID_URL = False
+    _WEBPAGE_TESTS = [
+        {
+            'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
+            'playlist': [{
+                'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
+                'info_dict': {
+                    'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
+                    'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
+                    'ext': 'flv',
+                    'duration': 2235.90,
+                }
+            }, {
+                'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
+                'info_dict': {
+                    'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
+                    'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
+                    'ext': 'flv',
+                    'duration': 2235.93,
+                }
+            }],
+            'info_dict': {
+                'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
+            },
+            'skip': 'webpage dead'
+        },
+
+    ]
+
+    def _extract_from_webpage(self, url, webpage):
+        camtasia_cfg = self._search_regex(
+            r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
+            webpage, 'camtasia configuration file', default=None)
+        if camtasia_cfg is None:
+            return None
+
+        title = self._html_search_meta('DC.title', webpage, fatal=True)
+
+        camtasia_url = urllib.parse.urljoin(url, camtasia_cfg)
+        camtasia_cfg = self._download_xml(
+            camtasia_url, self._generic_id(url),
+            note='Downloading camtasia configuration',
+            errnote='Failed to download camtasia configuration')
+        fileset_node = camtasia_cfg.find('./playlist/array/fileset')
+
+        entries = []
+        for n in fileset_node.getchildren():
+            url_n = n.find('./uri')
+            if url_n is None:
+                continue
+
+            entries.append({
+                'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
+                'title': f'{title} - {n.tag}',
+                'url': urllib.parse.urljoin(url, url_n.text),
+                'duration': float_or_none(n.find('./duration').text),
+            })
+
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'title': title,
+        }

yt_dlp/extractor/channel9.py

@@ -14,6 +14,7 @@ class Channel9IE(InfoExtractor):
     IE_DESC = 'Channel 9'
     IE_NAME = 'channel9'
     _VALID_URL = r'https?://(?:www\.)?(?:channel9\.msdn\.com|s\.ch9\.ms)/(?P<contentpath>.+?)(?P<rss>/RSS)?/?(?:[?#&]|$)'
+    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>https?://channel9\.msdn\.com/(?:[^/]+/)+)player\b']
 
     _TESTS = [{
         'url': 'http://channel9.msdn.com/Events/TechEd/Australia/2013/KOS002',
@@ -78,12 +79,6 @@ class Channel9IE(InfoExtractor):
 
     _RSS_URL = 'http://channel9.msdn.com/%s/RSS'
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return re.findall(
-            r'<iframe[^>]+src=["\'](https?://channel9\.msdn\.com/(?:[^/]+/)+)player\b',
-            webpage)
-
     def _extract_list(self, video_id, rss_url=None):
         if not rss_url:
             rss_url = self._RSS_URL % video_id

yt_dlp/extractor/cinchcast.py

@@ -7,6 +7,8 @@ from ..utils import (
 
 class CinchcastIE(InfoExtractor):
     _VALID_URL = r'https?://player\.cinchcast\.com/.*?(?:assetId|show_id)=(?P<id>[0-9]+)'
+    _EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1']
+
     _TESTS = [{
         'url': 'http://player.cinchcast.com/?show_id=5258197&platformId=1&assetType=single',
         'info_dict': {

yt_dlp/extractor/cloudflarestream.py

@@ -1,5 +1,4 @@
 import base64
-import re
 
 from .common import InfoExtractor
 
@@ -16,6 +15,7 @@ class CloudflareStreamIE(InfoExtractor):
                         )
                         (?P<id>%s)
                     ''' % (_DOMAIN_RE, _EMBED_RE, _ID_RE)
+    _EMBED_REGEX = [fr'<script[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//{_EMBED_RE}(?:{_ID_RE}).*?)\1']
     _TESTS = [{
         'url': 'https://embed.cloudflarestream.com/embed/we4g.fla9.latest.js?video=31c9291ab41fac05471db4e73aa11717',
         'info_dict': {
@@ -37,21 +37,13 @@ class CloudflareStreamIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return [
-            mobj.group('url')
-            for mobj in re.finditer(
-                r'<script[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//%s(?:%s).*?)\1' % (CloudflareStreamIE._EMBED_RE, CloudflareStreamIE._ID_RE),
-                webpage)]
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         domain = 'bytehighway.net' if 'bytehighway.net/' in url else 'videodelivery.net'
         base_url = 'https://%s/%s/' % (domain, video_id)
         if '.' in video_id:
             video_id = self._parse_json(base64.urlsafe_b64decode(
-                video_id.split('.')[1]), video_id)['sub']
+                video_id.split('.')[1] + '==='), video_id)['sub']
         manifest_base_url = base_url + 'manifest/video.'
 
         formats = self._extract_m3u8_formats(

yt_dlp/extractor/common.py

@@ -11,17 +11,20 @@ import math
 import netrc
 import os
 import random
+import re
 import sys
 import time
+import types
 import urllib.parse
 import urllib.request
 import xml.etree.ElementTree
 
-from ..compat import functools, re  # isort: split
+from ..compat import functools  # isort: split
 from ..compat import compat_etree_fromstring, compat_expanduser, compat_os_name
 from ..downloader import FileDownloader
 from ..downloader.f4m import get_base_url, remove_encrypted_media
 from ..utils import (
+    IDENTITY,
     JSON_LD_RE,
     NO_DEFAULT,
     ExtractorError,
@@ -29,6 +32,7 @@ from ..utils import (
     GeoUtils,
     LenientJSONDecoder,
     RegexNotFoundError,
+    RetryManager,
     UnsupportedError,
     age_restricted,
     base_url,
@@ -58,6 +62,7 @@ from ..utils import (
     parse_m3u8_attributes,
     parse_resolution,
     sanitize_filename,
+    sanitize_url,
     sanitized_Request,
     str_or_none,
     str_to_int,
@@ -149,6 +154,7 @@ class InfoExtractor:
                     * abr        Average audio bitrate in KBit/s
                     * acodec     Name of the audio codec in use
                     * asr        Audio sampling rate in Hertz
+                    * audio_channels  Number of audio channels
                     * vbr        Average video bitrate in KBit/s
                     * fps        Frame rate
                     * vcodec     Name of the video codec in use
@@ -311,7 +317,8 @@ class InfoExtractor:
                     live stream that goes on instead of a fixed-length video.
     was_live:       True, False, or None (=unknown). Whether this video was
                     originally a live stream.
-    live_status:    'is_live', 'is_upcoming', 'was_live', 'not_live' or None (=unknown)
+    live_status:    None (=unknown), 'is_live', 'is_upcoming', 'was_live', 'not_live',
+                    or 'post_live' (was live, but VOD is not yet processed)
                     If absent, automatically set from is_live, was_live
     start_time:     Time in seconds where the reproduction should start, as
                     specified in the URL.
@@ -324,11 +331,12 @@ class InfoExtractor:
     playable_in_embed: Whether this video is allowed to play in embedded
                     players on other sites. Can be True (=always allowed),
                     False (=never allowed), None (=unknown), or a string
-                    specifying the criteria for embedability (Eg: 'whitelist')
+                    specifying the criteria for embedability; e.g. 'whitelist'
     availability:   Under what condition the video is available. One of
                     'private', 'premium_only', 'subscriber_only', 'needs_auth',
                     'unlisted' or 'public'. Use 'InfoExtractor._availability'
                     to set it
+    _old_archive_ids: A list of old archive ids needed for backward compatibility
     __post_extractor: A function to be called just before the metadata is
                     written to either disk, logger or console. The function
                     must return a dict which will be added to the info_dict.
@@ -429,14 +437,26 @@ class InfoExtractor:
     title, description etc.
 
 
-    Subclasses of this should define a _VALID_URL regexp and, re-define the
-    _real_extract() and (optionally) _real_initialize() methods.
-    Probably, they should also be added to the list of extractors.
+    Subclasses of this should also be added to the list of extractors and
+    should define a _VALID_URL regexp and, re-define the _real_extract() and
+    (optionally) _real_initialize() methods.
 
     Subclasses may also override suitable() if necessary, but ensure the function
     signature is preserved and that this function imports everything it needs
     (except other extractors), so that lazy_extractors works correctly.
 
+    Subclasses can define a list of _EMBED_REGEX, which will be searched for in
+    the HTML of Generic webpages. It may also override _extract_embed_urls
+    or _extract_from_webpage as necessary. While these are normally classmethods,
+    _extract_from_webpage is allowed to be an instance method.
+
+    _extract_from_webpage may raise self.StopExtraction() to stop further
+    processing of the webpage and obtain exclusive rights to it. This is useful
+    when the extractor cannot reliably be matched using just the URL,
+    e.g. invidious/peertube instances
+
+    Embed-only extractors can be defined by setting _VALID_URL = False.
+
     To support username + password (or netrc) login, the extractor must define a
     _NETRC_MACHINE and re-define _perform_login(username, password) and
     (optionally) _initialize_pre_login() methods. The _perform_login method will
@@ -460,6 +480,9 @@ class InfoExtractor:
     will be used by geo restriction bypass mechanism similarly
     to _GEO_COUNTRIES.
 
+    The _ENABLED attribute should be set to False for IEs that
+    are disabled by default and must be explicitly enabled.
+
     The _WORKING attribute should be set to False for broken IEs
     in order to warn the users and skip the tests.
     """
@@ -471,9 +494,12 @@ class InfoExtractor:
     _GEO_COUNTRIES = None
     _GEO_IP_BLOCKS = None
     _WORKING = True
+    _ENABLED = True
     _NETRC_MACHINE = None
     IE_DESC = None
     SEARCH_KEY = None
+    _VALID_URL = None
+    _EMBED_REGEX = []
 
     def _login_hint(self, method=NO_DEFAULT, netrc=None):
         password_hint = f'--username and --password, or --netrc ({netrc or self._NETRC_MACHINE}) to provide account credentials'
@@ -497,12 +523,12 @@ class InfoExtractor:
 
     @classmethod
     def _match_valid_url(cls, url):
+        if cls._VALID_URL is False:
+            return None
         # This does not use has/getattr intentionally - we want to know whether
         # we have cached the regexp for *this* class, whereas getattr would also
         # match the superclass
         if '_VALID_URL_RE' not in cls.__dict__:
-            if '_VALID_URL' not in cls.__dict__:
-                cls._VALID_URL = cls._make_valid_url()
             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
         return cls._VALID_URL_RE.match(url)
 
@@ -646,10 +672,10 @@ class InfoExtractor:
                         return None
                     if self._x_forwarded_for_ip:
                         ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip
-                    subtitles = ie_result.get('subtitles')
-                    if (subtitles and 'live_chat' in subtitles
-                            and 'no-live-chat' in self.get_param('compat_opts', [])):
-                        del subtitles['live_chat']
+                    subtitles = ie_result.get('subtitles') or {}
+                    if 'no-live-chat' in self.get_param('compat_opts'):
+                        for lang in ('live_chat', 'comments', 'danmaku'):
+                            subtitles.pop(lang, None)
                     return ie_result
                 except GeoRestrictedError as e:
                     if self.__maybe_fake_ip_and_retry(e.countries):
@@ -1141,10 +1167,12 @@ class InfoExtractor:
             'url': url,
         }
 
-    def playlist_from_matches(self, matches, playlist_id=None, playlist_title=None, getter=None, ie=None, video_kwargs=None, **kwargs):
-        urls = (self.url_result(self._proto_relative_url(m), ie, **(video_kwargs or {}))
-                for m in orderedSet(map(getter, matches) if getter else matches))
-        return self.playlist_result(urls, playlist_id, playlist_title, **kwargs)
+    @classmethod
+    def playlist_from_matches(cls, matches, playlist_id=None, playlist_title=None,
+                              getter=IDENTITY, ie=None, video_kwargs=None, **kwargs):
+        return cls.playlist_result(
+            (cls.url_result(m, ie, **(video_kwargs or {})) for m in orderedSet(map(getter, matches), lazy=True)),
+            playlist_id, playlist_title, **kwargs)
 
     @staticmethod
     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None, *, multi_video=False, **kwargs):
@@ -1351,12 +1379,20 @@ class InfoExtractor:
     def _dc_search_uploader(self, html):
         return self._html_search_meta('dc.creator', html, 'uploader')
 
-    def _rta_search(self, html):
+    @staticmethod
+    def _rta_search(html):
         # See http://www.rtalabel.org/index.php?content=howtofaq#single
         if re.search(r'(?ix)<meta\s+name="rating"\s+'
                      r'     content="RTA-5042-1996-1400-1577-RTA"',
                      html):
             return 18
+
+        # And then there are the jokers who advertise that they use RTA, but actually don't.
+        AGE_LIMIT_MARKERS = [
+            r'Proudly Labeled <a href="http://www\.rtalabel\.org/" title="Restricted to Adults">RTA</a>',
+        ]
+        if any(re.search(marker, html) for marker in AGE_LIMIT_MARKERS):
+            return 18
         return 0
 
     def _media_rating_search(self, html):
@@ -1637,8 +1673,8 @@ class InfoExtractor:
         regex = r' *((?P<reverse>\+)?(?P<field>[a-zA-Z0-9_]+)((?P<separator>[~:])(?P<limit>.*?))?)? *$'
 
         default = ('hidden', 'aud_or_vid', 'hasvid', 'ie_pref', 'lang', 'quality',
-                   'res', 'fps', 'hdr:12', 'codec:vp9.2', 'size', 'br', 'asr',
-                   'proto', 'ext', 'hasaud', 'source', 'id')  # These must not be aliases
+                   'res', 'fps', 'hdr:12', 'vcodec:vp9.2', 'channels', 'acodec',
+                   'size', 'br', 'asr', 'proto', 'ext', 'hasaud', 'source', 'id')  # These must not be aliases
         ytdl_default = ('hasaud', 'lang', 'quality', 'tbr', 'filesize', 'vbr',
                         'height', 'width', 'proto', 'vext', 'abr', 'aext',
                         'fps', 'fs_approx', 'source', 'id')
@@ -1657,7 +1693,7 @@ class InfoExtractor:
                      'order_free': ('webm', 'mp4', 'flv', '', 'none')},
             'aext': {'type': 'ordered', 'field': 'audio_ext',
                      'order': ('m4a', 'aac', 'mp3', 'ogg', 'opus', 'webm', '', 'none'),
-                     'order_free': ('opus', 'ogg', 'webm', 'm4a', 'mp3', 'aac', '', 'none')},
+                     'order_free': ('ogg', 'opus', 'webm', 'mp3', 'm4a', 'aac', '', 'none')},
             'hidden': {'visible': False, 'forced': True, 'type': 'extractor', 'max': -1000},
             'aud_or_vid': {'visible': False, 'forced': True, 'type': 'multiple',
                            'field': ('vcodec', 'acodec'),
@@ -1673,6 +1709,7 @@ class InfoExtractor:
             'height': {'convert': 'float_none'},
             'width': {'convert': 'float_none'},
             'fps': {'convert': 'float_none'},
+            'channels': {'convert': 'float_none', 'field': 'audio_channels'},
             'tbr': {'convert': 'float_none'},
             'vbr': {'convert': 'float_none'},
             'abr': {'convert': 'float_none'},
@@ -1686,13 +1723,14 @@ class InfoExtractor:
             'res': {'type': 'multiple', 'field': ('height', 'width'),
                     'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))},
 
-            # For compatibility with youtube-dl
+            # Actual field names
             'format_id': {'type': 'alias', 'field': 'id'},
             'preference': {'type': 'alias', 'field': 'ie_pref'},
             'language_preference': {'type': 'alias', 'field': 'lang'},
             'source_preference': {'type': 'alias', 'field': 'source'},
             'protocol': {'type': 'alias', 'field': 'proto'},
             'filesize_approx': {'type': 'alias', 'field': 'fs_approx'},
+            'audio_channels': {'type': 'alias', 'field': 'channels'},
 
             # Deprecated
             'dimension': {'type': 'alias', 'field': 'res', 'deprecated': True},
@@ -1728,9 +1766,8 @@ class InfoExtractor:
             if field not in self.settings:
                 if key in ('forced', 'priority'):
                     return False
-                self.ydl.deprecation_warning(
-                    f'Using arbitrary fields ({field}) for format sorting is deprecated '
-                    'and may be removed in a future version')
+                self.ydl.deprecated_feature(f'Using arbitrary fields ({field}) for format sorting is '
+                                            'deprecated and may be removed in a future version')
                 self.settings[field] = {}
             propObj = self.settings[field]
             if key not in propObj:
@@ -1815,9 +1852,8 @@ class InfoExtractor:
                 if self._get_field_setting(field, 'type') == 'alias':
                     alias, field = field, self._get_field_setting(field, 'field')
                     if self._get_field_setting(alias, 'deprecated'):
-                        self.ydl.deprecation_warning(
-                            f'Format sorting alias {alias} is deprecated '
-                            f'and may be removed in a future version. Please use {field} instead')
+                        self.ydl.deprecated_feature(f'Format sorting alias {alias} is deprecated and may '
+                                                    'be removed in a future version. Please use {field} instead')
                 reverse = match.group('reverse') is not None
                 closest = match.group('separator') == '~'
                 limit_text = match.group('limit')
@@ -1963,14 +1999,9 @@ class InfoExtractor:
             else 'https:')
 
     def _proto_relative_url(self, url, scheme=None):
-        if url is None:
-            return url
-        if url.startswith('//'):
-            if scheme is None:
-                scheme = self.http_scheme()
-            return scheme + url
-        else:
-            return url
+        scheme = scheme or self.http_scheme()
+        assert scheme.endswith(':')
+        return sanitize_url(url, scheme=scheme[:-1])
 
     def _sleep(self, timeout, video_id, msg_template=None):
         if msg_template is None:
@@ -2338,7 +2369,7 @@ class InfoExtractor:
                     audio_group_id = last_stream_inf.get('AUDIO')
                     # As per [1, 4.3.4.1.1] any EXT-X-STREAM-INF tag which
                     # references a rendition group MUST have a CODECS attribute.
-                    # However, this is not always respected, for example, [2]
+                    # However, this is not always respected. E.g. [2]
                     # contains EXT-X-STREAM-INF tag which references AUDIO
                     # rendition group but does not have CODECS and despite
                     # referencing an audio group it represents a complete
@@ -2974,8 +3005,8 @@ class InfoExtractor:
                                     segment_number += 1
                                 segment_time += segment_d
                     elif 'segment_urls' in representation_ms_info and 's' in representation_ms_info:
-                        # No media template
-                        # Example: https://www.youtube.com/watch?v=iXZV5uAYMJI
+                        # No media template,
+                        # e.g. https://www.youtube.com/watch?v=iXZV5uAYMJI
                         # or any YouTube dashsegments video
                         fragments = []
                         segment_index = 0
@@ -2992,7 +3023,7 @@ class InfoExtractor:
                         representation_ms_info['fragments'] = fragments
                     elif 'segment_urls' in representation_ms_info:
                         # Segment URLs with no SegmentTimeline
-                        # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
+                        # E.g. https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
                         # https://github.com/ytdl-org/youtube-dl/pull/14844
                         fragments = []
                         segment_duration = float_or_none(
@@ -3220,8 +3251,8 @@ class InfoExtractor:
         media_tags.extend(re.findall(
             # We only allow video|audio followed by a whitespace or '>'.
             # Allowing more characters may end up in significant slow down (see
-            # https://github.com/ytdl-org/youtube-dl/issues/11979, example URL:
-            # http://www.porntrex.com/maps/videositemap.xml).
+            # https://github.com/ytdl-org/youtube-dl/issues/11979,
+            # e.g. http://www.porntrex.com/maps/videositemap.xml).
             r'(?s)(<(?P<tag>%s)(?:\s+[^>]*)?>)(.*?)</(?P=tag)>' % _MEDIA_TAG_NAME_RE, webpage))
         for media_tag, _, media_type, media_content in media_tags:
             media_info = {
@@ -3229,7 +3260,7 @@ class InfoExtractor:
                 'subtitles': {},
             }
             media_attributes = extract_attributes(media_tag)
-            src = strip_or_none(media_attributes.get('src'))
+            src = strip_or_none(dict_get(media_attributes, ('src', 'data-video-src', 'data-src', 'data-source')))
             if src:
                 f = parse_content_type(media_attributes.get('type'))
                 _, formats = _media_formats(src, media_type, f)
@@ -3240,7 +3271,7 @@ class InfoExtractor:
                     s_attr = extract_attributes(source_tag)
                     # data-video-src and data-src are non standard but seen
                     # several times in the wild
-                    src = strip_or_none(dict_get(s_attr, ('src', 'data-video-src', 'data-src')))
+                    src = strip_or_none(dict_get(s_attr, ('src', 'data-video-src', 'data-src', 'data-source')))
                     if not src:
                         continue
                     f = parse_content_type(s_attr.get('type'))
@@ -3641,11 +3672,18 @@ class InfoExtractor:
             t['name'] = cls.ie_key()
             yield t
 
+    @classmethod
+    def get_webpage_testcases(cls):
+        tests = getattr(cls, '_WEBPAGE_TESTS', [])
+        for t in tests:
+            t['name'] = cls.ie_key()
+        return tests
+
     @classproperty
     def age_limit(cls):
         """Get age limit from the testcases"""
         return max(traverse_obj(
-            tuple(cls.get_testcases(include_onlymatching=False)),
+            (*cls.get_testcases(include_onlymatching=False), *cls.get_webpage_testcases()),
             (..., (('playlist', 0), None), 'info_dict', 'age_limit')) or [0])
 
     @classmethod
@@ -3670,7 +3708,7 @@ class InfoExtractor:
             desc += f'; "{cls.SEARCH_KEY}:" prefix'
             if search_examples:
                 _COUNTS = ('', '5', '10', 'all')
-                desc += f' (Example: "{cls.SEARCH_KEY}{random.choice(_COUNTS)}:{random.choice(search_examples)}")'
+                desc += f' (e.g. "{cls.SEARCH_KEY}{random.choice(_COUNTS)}:{random.choice(search_examples)}")'
         if not cls.working():
             desc += ' (**Currently broken**)' if markdown else ' (Currently broken)'
 
@@ -3765,10 +3803,12 @@ class InfoExtractor:
             headers['Ytdl-request-proxy'] = geo_verification_proxy
         return headers
 
-    def _generic_id(self, url):
+    @staticmethod
+    def _generic_id(url):
         return urllib.parse.unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
 
-    def _generic_title(self, url):
+    @staticmethod
+    def _generic_title(url):
         return urllib.parse.unquote(os.path.splitext(url_basename(url))[0])
 
     @staticmethod
@@ -3814,6 +3854,52 @@ class InfoExtractor:
         self.to_screen(f'Downloading {playlist_label}{playlist_id} - add --no-playlist to download just the {video_label}{video_id}')
         return True
 
+    def _error_or_warning(self, err, _count=None, _retries=0, *, fatal=True):
+        RetryManager.report_retry(err, _count or int(fatal), _retries, info=self.to_screen, warn=self.report_warning,
+                                  sleep_func=self.get_param('retry_sleep_functions', {}).get('extractor'))
+
+    def RetryManager(self, **kwargs):
+        return RetryManager(self.get_param('extractor_retries', 3), self._error_or_warning, **kwargs)
+
+    @classmethod
+    def extract_from_webpage(cls, ydl, url, webpage):
+        ie = (cls if isinstance(cls._extract_from_webpage, types.MethodType)
+              else ydl.get_info_extractor(cls.ie_key()))
+        for info in ie._extract_from_webpage(url, webpage) or []:
+            # url = None since we do not want to set (webpage/original)_url
+            ydl.add_default_extra_info(info, ie, None)
+            yield info
+
+    @classmethod
+    def _extract_from_webpage(cls, url, webpage):
+        for embed_url in orderedSet(
+                cls._extract_embed_urls(url, webpage) or [], lazy=True):
+            yield cls.url_result(embed_url, None if cls._VALID_URL is False else cls)
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        """@returns all the embed urls on the webpage"""
+        if '_EMBED_URL_RE' not in cls.__dict__:
+            assert isinstance(cls._EMBED_REGEX, (list, tuple))
+            for idx, regex in enumerate(cls._EMBED_REGEX):
+                assert regex.count('(?P<url>') == 1, \
+                    f'{cls.__name__}._EMBED_REGEX[{idx}] must have exactly 1 url group\n\t{regex}'
+            cls._EMBED_URL_RE = tuple(map(re.compile, cls._EMBED_REGEX))
+
+        for regex in cls._EMBED_URL_RE:
+            for mobj in regex.finditer(webpage):
+                embed_url = urllib.parse.urljoin(url, unescapeHTML(mobj.group('url')))
+                if cls._VALID_URL is False or cls.suitable(embed_url):
+                    yield embed_url
+
+    class StopExtraction(Exception):
+        pass
+
+    @classmethod
+    def _extract_url(cls, webpage):  # TODO: Remove
+        """Only for compatibility with some older extractors"""
+        return next(iter(cls._extract_embed_urls(None, webpage) or []), None)
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
@@ -3824,8 +3910,8 @@ class SearchInfoExtractor(InfoExtractor):
 
     _MAX_RESULTS = float('inf')
 
-    @classmethod
-    def _make_valid_url(cls):
+    @classproperty
+    def _VALID_URL(cls):
         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
 
     def _real_extract(self, query):
@@ -3857,3 +3943,12 @@ class SearchInfoExtractor(InfoExtractor):
     @classproperty
     def SEARCH_KEY(cls):
         return cls._SEARCH_KEY
+
+
+class UnsupportedURLIE(InfoExtractor):
+    _VALID_URL = '.*'
+    _ENABLED = False
+    IE_DESC = False
+
+    def _real_extract(self, url):
+        raise UnsupportedError(url)

yt_dlp/extractor/commonmistakes.py

@@ -4,9 +4,7 @@ from ..utils import ExtractorError
 
 class CommonMistakesIE(InfoExtractor):
     IE_DESC = False  # Do not list
-    _VALID_URL = r'''(?x)
-        (?:url|URL)$
-    '''
+    _VALID_URL = r'(?:url|URL|yt-dlp)$'
 
     _TESTS = [{
         'url': 'url',

yt_dlp/extractor/condenast.py

@@ -58,7 +58,10 @@ class CondeNastIE(InfoExtractor):
         )''' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
-    EMBED_URL = r'(?:https?:)?//player(?:-backend)?\.(?:%s)\.com/(?:embed(?:js)?|(?:script|inline)/video)/.+?' % '|'.join(_SITES.keys())
+    _EMBED_REGEX = [r'''(?x)
+        <(?:iframe|script)[^>]+?src=(["\'])(?P<url>
+            (?:https?:)?//player(?:-backend)?\.(?:%s)\.com/(?:embed(?:js)?|(?:script|inline)/video)/.+?
+        )\1''' % '|'.join(_SITES.keys())]
 
     _TESTS = [{
         'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',

yt_dlp/extractor/crooksandliars.py

@@ -7,6 +7,8 @@ from ..utils import (
 
 class CrooksAndLiarsIE(InfoExtractor):
     _VALID_URL = r'https?://embed\.crooksandliars\.com/(?:embed|v)/(?P<id>[A-Za-z0-9]+)'
+    _EMBED_REGEX = [r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1']
+
     _TESTS = [{
         'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
         'info_dict': {

yt_dlp/extractor/crunchyroll.py

@@ -28,6 +28,7 @@ from ..utils import (
     join_nonempty,
     lowercase_escape,
     merge_dicts,
+    parse_iso8601,
     qualities,
     remove_end,
     sanitized_Request,
@@ -113,7 +114,14 @@ class CrunchyrollBaseIE(InfoExtractor):
 
 class CrunchyrollIE(CrunchyrollBaseIE, VRVBaseIE):
     IE_NAME = 'crunchyroll'
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|(?!series/|watch/)(?:[^/]+/){1,2}[^/?&]*?)(?P<id>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'''(?x)
+        https?://(?:(?P<prefix>www|m)\.)?(?P<url>
+            crunchyroll\.(?:com|fr)/(?:
+                media(?:-|/\?id=)|
+                (?!series/|watch/)(?:[^/]+/){1,2}[^/?&#]*?
+            )(?P<id>[0-9]+)
+        )(?:[/?&#]|$)'''
+
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {
@@ -649,7 +657,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
 class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
     IE_NAME = 'crunchyroll:playlist'
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:\w{1,2}/)?(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login|media-\d+))(?P<id>[\w\-]+))/?(?:\?|$)'
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:\w{2}(?:-\w{2})?/)?(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login|media-\d+))(?P<id>[\w\-]+))/?(?:\?|$)'
 
     _TESTS = [{
         'url': 'https://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
@@ -712,15 +720,20 @@ class CrunchyrollBetaBaseIE(CrunchyrollBaseIE):
 
     def _get_params(self, lang):
         if not CrunchyrollBetaBaseIE.params:
+            if self._get_cookies(f'https://beta.crunchyroll.com/{lang}').get('etp_rt'):
+                grant_type, key = 'etp_rt_cookie', 'accountAuthClientId'
+            else:
+                grant_type, key = 'client_id', 'anonClientId'
+
             initial_state, app_config = self._get_beta_embedded_json(self._download_webpage(
                 f'https://beta.crunchyroll.com/{lang}', None, note='Retrieving main page'), None)
             api_domain = app_config['cxApiParams']['apiDomain']
-            basic_token = str(base64.b64encode(('%s:' % app_config['cxApiParams']['accountAuthClientId']).encode('ascii')), 'ascii')
+
             auth_response = self._download_json(
-                f'{api_domain}/auth/v1/token', None, note='Authenticating with cookie',
+                f'{api_domain}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
                 headers={
-                    'Authorization': 'Basic ' + basic_token
-                }, data='grant_type=etp_rt_cookie'.encode('ascii'))
+                    'Authorization': 'Basic ' + str(base64.b64encode(('%s:' % app_config['cxApiParams'][key]).encode('ascii')), 'ascii')
+                }, data=f'grant_type={grant_type}'.encode('ascii'))
             policy_response = self._download_json(
                 f'{api_domain}/index/v2', None, note='Retrieving signed policy',
                 headers={
@@ -739,130 +752,80 @@ class CrunchyrollBetaBaseIE(CrunchyrollBaseIE):
             CrunchyrollBetaBaseIE.params = (api_domain, bucket, params)
         return CrunchyrollBetaBaseIE.params
 
-    def _redirect_from_beta(self, url, lang, internal_id, display_id, is_episode, iekey):
-        initial_state, app_config = self._get_beta_embedded_json(self._download_webpage(url, display_id), display_id)
-        content_data = initial_state['content']['byId'][internal_id]
-        if is_episode:
-            video_id = content_data['external_id'].split('.')[1]
-            series_id = content_data['episode_metadata']['series_slug_title']
-        else:
-            series_id = content_data['slug_title']
-        series_id = re.sub(r'-{2,}', '-', series_id)
-        url = f'https://www.crunchyroll.com/{lang}{series_id}'
-        if is_episode:
-            url = url + f'/{display_id}-{video_id}'
-        self.to_screen(f'{display_id}: Not logged in. Redirecting to non-beta site - {url}')
-        return self.url_result(url, iekey, display_id)
-
 
 class CrunchyrollBetaIE(CrunchyrollBetaBaseIE):
     IE_NAME = 'crunchyroll:beta'
-    _VALID_URL = r'https?://beta\.crunchyroll\.com/(?P<lang>(?:\w{1,2}/)?)watch/(?P<id>\w+)/(?P<display_id>[\w\-]*)/?(?:\?|$)'
+    _VALID_URL = r'''(?x)
+        https?://beta\.crunchyroll\.com/
+        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        watch/(?P<id>\w+)
+        (?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)'''
     _TESTS = [{
         'url': 'https://beta.crunchyroll.com/watch/GY2P1Q98Y/to-the-future',
         'info_dict': {
-            'id': '696363',
+            'id': 'GY2P1Q98Y',
             'ext': 'mp4',
-            'timestamp': 1459610100,
+            'duration': 1380.241,
+            'timestamp': 1459632600,
             'description': 'md5:a022fbec4fbb023d43631032c91ed64b',
-            'uploader': 'Toei Animation',
             'title': 'World Trigger Episode 73 – To the Future',
             'upload_date': '20160402',
-            'episode_number': 73,
             'series': 'World Trigger',
-            'average_rating': 4.9,
-            'episode': 'To the Future',
+            'series_id': 'GR757DMKY',
             'season': 'World Trigger',
-            'thumbnail': 'https://img1.ak.crunchyroll.com/i/spire3-tmb/c870dedca1a83137c2d3d144984155ed1459527119_main.jpg',
+            'season_id': 'GR9P39NJ6',
             'season_number': 1,
+            'episode': 'To the Future',
+            'episode_number': 73,
+            'thumbnail': r're:^https://beta.crunchyroll.com/imgsrv/.*\.jpeg$',
         },
         'params': {'skip_download': 'm3u8'},
-        'expected_warnings': ['Unable to download XML']
     }, {
-        'url': 'https://beta.crunchyroll.com/watch/GYK53DMPR/wicked-lord-shingan-reborn',
-        'info_dict': {
-            'id': '648781',
-            'ext': 'mp4',
-            'episode_number': 1,
-            'timestamp': 1389173400,
-            'series': 'Love, Chunibyo & Other Delusions - Heart Throb -',
-            'description': 'md5:5579d1a0355cc618558ba23d27067a62',
-            'uploader': 'TBS',
-            'episode': 'Wicked Lord Shingan... Reborn',
-            'average_rating': 4.9,
-            'season': 'Love, Chunibyo & Other Delusions - Heart Throb -',
-            'thumbnail': 'https://img1.ak.crunchyroll.com/i/spire3-tmb/2ba0384e225a5370d5f0ee9496d91ea51389046521_main.jpg',
-            'title': 'Love, Chunibyo & Other Delusions - Heart Throb - Episode 1 – Wicked Lord Shingan... Reborn',
-            'season_number': 2,
-            'upload_date': '20140108',
-        },
-        'params': {'skip_download': 'm3u8'},
-        'expected_warnings': ['Unable to download XML']
+        'url': 'https://beta.crunchyroll.com/watch/GY2P1Q98Y',
+        'only_matching': True,
     }, {
-        'url': 'https://beta.crunchyroll.com/watch/GY2P1Q98Y/',
+        'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
         lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
-
-        if not self._get_cookies(url).get('etp_rt'):
-            return self._redirect_from_beta(url, lang, internal_id, display_id, True, CrunchyrollIE.ie_key())
-
         api_domain, bucket, params = self._get_params(lang)
 
         episode_response = self._download_json(
             f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
-            note='Retrieving episode metadata',
-            query=params)
+            note='Retrieving episode metadata', query=params)
         if episode_response.get('is_premium_only') and not episode_response.get('playback'):
             raise ExtractorError('This video is for premium members only.', expected=True)
-        stream_response = self._download_json(
-            episode_response['playback'], display_id,
-            note='Retrieving stream info')
 
-        thumbnails = []
-        for thumbnails_data in traverse_obj(episode_response, ('images', 'thumbnail')):
-            for thumbnail_data in thumbnails_data:
-                thumbnails.append({
-                    'url': thumbnail_data.get('source'),
-                    'width': thumbnail_data.get('width'),
-                    'height': thumbnail_data.get('height'),
-                })
-        subtitles = {}
-        for lang, subtitle_data in stream_response.get('subtitles').items():
-            subtitles[lang] = [{
-                'url': subtitle_data.get('url'),
-                'ext': subtitle_data.get('format')
-            }]
+        stream_response = self._download_json(
+            f'{api_domain}{episode_response["__links__"]["streams"]["href"]}', display_id,
+            note='Retrieving stream info', query=params)
+        get_streams = lambda name: (traverse_obj(stream_response, name) or {}).items()
 
         requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
         hardsub_preference = qualities(requested_hardsubs[::-1])
         requested_formats = self._configuration_arg('format') or ['adaptive_hls']
 
         formats = []
-        for stream_type, streams in stream_response.get('streams', {}).items():
+        for stream_type, streams in get_streams('streams'):
             if stream_type not in requested_formats:
                 continue
             for stream in streams.values():
                 hardsub_lang = stream.get('hardsub_locale') or ''
                 if hardsub_lang.lower() not in requested_hardsubs:
                     continue
-                format_id = join_nonempty(
-                    stream_type,
-                    format_field(stream, 'hardsub_locale', 'hardsub-%s'))
+                format_id = join_nonempty(stream_type, format_field(stream, 'hardsub_locale', 'hardsub-%s'))
                 if not stream.get('url'):
                     continue
-                if stream_type.split('_')[-1] == 'hls':
+                if stream_type.endswith('hls'):
                     adaptive_formats = self._extract_m3u8_formats(
                         stream['url'], display_id, 'mp4', m3u8_id=format_id,
-                        note='Downloading %s information' % format_id,
-                        fatal=False)
-                elif stream_type.split('_')[-1] == 'dash':
+                        fatal=False, note=f'Downloading {format_id} HLS manifest')
+                elif stream_type.endswith('dash'):
                     adaptive_formats = self._extract_mpd_formats(
                         stream['url'], display_id, mpd_id=format_id,
-                        note='Downloading %s information' % format_id,
-                        fatal=False)
+                        fatal=False, note=f'Downloading {format_id} MPD manifest')
                 for f in adaptive_formats:
                     if f.get('acodec') != 'none':
                         f['language'] = stream_response.get('audio_locale')
@@ -872,10 +835,11 @@ class CrunchyrollBetaIE(CrunchyrollBetaBaseIE):
 
         return {
             'id': internal_id,
-            'title': '%s Episode %s – %s' % (episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
-            'description': episode_response.get('description').replace(r'\r\n', '\n'),
+            'title': '%s Episode %s – %s' % (
+                episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
+            'description': try_get(episode_response, lambda x: x['description'].replace(r'\r\n', '\n')),
             'duration': float_or_none(episode_response.get('duration_ms'), 1000),
-            'thumbnails': thumbnails,
+            'timestamp': parse_iso8601(episode_response.get('upload_date')),
             'series': episode_response.get('series_title'),
             'series_id': episode_response.get('series_id'),
             'season': episode_response.get('season_title'),
@@ -883,39 +847,42 @@ class CrunchyrollBetaIE(CrunchyrollBetaBaseIE):
             'season_number': episode_response.get('season_number'),
             'episode': episode_response.get('title'),
             'episode_number': episode_response.get('sequence_number'),
-            'subtitles': subtitles,
-            'formats': formats
+            'formats': formats,
+            'thumbnails': [{
+                'url': thumb.get('source'),
+                'width': thumb.get('width'),
+                'height': thumb.get('height'),
+            } for thumb in traverse_obj(episode_response, ('images', 'thumbnail', ..., ...)) or []],
+            'subtitles': {
+                lang: [{
+                    'url': subtitle_data.get('url'),
+                    'ext': subtitle_data.get('format')
+                }] for lang, subtitle_data in get_streams('subtitles')
+            },
         }
 
 
 class CrunchyrollBetaShowIE(CrunchyrollBetaBaseIE):
     IE_NAME = 'crunchyroll:playlist:beta'
-    _VALID_URL = r'https?://beta\.crunchyroll\.com/(?P<lang>(?:\w{1,2}/)?)series/(?P<id>\w+)/(?P<display_id>[\w\-]*)/?(?:\?|$)'
+    _VALID_URL = r'''(?x)
+        https?://beta\.crunchyroll\.com/
+        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        series/(?P<id>\w+)
+        (?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)'''
     _TESTS = [{
         'url': 'https://beta.crunchyroll.com/series/GY19NQ2QR/Girl-Friend-BETA',
         'info_dict': {
-            'id': 'girl-friend-beta',
+            'id': 'GY19NQ2QR',
             'title': 'Girl Friend BETA',
         },
         'playlist_mincount': 10,
     }, {
-        'url': 'https://beta.crunchyroll.com/series/GYJQV73V6/love-chunibyo--other-delusions---heart-throb--',
-        'info_dict': {
-            'id': 'love-chunibyo-other-delusions-heart-throb-',
-            'title': 'Love, Chunibyo & Other Delusions - Heart Throb -',
-        },
-        'playlist_mincount': 10,
-    }, {
-        'url': 'https://beta.crunchyroll.com/it/series/GY19NQ2QR/Girl-Friend-BETA',
+        'url': 'https://beta.crunchyroll.com/it/series/GY19NQ2QR',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
         lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
-
-        if not self._get_cookies(url).get('etp_rt'):
-            return self._redirect_from_beta(url, lang, internal_id, display_id, False, CrunchyrollShowPlaylistIE.ie_key())
-
         api_domain, bucket, params = self._get_params(lang)
 
         series_response = self._download_json(

yt_dlp/extractor/cspan.py

@@ -163,7 +163,7 @@ class CSpanIE(InfoExtractor):
                 video_id = m.group('id')
                 video_type = 'program' if m.group('type') == 'prog' else 'clip'
             else:
-                senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+                senate_isvp_url = SenateISVPIE._extract_url(webpage)
                 if senate_isvp_url:
                     title = self._og_search_title(webpage)
                     surl = smuggle_url(senate_isvp_url, {'force_title': title})

yt_dlp/extractor/dailymail.py

@@ -1,5 +1,3 @@
-import re
-
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -12,6 +10,7 @@ from ..utils import (
 
 class DailyMailIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dailymail\.co\.uk/(?:video/[^/]+/video-|embed/video/)(?P<id>[0-9]+)'
+    _EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?dailymail\.co\.uk/embed/video/\d+\.html)']
     _TESTS = [{
         'url': 'http://www.dailymail.co.uk/video/tvshowbiz/video-1295863/The-Mountain-appears-sparkling-water-ad-Heavy-Bubbles.html',
         'md5': 'f6129624562251f628296c3a9ffde124',
@@ -26,12 +25,6 @@ class DailyMailIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return re.findall(
-            r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?dailymail\.co\.uk/embed/video/\d+\.html)',
-            webpage)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)