diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml index 20e5e944f..c8d3de06b 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.yml +++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml @@ -2,13 +2,11 @@ name: Broken site support description: Report issue with yt-dlp on a supported site labels: [triage, site-bug] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -24,9 +22,7 @@ body: required: true - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar issues **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and I'm willing to share it if required - type: input @@ -47,6 +43,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -78,11 +76,3 @@ body: render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml index 4aeff7dc6..a9564c0c2 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml @@ -2,13 +2,11 @@ name: Site support request description: Request support for a new site labels: [triage, site-request] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -24,9 +22,7 @@ body: required: true - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and am willing to share it if required - type: input @@ -59,6 +55,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -90,11 +88,3 @@ body: render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml index 2f516ebb7..6e2380fae 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml @@ -1,14 +1,12 @@ name: Site feature request -description: Request a new functionality for a supported site +description: Request new functionality for a site supported by yt-dlp labels: [triage, site-enhancement] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -22,9 +20,7 @@ body: required: true - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and I'm willing to share it if required - type: input @@ -55,6 +51,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -86,11 +84,3 @@ body: render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml index 201586e9d..6fc523be0 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml @@ -2,13 +2,11 @@ name: Core bug report description: Report a bug unrelated to any particular site or extractor labels: [triage, bug] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -20,13 +18,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details - required: true - - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) - required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar issues **including closed ones**. DO NOT post duplicates required: true - type: textarea id: description @@ -40,6 +32,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -71,11 +65,3 @@ body: render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml index 765de86a2..57a33bb71 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml @@ -1,14 +1,12 @@ name: Feature request -description: Request a new functionality unrelated to any particular site or extractor +description: Request a new feature unrelated to any particular site or extractor labels: [triage, enhancement] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -22,9 +20,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - type: textarea id: description @@ -38,6 +34,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" @@ -65,11 +63,3 @@ body: [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc render: shell - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml index 198e21bec..28ec7cbe0 100644 --- a/.github/ISSUE_TEMPLATE/6_question.yml +++ b/.github/ISSUE_TEMPLATE/6_question.yml @@ -1,14 +1,12 @@ name: Ask question -description: Ask yt-dlp related question +description: Ask a question about using yt-dlp labels: [question] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: markdown attributes: value: | @@ -28,9 +26,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar questions **including closed ones**. DO NOT post duplicates required: true - type: textarea id: question @@ -44,6 +40,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" @@ -71,11 +69,3 @@ body: [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc render: shell - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 9cdffa4b1..0131631bb 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,5 @@ blank_issues_enabled: false contact_links: - - name: Get help from the community on Discord + - name: Get help on Discord url: https://discord.gg/H5MNcFW63r - about: Join the yt-dlp Discord for community-powered support! - - name: Matrix Bridge to the Discord server - url: https://matrix.to/#/#yt-dlp:matrix.org - about: For those who do not want to use Discord + about: Join the yt-dlp Discord server for support and discussion diff --git a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml index bff28ae4e..f1a2d3090 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml @@ -18,9 +18,7 @@ body: required: true - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar issues **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and I'm willing to share it if required - type: input diff --git a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml index 2bffe738d..31b89b683 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml @@ -18,9 +18,7 @@ body: required: true - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and am willing to share it if required - type: input diff --git a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml index 6c3127983..421766a75 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml @@ -1,5 +1,5 @@ name: Site feature request -description: Request a new functionality for a supported site +description: Request new functionality for a site supported by yt-dlp labels: [triage, site-enhancement] body: %(no_skip)s @@ -16,9 +16,7 @@ body: required: true - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and I'm willing to share it if required - type: input diff --git a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml index 5f357d96e..31a19b292 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml @@ -14,13 +14,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details - required: true - - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) - required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar issues **including closed ones**. DO NOT post duplicates required: true - type: textarea id: description diff --git a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml index 99107ff58..b8ab6610b 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml @@ -1,5 +1,5 @@ name: Feature request -description: Request a new functionality unrelated to any particular site or extractor +description: Request a new feature unrelated to any particular site or extractor labels: [triage, enhancement] body: %(no_skip)s @@ -16,9 +16,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - type: textarea id: description diff --git a/.github/ISSUE_TEMPLATE_tmpl/6_question.yml b/.github/ISSUE_TEMPLATE_tmpl/6_question.yml index bd742109a..062e96321 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/6_question.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/6_question.yml @@ -1,5 +1,5 @@ name: Ask question -description: Ask yt-dlp related question +description: Ask a question about using yt-dlp labels: [question] body: %(no_skip)s @@ -22,9 +22,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar questions **including closed ones**. DO NOT post duplicates required: true - type: textarea id: question diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4deee572f..4dcfcc48c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,14 +1,17 @@ -**IMPORTANT**: PRs without the template will be CLOSED + ### Description of your *pull request* and other information - - -ADD DESCRIPTION HERE +ADD DETAILED DESCRIPTION HERE Fixes # @@ -16,24 +19,22 @@ ### Description of your *pull request* and other information
Template ### Before submitting a *pull request* make sure you have: - [ ] At least skimmed through [contributing guidelines](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions) including [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions) - [ ] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests -### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check all of the following options that apply: -- [ ] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/) -- [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence) +### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check those that apply and remove the others: +- [ ] I am the original author of the code in this PR, and I am willing to release it under [Unlicense](http://unlicense.org/) +- [ ] I am not the original author of the code in this PR, but it is in the public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence) -### What is the purpose of your *pull request*? +### What is the purpose of your *pull request*? Check those that apply and remove the others: - [ ] Fix or improvement to an extractor (Make sure to add/update tests) - [ ] New extractor ([Piracy websites will not be accepted](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy)) - [ ] Core bug fix/improvement diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 170a6ac19..910c409ef 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -33,7 +33,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -47,7 +47,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -60,6 +60,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" diff --git a/.gitignore b/.gitignore index fdd904f7f..8fcd0de64 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,7 @@ updates_key.pem *.class *.isorted *.stackdump +uv.lock # Generated AUTHORS diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 4b6964260..f22625e3f 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -713,3 +713,50 @@ xiaomac wesson09 Crypto90 MutantPiggieGolem1 +Sanceilaks +Strkmn +0x9fff00 +4ft35t +7x11x13 +b5i +cotko +d3d9 +Dioarya +finch71 +hexahigh +InvalidUsernameException +jixunmoe +knackku +krandor +kvk-2015 +lonble +msm595 +n10dollar +NecroRomnt +pjrobertson +subsense +test20140 +arantius +entourage8 +lfavole +mp3butcher +slipinthedove +YoshiTabletopGamer +Arc8ne +benfaerber +chrisellsworth +fries1234 +Kenshin9977 +MichaelDeBoey +msikma +pedro +pferreir +red-acid +refack +rysson +somini +thedenv +vallovic +arabcoders +mireq +mlabeeb03 diff --git a/Changelog.md b/Changelog.md index 75e824033..7aec62771 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,279 @@ # Changelog # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master --> +### 2025.03.31 + +#### Core changes +- [Add `--compat-options 2024`](https://github.com/yt-dlp/yt-dlp/commit/22e34adbd741e1c7072015debd615dc3fb71c401) ([#12789](https://github.com/yt-dlp/yt-dlp/issues/12789)) by [seproDev](https://github.com/seproDev) + +#### Extractor changes +- **francaisfacile**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/bb321cfdc3fd4400598ddb12a15862bc2ac8fc10) ([#12787](https://github.com/yt-dlp/yt-dlp/issues/12787)) by [mlabeeb03](https://github.com/mlabeeb03) +- **generic**: [Validate response before checking m3u8 live status](https://github.com/yt-dlp/yt-dlp/commit/9a1ec1d36e172d252714cef712a6d091e0a0c4f2) ([#12784](https://github.com/yt-dlp/yt-dlp/issues/12784)) by [bashonly](https://github.com/bashonly) +- **microsoftlearnepisode**: [Extract more formats](https://github.com/yt-dlp/yt-dlp/commit/d63696f23a341ee36a3237ccb5d5e14b34c2c579) ([#12799](https://github.com/yt-dlp/yt-dlp/issues/12799)) by [bashonly](https://github.com/bashonly) +- **mlbtv**: [Fix radio-only extraction](https://github.com/yt-dlp/yt-dlp/commit/f033d86b96b36f8c5289dd7c3304f42d4d9f6ff4) ([#12792](https://github.com/yt-dlp/yt-dlp/issues/12792)) by [bashonly](https://github.com/bashonly) +- **on24**: [Support `mainEvent` URLs](https://github.com/yt-dlp/yt-dlp/commit/e465b078ead75472fcb7b86f6ccaf2b5d3bc4c21) ([#12800](https://github.com/yt-dlp/yt-dlp/issues/12800)) by [bashonly](https://github.com/bashonly) +- **sbs**: [Fix subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/29560359120f28adaaac67c86fa8442eb72daa0d) ([#12785](https://github.com/yt-dlp/yt-dlp/issues/12785)) by [bashonly](https://github.com/bashonly) +- **stvr**: [Rename extractor from RTVS to STVR](https://github.com/yt-dlp/yt-dlp/commit/5fc521cbd0ce7b2410d0935369558838728e205d) ([#12788](https://github.com/yt-dlp/yt-dlp/issues/12788)) by [mireq](https://github.com/mireq) +- **twitch**: clips: [Extract portrait formats](https://github.com/yt-dlp/yt-dlp/commit/61046c31612b30c749cbdae934b7fe26abe659d7) ([#12763](https://github.com/yt-dlp/yt-dlp/issues/12763)) by [DmitryScaletta](https://github.com/DmitryScaletta) +- **youtube** + - [Add `player_js_variant` extractor-arg](https://github.com/yt-dlp/yt-dlp/commit/07f04005e40ebdb368920c511e36e98af0077ed3) ([#12767](https://github.com/yt-dlp/yt-dlp/issues/12767)) by [bashonly](https://github.com/bashonly) + - tab: [Fix playlist continuation extraction](https://github.com/yt-dlp/yt-dlp/commit/6a6d97b2cbc78f818de05cc96edcdcfd52caa259) ([#12777](https://github.com/yt-dlp/yt-dlp/issues/12777)) by [coletdjnz](https://github.com/coletdjnz) + +#### Misc. changes +- **cleanup**: Miscellaneous: [5e457af](https://github.com/yt-dlp/yt-dlp/commit/5e457af57fae9645b1b8fa0ed689229c8fb9656b) by [bashonly](https://github.com/bashonly) + +### 2025.03.27 + +#### Core changes +- **jsinterp**: [Fix nested attributes and object extraction](https://github.com/yt-dlp/yt-dlp/commit/a8b9ff3c2a0ae25735e580173becc78545b92572) ([#12760](https://github.com/yt-dlp/yt-dlp/issues/12760)) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev) + +#### Extractor changes +- **youtube**: [Make signature and nsig extraction more robust](https://github.com/yt-dlp/yt-dlp/commit/48be862b32648bff5b3e553e40fca4dcc6e88b28) ([#12761](https://github.com/yt-dlp/yt-dlp/issues/12761)) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev) + +### 2025.03.26 + +#### Extractor changes +- **youtube** + - [Fix signature and nsig extraction for player `4fcd6e4a`](https://github.com/yt-dlp/yt-dlp/commit/a550dfc904a02843a26369ae50dbb7c0febfb30e) ([#12748](https://github.com/yt-dlp/yt-dlp/issues/12748)) by [seproDev](https://github.com/seproDev) + - [Only cache nsig code on successful decoding](https://github.com/yt-dlp/yt-dlp/commit/ecee97b4fa90d51c48f9154c3a6d5a8ffe46cd5c) ([#12750](https://github.com/yt-dlp/yt-dlp/issues/12750)) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev) + +### 2025.03.25 + +#### Core changes +- [Fix attribute error on failed VT init](https://github.com/yt-dlp/yt-dlp/commit/b872ffec50fd50f790a5a490e006a369a28a3df3) ([#12696](https://github.com/yt-dlp/yt-dlp/issues/12696)) by [Grub4K](https://github.com/Grub4K) +- **utils**: `js_to_json`: [Make function less fatal](https://github.com/yt-dlp/yt-dlp/commit/9491b44032b330e05bd5eaa546187005d1e8538e) ([#12715](https://github.com/yt-dlp/yt-dlp/issues/12715)) by [seproDev](https://github.com/seproDev) + +#### Extractor changes +- [Fix sorting of HLS audio formats by `GROUP-ID`](https://github.com/yt-dlp/yt-dlp/commit/86ab79e1a5182092321102adf6ca34195803b878) ([#12714](https://github.com/yt-dlp/yt-dlp/issues/12714)) by [bashonly](https://github.com/bashonly) +- **17live**: vod: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/3396eb50dcd245b49c0f4aecd6e80ec914095d16) ([#12723](https://github.com/yt-dlp/yt-dlp/issues/12723)) by [subrat-lima](https://github.com/subrat-lima) +- **9now.com.au**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/9d5e6de2e7a47226d1f72c713ad45c88ba01db68) ([#12702](https://github.com/yt-dlp/yt-dlp/issues/12702)) by [bashonly](https://github.com/bashonly) +- **chzzk**: video: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/e2dfccaf808b406d5bcb7dd04ae9ce420752dd6f) ([#12692](https://github.com/yt-dlp/yt-dlp/issues/12692)) by [bashonly](https://github.com/bashonly), [dirkf](https://github.com/dirkf) +- **deezer**: [Remove extractors](https://github.com/yt-dlp/yt-dlp/commit/be5af3f9e91747768c2b41157851bfbe14c663f7) ([#12704](https://github.com/yt-dlp/yt-dlp/issues/12704)) by [seproDev](https://github.com/seproDev) +- **generic**: [Fix MPD base URL parsing](https://github.com/yt-dlp/yt-dlp/commit/5086d4aed6aeb3908c62f49e2d8f74cc0cb05110) ([#12718](https://github.com/yt-dlp/yt-dlp/issues/12718)) by [fireattack](https://github.com/fireattack) +- **streaks**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/801afeac91f97dc0b58cd39cc7e8c50f619dc4e1) ([#12679](https://github.com/yt-dlp/yt-dlp/issues/12679)) by [doe1080](https://github.com/doe1080) +- **tver**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/66e0bab814e4a52ef3e12d81123ad992a29df50e) ([#12659](https://github.com/yt-dlp/yt-dlp/issues/12659)) by [arabcoders](https://github.com/arabcoders), [bashonly](https://github.com/bashonly) +- **viki**: [Remove extractors](https://github.com/yt-dlp/yt-dlp/commit/fe4f14b8369038e7c58f7de546d76de1ce3a91ce) ([#12703](https://github.com/yt-dlp/yt-dlp/issues/12703)) by [seproDev](https://github.com/seproDev) +- **vrsquare**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/b7fbb5a0a16a8e8d3e29c29e26ebed677d0d6ea3) ([#12515](https://github.com/yt-dlp/yt-dlp/issues/12515)) by [doe1080](https://github.com/doe1080) +- **youtube** + - [Fix PhantomJS nsig fallback](https://github.com/yt-dlp/yt-dlp/commit/4054a2b623bd1e277b49d2e9abc3d112a4b1c7be) ([#12728](https://github.com/yt-dlp/yt-dlp/issues/12728)) by [bashonly](https://github.com/bashonly) + - [Fix signature and nsig extraction for player `363db69b`](https://github.com/yt-dlp/yt-dlp/commit/b9c979461b244713bf42691a5bc02834e2ba4b2c) ([#12725](https://github.com/yt-dlp/yt-dlp/issues/12725)) by [bashonly](https://github.com/bashonly) + +#### Networking changes +- **Request Handler**: curl_cffi: [Support `curl_cffi` 0.10.x](https://github.com/yt-dlp/yt-dlp/commit/9bf23902ceb948b9685ce1dab575491571720fc6) ([#12670](https://github.com/yt-dlp/yt-dlp/issues/12670)) by [Grub4K](https://github.com/Grub4K) + +#### Misc. changes +- **cleanup**: Miscellaneous: [9dde546](https://github.com/yt-dlp/yt-dlp/commit/9dde546e7ee3e1515d88ee3af08b099351455dc0) by [seproDev](https://github.com/seproDev) + +### 2025.03.21 + +#### Core changes +- [Fix external downloader availability when using `--ffmpeg-location`](https://github.com/yt-dlp/yt-dlp/commit/9f77e04c76e36e1cbbf49bc9eb385fa6ef804b67) ([#12318](https://github.com/yt-dlp/yt-dlp/issues/12318)) by [Kenshin9977](https://github.com/Kenshin9977) +- [Load plugins on demand](https://github.com/yt-dlp/yt-dlp/commit/4445f37a7a66b248dbd8376c43137e6e441f138e) ([#11305](https://github.com/yt-dlp/yt-dlp/issues/11305)) by [coletdjnz](https://github.com/coletdjnz), [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan) (With fixes in [c034d65](https://github.com/yt-dlp/yt-dlp/commit/c034d655487be668222ef9476a16f374584e49a7)) +- [Support emitting ConEmu progress codes](https://github.com/yt-dlp/yt-dlp/commit/f7a1f2d8132967a62b0f6d5665c6d2dde2d42c09) ([#10649](https://github.com/yt-dlp/yt-dlp/issues/10649)) by [Grub4K](https://github.com/Grub4K) + +#### Extractor changes +- **azmedien**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/26a502fc727d0e91b2db6bf4a112823bcc672e85) ([#12375](https://github.com/yt-dlp/yt-dlp/issues/12375)) by [goggle](https://github.com/goggle) +- **bilibiliplaylist**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/f5fb2229e66cf59d5bf16065bc041b42a28354a0) ([#12690](https://github.com/yt-dlp/yt-dlp/issues/12690)) by [bashonly](https://github.com/bashonly) +- **bunnycdn**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/3a1583ca75fb523cbad0e5e174387ea7b477d175) ([#11586](https://github.com/yt-dlp/yt-dlp/issues/11586)) by [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev) +- **canalsurmas**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/01a8be4c23f186329d85f9c78db34a55f3294ac5) ([#12497](https://github.com/yt-dlp/yt-dlp/issues/12497)) by [Arc8ne](https://github.com/Arc8ne) +- **cda**: [Fix login support](https://github.com/yt-dlp/yt-dlp/commit/be0d819e1103195043f6743650781f0d4d343f6d) ([#12552](https://github.com/yt-dlp/yt-dlp/issues/12552)) by [rysson](https://github.com/rysson) +- **cultureunplugged**: [Extend `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/3042afb5fe342d3a00de76704cd7de611acc350e) ([#12486](https://github.com/yt-dlp/yt-dlp/issues/12486)) by [seproDev](https://github.com/seproDev) +- **dailymotion**: [Improve embed detection](https://github.com/yt-dlp/yt-dlp/commit/ad60137c141efa5023fbc0ac8579eaefe8b3d8cc) ([#12464](https://github.com/yt-dlp/yt-dlp/issues/12464)) by [seproDev](https://github.com/seproDev) +- **gem.cbc.ca**: [Fix login support](https://github.com/yt-dlp/yt-dlp/commit/eb1417786a3027b1e7290ec37ef6aaece50ebed0) ([#12414](https://github.com/yt-dlp/yt-dlp/issues/12414)) by [bashonly](https://github.com/bashonly) +- **globo**: [Fix subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/0e1697232fcbba7551f983fd1ba93bb445cbb08b) ([#12270](https://github.com/yt-dlp/yt-dlp/issues/12270)) by [pedro](https://github.com/pedro) +- **instagram** + - [Add `app_id` extractor-arg](https://github.com/yt-dlp/yt-dlp/commit/a90641c8363fa0c10800b36eb6b01ee22d3a9409) ([#12359](https://github.com/yt-dlp/yt-dlp/issues/12359)) by [chrisellsworth](https://github.com/chrisellsworth) + - [Fix extraction of older private posts](https://github.com/yt-dlp/yt-dlp/commit/a59abe0636dc49b22a67246afe35613571b86f05) ([#12451](https://github.com/yt-dlp/yt-dlp/issues/12451)) by [bashonly](https://github.com/bashonly) + - [Improve error handling](https://github.com/yt-dlp/yt-dlp/commit/480125560a3b9972d29ae0da850aba8109e6bd41) ([#12410](https://github.com/yt-dlp/yt-dlp/issues/12410)) by [bashonly](https://github.com/bashonly) + - story: [Support `--no-playlist`](https://github.com/yt-dlp/yt-dlp/commit/65c3c58c0a67463a150920203cec929045c95a24) ([#12397](https://github.com/yt-dlp/yt-dlp/issues/12397)) by [fireattack](https://github.com/fireattack) +- **jamendo**: [Fix thumbnail extraction](https://github.com/yt-dlp/yt-dlp/commit/89a68c4857ddbaf937ff22f12648baaf6b5af840) ([#12622](https://github.com/yt-dlp/yt-dlp/issues/12622)) by [bashonly](https://github.com/bashonly), [JChris246](https://github.com/JChris246) +- **ketnet**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/bbada3ec0779422cde34f1ce3dcf595da463b493) ([#12628](https://github.com/yt-dlp/yt-dlp/issues/12628)) by [MichaelDeBoey](https://github.com/MichaelDeBoey) +- **lbry** + - [Make m3u8 format extraction non-fatal](https://github.com/yt-dlp/yt-dlp/commit/9807181cfbf87bfa732f415c30412bdbd77cbf81) ([#12463](https://github.com/yt-dlp/yt-dlp/issues/12463)) by [bashonly](https://github.com/bashonly) + - [Raise appropriate error for non-media files](https://github.com/yt-dlp/yt-dlp/commit/7126b472601814b7fd8c9de02069e8fff1764891) ([#12462](https://github.com/yt-dlp/yt-dlp/issues/12462)) by [bashonly](https://github.com/bashonly) +- **loco**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/983095485c731240aae27c950cb8c24a50827b56) ([#12667](https://github.com/yt-dlp/yt-dlp/issues/12667)) by [DTrombett](https://github.com/DTrombett) +- **magellantv**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/172d5fcd778bf2605db7647ebc56b29ed18d24ac) ([#12505](https://github.com/yt-dlp/yt-dlp/issues/12505)) by [seproDev](https://github.com/seproDev) +- **mitele**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/7223d29569a48a35ad132a508c115973866838d3) ([#12689](https://github.com/yt-dlp/yt-dlp/issues/12689)) by [bashonly](https://github.com/bashonly) +- **msn**: [Rework extractor](https://github.com/yt-dlp/yt-dlp/commit/4815dac131d42c51e12c1d05232db0bbbf607329) ([#12513](https://github.com/yt-dlp/yt-dlp/issues/12513)) by [seproDev](https://github.com/seproDev), [thedenv](https://github.com/thedenv) +- **n1**: [Fix extraction of newer articles](https://github.com/yt-dlp/yt-dlp/commit/9d70abe4de401175cbbaaa36017806f16b2df9af) ([#12514](https://github.com/yt-dlp/yt-dlp/issues/12514)) by [u-spec-png](https://github.com/u-spec-png) +- **nbcstations**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/ebac65aa9e0bf9a97c24d00f7977900d2577364b) ([#12534](https://github.com/yt-dlp/yt-dlp/issues/12534)) by [refack](https://github.com/refack) +- **niconico** + - [Fix format sorting](https://github.com/yt-dlp/yt-dlp/commit/7508e34f203e97389f1d04db92140b13401dd724) ([#12442](https://github.com/yt-dlp/yt-dlp/issues/12442)) by [xpadev-net](https://github.com/xpadev-net) + - live: [Fix thumbnail extraction](https://github.com/yt-dlp/yt-dlp/commit/c2e6e1d5f77f3b720a6266f2869eb750d20e5dc1) ([#12419](https://github.com/yt-dlp/yt-dlp/issues/12419)) by [bashonly](https://github.com/bashonly) +- **openrec**: [Fix `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/17504f253564cfad86244de2b6346d07d2300ca5) ([#12608](https://github.com/yt-dlp/yt-dlp/issues/12608)) by [fireattack](https://github.com/fireattack) +- **pinterest**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/bd0a66816934de70312eea1e71c59c13b401dc3a) ([#12538](https://github.com/yt-dlp/yt-dlp/issues/12538)) by [mikf](https://github.com/mikf) +- **playsuisse**: [Fix login support](https://github.com/yt-dlp/yt-dlp/commit/6933f5670cea9c3e2fb16c1caa1eda54d13122c5) ([#12444](https://github.com/yt-dlp/yt-dlp/issues/12444)) by [bashonly](https://github.com/bashonly) +- **reddit**: [Truncate title](https://github.com/yt-dlp/yt-dlp/commit/d9a53cc1e6fd912daf500ca4f19e9ca88994dbf9) ([#12567](https://github.com/yt-dlp/yt-dlp/issues/12567)) by [seproDev](https://github.com/seproDev) +- **rtp**: [Rework extractor](https://github.com/yt-dlp/yt-dlp/commit/8eb9c1bf3b9908cca22ef043602aa24fb9f352c6) ([#11638](https://github.com/yt-dlp/yt-dlp/issues/11638)) by [pferreir](https://github.com/pferreir), [red-acid](https://github.com/red-acid), [seproDev](https://github.com/seproDev), [somini](https://github.com/somini), [vallovic](https://github.com/vallovic) +- **softwhiteunderbelly**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/652827d5a076c9483c36654ad2cf3fe46219baf4) ([#12281](https://github.com/yt-dlp/yt-dlp/issues/12281)) by [benfaerber](https://github.com/benfaerber) +- **soop**: [Fix timestamp extraction](https://github.com/yt-dlp/yt-dlp/commit/8305df00012ff8138a6ff95279d06b54ac607f63) ([#12609](https://github.com/yt-dlp/yt-dlp/issues/12609)) by [msikma](https://github.com/msikma) +- **soundcloud** + - [Extract tags](https://github.com/yt-dlp/yt-dlp/commit/9deed13d7cce6d3647379e50589c92de89227509) ([#12420](https://github.com/yt-dlp/yt-dlp/issues/12420)) by [bashonly](https://github.com/bashonly) + - [Fix thumbnail extraction](https://github.com/yt-dlp/yt-dlp/commit/6deeda5c11f34f613724fa0627879f0d607ba1b4) ([#12447](https://github.com/yt-dlp/yt-dlp/issues/12447)) by [bashonly](https://github.com/bashonly) +- **tiktok** + - [Improve error handling](https://github.com/yt-dlp/yt-dlp/commit/99ea2978757a431eeb2a265b3395ccbe4ce202cf) ([#12445](https://github.com/yt-dlp/yt-dlp/issues/12445)) by [bashonly](https://github.com/bashonly) + - [Truncate title](https://github.com/yt-dlp/yt-dlp/commit/83b119dadb0f267f1fb66bf7ed74c097349de79e) ([#12566](https://github.com/yt-dlp/yt-dlp/issues/12566)) by [seproDev](https://github.com/seproDev) +- **tv8.it**: [Add live and playlist extractors](https://github.com/yt-dlp/yt-dlp/commit/2ee3a0aff9be2be3bea60640d3d8a0febaf0acb6) ([#12569](https://github.com/yt-dlp/yt-dlp/issues/12569)) by [DTrombett](https://github.com/DTrombett) +- **tvw**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/42b7440963866e31ff84a5b89030d1c596fa2e6e) ([#12271](https://github.com/yt-dlp/yt-dlp/issues/12271)) by [fries1234](https://github.com/fries1234) +- **twitter** + - [Fix syndication token generation](https://github.com/yt-dlp/yt-dlp/commit/b8b47547049f5ebc3dd680fc7de70ed0ca9c0d70) ([#12537](https://github.com/yt-dlp/yt-dlp/issues/12537)) by [bashonly](https://github.com/bashonly) + - [Truncate title](https://github.com/yt-dlp/yt-dlp/commit/06f6de78db2eceeabd062ab1a3023e0ff9d4df53) ([#12560](https://github.com/yt-dlp/yt-dlp/issues/12560)) by [seproDev](https://github.com/seproDev) +- **vk**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9) ([#12510](https://github.com/yt-dlp/yt-dlp/issues/12510)) by [seproDev](https://github.com/seproDev) +- **vrtmax**: [Rework extractor](https://github.com/yt-dlp/yt-dlp/commit/df9ebeec00d658693252978d1ffb885e67aa6ab6) ([#12479](https://github.com/yt-dlp/yt-dlp/issues/12479)) by [bergoid](https://github.com/bergoid), [MichaelDeBoey](https://github.com/MichaelDeBoey), [seproDev](https://github.com/seproDev) +- **weibo**: [Support playlists](https://github.com/yt-dlp/yt-dlp/commit/0bb39788626002a8a67e925580227952c563c8b9) ([#12284](https://github.com/yt-dlp/yt-dlp/issues/12284)) by [4ft35t](https://github.com/4ft35t) +- **wsj**: [Support opinion URLs and impersonation](https://github.com/yt-dlp/yt-dlp/commit/7f3006eb0c0659982bb956d71b0bc806bcb0a5f2) ([#12431](https://github.com/yt-dlp/yt-dlp/issues/12431)) by [refack](https://github.com/refack) +- **youtube** + - [Fix nsig and signature extraction for player `643afba4`](https://github.com/yt-dlp/yt-dlp/commit/9b868518a15599f3d7ef5a1c730dda164c30da9b) ([#12684](https://github.com/yt-dlp/yt-dlp/issues/12684)) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev) + - [Player client maintenance](https://github.com/yt-dlp/yt-dlp/commit/3380febe9984c21c79c3147c1d390a4cf339bc4c) ([#12603](https://github.com/yt-dlp/yt-dlp/issues/12603)) by [seproDev](https://github.com/seproDev) + - [Split into package](https://github.com/yt-dlp/yt-dlp/commit/4432a9390c79253ac830702b226d2e558b636725) ([#12557](https://github.com/yt-dlp/yt-dlp/issues/12557)) by [coletdjnz](https://github.com/coletdjnz) + - [Warn on DRM formats](https://github.com/yt-dlp/yt-dlp/commit/e67d786c7cc87bd449d22e0ddef08306891c1173) ([#12593](https://github.com/yt-dlp/yt-dlp/issues/12593)) by [coletdjnz](https://github.com/coletdjnz) + - [Warn on missing formats due to SSAP](https://github.com/yt-dlp/yt-dlp/commit/79ec2fdff75c8c1bb89b550266849ad4dec48dd3) ([#12483](https://github.com/yt-dlp/yt-dlp/issues/12483)) by [coletdjnz](https://github.com/coletdjnz) + +#### Networking changes +- [Add `keep_header_casing` extension](https://github.com/yt-dlp/yt-dlp/commit/7d18fed8f1983fe6de4ddc810dfb2761ba5744ac) ([#11652](https://github.com/yt-dlp/yt-dlp/issues/11652)) by [coletdjnz](https://github.com/coletdjnz), [Grub4K](https://github.com/Grub4K) +- [Always add unsupported suffix on version mismatch](https://github.com/yt-dlp/yt-dlp/commit/95f8df2f796d0048119615200758199aedcd7cf4) ([#12626](https://github.com/yt-dlp/yt-dlp/issues/12626)) by [Grub4K](https://github.com/Grub4K) + +#### Misc. changes +- **cleanup**: Miscellaneous: [f36e4b6](https://github.com/yt-dlp/yt-dlp/commit/f36e4b6e65cb8403791aae2f520697115cb88dec) by [dirkf](https://github.com/dirkf), [gamer191](https://github.com/gamer191), [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev) +- **test**: [Show all differences for `expect_value` and `expect_dict`](https://github.com/yt-dlp/yt-dlp/commit/a3e0c7d3b267abdf3933b709704a28d43bb46503) ([#12334](https://github.com/yt-dlp/yt-dlp/issues/12334)) by [Grub4K](https://github.com/Grub4K) + +### 2025.02.19 + +#### Core changes +- **jsinterp** + - [Add `js_number_to_string`](https://github.com/yt-dlp/yt-dlp/commit/0d9f061d38c3a4da61972e2adad317079f2f1c84) ([#12110](https://github.com/yt-dlp/yt-dlp/issues/12110)) by [Grub4K](https://github.com/Grub4K) + - [Improve zeroise](https://github.com/yt-dlp/yt-dlp/commit/4ca8c44a073d5aa3a3e3112c35b2b23d6ce25ac6) ([#12313](https://github.com/yt-dlp/yt-dlp/issues/12313)) by [seproDev](https://github.com/seproDev) + +#### Extractor changes +- **acast**: [Support shows.acast.com URLs](https://github.com/yt-dlp/yt-dlp/commit/57c717fee4bfbc9309845bbb48901b72e4b69304) ([#12223](https://github.com/yt-dlp/yt-dlp/issues/12223)) by [barsnick](https://github.com/barsnick) +- **cwtv** + - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/18a28514e306e822eab4f3a79c76d515bf076406) ([#12207](https://github.com/yt-dlp/yt-dlp/issues/12207)) by [arantius](https://github.com/arantius) + - movie: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/03c3d705778c07739e0034b51490877cffdc0983) ([#12227](https://github.com/yt-dlp/yt-dlp/issues/12227)) by [bashonly](https://github.com/bashonly) +- **digiview**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/f53553087d3fde9dcd61d6e9f98caf09db1d8ef2) ([#9902](https://github.com/yt-dlp/yt-dlp/issues/9902)) by [lfavole](https://github.com/lfavole) +- **dropbox**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/861aeec449c8f3c062d962945b234ff0341f61f3) ([#12228](https://github.com/yt-dlp/yt-dlp/issues/12228)) by [bashonly](https://github.com/bashonly) +- **francetv** + - site + - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/817483ccc68aed6049ed9c4a2ffae44ca82d2b1c) ([#12236](https://github.com/yt-dlp/yt-dlp/issues/12236)) by [bashonly](https://github.com/bashonly) + - [Fix livestream extraction](https://github.com/yt-dlp/yt-dlp/commit/1295bbedd45fa8d9bc3f7a194864ae280297848e) ([#12316](https://github.com/yt-dlp/yt-dlp/issues/12316)) by [bashonly](https://github.com/bashonly) +- **francetvinfo.fr**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/5c4c2ddfaa47988b4d50c1ad4988badc0b4f30c2) ([#12402](https://github.com/yt-dlp/yt-dlp/issues/12402)) by [bashonly](https://github.com/bashonly) +- **gem.cbc.ca**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/5271ef48c6f61c145e03e18e960995d2e651d205) ([#12404](https://github.com/yt-dlp/yt-dlp/issues/12404)) by [bashonly](https://github.com/bashonly), [dirkf](https://github.com/dirkf) +- **generic**: [Extract `live_status` for DASH manifest URLs](https://github.com/yt-dlp/yt-dlp/commit/19edaa44fcd375f54e63d6227b092f5252d3e889) ([#12256](https://github.com/yt-dlp/yt-dlp/issues/12256)) by [mp3butcher](https://github.com/mp3butcher) +- **globo**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/f8d0161455f00add65585ca1a476a7b5d56f5f96) ([#11795](https://github.com/yt-dlp/yt-dlp/issues/11795)) by [slipinthedove](https://github.com/slipinthedove), [YoshiTabletopGamer](https://github.com/YoshiTabletopGamer) +- **goplay**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/d59f14a0a7a8b55e6bf468237def62b73ab4a517) ([#12237](https://github.com/yt-dlp/yt-dlp/issues/12237)) by [alard](https://github.com/alard) +- **pbs**: [Support www.thirteen.org URLs](https://github.com/yt-dlp/yt-dlp/commit/9fb8ab2ff67fb699f60cce09163a580976e90c0e) ([#11191](https://github.com/yt-dlp/yt-dlp/issues/11191)) by [rohieb](https://github.com/rohieb) +- **reddit**: [Bypass gated subreddit warning](https://github.com/yt-dlp/yt-dlp/commit/6ca23ffaa4663cb552f937f0b1e9769b66db11bd) ([#12335](https://github.com/yt-dlp/yt-dlp/issues/12335)) by [bashonly](https://github.com/bashonly) +- **twitter**: [Fix syndication token generation](https://github.com/yt-dlp/yt-dlp/commit/14cd7f3443c6da4d49edaefcc12da9dee86e243e) ([#12107](https://github.com/yt-dlp/yt-dlp/issues/12107)) by [Grub4K](https://github.com/Grub4K), [pjrobertson](https://github.com/pjrobertson) +- **youtube** + - [Retry on more critical requests](https://github.com/yt-dlp/yt-dlp/commit/d48e612609d012abbea3785be4d26d78a014abb2) ([#12339](https://github.com/yt-dlp/yt-dlp/issues/12339)) by [coletdjnz](https://github.com/coletdjnz) + - [nsig workaround for `tce` player JS](https://github.com/yt-dlp/yt-dlp/commit/ec17fb16e8d69d4e3e10fb73bf3221be8570dfee) ([#12401](https://github.com/yt-dlp/yt-dlp/issues/12401)) by [bashonly](https://github.com/bashonly) +- **zdf**: [Extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/241ace4f104d50fdf7638f9203927aefcf57a1f7) ([#9565](https://github.com/yt-dlp/yt-dlp/issues/9565)) by [StefanLobbenmeier](https://github.com/StefanLobbenmeier) (With fixes in [e7882b6](https://github.com/yt-dlp/yt-dlp/commit/e7882b682b959e476d8454911655b3e9b14c79b2) by [bashonly](https://github.com/bashonly)) + +#### Downloader changes +- **hls** + - [Fix `BYTERANGE` logic](https://github.com/yt-dlp/yt-dlp/commit/10b7ff68e98f17655e31952f6e17120b2d7dda96) ([#11972](https://github.com/yt-dlp/yt-dlp/issues/11972)) by [entourage8](https://github.com/entourage8) + - [Support `--write-pages` for m3u8 media playlists](https://github.com/yt-dlp/yt-dlp/commit/be69468752ff598cacee57bb80533deab2367a5d) ([#12333](https://github.com/yt-dlp/yt-dlp/issues/12333)) by [bashonly](https://github.com/bashonly) + - [Support `hls_media_playlist_data` format field](https://github.com/yt-dlp/yt-dlp/commit/c987be0acb6872c6561f28aa28171e803393d851) ([#12322](https://github.com/yt-dlp/yt-dlp/issues/12322)) by [bashonly](https://github.com/bashonly) + +#### Misc. changes +- [Improve Issue/PR templates](https://github.com/yt-dlp/yt-dlp/commit/517ddf3c3f12560ab93e3d36244dc82db9f97818) ([#11499](https://github.com/yt-dlp/yt-dlp/issues/11499)) by [seproDev](https://github.com/seproDev) (With fixes in [4ecb833](https://github.com/yt-dlp/yt-dlp/commit/4ecb833472c90e078567b561fb7c089f1aa9587b) by [bashonly](https://github.com/bashonly)) +- **cleanup**: Miscellaneous: [4985a40](https://github.com/yt-dlp/yt-dlp/commit/4985a4041770eaa0016271809a1fd950dc809a55) by [dirkf](https://github.com/dirkf), [Grub4K](https://github.com/Grub4K), [StefanLobbenmeier](https://github.com/StefanLobbenmeier) +- **docs**: [Add note to `supportedsites.md`](https://github.com/yt-dlp/yt-dlp/commit/01a63629a21781458dcbd38779898e117678f5ff) ([#12382](https://github.com/yt-dlp/yt-dlp/issues/12382)) by [seproDev](https://github.com/seproDev) +- **test**: download: [Validate and sort info dict fields](https://github.com/yt-dlp/yt-dlp/commit/208163447408c78673b08c172beafe5c310fb167) ([#12299](https://github.com/yt-dlp/yt-dlp/issues/12299)) by [bashonly](https://github.com/bashonly), [pzhlkj6612](https://github.com/pzhlkj6612) + +### 2025.01.26 + +#### Core changes +- [Fix float comparison values in format filters](https://github.com/yt-dlp/yt-dlp/commit/f7d071e8aa3bf67ed7e0f881e749ca9ab50b3f8f) ([#11880](https://github.com/yt-dlp/yt-dlp/issues/11880)) by [bashonly](https://github.com/bashonly), [Dioarya](https://github.com/Dioarya) +- **utils**: `sanitize_path`: [Fix some incorrect behavior](https://github.com/yt-dlp/yt-dlp/commit/fc12e724a3b4988cfc467d2981887dde48c26b69) ([#11923](https://github.com/yt-dlp/yt-dlp/issues/11923)) by [Grub4K](https://github.com/Grub4K) + +#### Extractor changes +- **1tv**: [Support sport1tv.ru domain](https://github.com/yt-dlp/yt-dlp/commit/61ae5dc34ac775d6c122575e21ef2153b1273a2b) ([#11889](https://github.com/yt-dlp/yt-dlp/issues/11889)) by [kvk-2015](https://github.com/kvk-2015) +- **abematv**: [Support season extraction](https://github.com/yt-dlp/yt-dlp/commit/c709cc41cbc16edc846e0a431cfa8508396d4cb6) ([#11771](https://github.com/yt-dlp/yt-dlp/issues/11771)) by [middlingphys](https://github.com/middlingphys) +- **bilibili** + - [Support space `/lists/` URLs](https://github.com/yt-dlp/yt-dlp/commit/465167910407449354eb48e9861efd0819f53eb5) ([#11964](https://github.com/yt-dlp/yt-dlp/issues/11964)) by [c-basalt](https://github.com/c-basalt) + - [Support space video list extraction without login](https://github.com/yt-dlp/yt-dlp/commit/78912ed9c81f109169b828c397294a6cf8eacf41) ([#12089](https://github.com/yt-dlp/yt-dlp/issues/12089)) by [grqz](https://github.com/grqz) +- **bilibilidynamic**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/9676b05715b61c8c5dd5598871e60d8807fb1a86) ([#11838](https://github.com/yt-dlp/yt-dlp/issues/11838)) by [finch71](https://github.com/finch71), [grqz](https://github.com/grqz) +- **bluesky**: [Prefer source format](https://github.com/yt-dlp/yt-dlp/commit/ccda63934df7de2823f0834218c4254c7c4d2e4c) ([#12154](https://github.com/yt-dlp/yt-dlp/issues/12154)) by [0x9fff00](https://github.com/0x9fff00) +- **crunchyroll**: [Remove extractors](https://github.com/yt-dlp/yt-dlp/commit/ff44ed53061e065804da6275d182d7928cc03a5e) ([#12195](https://github.com/yt-dlp/yt-dlp/issues/12195)) by [seproDev](https://github.com/seproDev) +- **dropout**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/164368610456e2d96b279f8b120dea08f7b1d74f) ([#12102](https://github.com/yt-dlp/yt-dlp/issues/12102)) by [bashonly](https://github.com/bashonly) +- **eggs**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/20c765d02385a105c8ef13b6f7a737491d29c19a) ([#11904](https://github.com/yt-dlp/yt-dlp/issues/11904)) by [seproDev](https://github.com/seproDev), [subsense](https://github.com/subsense) +- **funimation**: [Remove extractors](https://github.com/yt-dlp/yt-dlp/commit/cdcf1e86726b8fa44f7e7126bbf1c18e1798d25c) ([#12167](https://github.com/yt-dlp/yt-dlp/issues/12167)) by [doe1080](https://github.com/doe1080) +- **goodgame**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/e7cc02b14d8d323f805d14325a9c95593a170d28) ([#12173](https://github.com/yt-dlp/yt-dlp/issues/12173)) by [NecroRomnt](https://github.com/NecroRomnt) +- **lbry**: [Support signed URLs](https://github.com/yt-dlp/yt-dlp/commit/de30f652ffb7623500215f5906844f2ae0d92c7b) ([#12138](https://github.com/yt-dlp/yt-dlp/issues/12138)) by [seproDev](https://github.com/seproDev) +- **naver**: [Fix m3u8 formats extraction](https://github.com/yt-dlp/yt-dlp/commit/b3007c44cdac38187fc6600de76959a7079a44d1) ([#12037](https://github.com/yt-dlp/yt-dlp/issues/12037)) by [kclauhk](https://github.com/kclauhk) +- **nest**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/1ef3ee7500c4ab8c26f7fdc5b0ad1da4d16eec8e) ([#11747](https://github.com/yt-dlp/yt-dlp/issues/11747)) by [pabs3](https://github.com/pabs3), [seproDev](https://github.com/seproDev) +- **niconico**: series: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/bc88b904cd02314da41ce1b2fdf046d0680fe965) ([#11822](https://github.com/yt-dlp/yt-dlp/issues/11822)) by [test20140](https://github.com/test20140) +- **nrk** + - [Extract more formats](https://github.com/yt-dlp/yt-dlp/commit/89198bb23b4d03e0473ac408bfb50d67c2f71165) ([#12069](https://github.com/yt-dlp/yt-dlp/issues/12069)) by [hexahigh](https://github.com/hexahigh) + - [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/45732e2590a1bd0bc9608f5eb68c59341ca84f02) ([#12193](https://github.com/yt-dlp/yt-dlp/issues/12193)) by [hexahigh](https://github.com/hexahigh) +- **patreon**: [Extract attachment filename as `alt_title`](https://github.com/yt-dlp/yt-dlp/commit/e2e73b5c65593ec0a5e685663e6ec0f4aaffc1f1) ([#12000](https://github.com/yt-dlp/yt-dlp/issues/12000)) by [msm595](https://github.com/msm595) +- **pbs**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/13825ab77815ee6e1603abbecbb9f3795057b93c) ([#12024](https://github.com/yt-dlp/yt-dlp/issues/12024)) by [dirkf](https://github.com/dirkf), [krandor](https://github.com/krandor), [n10dollar](https://github.com/n10dollar) +- **piramidetv**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/af2c821d74049b519895288aca23cee81fc4b049) ([#10777](https://github.com/yt-dlp/yt-dlp/issues/10777)) by [HobbyistDev](https://github.com/HobbyistDev), [kclauhk](https://github.com/kclauhk), [seproDev](https://github.com/seproDev) +- **redgifs**: [Support `/ifr/` URLs](https://github.com/yt-dlp/yt-dlp/commit/4850ce91d163579fa615c3c0d44c9bd64682c22b) ([#11805](https://github.com/yt-dlp/yt-dlp/issues/11805)) by [invertico](https://github.com/invertico) +- **rtvslo.si**: show: [Extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/3fc46086562857d5493cbcff687f76e4e4ed303f) ([#12136](https://github.com/yt-dlp/yt-dlp/issues/12136)) by [cotko](https://github.com/cotko) +- **senategov**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/68221ecc87c6a3f3515757bac2a0f9674a38e3f2) ([#9361](https://github.com/yt-dlp/yt-dlp/issues/9361)) by [Grabien](https://github.com/Grabien), [seproDev](https://github.com/seproDev) +- **soundcloud** + - [Extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/6d304133ab32bcd1eb78ff1467f1a41dd9b66c33) ([#11945](https://github.com/yt-dlp/yt-dlp/issues/11945)) by [7x11x13](https://github.com/7x11x13) + - user: [Add `/comments` page support](https://github.com/yt-dlp/yt-dlp/commit/7bfb4f72e490310d2681c7f4815218a2ebbc73ee) ([#11999](https://github.com/yt-dlp/yt-dlp/issues/11999)) by [7x11x13](https://github.com/7x11x13) +- **subsplash**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/5d904b077d2f58ae44bdf208d2dcfcc3ff8347f5) ([#11054](https://github.com/yt-dlp/yt-dlp/issues/11054)) by [seproDev](https://github.com/seproDev), [subrat-lima](https://github.com/subrat-lima) +- **theatercomplextownppv**: [Support `live` URLs](https://github.com/yt-dlp/yt-dlp/commit/797d2472a299692e01ad1500e8c3b7bc1daa7fe4) ([#11720](https://github.com/yt-dlp/yt-dlp/issues/11720)) by [bashonly](https://github.com/bashonly) +- **vimeo**: [Fix thumbnail extraction](https://github.com/yt-dlp/yt-dlp/commit/9ff330948c92f6b2e1d9c928787362ab19cd6c62) ([#12142](https://github.com/yt-dlp/yt-dlp/issues/12142)) by [jixunmoe](https://github.com/jixunmoe) +- **vimp**: Playlist: [Add support for tags](https://github.com/yt-dlp/yt-dlp/commit/d4f5be1735c8feaeb3308666e0b878e9782f529d) ([#11688](https://github.com/yt-dlp/yt-dlp/issues/11688)) by [FestplattenSchnitzel](https://github.com/FestplattenSchnitzel) +- **weibo**: [Extend `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/a567f97b62ae9f6d6f5a9376c361512ab8dceda2) ([#12088](https://github.com/yt-dlp/yt-dlp/issues/12088)) by [4ft35t](https://github.com/4ft35t) +- **xhamster**: [Various improvements](https://github.com/yt-dlp/yt-dlp/commit/3b99a0f0e07f0120ab416f34a8f5ab75d4fdf1d1) ([#11738](https://github.com/yt-dlp/yt-dlp/issues/11738)) by [knackku](https://github.com/knackku) +- **xiaohongshu**: [Extract more formats](https://github.com/yt-dlp/yt-dlp/commit/f9f24ae376a9eaca777816479a4a29f6f0ce7681) ([#12147](https://github.com/yt-dlp/yt-dlp/issues/12147)) by [seproDev](https://github.com/seproDev) +- **youtube** + - [Download `tv` client Innertube config](https://github.com/yt-dlp/yt-dlp/commit/326fb1ffaf4e8349f1fe8ba2a81839652e044bff) ([#12168](https://github.com/yt-dlp/yt-dlp/issues/12168)) by [coletdjnz](https://github.com/coletdjnz) + - [Extract `media_type` for livestreams](https://github.com/yt-dlp/yt-dlp/commit/421bc72103d1faed473a451299cd17d6abb433bb) ([#11605](https://github.com/yt-dlp/yt-dlp/issues/11605)) by [nosoop](https://github.com/nosoop) + - [Restore convenience workarounds](https://github.com/yt-dlp/yt-dlp/commit/f0d4b8a5d6354b294bc9631cf15a7160b7bad5de) ([#12181](https://github.com/yt-dlp/yt-dlp/issues/12181)) by [bashonly](https://github.com/bashonly) + - [Update `ios` player client](https://github.com/yt-dlp/yt-dlp/commit/de82acf8769282ce321a86737ecc1d4bef0e82a7) ([#12155](https://github.com/yt-dlp/yt-dlp/issues/12155)) by [b5i](https://github.com/b5i) + - [Use different PO token for GVS and Player](https://github.com/yt-dlp/yt-dlp/commit/6b91d232e316efa406035915532eb126fbaeea38) ([#12090](https://github.com/yt-dlp/yt-dlp/issues/12090)) by [coletdjnz](https://github.com/coletdjnz) + - tab: [Improve shorts title extraction](https://github.com/yt-dlp/yt-dlp/commit/76ac023ff02f06e8c003d104f02a03deeddebdcd) ([#11997](https://github.com/yt-dlp/yt-dlp/issues/11997)) by [bashonly](https://github.com/bashonly), [d3d9](https://github.com/d3d9) +- **zdf**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/bb69f5dab79fb32c4ec0d50e05f7fa26d05d54ba) ([#11041](https://github.com/yt-dlp/yt-dlp/issues/11041)) by [InvalidUsernameException](https://github.com/InvalidUsernameException) + +#### Misc. changes +- **cleanup**: Miscellaneous: [3b45319](https://github.com/yt-dlp/yt-dlp/commit/3b4531934465580be22937fecbb6e1a3a9e2334f) by [bashonly](https://github.com/bashonly), [lonble](https://github.com/lonble), [pjrobertson](https://github.com/pjrobertson), [seproDev](https://github.com/seproDev) + +### 2025.01.15 + +#### Extractor changes +- **youtube**: [Do not use `web_creator` as a default client](https://github.com/yt-dlp/yt-dlp/commit/c8541f8b13e743fcfa06667530d13fee8686e22a) ([#12087](https://github.com/yt-dlp/yt-dlp/issues/12087)) by [bashonly](https://github.com/bashonly) + +### 2025.01.12 + +#### Core changes +- [Fix filename sanitization with `--no-windows-filenames`](https://github.com/yt-dlp/yt-dlp/commit/8346b549150003df988538e54c9d8bc4de568979) ([#11988](https://github.com/yt-dlp/yt-dlp/issues/11988)) by [bashonly](https://github.com/bashonly) +- [Validate retries values are non-negative](https://github.com/yt-dlp/yt-dlp/commit/1f4e1e85a27c5b43e34d7706cfd88ffce1b56a4a) ([#11927](https://github.com/yt-dlp/yt-dlp/issues/11927)) by [Strkmn](https://github.com/Strkmn) + +#### Extractor changes +- **drtalks**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/1f489f4a45691cac3f9e787d22a3a8a086229ba6) ([#10831](https://github.com/yt-dlp/yt-dlp/issues/10831)) by [pzhlkj6612](https://github.com/pzhlkj6612), [seproDev](https://github.com/seproDev) +- **plvideo**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/3c14e9191f3035b9a729d1d87bc0381f42de57cf) ([#10657](https://github.com/yt-dlp/yt-dlp/issues/10657)) by [Sanceilaks](https://github.com/Sanceilaks), [seproDev](https://github.com/seproDev) +- **vine**: [Remove extractors](https://github.com/yt-dlp/yt-dlp/commit/e2ef4fece6c9742d1733e3bae408c4787765f78c) ([#11700](https://github.com/yt-dlp/yt-dlp/issues/11700)) by [allendema](https://github.com/allendema) +- **xiaohongshu**: [Extend `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/763ed06ee69f13949397897bd42ff2ec3dc3d384) ([#11806](https://github.com/yt-dlp/yt-dlp/issues/11806)) by [HobbyistDev](https://github.com/HobbyistDev) +- **youtube** + - [Fix DASH formats incorrectly skipped in some situations](https://github.com/yt-dlp/yt-dlp/commit/0b6b7742c2e7f2a1fcb0b54ef3dd484bab404b3f) ([#11910](https://github.com/yt-dlp/yt-dlp/issues/11910)) by [coletdjnz](https://github.com/coletdjnz) + - [Refactor cookie auth](https://github.com/yt-dlp/yt-dlp/commit/75079f4e3f7dce49b61ef01da7adcd9876a0ca3b) ([#11989](https://github.com/yt-dlp/yt-dlp/issues/11989)) by [coletdjnz](https://github.com/coletdjnz) + - [Use `tv` instead of `mweb` client by default](https://github.com/yt-dlp/yt-dlp/commit/712d2abb32f59b2d246be2901255f84f1a4c30b3) ([#12059](https://github.com/yt-dlp/yt-dlp/issues/12059)) by [coletdjnz](https://github.com/coletdjnz) + +#### Misc. changes +- **cleanup**: Miscellaneous: [dade5e3](https://github.com/yt-dlp/yt-dlp/commit/dade5e35c89adaad04408bfef766820dbca06ebe) by [grqz](https://github.com/grqz), [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev) + +### 2024.12.23 + +#### Core changes +- [Don't sanitize filename on Unix when `--no-windows-filenames`](https://github.com/yt-dlp/yt-dlp/commit/6fc85f617a5850307fd5b258477070e6ee177796) ([#9591](https://github.com/yt-dlp/yt-dlp/issues/9591)) by [pukkandan](https://github.com/pukkandan) +- **update** + - [Check 64-bitness when upgrading ARM builds](https://github.com/yt-dlp/yt-dlp/commit/b91c3925c2059970daa801cb131c0c2f4f302e72) ([#11819](https://github.com/yt-dlp/yt-dlp/issues/11819)) by [bashonly](https://github.com/bashonly) + - [Fix endless update loop for `linux_exe` builds](https://github.com/yt-dlp/yt-dlp/commit/3d3ee458c1fe49dd5ebd7651a092119d23eb7000) ([#11827](https://github.com/yt-dlp/yt-dlp/issues/11827)) by [bashonly](https://github.com/bashonly) + +#### Extractor changes +- **soundcloud**: [Various fixes](https://github.com/yt-dlp/yt-dlp/commit/d298693b1b266d198e8eeecb90ea17c4a031268f) ([#11820](https://github.com/yt-dlp/yt-dlp/issues/11820)) by [bashonly](https://github.com/bashonly) +- **youtube** + - [Add age-gate workaround for some embeddable videos](https://github.com/yt-dlp/yt-dlp/commit/09a6c687126f04e243fcb105a828787efddd1030) ([#11821](https://github.com/yt-dlp/yt-dlp/issues/11821)) by [bashonly](https://github.com/bashonly) + - [Fix `uploader_id` extraction](https://github.com/yt-dlp/yt-dlp/commit/1a8851b689763e5173b96f70f8a71df0e4a44b66) ([#11818](https://github.com/yt-dlp/yt-dlp/issues/11818)) by [bashonly](https://github.com/bashonly) + - [Player client maintenance](https://github.com/yt-dlp/yt-dlp/commit/65cf46cddd873fd229dbb0fc0689bca4c201c6b6) ([#11893](https://github.com/yt-dlp/yt-dlp/issues/11893)) by [bashonly](https://github.com/bashonly) + - [Skip iOS formats that require PO Token](https://github.com/yt-dlp/yt-dlp/commit/9f42e68a74f3f00b0253fe70763abd57cac4237b) ([#11890](https://github.com/yt-dlp/yt-dlp/issues/11890)) by [coletdjnz](https://github.com/coletdjnz) + ### 2024.12.13 #### Extractor changes diff --git a/README.md b/README.md index 7c80fbbda..42209ecbb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ [![Release version](https://img.shields.io/github/v/release/yt-dlp/yt-dlp?color=brightgreen&label=Download&style=for-the-badge)](#installation "Installation") [![PyPI](https://img.shields.io/badge/-PyPI-blue.svg?logo=pypi&labelColor=555555&style=for-the-badge)](https://pypi.org/project/yt-dlp "PyPI") [![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)](Collaborators.md#collaborators "Donate") -[![Matrix](https://img.shields.io/matrix/yt-dlp:matrix.org?color=brightgreen&labelColor=555555&label=&logo=element&style=for-the-badge)](https://matrix.to/#/#yt-dlp:matrix.org "Matrix") [![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)](https://discord.gg/H5MNcFW63r "Discord") [![Supported Sites](https://img.shields.io/badge/-Supported_Sites-brightgreen.svg?style=for-the-badge)](supportedsites.md "Supported Sites") [![License: Unlicense](https://img.shields.io/badge/-Unlicense-blue.svg?style=for-the-badge)](LICENSE "License") @@ -338,10 +337,11 @@ ## General Options: --plugin-dirs PATH Path to an additional directory to search for plugins. This option can be used multiple times to add multiple directories. - Note that this currently only works for - extractor plugins; postprocessor plugins can - only be loaded from the default plugin - directories + Use "default" to search the default plugin + directories (default) + --no-plugin-dirs Clear plugin directories to search, + including defaults and those provided by + previous --plugin-dirs --flat-playlist Do not extract a playlist's URL result entries; some entry metadata may be missing and downloading may be bypassed @@ -613,8 +613,7 @@ ## Filesystem Options: --no-restrict-filenames Allow Unicode characters, "&" and spaces in filenames (default) --windows-filenames Force filenames to be Windows-compatible - --no-windows-filenames Make filenames Windows-compatible only if - using Windows (default) + --no-windows-filenames Sanitize filenames only minimally --trim-filenames LENGTH Limit the filename length (excluding extension) to the specified number of characters @@ -1527,7 +1526,7 @@ ## Sorting Formats - `hasvid`: Gives priority to formats that have a video stream - `hasaud`: Gives priority to formats that have an audio stream - `ie_pref`: The format preference - - `lang`: The language preference + - `lang`: The language preference as determined by the extractor (e.g. original language preferred over audio description) - `quality`: The quality of the format - `source`: The preference of the source - `proto`: Protocol used for download (`https`/`ftps` > `http`/`ftp` > `m3u8_native`/`m3u8` > `http_dash_segments`> `websocket_frag` > `mms`/`rtsp` > `f4f`/`f4m`) @@ -1761,7 +1760,7 @@ # Replace all spaces and "_" in title and uploader with a `-` # EXTRACTOR ARGUMENTS -Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. E.g. `--extractor-args "youtube:player-client=tv,mweb;formats=incomplete" --extractor-args "funimation:version=uncut"` +Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. E.g. `--extractor-args "youtube:player-client=tv,mweb;formats=incomplete" --extractor-args "twitter:api=syndication"` Note: In CLI, `ARG` can use `-` instead of `_`; e.g. `youtube:player-client"` becomes `youtube:player_client"` @@ -1770,19 +1769,20 @@ # EXTRACTOR ARGUMENTS #### youtube * `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube.py](https://github.com/yt-dlp/yt-dlp/blob/c26f9b991a0681fd3ea548d535919cec1fbbd430/yt_dlp/extractor/youtube.py#L381-L390) for list of supported content language codes * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively -* `player_client`: Clients to extract video data from. The main clients are `web`, `ios` and `android`, with variants `_music` and `_creator` (e.g. `ios_creator`); and `mweb`, `android_vr`, `web_safari`, `web_embedded`, `tv` and `tv_embedded` with no variants. By default, `ios,mweb` is used, or `web_creator,mweb` is used when authenticating with cookies. The `_music` variants are added for `music.youtube.com` URLs. Some clients, such as `web` and `android`, require a `po_token` for their formats to be downloadable. Some clients, such as the `_creator` variants, will only work with authentication. Not all clients support authentication via cookies. You can use `all` to use all the clients, and `default` for the default clients. You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=all,-web` -* `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details +* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `android`, `android_vr`, `tv` and `tv_embedded`. By default, `tv,ios,web` is used, or `tv,web` is used when authenticating with cookies. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `tv_embedded` and `web_creator` clients are added for age-restricted videos if account age-verification is required. Some clients, such as `web` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-ios` +* `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player), `initial_data` (skip initial data/next ep request). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause issues such as missing formats or metadata. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) and [#12826](https://github.com/yt-dlp/yt-dlp/issues/12826) for more details * `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp. * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side) * `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all` * E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total -* `formats`: Change the types of formats to return. `dashy` (convert HTTP to DASH), `duplicate` (identical content but different URLs or protocol; includes `dashy`), `incomplete` (cannot be downloaded completely - live dash and post-live m3u8) +* `formats`: Change the types of formats to return. `dashy` (convert HTTP to DASH), `duplicate` (identical content but different URLs or protocol; includes `dashy`), `incomplete` (cannot be downloaded completely - live dash and post-live m3u8), `missing_pot` (include formats that require a PO Token but are missing one) * `innertube_host`: Innertube API host to use for all API requests; e.g. `studio.youtube.com`, `youtubei.googleapis.com`. Note that cookies exported from one subdomain will not work on others * `innertube_key`: Innertube API key to use for all API requests. By default, no API key is used * `raise_incomplete_data`: `Incomplete Data Received` raises an error instead of reporting a warning * `data_sync_id`: Overrides the account Data Sync ID used in Innertube API requests. This may be needed if you are using an account with `youtube:player_skip=webpage,configs` or `youtubetab:skip=webpage` * `visitor_data`: Overrides the Visitor Data used in Innertube API requests. This should be used with `player_skip=webpage,configs` and without cookies. Note: this may have adverse effects if used improperly. If a session from a browser is wanted, you should pass cookies instead (which contain the Visitor ID) -* `po_token`: Proof of Origin (PO) Token(s) to use for requesting video playback. Comma seperated list of PO Tokens in the format `CLIENT+PO_TOKEN`, e.g. `youtube:po_token=web+XXX,android+YYY` +* `po_token`: Proof of Origin (PO) Token(s) to use. Comma seperated list of PO Tokens in the format `CLIENT.CONTEXT+PO_TOKEN`, e.g. `youtube:po_token=web.gvs+XXX,web.player=XXX,web_safari.gvs+YYY`. Context can be either `gvs` (Google Video Server URLs) or `player` (Innertube player request) +* `player_js_variant`: The player javascript variant to use for signature and nsig deciphering. The known variants are: `main`, `tce`, `tv`, `tv_es6`, `phone`, `tablet`. Only `main` is recommended as a possible workaround; the others are for debugging purposes. The default is to use what is prescribed by the site, and can be selected with `actual` #### youtubetab (YouTube playlists, channels, feeds, etc.) * `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details) @@ -1796,13 +1796,6 @@ #### generic * `is_live`: Bypass live HLS detection and manually set `live_status` - a value of `false` will set `not_live`, any other value (or no value) will set `is_live` * `impersonate`: Target(s) to try and impersonate with the initial webpage request; e.g. `generic:impersonate=safari,chrome-110`. Use `generic:impersonate` to impersonate any available target, and use `generic:impersonate=false` to disable impersonation (default) -#### funimation -* `language`: Audio languages to extract, e.g. `funimation:language=english,japanese` -* `version`: The video version to extract - `uncut` or `simulcast` - -#### crunchyrollbeta (Crunchyroll) -* `hardsub`: One or more hardsub versions to extract (in order of preference), or `all` (default: `None` = no hardsubs will be extracted), e.g. `crunchyrollbeta:hardsub=en-US,de-DE` - #### vikichannel * `video_types`: Types of videos to download - one or more of `episodes`, `movies`, `clips`, `trailers` @@ -1820,6 +1813,9 @@ #### hotstar * `vcodec`: vcodec to ignore - one or more of `h264`, `h265`, `dvh265` * `dr`: dynamic range to ignore - one or more of `sdr`, `hdr10`, `dv` +#### instagram +* `app_id`: The value of the `X-IG-App-ID` header used for API requests. Default is the web app ID, `936619743392459` + #### niconicochannelplus * `max_comments`: Maximum number of comments to extract - default is `120` @@ -1871,6 +1867,9 @@ #### bilibili #### sonylivseries * `sort_order`: Episode sort order for series extraction - one of `asc` (ascending, oldest first) or `desc` (descending, newest first). Default is `asc` +#### tver +* `backend`: Backend API to use for extraction - one of `streaks` (default) or `brightcove` (deprecated) + **Note**: These options may be changed/removed in the future without concern for backward compatibility @@ -2220,7 +2219,7 @@ ### Differences in default behavior * Live chats (if available) are considered as subtitles. Use `--sub-langs all,-live_chat` to download all subtitles except live chat. You can also use `--compat-options no-live-chat` to prevent any live chat/danmaku from downloading * YouTube channel URLs download all uploads of the channel. To download only the videos in a specific tab, pass the tab's URL. If the channel does not show the requested tab, an error will be raised. Also, `/live` URLs raise an error if there are no live videos instead of silently downloading the entire channel. You may use `--compat-options no-youtube-channel-redirect` to revert all these redirections * Unavailable videos are also listed for YouTube playlists. Use `--compat-options no-youtube-unavailable-videos` to remove this -* The upload dates extracted from YouTube are in UTC [when available](https://github.com/yt-dlp/yt-dlp/blob/89e4d86171c7b7c997c77d4714542e0383bf0db0/yt_dlp/extractor/youtube.py#L3898-L3900). Use `--compat-options no-youtube-prefer-utc-upload-date` to prefer the non-UTC upload date. +* The upload dates extracted from YouTube are in UTC. * If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this * Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead * Some internal metadata such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this @@ -2239,9 +2238,10 @@ ### Differences in default behavior * `--compat-options all`: Use all compat options (**Do NOT use this!**) * `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort` * `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort` -* `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date` +* `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization` * `--compat-options 2022`: Same as `--compat-options 2023,playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx` -* `--compat-options 2023`: Same as `--compat-options prefer-vp9-sort`. Use this to enable all future compat options +* `--compat-options 2023`: Same as `--compat-options 2024,prefer-vp9-sort` +* `--compat-options 2024`: Currently does nothing. Use this to enable all future compat options The following compat options restore vulnerable behavior from before security patches: diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json index 079e2f729..8aa7b7e2b 100644 --- a/devscripts/changelog_override.json +++ b/devscripts/changelog_override.json @@ -239,5 +239,11 @@ "action": "add", "when": "52c0ffe40ad6e8404d93296f575007b05b04c686", "short": "[priority] **Login with OAuth is no longer supported for YouTube**\nDue to a change made by the site, yt-dlp is no longer able to support OAuth login for YouTube. [Read more](https://github.com/yt-dlp/yt-dlp/issues/11462#issuecomment-2471703090)" + }, + { + "action": "change", + "when": "76ac023ff02f06e8c003d104f02a03deeddebdcd", + "short": "[ie/youtube:tab] Improve shorts title extraction (#11997)", + "authors": ["bashonly", "d3d9"] } ] diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 2a418ddbf..110fcc245 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -11,11 +11,13 @@ from devscripts.utils import get_filename_args, read_file, write_file -VERBOSE_TMPL = ''' +VERBOSE = ''' - type: checkboxes id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -47,31 +49,23 @@ render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. '''.strip() NO_SKIP = ''' - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. '''.strip() def main(): - fields = {'no_skip': NO_SKIP} - fields['verbose'] = VERBOSE_TMPL % fields - fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose']) + fields = { + 'no_skip': NO_SKIP, + 'verbose': VERBOSE, + 'verbose_optional': re.sub(r'(\n\s+validations:)?\n\s+required: true', '', VERBOSE), + } infile, outfile = get_filename_args(has_infile=True) write_file(outfile, read_file(infile) % fields) diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index d288d8429..0ce773e82 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -10,6 +10,9 @@ from inspect import getsource from devscripts.utils import get_filename_args, read_file, write_file +from yt_dlp.extractor import import_extractors +from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor +from yt_dlp.globals import extractors NO_ATTR = object() STATIC_CLASS_PROPERTIES = [ @@ -38,8 +41,7 @@ def main(): lazy_extractors_filename = get_filename_args(default_outfile='yt_dlp/extractor/lazy_extractors.py') - from yt_dlp.extractor.extractors import _ALL_CLASSES - from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor + import_extractors() DummyInfoExtractor = type('InfoExtractor', (InfoExtractor,), {'IE_NAME': NO_ATTR}) module_src = '\n'.join(( @@ -47,7 +49,7 @@ def main(): ' _module = None', *extra_ie_code(DummyInfoExtractor), '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n', - *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor), + *build_ies(list(extractors.value.values()), (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor), )) write_file(lazy_extractors_filename, f'{module_src}\n') @@ -73,7 +75,7 @@ def build_ies(ies, bases, attr_base): if ie in ies: names.append(ie.__name__) - yield f'\n_ALL_CLASSES = [{", ".join(names)}]' + yield '\n_CLASS_LOOKUP = {%s}' % ', '.join(f'{name!r}: {name}' for name in names) def sort_ies(ies, ignored_bases): diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 01548ef97..145f6d47f 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -10,10 +10,21 @@ from devscripts.utils import get_filename_args, write_file from yt_dlp.extractor import list_extractor_classes +TEMPLATE = '''\ +# Supported sites + +Below is a list of all extractors that are currently included with yt-dlp. +If a site is not listed here, it might still be supported by yt-dlp's embed extraction or generic extractor. +Not all sites listed here are guaranteed to work; websites are constantly changing and sometimes this breaks yt-dlp's support for them. +The only reliable way to check if a site is supported is to try it. + +{ie_list} +''' + def main(): out = '\n'.join(ie.description() for ie in list_extractor_classes() if ie.IE_DESC is not False) - write_file(get_filename_args(), f'# Supported sites\n{out}\n') + write_file(get_filename_args(), TEMPLATE.format(ie_list=out)) if __name__ == '__main__': diff --git a/devscripts/run_tests.py b/devscripts/run_tests.py index eb614fe59..ebb3500b6 100755 --- a/devscripts/run_tests.py +++ b/devscripts/run_tests.py @@ -25,7 +25,8 @@ def parse_args(): def run_tests(*tests, pattern=None, ci=False): - run_core = 'core' in tests or (not pattern and not tests) + # XXX: hatch uses `tests` if no arguments are passed + run_core = 'core' in tests or 'tests' in tests or (not pattern and not tests) run_download = 'download' in tests pytest_args = args.pytest_args or os.getenv('HATCH_TEST_ARGS', '') diff --git a/pyproject.toml b/pyproject.toml index 96e2d669a..5e987a6fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,8 +55,7 @@ default = [ "websockets>=13.0", ] curl-cffi = [ - "curl-cffi==0.5.10; os_name=='nt' and implementation_name=='cpython'", - "curl-cffi>=0.5.10,!=0.6.*,<0.7.2; os_name!='nt' and implementation_name=='cpython'", + "curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.11; implementation_name=='cpython'", ] secretstorage = [ "cffi", @@ -76,7 +75,7 @@ dev = [ ] static-analysis = [ "autopep8~=2.0", - "ruff~=0.8.0", + "ruff~=0.11.0", ] test = [ "pytest~=8.1", @@ -195,6 +194,7 @@ ignore = [ "B023", # function-uses-loop-variable (false positives) "B028", # no-explicit-stacklevel "B904", # raise-without-from-inside-except + "A005", # stdlib-module-shadowing "C401", # unnecessary-generator-set "C402", # unnecessary-generator-dict "PIE790", # unnecessary-placeholder @@ -383,9 +383,14 @@ select = [ "W391", "W504", ] +exclude = "*/extractor/lazy_extractors.py,*venv*,*/test/testdata/sigs/player-*.js,.idea,.vscode" [tool.pytest.ini_options] -addopts = "-ra -v --strict-markers" +addopts = [ + "-ra", # summary: all except passed + "--verbose", + "--strict-markers", +] markers = [ "download", ] diff --git a/supportedsites.md b/supportedsites.md index 916735e08..44c6ef5a1 100644 --- a/supportedsites.md +++ b/supportedsites.md @@ -1,6 +1,13 @@ # Supported sites + +Below is a list of all extractors that are currently included with yt-dlp. +If a site is not listed here, it might still be supported by yt-dlp's embed extraction or generic extractor. +Not all sites listed here are guaranteed to work; websites are constantly changing and sometimes this breaks yt-dlp's support for them. +The only reliable way to check if a site is supported is to try it. + - **17live** - **17live:clip** + - **17live:vod** - **1News**: 1news.co.nz article videos - **1tv**: Первый канал - **20min** @@ -171,6 +178,7 @@ # Supported sites - **BilibiliCheese** - **BilibiliCheeseSeason** - **BilibiliCollectionList** + - **BiliBiliDynamic** - **BilibiliFavoritesList** - **BiliBiliPlayer** - **BilibiliPlaylist** @@ -193,7 +201,7 @@ # Supported sites - **blogger.com** - **Bloomberg** - **Bluesky** - - **BokeCC** + - **BokeCC**: CC视频 - **BongaCams** - **Boosty** - **BostonGlobe** @@ -217,6 +225,7 @@ # Supported sites - **bt:vestlendingen**: Bergens Tidende - Vestlendingen - **Bundesliga** - **Bundestag** + - **BunnyCdn** - **BusinessInsider** - **BuzzFeed** - **BYUtv**: (**Currently broken**) @@ -235,6 +244,7 @@ # Supported sites - **CanalAlpha** - **canalc2.tv** - **Canalplus**: mycanal.fr and piwiplus.fr + - **Canalsurmas** - **CaracolTvPlay**: [*caracoltv-play*](## "netrc machine") - **CartoonNetwork** - **cbc.ca** @@ -303,10 +313,6 @@ # Supported sites - **CrowdBunker** - **CrowdBunkerChannel** - **Crtvg** - - **crunchyroll**: [*crunchyroll*](## "netrc machine") - - **crunchyroll:artist**: [*crunchyroll*](## "netrc machine") - - **crunchyroll:music**: [*crunchyroll*](## "netrc machine") - - **crunchyroll:playlist**: [*crunchyroll*](## "netrc machine") - **CSpan**: C-SPAN - **CSpanCongress** - **CtsNews**: 華視新聞 @@ -317,7 +323,8 @@ # Supported sites - **curiositystream**: [*curiositystream*](## "netrc machine") - **curiositystream:collections**: [*curiositystream*](## "netrc machine") - **curiositystream:series**: [*curiositystream*](## "netrc machine") - - **CWTV** + - **cwtv** + - **cwtv:movie** - **Cybrary**: [*cybrary*](## "netrc machine") - **CybraryCourse**: [*cybrary*](## "netrc machine") - **DacastPlaylist** @@ -341,8 +348,6 @@ # Supported sites - **daystar:clip** - **DBTV** - **DctpTv** - - **DeezerAlbum** - - **DeezerPlaylist** - **democracynow** - **DestinationAmerica** - **DetikEmbed** @@ -352,6 +357,7 @@ # Supported sites - **DigitalConcertHall**: [*digitalconcerthall*](## "netrc machine") DigitalConcertHall extractor - **DigitallySpeaking** - **Digiteka** + - **Digiview** - **DiscogsReleasePlaylist** - **DiscoveryLife** - **DiscoveryNetworksDe** @@ -374,6 +380,7 @@ # Supported sites - **Dropbox** - **Dropout**: [*dropout*](## "netrc machine") - **DropoutSeason** + - **DrTalks** - **DrTuber** - **drtv** - **drtv:live** @@ -392,6 +399,8 @@ # Supported sites - **Ebay** - **egghead:course**: egghead.io course - **egghead:lesson**: egghead.io lesson + - **eggs:artist** + - **eggs:single** - **EinsUndEinsTV**: [*1und1tv*](## "netrc machine") - **EinsUndEinsTVLive**: [*1und1tv*](## "netrc machine") - **EinsUndEinsTVRecordings**: [*1und1tv*](## "netrc machine") @@ -463,11 +472,12 @@ # Supported sites - **FoxNewsVideo** - **FoxSports** - **fptplay**: fptplay.vn + - **FrancaisFacile** - **FranceCulture** - **FranceInter** - - **FranceTV** + - **francetv** + - **francetv:site** - **francetvinfo.fr** - - **FranceTVSite** - **Freesound** - **freespeech.org** - **freetv:series** @@ -476,9 +486,6 @@ # Supported sites - **FrontendMastersCourse**: [*frontendmasters*](## "netrc machine") - **FrontendMastersLesson**: [*frontendmasters*](## "netrc machine") - **FujiTVFODPlus7** - - **Funimation**: [*funimation*](## "netrc machine") - - **funimation:page**: [*funimation*](## "netrc machine") - - **funimation:show**: [*funimation*](## "netrc machine") - **Funk** - **Funker530** - **Fux** @@ -502,7 +509,7 @@ # Supported sites - **GediDigital** - **gem.cbc.ca**: [*cbcgem*](## "netrc machine") - **gem.cbc.ca:live** - - **gem.cbc.ca:playlist** + - **gem.cbc.ca:playlist**: [*cbcgem*](## "netrc machine") - **Genius** - **GeniusLyrics** - **Germanupa**: germanupa.de @@ -604,10 +611,10 @@ # Supported sites - **Inc** - **IndavideoEmbed** - **InfoQ** - - **Instagram**: [*instagram*](## "netrc machine") - - **instagram:story**: [*instagram*](## "netrc machine") - - **instagram:tag**: [*instagram*](## "netrc machine") Instagram hashtag search URLs - - **instagram:user**: [*instagram*](## "netrc machine") Instagram user profile (**Currently broken**) + - **Instagram** + - **instagram:story** + - **instagram:tag**: Instagram hashtag search URLs + - **instagram:user**: Instagram user profile (**Currently broken**) - **InstagramIOS**: IOS instagram:// URL - **Internazionale** - **InternetVideoArchive** @@ -656,7 +663,6 @@ # Supported sites - **KelbyOne**: (**Currently broken**) - **Kenh14Playlist** - **Kenh14Video** - - **Ketnet** - **khanacademy** - **khanacademy:unit** - **kick:clips** @@ -728,6 +734,7 @@ # Supported sites - **Livestreamfails** - **Lnk** - **loc**: Library of Congress + - **Loco** - **loom** - **loom:folder** - **LoveHomePorn** @@ -822,11 +829,11 @@ # Supported sites - **MotherlessUploader** - **Motorsport**: motorsport.com (**Currently broken**) - **MovieFap** - - **Moviepilot** + - **moviepilot**: Moviepilot trailer - **MoviewPlay** - **Moviezine** - **MovingImage** - - **MSN**: (**Currently broken**) + - **MSN** - **mtg**: MTG services - **mtv** - **mtv.de**: (**Currently broken**) @@ -891,6 +898,8 @@ # Supported sites - **nebula:video**: [*watchnebula*](## "netrc machine") - **NekoHacker** - **NerdCubedFeed** + - **Nest** + - **NestClip** - **netease:album**: 网易云音乐 - 专辑 - **netease:djradio**: 网易云音乐 - 电台 - **netease:mv**: 网易云音乐 - MV @@ -1070,6 +1079,8 @@ # Supported sites - **Pinkbike** - **Pinterest** - **PinterestCollection** + - **PiramideTV** + - **PiramideTVChannel** - **pixiv:sketch** - **pixiv:​sketch:user** - **Pladform** @@ -1086,6 +1097,7 @@ # Supported sites - **pluralsight**: [*pluralsight*](## "netrc machine") - **pluralsight:course** - **PlutoTV**: (**Currently broken**) + - **PlVideo**: Платформа - **PodbayFM** - **PodbayFMChannel** - **Podchaser** @@ -1240,7 +1252,6 @@ # Supported sites - **rtve.es:infantil**: RTVE infantil - **rtve.es:live**: RTVE.es live streams - **rtve.es:television** - - **RTVS** - **rtvslo.si** - **rtvslo.si:show** - **RudoVideo** @@ -1295,8 +1306,8 @@ # Supported sites - **sejm** - **Sen** - **SenalColombiaLive**: (**Currently broken**) - - **SenateGov** - - **SenateISVP** + - **senate.gov** + - **senate.gov:isvp** - **SendtoNews**: (**Currently broken**) - **Servus** - **Sexu**: (**Currently broken**) @@ -1332,6 +1343,7 @@ # Supported sites - **Smotrim** - **SnapchatSpotlight** - **Snotr** + - **SoftWhiteUnderbelly**: [*softwhiteunderbelly*](## "netrc machine") - **Sohu** - **SohuV** - **SonyLIV**: [*sonyliv*](## "netrc machine") @@ -1388,12 +1400,16 @@ # Supported sites - **StoryFire** - **StoryFireSeries** - **StoryFireUser** + - **Streaks** - **Streamable** - **StreamCZ** - **StreetVoice** - **StretchInternet** - **Stripchat** - **stv:player** + - **stvr**: Slovak Television and Radio (formerly RTVS) + - **Subsplash** + - **subsplash:playlist** - **Substack** - **SunPorno** - **sverigesradio:episode** @@ -1524,6 +1540,8 @@ # Supported sites - **tv5unis** - **tv5unis:video** - **tv8.it** + - **tv8.it:live**: TV8 Live + - **tv8.it:playlist**: TV8 Playlist - **TVANouvelles** - **TVANouvellesArticle** - **tvaplus**: TVA+ @@ -1544,6 +1562,7 @@ # Supported sites - **tvp:​vod:series** - **TVPlayer** - **TVPlayHome** + - **Tvw** - **Tweakers** - **TwitCasting** - **TwitCastingLive** @@ -1625,8 +1644,6 @@ # Supported sites - **viewlift** - **viewlift:embed** - **Viidea** - - **viki**: [*viki*](## "netrc machine") - - **viki:channel**: [*viki*](## "netrc machine") - **vimeo**: [*vimeo*](## "netrc machine") - **vimeo:album**: [*vimeo*](## "netrc machine") - **vimeo:channel**: [*vimeo*](## "netrc machine") @@ -1641,8 +1658,6 @@ # Supported sites - **Vimm:stream** - **ViMP** - **ViMP:Playlist** - - **Vine** - - **vine:user** - **Viously** - **Viqeo**: (**Currently broken**) - **Viu** @@ -1666,8 +1681,12 @@ # Supported sites - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **vqq:series** - **vqq:video** + - **vrsquare**: VR SQUARE + - **vrsquare:channel** + - **vrsquare:search** + - **vrsquare:section** - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza - - **VrtNU**: [*vrtnu*](## "netrc machine") VRT MAX + - **vrtmax**: [*vrtnu*](## "netrc machine") VRT MAX (formerly VRT NU) - **VTM**: (**Currently broken**) - **VTV** - **VTVGo** diff --git a/test/helper.py b/test/helper.py index c776e70b7..e4cb478e2 100644 --- a/test/helper.py +++ b/test/helper.py @@ -101,87 +101,109 @@ def getwebpagetestcases(): md5 = lambda s: hashlib.md5(s.encode()).hexdigest() -def expect_value(self, got, expected, field): - if isinstance(expected, str) and expected.startswith('re:'): - match_str = expected[len('re:'):] - match_rex = re.compile(match_str) +def _iter_differences(got, expected, field): + if isinstance(expected, str): + op, _, val = expected.partition(':') + if op in ('mincount', 'maxcount', 'count'): + if not isinstance(got, (list, dict)): + yield field, f'expected either {list.__name__} or {dict.__name__}, got {type(got).__name__}' + return - self.assertTrue( - isinstance(got, str), - f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}') - self.assertTrue( - match_rex.match(got), - f'field {field} (value: {got!r}) should match {match_str!r}') - elif isinstance(expected, str) and expected.startswith('startswith:'): - start_str = expected[len('startswith:'):] - self.assertTrue( - isinstance(got, str), - f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}') - self.assertTrue( - got.startswith(start_str), - f'field {field} (value: {got!r}) should start with {start_str!r}') - elif isinstance(expected, str) and expected.startswith('contains:'): - contains_str = expected[len('contains:'):] - self.assertTrue( - isinstance(got, str), - f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}') - self.assertTrue( - contains_str in got, - f'field {field} (value: {got!r}) should contain {contains_str!r}') - elif isinstance(expected, type): - self.assertTrue( - isinstance(got, expected), - f'Expected type {expected!r} for field {field}, but got value {got!r} of type {type(got)!r}') - elif isinstance(expected, dict) and isinstance(got, dict): - expect_dict(self, got, expected) - elif isinstance(expected, list) and isinstance(got, list): - self.assertEqual( - len(expected), len(got), - f'Expect a list of length {len(expected)}, but got a list of length {len(got)} for field {field}') - for index, (item_got, item_expected) in enumerate(zip(got, expected)): - type_got = type(item_got) - type_expected = type(item_expected) - self.assertEqual( - type_expected, type_got, - f'Type mismatch for list item at index {index} for field {field}, ' - f'expected {type_expected!r}, got {type_got!r}') - expect_value(self, item_got, item_expected, field) - else: - if isinstance(expected, str) and expected.startswith('md5:'): - self.assertTrue( - isinstance(got, str), - f'Expected field {field} to be a unicode object, but got value {got!r} of type {type(got)!r}') - got = 'md5:' + md5(got) - elif isinstance(expected, str) and re.match(r'^(?:min|max)?count:\d+', expected): - self.assertTrue( - isinstance(got, (list, dict)), - f'Expected field {field} to be a list or a dict, but it is of type {type(got).__name__}') - op, _, expected_num = expected.partition(':') - expected_num = int(expected_num) + expected_num = int(val) + got_num = len(got) if op == 'mincount': - assert_func = assertGreaterEqual - msg_tmpl = 'Expected %d items in field %s, but only got %d' - elif op == 'maxcount': - assert_func = assertLessEqual - msg_tmpl = 'Expected maximum %d items in field %s, but got %d' - elif op == 'count': - assert_func = assertEqual - msg_tmpl = 'Expected exactly %d items in field %s, but got %d' - else: - assert False - assert_func( - self, len(got), expected_num, - msg_tmpl % (expected_num, field, len(got))) + if got_num < expected_num: + yield field, f'expected at least {val} items, got {got_num}' + return + + if op == 'maxcount': + if got_num > expected_num: + yield field, f'expected at most {val} items, got {got_num}' + return + + assert op == 'count' + if got_num != expected_num: + yield field, f'expected exactly {val} items, got {got_num}' return - self.assertEqual( - expected, got, - f'Invalid value for field {field}, expected {expected!r}, got {got!r}') + + if not isinstance(got, str): + yield field, f'expected {str.__name__}, got {type(got).__name__}' + return + + if op == 're': + if not re.match(val, got): + yield field, f'should match {val!r}, got {got!r}' + return + + if op == 'startswith': + if not got.startswith(val): + yield field, f'should start with {val!r}, got {got!r}' + return + + if op == 'contains': + if not val.startswith(got): + yield field, f'should contain {val!r}, got {got!r}' + return + + if op == 'md5': + hash_val = md5(got) + if hash_val != val: + yield field, f'expected hash {val}, got {hash_val}' + return + + if got != expected: + yield field, f'expected {expected!r}, got {got!r}' + return + + if isinstance(expected, dict) and isinstance(got, dict): + for key, expected_val in expected.items(): + if key not in got: + yield field, f'missing key: {key!r}' + continue + + field_name = key if field is None else f'{field}.{key}' + yield from _iter_differences(got[key], expected_val, field_name) + return + + if isinstance(expected, type): + if not isinstance(got, expected): + yield field, f'expected {expected.__name__}, got {type(got).__name__}' + return + + if isinstance(expected, list) and isinstance(got, list): + # TODO: clever diffing algorithm lmao + if len(expected) != len(got): + yield field, f'expected length of {len(expected)}, got {len(got)}' + return + + for index, (got_val, expected_val) in enumerate(zip(got, expected)): + field_name = str(index) if field is None else f'{field}.{index}' + yield from _iter_differences(got_val, expected_val, field_name) + return + + if got != expected: + yield field, f'expected {expected!r}, got {got!r}' + + +def _expect_value(message, got, expected, field): + mismatches = list(_iter_differences(got, expected, field)) + if not mismatches: + return + + fields = [field for field, _ in mismatches if field is not None] + return ''.join(( + message, f' ({", ".join(fields)})' if fields else '', + *(f'\n\t{field}: {message}' for field, message in mismatches))) + + +def expect_value(self, got, expected, field): + if message := _expect_value('values differ', got, expected, field): + self.fail(message) def expect_dict(self, got_dict, expected_dict): - for info_field, expected in expected_dict.items(): - got = got_dict.get(info_field) - expect_value(self, got, expected, info_field) + if message := _expect_value('dictionaries differ', got_dict, expected_dict, None): + self.fail(message) def sanitize_got_info_dict(got_dict): @@ -237,6 +259,20 @@ def sanitize(key, value): def expect_info_dict(self, got_dict, expected_dict): + ALLOWED_KEYS_SORT_ORDER = ( + # NB: Keep in sync with the docstring of extractor/common.py + 'id', 'ext', 'direct', 'display_id', 'title', 'alt_title', 'description', 'media_type', + 'uploader', 'uploader_id', 'uploader_url', 'channel', 'channel_id', 'channel_url', 'channel_is_verified', + 'channel_follower_count', 'comment_count', 'view_count', 'concurrent_view_count', + 'like_count', 'dislike_count', 'repost_count', 'average_rating', 'age_limit', 'duration', 'thumbnail', 'heatmap', + 'chapters', 'chapter', 'chapter_number', 'chapter_id', 'start_time', 'end_time', 'section_start', 'section_end', + 'categories', 'tags', 'cast', 'composers', 'artists', 'album_artists', 'creators', 'genres', + 'track', 'track_number', 'track_id', 'album', 'album_type', 'disc_number', + 'series', 'series_id', 'season', 'season_number', 'season_id', 'episode', 'episode_number', 'episode_id', + 'timestamp', 'upload_date', 'release_timestamp', 'release_date', 'release_year', 'modified_timestamp', 'modified_date', + 'playable_in_embed', 'availability', 'live_status', 'location', 'license', '_old_archive_ids', + ) + expect_dict(self, got_dict, expected_dict) # Check for the presence of mandatory fields if got_dict.get('_type') not in ('playlist', 'multi_video'): @@ -252,7 +288,13 @@ def expect_info_dict(self, got_dict, expected_dict): test_info_dict = sanitize_got_info_dict(got_dict) - missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys()) + # Check for invalid/misspelled field names being returned by the extractor + invalid_keys = sorted(test_info_dict.keys() - ALLOWED_KEYS_SORT_ORDER) + self.assertFalse(invalid_keys, f'Invalid fields returned by the extractor: {", ".join(invalid_keys)}') + + missing_keys = sorted( + test_info_dict.keys() - expected_dict.keys(), + key=lambda x: ALLOWED_KEYS_SORT_ORDER.index(x)) if missing_keys: def _repr(v): if isinstance(v, str): diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index 54f35ef55..c6ff6209a 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -638,6 +638,7 @@ def test_parse_m3u8_formats(self): 'img_bipbop_adv_example_fmp4', 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8', [{ + # 60kbps (bitrate not provided in m3u8); sorted as worst because it's grouped with lowest bitrate video track 'format_id': 'aud1-English', 'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a1/prog_index.m3u8', 'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8', @@ -645,15 +646,9 @@ def test_parse_m3u8_formats(self): 'ext': 'mp4', 'protocol': 'm3u8_native', 'audio_ext': 'mp4', + 'source_preference': 0, }, { - 'format_id': 'aud2-English', - 'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a2/prog_index.m3u8', - 'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8', - 'language': 'en', - 'ext': 'mp4', - 'protocol': 'm3u8_native', - 'audio_ext': 'mp4', - }, { + # 192kbps (bitrate not provided in m3u8) 'format_id': 'aud3-English', 'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a3/prog_index.m3u8', 'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8', @@ -661,6 +656,17 @@ def test_parse_m3u8_formats(self): 'ext': 'mp4', 'protocol': 'm3u8_native', 'audio_ext': 'mp4', + 'source_preference': 1, + }, { + # 384kbps (bitrate not provided in m3u8); sorted as best because it's grouped with the highest bitrate video track + 'format_id': 'aud2-English', + 'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a2/prog_index.m3u8', + 'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8', + 'language': 'en', + 'ext': 'mp4', + 'protocol': 'm3u8_native', + 'audio_ext': 'mp4', + 'source_preference': 2, }, { 'format_id': '530', 'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v2/prog_index.m3u8', diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 966d27a49..708a04f92 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -6,6 +6,8 @@ import unittest from unittest.mock import patch +from yt_dlp.globals import all_plugins_loaded + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -486,11 +488,11 @@ def assert_syntax_error(format_spec): def test_format_filtering(self): formats = [ - {'format_id': 'A', 'filesize': 500, 'width': 1000}, - {'format_id': 'B', 'filesize': 1000, 'width': 500}, - {'format_id': 'C', 'filesize': 1000, 'width': 400}, - {'format_id': 'D', 'filesize': 2000, 'width': 600}, - {'format_id': 'E', 'filesize': 3000}, + {'format_id': 'A', 'filesize': 500, 'width': 1000, 'aspect_ratio': 1.0}, + {'format_id': 'B', 'filesize': 1000, 'width': 500, 'aspect_ratio': 1.33}, + {'format_id': 'C', 'filesize': 1000, 'width': 400, 'aspect_ratio': 1.5}, + {'format_id': 'D', 'filesize': 2000, 'width': 600, 'aspect_ratio': 1.78}, + {'format_id': 'E', 'filesize': 3000, 'aspect_ratio': 0.56}, {'format_id': 'F'}, {'format_id': 'G', 'filesize': 1000000}, ] @@ -549,6 +551,31 @@ def test_format_filtering(self): ydl.process_ie_result(info_dict) self.assertEqual(ydl.downloaded_info_dicts, []) + ydl = YDL({'format': 'best[aspect_ratio=1]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'A') + + ydl = YDL({'format': 'all[aspect_ratio > 1.00]'}) + ydl.process_ie_result(info_dict) + downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts] + self.assertEqual(downloaded_ids, ['D', 'C', 'B']) + + ydl = YDL({'format': 'all[aspect_ratio < 1.00]'}) + ydl.process_ie_result(info_dict) + downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts] + self.assertEqual(downloaded_ids, ['E']) + + ydl = YDL({'format': 'best[aspect_ratio=1.5]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'C') + + ydl = YDL({'format': 'all[aspect_ratio!=1]'}) + ydl.process_ie_result(info_dict) + downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts] + self.assertEqual(downloaded_ids, ['E', 'D', 'C', 'B']) + @patch('yt_dlp.postprocessor.ffmpeg.FFmpegMergerPP.available', False) def test_default_format_spec_without_ffmpeg(self): ydl = YDL({}) @@ -761,6 +788,13 @@ def test(tmpl, expected, *, info=None, **params): test('%(width)06d.%%(ext)s', 'NA.%(ext)s') test('%%(width)06d.%(ext)s', '%(width)06d.mp4') + # Sanitization options + test('%(title3)s', (None, 'foo⧸bar⧹test')) + test('%(title5)s', (None, 'aei_A'), restrictfilenames=True) + test('%(title3)s', (None, 'foo_bar_test'), windowsfilenames=False, restrictfilenames=True) + if sys.platform != 'win32': + test('%(title3)s', (None, 'foo⧸bar\\test'), windowsfilenames=False) + # ID sanitization test('%(id)s', '_abcd', info={'id': '_abcd'}) test('%(some_id)s', '_abcd', info={'some_id': '_abcd'}) @@ -1395,6 +1429,12 @@ def check_for_cookie_header(result): self.assertFalse(result.get('cookies'), msg='Cookies set in cookies field for wrong domain') self.assertFalse(ydl.cookiejar.get_cookie_header(fmt['url']), msg='Cookies set in cookiejar for wrong domain') + def test_load_plugins_compat(self): + # Should try to reload plugins if they haven't already been loaded + all_plugins_loaded.value = False + FakeYDL().close() + assert all_plugins_loaded.value + if __name__ == '__main__': unittest.main() diff --git a/test/test_http_proxy.py b/test/test_http_proxy.py index 2435c878a..e903ff8be 100644 --- a/test/test_http_proxy.py +++ b/test/test_http_proxy.py @@ -331,10 +331,6 @@ def test_http_connect_auth(self, handler, ctx): assert proxy_info['proxy'] == server_address assert 'Proxy-Authorization' in proxy_info['headers'] - @pytest.mark.skip_handler( - 'Requests', - 'bug in urllib3 causes unclosed socket: https://github.com/urllib3/urllib3/issues/3374', - ) def test_http_connect_bad_auth(self, handler, ctx): with ctx.http_server(HTTPConnectProxyHandler, username='test', password='test') as server_address: with handler(verify=False, proxies={ctx.REQUEST_PROTO: f'http://test:bad@{server_address}'}) as rh: diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index 06840ed85..b14069ccc 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -9,7 +9,7 @@ import math -from yt_dlp.jsinterp import JS_Undefined, JSInterpreter +from yt_dlp.jsinterp import JS_Undefined, JSInterpreter, js_number_to_string class NaN: @@ -93,6 +93,16 @@ def test_operators(self): self._test('function f(){return 0 ?? 42;}', 0) self._test('function f(){return "life, the universe and everything" < 42;}', False) self._test('function f(){return 0 - 7 * - 6;}', 42) + self._test('function f(){return true << "5";}', 32) + self._test('function f(){return true << true;}', 2) + self._test('function f(){return "19" & "21.9";}', 17) + self._test('function f(){return "19" & false;}', 0) + self._test('function f(){return "11.0" >> "2.1";}', 2) + self._test('function f(){return 5 ^ 9;}', 12) + self._test('function f(){return 0.0 << NaN}', 0) + self._test('function f(){return null << undefined}', 0) + # TODO: Does not work due to number too large + # self._test('function f(){return 21 << 4294967297}', 42) def test_array_access(self): self._test('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}', [5, 2, 7]) @@ -108,6 +118,7 @@ def test_assignments(self): self._test('function f(){var x = 20; x = 30 + 1; return x;}', 31) self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51) self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11) + self._test('function f(){var x = 2; var y = ["a", "b"]; y[x%y["length"]]="z"; return y}', ['z', 'b']) @unittest.skip('Not implemented') def test_comments(self): @@ -374,7 +385,7 @@ def test_negative(self): @unittest.skip('Not implemented') def test_packed(self): jsi = JSInterpreter('''function f(p,a,c,k,e,d){while(c--)if(k[c])p=p.replace(new RegExp('\\b'+c.toString(a)+'\\b','g'),k[c]);return p}''') - self.assertEqual(jsi.call_function('f', '''h 7=g("1j");7.7h({7g:[{33:"w://7f-7e-7d-7c.v.7b/7a/79/78/77/76.74?t=73&s=2s&e=72&f=2t&71=70.0.0.1&6z=6y&6x=6w"}],6v:"w://32.v.u/6u.31",16:"r%",15:"r%",6t:"6s",6r:"",6q:"l",6p:"l",6o:"6n",6m:\'6l\',6k:"6j",9:[{33:"/2u?b=6i&n=50&6h=w://32.v.u/6g.31",6f:"6e"}],1y:{6d:1,6c:\'#6b\',6a:\'#69\',68:"67",66:30,65:r,},"64":{63:"%62 2m%m%61%5z%5y%5x.u%5w%5v%5u.2y%22 2k%m%1o%22 5t%m%1o%22 5s%m%1o%22 2j%m%5r%22 16%m%5q%22 15%m%5p%22 5o%2z%5n%5m%2z",5l:"w://v.u/d/1k/5k.2y",5j:[]},\'5i\':{"5h":"5g"},5f:"5e",5d:"w://v.u",5c:{},5b:l,1x:[0.25,0.50,0.75,1,1.25,1.5,2]});h 1m,1n,5a;h 59=0,58=0;h 7=g("1j");h 2x=0,57=0,56=0;$.55({54:{\'53-52\':\'2i-51\'}});7.j(\'4z\',6(x){c(5>0&&x.1l>=5&&1n!=1){1n=1;$(\'q.4y\').4x(\'4w\')}});7.j(\'13\',6(x){2x=x.1l});7.j(\'2g\',6(x){2w(x)});7.j(\'4v\',6(){$(\'q.2v\').4u()});6 2w(x){$(\'q.2v\').4t();c(1m)19;1m=1;17=0;c(4s.4r===l){17=1}$.4q(\'/2u?b=4p&2l=1k&4o=2t-4n-4m-2s-4l&4k=&4j=&4i=&17=\'+17,6(2r){$(\'#4h\').4g(2r)});$(\'.3-8-4f-4e:4d("4c")\').2h(6(e){2q();g().4b(0);g().4a(l)});6 2q(){h $14=$("").2p({1l:"49",16:"r%",15:"r%",48:0,2n:0,2o:47,46:"45(10%, 10%, 10%, 0.4)","44-43":"42"});$("<41 />").2p({16:"60%",15:"60%",2o:40,"3z-2n":"3y"}).3x({\'2m\':\'/?b=3w&2l=1k\',\'2k\':\'0\',\'2j\':\'2i\'}).2f($14);$14.2h(6(){$(3v).3u();g().2g()});$14.2f($(\'#1j\'))}g().13(0);}6 3t(){h 9=7.1b(2e);2d.2c(9);c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==2e){2d.2c(\'!!=\'+i);7.1p(i)}}}}7.j(\'3s\',6(){g().1h("/2a/3r.29","3q 10 28",6(){g().13(g().27()+10)},"2b");$("q[26=2b]").23().21(\'.3-20-1z\');g().1h("/2a/3p.29","3o 10 28",6(){h 12=g().27()-10;c(12<0)12=0;g().13(12)},"24");$("q[26=24]").23().21(\'.3-20-1z\');});6 1i(){}7.j(\'3n\',6(){1i()});7.j(\'3m\',6(){1i()});7.j("k",6(y){h 9=7.1b();c(9.n<2)19;$(\'.3-8-3l-3k\').3j(6(){$(\'#3-8-a-k\').1e(\'3-8-a-z\');$(\'.3-a-k\').p(\'o-1f\',\'11\')});7.1h("/3i/3h.3g","3f 3e",6(){$(\'.3-1w\').3d(\'3-8-1v\');$(\'.3-8-1y, .3-8-1x\').p(\'o-1g\',\'11\');c($(\'.3-1w\').3c(\'3-8-1v\')){$(\'.3-a-k\').p(\'o-1g\',\'l\');$(\'.3-a-k\').p(\'o-1f\',\'l\');$(\'.3-8-a\').1e(\'3-8-a-z\');$(\'.3-8-a:1u\').3b(\'3-8-a-z\')}3a{$(\'.3-a-k\').p(\'o-1g\',\'11\');$(\'.3-a-k\').p(\'o-1f\',\'11\');$(\'.3-8-a:1u\').1e(\'3-8-a-z\')}},"39");7.j("38",6(y){1d.37(\'1c\',y.9[y.36].1a)});c(1d.1t(\'1c\')){35("1s(1d.1t(\'1c\'));",34)}});h 18;6 1s(1q){h 9=7.1b();c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==1q){c(i==18){19}18=i;7.1p(i)}}}}',36,270,'|||jw|||function|player|settings|tracks|submenu||if||||jwplayer|var||on|audioTracks|true|3D|length|aria|attr|div|100|||sx|filemoon|https||event|active||false|tt|seek|dd|height|width|adb|current_audio|return|name|getAudioTracks|default_audio|localStorage|removeClass|expanded|checked|addButton|callMeMaybe|vplayer|0fxcyc2ajhp1|position|vvplay|vvad|220|setCurrentAudioTrack|audio_name|for|audio_set|getItem|last|open|controls|playbackRates|captions|rewind|icon|insertAfter||detach|ff00||button|getPosition|sec|png|player8|ff11|log|console|track_name|appendTo|play|click|no|scrolling|frameborder|file_code|src|top|zIndex|css|showCCform|data|1662367683|383371|dl|video_ad|doPlay|prevt|mp4|3E||jpg|thumbs|file|300|setTimeout|currentTrack|setItem|audioTrackChanged|dualSound|else|addClass|hasClass|toggleClass|Track|Audio|svg|dualy|images|mousedown|buttons|topbar|playAttemptFailed|beforePlay|Rewind|fr|Forward|ff|ready|set_audio_track|remove|this|upload_srt|prop|50px|margin|1000001|iframe|center|align|text|rgba|background|1000000|left|absolute|pause|setCurrentCaptions|Upload|contains|item|content|html|fviews|referer|prem|embed|3e57249ef633e0d03bf76ceb8d8a4b65|216|83|hash|view|get|TokenZir|window|hide|show|complete|slow|fadeIn|video_ad_fadein|time||cache|Cache|Content|headers|ajaxSetup|v2done|tott|vastdone2|vastdone1|vvbefore|playbackRateControls|cast|aboutlink|FileMoon|abouttext|UHD|1870|qualityLabels|sites|GNOME_POWER|link|2Fiframe|3C|allowfullscreen|22360|22640|22no|marginheight|marginwidth|2FGNOME_POWER|2F0fxcyc2ajhp1|2Fe|2Ffilemoon|2F|3A||22https|3Ciframe|code|sharing|fontOpacity|backgroundOpacity|Tahoma|fontFamily|303030|backgroundColor|FFFFFF|color|userFontScale|thumbnails|kind|0fxcyc2ajhp10000|url|get_slides|start|startparam|none|preload|html5|primary|hlshtml|androidhls|duration|uniform|stretching|0fxcyc2ajhp1_xt|image|2048|sp|6871|asn|127|srv|43200|_g3XlBcu2lmD9oDexD2NLWSmah2Nu3XcDrl93m9PwXY|m3u8||master|0fxcyc2ajhp1_x|00076|01|hls2|to|s01|delivery|storage|moon|sources|setup'''.split('|'))) + self.assertEqual(jsi.call_function('f', '''h 7=g("1j");7.7h({7g:[{33:"w://7f-7e-7d-7c.v.7b/7a/79/78/77/76.74?t=73&s=2s&e=72&f=2t&71=70.0.0.1&6z=6y&6x=6w"}],6v:"w://32.v.u/6u.31",16:"r%",15:"r%",6t:"6s",6r:"",6q:"l",6p:"l",6o:"6n",6m:\'6l\',6k:"6j",9:[{33:"/2u?b=6i&n=50&6h=w://32.v.u/6g.31",6f:"6e"}],1y:{6d:1,6c:\'#6b\',6a:\'#69\',68:"67",66:30,65:r,},"64":{63:"%62 2m%m%61%5z%5y%5x.u%5w%5v%5u.2y%22 2k%m%1o%22 5t%m%1o%22 5s%m%1o%22 2j%m%5r%22 16%m%5q%22 15%m%5p%22 5o%2z%5n%5m%2z",5l:"w://v.u/d/1k/5k.2y",5j:[]},\'5i\':{"5h":"5g"},5f:"5e",5d:"w://v.u",5c:{},5b:l,1x:[0.25,0.50,0.75,1,1.25,1.5,2]});h 1m,1n,5a;h 59=0,58=0;h 7=g("1j");h 2x=0,57=0,56=0;$.55({54:{\'53-52\':\'2i-51\'}});7.j(\'4z\',6(x){c(5>0&&x.1l>=5&&1n!=1){1n=1;$(\'q.4y\').4x(\'4w\')}});7.j(\'13\',6(x){2x=x.1l});7.j(\'2g\',6(x){2w(x)});7.j(\'4v\',6(){$(\'q.2v\').4u()});6 2w(x){$(\'q.2v\').4t();c(1m)19;1m=1;17=0;c(4s.4r===l){17=1}$.4q(\'/2u?b=4p&2l=1k&4o=2t-4n-4m-2s-4l&4k=&4j=&4i=&17=\'+17,6(2r){$(\'#4h\').4g(2r)});$(\'.3-8-4f-4e:4d("4c")\').2h(6(e){2q();g().4b(0);g().4a(l)});6 2q(){h $14=$("").2p({1l:"49",16:"r%",15:"r%",48:0,2n:0,2o:47,46:"45(10%, 10%, 10%, 0.4)","44-43":"42"});$("<41 />").2p({16:"60%",15:"60%",2o:40,"3z-2n":"3y"}).3x({\'2m\':\'/?b=3w&2l=1k\',\'2k\':\'0\',\'2j\':\'2i\'}).2f($14);$14.2h(6(){$(3v).3u();g().2g()});$14.2f($(\'#1j\'))}g().13(0);}6 3t(){h 9=7.1b(2e);2d.2c(9);c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==2e){2d.2c(\'!!=\'+i);7.1p(i)}}}}7.j(\'3s\',6(){g().1h("/2a/3r.29","3q 10 28",6(){g().13(g().27()+10)},"2b");$("q[26=2b]").23().21(\'.3-20-1z\');g().1h("/2a/3p.29","3o 10 28",6(){h 12=g().27()-10;c(12<0)12=0;g().13(12)},"24");$("q[26=24]").23().21(\'.3-20-1z\');});6 1i(){}7.j(\'3n\',6(){1i()});7.j(\'3m\',6(){1i()});7.j("k",6(y){h 9=7.1b();c(9.n<2)19;$(\'.3-8-3l-3k\').3j(6(){$(\'#3-8-a-k\').1e(\'3-8-a-z\');$(\'.3-a-k\').p(\'o-1f\',\'11\')});7.1h("/3i/3h.3g","3f 3e",6(){$(\'.3-1w\').3d(\'3-8-1v\');$(\'.3-8-1y, .3-8-1x\').p(\'o-1g\',\'11\');c($(\'.3-1w\').3c(\'3-8-1v\')){$(\'.3-a-k\').p(\'o-1g\',\'l\');$(\'.3-a-k\').p(\'o-1f\',\'l\');$(\'.3-8-a\').1e(\'3-8-a-z\');$(\'.3-8-a:1u\').3b(\'3-8-a-z\')}3a{$(\'.3-a-k\').p(\'o-1g\',\'11\');$(\'.3-a-k\').p(\'o-1f\',\'11\');$(\'.3-8-a:1u\').1e(\'3-8-a-z\')}},"39");7.j("38",6(y){1d.37(\'1c\',y.9[y.36].1a)});c(1d.1t(\'1c\')){35("1s(1d.1t(\'1c\'));",34)}});h 18;6 1s(1q){h 9=7.1b();c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==1q){c(i==18){19}18=i;7.1p(i)}}}}',36,270,'|||jw|||function|player|settings|tracks|submenu||if||||jwplayer|var||on|audioTracks|true|3D|length|aria|attr|div|100|||sx|filemoon|https||event|active||false|tt|seek|dd|height|width|adb|current_audio|return|name|getAudioTracks|default_audio|localStorage|removeClass|expanded|checked|addButton|callMeMaybe|vplayer|0fxcyc2ajhp1|position|vvplay|vvad|220|setCurrentAudioTrack|audio_name|for|audio_set|getItem|last|open|controls|playbackRates|captions|rewind|icon|insertAfter||detach|ff00||button|getPosition|sec|png|player8|ff11|log|console|track_name|appendTo|play|click|no|scrolling|frameborder|file_code|src|top|zIndex|css|showCCform|data|1662367683|383371|dl|video_ad|doPlay|prevt|mp4|3E||jpg|thumbs|file|300|setTimeout|currentTrack|setItem|audioTrackChanged|dualSound|else|addClass|hasClass|toggleClass|Track|Audio|svg|dualy|images|mousedown|buttons|topbar|playAttemptFailed|beforePlay|Rewind|fr|Forward|ff|ready|set_audio_track|remove|this|upload_srt|prop|50px|margin|1000001|iframe|center|align|text|rgba|background|1000000|left|absolute|pause|setCurrentCaptions|Upload|contains|item|content|html|fviews|referer|prem|embed|3e57249ef633e0d03bf76ceb8d8a4b65|216|83|hash|view|get|TokenZir|window|hide|show|complete|slow|fadeIn|video_ad_fadein|time||cache|Cache|Content|headers|ajaxSetup|v2done|tott|vastdone2|vastdone1|vvbefore|playbackRateControls|cast|aboutlink|FileMoon|abouttext|UHD|1870|qualityLabels|sites|GNOME_POWER|link|2Fiframe|3C|allowfullscreen|22360|22640|22no|marginheight|marginwidth|2FGNOME_POWER|2F0fxcyc2ajhp1|2Fe|2Ffilemoon|2F|3A||22https|3Ciframe|code|sharing|fontOpacity|backgroundOpacity|Tahoma|fontFamily|303030|backgroundColor|FFFFFF|color|userFontScale|thumbnails|kind|0fxcyc2ajhp10000|url|get_slides|start|startparam|none|preload|html5|primary|hlshtml|androidhls|duration|uniform|stretching|0fxcyc2ajhp1_xt|image|2048|sp|6871|asn|127|srv|43200|_g3XlBcu2lmD9oDexD2NLWSmah2Nu3XcDrl93m9PwXY|m3u8||master|0fxcyc2ajhp1_x|00076|01|hls2|to|s01|delivery|storage|moon|sources|setup'''.split('|'))) # noqa: SIM905 def test_join(self): test_input = list('test') @@ -393,6 +404,8 @@ def test_split(self): test_result = list('test') tests = [ 'function f(a, b){return a.split(b)}', + 'function f(a, b){return a["split"](b)}', + 'function f(a, b){let x = ["split"]; return a[x[0]](b)}', 'function f(a, b){return String.prototype.split.call(a, b)}', 'function f(a, b){return String.prototype.split.apply(a, [b])}', ] @@ -431,6 +444,40 @@ def test_slice(self): self._test('function f(){return "012345678".slice(-1, 1)}', '') self._test('function f(){return "012345678".slice(-3, -1)}', '67') + def test_splice(self): + self._test('function f(){var T = ["0", "1", "2"]; T["splice"](2, 1, "0")[0]; return T }', ['0', '1', '0']) + + def test_js_number_to_string(self): + for test, radix, expected in [ + (0, None, '0'), + (-0, None, '0'), + (0.0, None, '0'), + (-0.0, None, '0'), + (math.nan, None, 'NaN'), + (-math.nan, None, 'NaN'), + (math.inf, None, 'Infinity'), + (-math.inf, None, '-Infinity'), + (10 ** 21.5, 8, '526665530627250154000000'), + (6, 2, '110'), + (254, 16, 'fe'), + (-10, 2, '-1010'), + (-0xff, 2, '-11111111'), + (0.1 + 0.2, 16, '0.4cccccccccccd'), + (1234.1234, 10, '1234.1234'), + # (1000000000000000128, 10, '1000000000000000100') + ]: + assert js_number_to_string(test, radix) == expected + + def test_extract_function(self): + jsi = JSInterpreter('function a(b) { return b + 1; }') + func = jsi.extract_function('a') + self.assertEqual(func([2]), 3) + + def test_extract_function_with_global_stack(self): + jsi = JSInterpreter('function c(d) { return d + e + f + g; }') + func = jsi.extract_function('c', {'e': 10}, {'f': 100, 'g': 1000}) + self.assertEqual(func([1]), 1111) + if __name__ == '__main__': unittest.main() diff --git a/test/test_networking.py b/test/test_networking.py index d96624af1..2f441fced 100644 --- a/test/test_networking.py +++ b/test/test_networking.py @@ -39,6 +39,7 @@ from yt_dlp.dependencies import brotli, curl_cffi, requests, urllib3 from yt_dlp.networking import ( HEADRequest, + PATCHRequest, PUTRequest, Request, RequestDirector, @@ -614,7 +615,6 @@ def test_source_address(self, handler): rh, Request(f'http://127.0.0.1:{self.http_port}/source_address')).read().decode() assert source_address == data - # Not supported by CurlCFFI @pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi') def test_gzip_trailing_garbage(self, handler): with handler() as rh: @@ -720,6 +720,15 @@ def test_allproxy(self, handler): rh, Request( f'http://127.0.0.1:{self.http_port}/headers', proxies={'all': 'http://10.255.255.255'})).close() + @pytest.mark.skip_handlers_if(lambda _, handler: handler not in ['Urllib', 'CurlCFFI'], 'handler does not support keep_header_casing') + def test_keep_header_casing(self, handler): + with handler() as rh: + res = validate_and_send( + rh, Request( + f'http://127.0.0.1:{self.http_port}/headers', headers={'X-test-heaDer': 'test'}, extensions={'keep_header_casing': True})).read().decode() + + assert 'X-test-heaDer: test' in res + @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True) class TestClientCertificate: @@ -1289,6 +1298,7 @@ class HTTPSupportedRH(ValidationRH): ({'legacy_ssl': False}, False), ({'legacy_ssl': True}, False), ({'legacy_ssl': 'notabool'}, AssertionError), + ({'keep_header_casing': True}, UnsupportedRequest), ]), ('Requests', 'http', [ ({'cookiejar': 'notacookiejar'}, AssertionError), @@ -1299,6 +1309,9 @@ class HTTPSupportedRH(ValidationRH): ({'legacy_ssl': False}, False), ({'legacy_ssl': True}, False), ({'legacy_ssl': 'notabool'}, AssertionError), + ({'keep_header_casing': False}, False), + ({'keep_header_casing': True}, False), + ({'keep_header_casing': 'notabool'}, AssertionError), ]), ('CurlCFFI', 'http', [ ({'cookiejar': 'notacookiejar'}, AssertionError), @@ -1844,6 +1857,7 @@ def test_method(self): def test_request_helpers(self): assert HEADRequest('http://example.com').method == 'HEAD' + assert PATCHRequest('http://example.com').method == 'PATCH' assert PUTRequest('http://example.com').method == 'PUT' def test_headers(self): diff --git a/test/test_plugins.py b/test/test_plugins.py index 77545d136..195726b18 100644 --- a/test/test_plugins.py +++ b/test/test_plugins.py @@ -10,22 +10,71 @@ sys.path.append(str(TEST_DATA_DIR)) importlib.invalidate_caches() -from yt_dlp.utils import Config -from yt_dlp.plugins import PACKAGE_NAME, directories, load_plugins +from yt_dlp.plugins import ( + PACKAGE_NAME, + PluginSpec, + directories, + load_plugins, + load_all_plugins, + register_plugin_spec, +) + +from yt_dlp.globals import ( + extractors, + postprocessors, + plugin_dirs, + plugin_ies, + plugin_pps, + all_plugins_loaded, + plugin_specs, +) + + +EXTRACTOR_PLUGIN_SPEC = PluginSpec( + module_name='extractor', + suffix='IE', + destination=extractors, + plugin_destination=plugin_ies, +) + +POSTPROCESSOR_PLUGIN_SPEC = PluginSpec( + module_name='postprocessor', + suffix='PP', + destination=postprocessors, + plugin_destination=plugin_pps, +) + + +def reset_plugins(): + plugin_ies.value = {} + plugin_pps.value = {} + plugin_dirs.value = ['default'] + plugin_specs.value = {} + all_plugins_loaded.value = False + # Clearing override plugins is probably difficult + for module_name in tuple(sys.modules): + for plugin_type in ('extractor', 'postprocessor'): + if module_name.startswith(f'{PACKAGE_NAME}.{plugin_type}.'): + del sys.modules[module_name] + + importlib.invalidate_caches() class TestPlugins(unittest.TestCase): TEST_PLUGIN_DIR = TEST_DATA_DIR / PACKAGE_NAME + def setUp(self): + reset_plugins() + + def tearDown(self): + reset_plugins() + def test_directories_containing_plugins(self): self.assertIn(self.TEST_PLUGIN_DIR, map(Path, directories())) def test_extractor_classes(self): - for module_name in tuple(sys.modules): - if module_name.startswith(f'{PACKAGE_NAME}.extractor'): - del sys.modules[module_name] - plugins_ie = load_plugins('extractor', 'IE') + plugins_ie = load_plugins(EXTRACTOR_PLUGIN_SPEC) self.assertIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys()) self.assertIn('NormalPluginIE', plugins_ie.keys()) @@ -35,17 +84,29 @@ def test_extractor_classes(self): f'{PACKAGE_NAME}.extractor._ignore' in sys.modules, 'loaded module beginning with underscore') self.assertNotIn('IgnorePluginIE', plugins_ie.keys()) + self.assertNotIn('IgnorePluginIE', plugin_ies.value) # Don't load extractors with underscore prefix self.assertNotIn('_IgnoreUnderscorePluginIE', plugins_ie.keys()) + self.assertNotIn('_IgnoreUnderscorePluginIE', plugin_ies.value) # Don't load extractors not specified in __all__ (if supplied) self.assertNotIn('IgnoreNotInAllPluginIE', plugins_ie.keys()) + self.assertNotIn('IgnoreNotInAllPluginIE', plugin_ies.value) self.assertIn('InAllPluginIE', plugins_ie.keys()) + self.assertIn('InAllPluginIE', plugin_ies.value) + + # Don't load override extractors + self.assertNotIn('OverrideGenericIE', plugins_ie.keys()) + self.assertNotIn('OverrideGenericIE', plugin_ies.value) + self.assertNotIn('_UnderscoreOverrideGenericIE', plugins_ie.keys()) + self.assertNotIn('_UnderscoreOverrideGenericIE', plugin_ies.value) def test_postprocessor_classes(self): - plugins_pp = load_plugins('postprocessor', 'PP') + plugins_pp = load_plugins(POSTPROCESSOR_PLUGIN_SPEC) self.assertIn('NormalPluginPP', plugins_pp.keys()) + self.assertIn(f'{PACKAGE_NAME}.postprocessor.normal', sys.modules.keys()) + self.assertIn('NormalPluginPP', plugin_pps.value) def test_importing_zipped_module(self): zip_path = TEST_DATA_DIR / 'zipped_plugins.zip' @@ -58,10 +119,10 @@ def test_importing_zipped_module(self): package = importlib.import_module(f'{PACKAGE_NAME}.{plugin_type}') self.assertIn(zip_path / PACKAGE_NAME / plugin_type, map(Path, package.__path__)) - plugins_ie = load_plugins('extractor', 'IE') + plugins_ie = load_plugins(EXTRACTOR_PLUGIN_SPEC) self.assertIn('ZippedPluginIE', plugins_ie.keys()) - plugins_pp = load_plugins('postprocessor', 'PP') + plugins_pp = load_plugins(POSTPROCESSOR_PLUGIN_SPEC) self.assertIn('ZippedPluginPP', plugins_pp.keys()) finally: @@ -69,23 +130,116 @@ def test_importing_zipped_module(self): os.remove(zip_path) importlib.invalidate_caches() # reset the import caches - def test_plugin_dirs(self): - # Internal plugin dirs hack for CLI --plugin-dirs - # To be replaced with proper system later - custom_plugin_dir = TEST_DATA_DIR / 'plugin_packages' - Config._plugin_dirs = [str(custom_plugin_dir)] - importlib.invalidate_caches() # reset the import caches + def test_reloading_plugins(self): + reload_plugins_path = TEST_DATA_DIR / 'reload_plugins' + load_plugins(EXTRACTOR_PLUGIN_SPEC) + load_plugins(POSTPROCESSOR_PLUGIN_SPEC) + # Remove default folder and add reload_plugin path + sys.path.remove(str(TEST_DATA_DIR)) + sys.path.append(str(reload_plugins_path)) + importlib.invalidate_caches() try: - package = importlib.import_module(f'{PACKAGE_NAME}.extractor') - self.assertIn(custom_plugin_dir / 'testpackage' / PACKAGE_NAME / 'extractor', map(Path, package.__path__)) + for plugin_type in ('extractor', 'postprocessor'): + package = importlib.import_module(f'{PACKAGE_NAME}.{plugin_type}') + self.assertIn(reload_plugins_path / PACKAGE_NAME / plugin_type, map(Path, package.__path__)) - plugins_ie = load_plugins('extractor', 'IE') - self.assertIn('PackagePluginIE', plugins_ie.keys()) + plugins_ie = load_plugins(EXTRACTOR_PLUGIN_SPEC) + self.assertIn('NormalPluginIE', plugins_ie.keys()) + self.assertTrue( + plugins_ie['NormalPluginIE'].REPLACED, + msg='Reloading has not replaced original extractor plugin') + self.assertTrue( + extractors.value['NormalPluginIE'].REPLACED, + msg='Reloading has not replaced original extractor plugin globally') + + plugins_pp = load_plugins(POSTPROCESSOR_PLUGIN_SPEC) + self.assertIn('NormalPluginPP', plugins_pp.keys()) + self.assertTrue(plugins_pp['NormalPluginPP'].REPLACED, + msg='Reloading has not replaced original postprocessor plugin') + self.assertTrue( + postprocessors.value['NormalPluginPP'].REPLACED, + msg='Reloading has not replaced original postprocessor plugin globally') finally: - Config._plugin_dirs = [] - importlib.invalidate_caches() # reset the import caches + sys.path.remove(str(reload_plugins_path)) + sys.path.append(str(TEST_DATA_DIR)) + importlib.invalidate_caches() + + def test_extractor_override_plugin(self): + load_plugins(EXTRACTOR_PLUGIN_SPEC) + + from yt_dlp.extractor.generic import GenericIE + + self.assertEqual(GenericIE.TEST_FIELD, 'override') + self.assertEqual(GenericIE.SECONDARY_TEST_FIELD, 'underscore-override') + + self.assertEqual(GenericIE.IE_NAME, 'generic+override+underscore-override') + importlib.invalidate_caches() + # test that loading a second time doesn't wrap a second time + load_plugins(EXTRACTOR_PLUGIN_SPEC) + from yt_dlp.extractor.generic import GenericIE + self.assertEqual(GenericIE.IE_NAME, 'generic+override+underscore-override') + + def test_load_all_plugin_types(self): + + # no plugin specs registered + load_all_plugins() + + self.assertNotIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys()) + self.assertNotIn(f'{PACKAGE_NAME}.postprocessor.normal', sys.modules.keys()) + + register_plugin_spec(EXTRACTOR_PLUGIN_SPEC) + register_plugin_spec(POSTPROCESSOR_PLUGIN_SPEC) + load_all_plugins() + self.assertTrue(all_plugins_loaded.value) + + self.assertIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys()) + self.assertIn(f'{PACKAGE_NAME}.postprocessor.normal', sys.modules.keys()) + + def test_no_plugin_dirs(self): + register_plugin_spec(EXTRACTOR_PLUGIN_SPEC) + register_plugin_spec(POSTPROCESSOR_PLUGIN_SPEC) + + plugin_dirs.value = [] + load_all_plugins() + + self.assertNotIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys()) + self.assertNotIn(f'{PACKAGE_NAME}.postprocessor.normal', sys.modules.keys()) + + def test_set_plugin_dirs(self): + custom_plugin_dir = str(TEST_DATA_DIR / 'plugin_packages') + plugin_dirs.value = [custom_plugin_dir] + + load_plugins(EXTRACTOR_PLUGIN_SPEC) + + self.assertIn(f'{PACKAGE_NAME}.extractor.package', sys.modules.keys()) + self.assertIn('PackagePluginIE', plugin_ies.value) + + def test_invalid_plugin_dir(self): + plugin_dirs.value = ['invalid_dir'] + with self.assertRaises(ValueError): + load_plugins(EXTRACTOR_PLUGIN_SPEC) + + def test_append_plugin_dirs(self): + custom_plugin_dir = str(TEST_DATA_DIR / 'plugin_packages') + + self.assertEqual(plugin_dirs.value, ['default']) + plugin_dirs.value.append(custom_plugin_dir) + self.assertEqual(plugin_dirs.value, ['default', custom_plugin_dir]) + + load_plugins(EXTRACTOR_PLUGIN_SPEC) + + self.assertIn(f'{PACKAGE_NAME}.extractor.package', sys.modules.keys()) + self.assertIn('PackagePluginIE', plugin_ies.value) + + def test_get_plugin_spec(self): + register_plugin_spec(EXTRACTOR_PLUGIN_SPEC) + register_plugin_spec(POSTPROCESSOR_PLUGIN_SPEC) + + self.assertEqual(plugin_specs.value.get('extractor'), EXTRACTOR_PLUGIN_SPEC) + self.assertEqual(plugin_specs.value.get('postprocessor'), POSTPROCESSOR_PLUGIN_SPEC) + self.assertIsNone(plugin_specs.value.get('invalid')) if __name__ == '__main__': diff --git a/test/test_subtitles.py b/test/test_subtitles.py index f3b005617..efd69b33d 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -23,7 +23,6 @@ TedTalkIE, ThePlatformFeedIE, ThePlatformIE, - VikiIE, VimeoIE, WallaIE, YoutubeIE, @@ -331,20 +330,6 @@ def test_subtitles_array_key(self): self.assertEqual(md5(subtitles['it']), '4b3264186fbb103508abe5311cfcb9cd') -@is_download_test -@unittest.skip('IE broken - DRM only') -class TestVikiSubtitles(BaseTestSubtitles): - url = 'http://www.viki.com/videos/1060846v-punch-episode-18' - IE = VikiIE - - def test_allsubtitles(self): - self.DL.params['writesubtitles'] = True - self.DL.params['allsubtitles'] = True - subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), {'en'}) - self.assertEqual(md5(subtitles['en']), '53cb083a5914b2d84ef1ab67b880d18a') - - @is_download_test class TestThePlatformSubtitles(BaseTestSubtitles): # from http://www.3playmedia.com/services-features/tools/integrations/theplatform/ diff --git a/test/test_utils.py b/test/test_utils.py index b3de14198..aedb565ec 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -3,19 +3,20 @@ # Allow direct execution import os import sys -import unittest -import unittest.mock -import warnings -import datetime as dt sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import contextlib +import datetime as dt import io import itertools import json +import pickle import subprocess +import unittest +import unittest.mock +import warnings import xml.etree.ElementTree from yt_dlp.compat import ( @@ -218,11 +219,8 @@ def test_sanitize_ids(self): self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw') self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI') + @unittest.mock.patch('sys.platform', 'win32') def test_sanitize_path(self): - with unittest.mock.patch('sys.platform', 'win32'): - self._test_sanitize_path() - - def _test_sanitize_path(self): self.assertEqual(sanitize_path('abc'), 'abc') self.assertEqual(sanitize_path('abc/def'), 'abc\\def') self.assertEqual(sanitize_path('abc\\def'), 'abc\\def') @@ -249,17 +247,34 @@ def _test_sanitize_path(self): self.assertEqual(sanitize_path('abc/def...'), 'abc\\def..#') self.assertEqual(sanitize_path('abc.../def'), 'abc..#\\def') self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#') - - self.assertEqual(sanitize_path('../abc'), '..\\abc') - self.assertEqual(sanitize_path('../../abc'), '..\\..\\abc') - self.assertEqual(sanitize_path('./abc'), 'abc') - self.assertEqual(sanitize_path('./../abc'), '..\\abc') - - self.assertEqual(sanitize_path('\\abc'), '\\abc') - self.assertEqual(sanitize_path('C:abc'), 'C:abc') - self.assertEqual(sanitize_path('C:abc\\..\\'), 'C:..') self.assertEqual(sanitize_path('C:\\abc:%(title)s.%(ext)s'), 'C:\\abc#%(title)s.%(ext)s') + # Check with nt._path_normpath if available + try: + from nt import _path_normpath as nt_path_normpath + except ImportError: + nt_path_normpath = None + + for test, expected in [ + ('C:\\', 'C:\\'), + ('../abc', '..\\abc'), + ('../../abc', '..\\..\\abc'), + ('./abc', 'abc'), + ('./../abc', '..\\abc'), + ('\\abc', '\\abc'), + ('C:abc', 'C:abc'), + ('C:abc\\..\\', 'C:'), + ('C:abc\\..\\def\\..\\..\\', 'C:..'), + ('C:\\abc\\xyz///..\\def\\', 'C:\\abc\\def'), + ('abc/../', '.'), + ('./abc/../', '.'), + ]: + result = sanitize_path(test) + assert result == expected, f'{test} was incorrectly resolved' + assert result == sanitize_path(result), f'{test} changed after sanitizing again' + if nt_path_normpath: + assert result == nt_path_normpath(test), f'{test} does not match nt._path_normpath' + def test_sanitize_url(self): self.assertEqual(sanitize_url('//foo.bar'), 'http://foo.bar') self.assertEqual(sanitize_url('httpss://foo.bar'), 'https://foo.bar') @@ -644,6 +659,8 @@ def test_url_or_none(self): self.assertEqual(url_or_none('mms://foo.de'), 'mms://foo.de') self.assertEqual(url_or_none('rtspu://foo.de'), 'rtspu://foo.de') self.assertEqual(url_or_none('ftps://foo.de'), 'ftps://foo.de') + self.assertEqual(url_or_none('ws://foo.de'), 'ws://foo.de') + self.assertEqual(url_or_none('wss://foo.de'), 'wss://foo.de') def test_parse_age_limit(self): self.assertEqual(parse_age_limit(None), None) @@ -1245,6 +1262,7 @@ def test_js_to_json_edgecases(self): def test_js_to_json_malformed(self): self.assertEqual(js_to_json('42a1'), '42"a1"') self.assertEqual(js_to_json('42a-1'), '42"a"-1') + self.assertEqual(js_to_json('{a: `${e("")}`}'), '{"a": "\\"e\\"(\\"\\")"}') def test_js_to_json_template_literal(self): self.assertEqual(js_to_json('`Hello ${name}`', {'name': '"world"'}), '"Hello world"') @@ -2068,21 +2086,26 @@ def test_http_header_dict(self): headers = HTTPHeaderDict() headers['ytdl-test'] = b'0' self.assertEqual(list(headers.items()), [('Ytdl-Test', '0')]) + self.assertEqual(list(headers.sensitive().items()), [('ytdl-test', '0')]) headers['ytdl-test'] = 1 self.assertEqual(list(headers.items()), [('Ytdl-Test', '1')]) + self.assertEqual(list(headers.sensitive().items()), [('ytdl-test', '1')]) headers['Ytdl-test'] = '2' self.assertEqual(list(headers.items()), [('Ytdl-Test', '2')]) + self.assertEqual(list(headers.sensitive().items()), [('Ytdl-test', '2')]) self.assertTrue('ytDl-Test' in headers) self.assertEqual(str(headers), str(dict(headers))) self.assertEqual(repr(headers), str(dict(headers))) headers.update({'X-dlp': 'data'}) self.assertEqual(set(headers.items()), {('Ytdl-Test', '2'), ('X-Dlp', 'data')}) + self.assertEqual(set(headers.sensitive().items()), {('Ytdl-test', '2'), ('X-dlp', 'data')}) self.assertEqual(dict(headers), {'Ytdl-Test': '2', 'X-Dlp': 'data'}) self.assertEqual(len(headers), 2) self.assertEqual(headers.copy(), headers) - headers2 = HTTPHeaderDict({'X-dlp': 'data3'}, **headers, **{'X-dlp': 'data2'}) + headers2 = HTTPHeaderDict({'X-dlp': 'data3'}, headers, **{'X-dlP': 'data2'}) self.assertEqual(set(headers2.items()), {('Ytdl-Test', '2'), ('X-Dlp', 'data2')}) + self.assertEqual(set(headers2.sensitive().items()), {('Ytdl-test', '2'), ('X-dlP', 'data2')}) self.assertEqual(len(headers2), 2) headers2.clear() self.assertEqual(len(headers2), 0) @@ -2090,16 +2113,23 @@ def test_http_header_dict(self): # ensure we prefer latter headers headers3 = HTTPHeaderDict({'Ytdl-TeSt': 1}, {'Ytdl-test': 2}) self.assertEqual(set(headers3.items()), {('Ytdl-Test', '2')}) + self.assertEqual(set(headers3.sensitive().items()), {('Ytdl-test', '2')}) del headers3['ytdl-tesT'] self.assertEqual(dict(headers3), {}) headers4 = HTTPHeaderDict({'ytdl-test': 'data;'}) self.assertEqual(set(headers4.items()), {('Ytdl-Test', 'data;')}) + self.assertEqual(set(headers4.sensitive().items()), {('ytdl-test', 'data;')}) # common mistake: strip whitespace from values # https://github.com/yt-dlp/yt-dlp/issues/8729 headers5 = HTTPHeaderDict({'ytdl-test': ' data; '}) self.assertEqual(set(headers5.items()), {('Ytdl-Test', 'data;')}) + self.assertEqual(set(headers5.sensitive().items()), {('ytdl-test', 'data;')}) + + # test if picklable + headers6 = HTTPHeaderDict(a=1, b=2) + self.assertEqual(pickle.loads(pickle.dumps(headers6)), headers6) def test_extract_basic_auth(self): assert extract_basic_auth('http://:foo.bar') == ('http://:foo.bar', None) diff --git a/test/test_websockets.py b/test/test_websockets.py index 06112cc0b..dead5fe5c 100644 --- a/test/test_websockets.py +++ b/test/test_websockets.py @@ -44,7 +44,7 @@ def websocket_handler(websocket): return websocket.send('2') elif isinstance(message, str): if message == 'headers': - return websocket.send(json.dumps(dict(websocket.request.headers))) + return websocket.send(json.dumps(dict(websocket.request.headers.raw_items()))) elif message == 'path': return websocket.send(websocket.request.path) elif message == 'source_address': @@ -266,18 +266,18 @@ def test_cookies(self, handler): with handler(cookiejar=cookiejar) as rh: ws = ws_validate_and_send(rh, Request(self.ws_base_url)) ws.send('headers') - assert json.loads(ws.recv())['cookie'] == 'test=ytdlp' + assert HTTPHeaderDict(json.loads(ws.recv()))['cookie'] == 'test=ytdlp' ws.close() with handler() as rh: ws = ws_validate_and_send(rh, Request(self.ws_base_url)) ws.send('headers') - assert 'cookie' not in json.loads(ws.recv()) + assert 'cookie' not in HTTPHeaderDict(json.loads(ws.recv())) ws.close() ws = ws_validate_and_send(rh, Request(self.ws_base_url, extensions={'cookiejar': cookiejar})) ws.send('headers') - assert json.loads(ws.recv())['cookie'] == 'test=ytdlp' + assert HTTPHeaderDict(json.loads(ws.recv()))['cookie'] == 'test=ytdlp' ws.close() @pytest.mark.skip_handler('Websockets', 'Set-Cookie not supported by websockets') @@ -287,7 +287,7 @@ def test_cookie_sync_only_cookiejar(self, handler): ws_validate_and_send(rh, Request(f'{self.ws_base_url}/get_cookie', extensions={'cookiejar': YoutubeDLCookieJar()})) ws = ws_validate_and_send(rh, Request(self.ws_base_url, extensions={'cookiejar': YoutubeDLCookieJar()})) ws.send('headers') - assert 'cookie' not in json.loads(ws.recv()) + assert 'cookie' not in HTTPHeaderDict(json.loads(ws.recv())) ws.close() @pytest.mark.skip_handler('Websockets', 'Set-Cookie not supported by websockets') @@ -298,12 +298,12 @@ def test_cookie_sync_delete_cookie(self, handler): ws_validate_and_send(rh, Request(f'{self.ws_base_url}/get_cookie')) ws = ws_validate_and_send(rh, Request(self.ws_base_url)) ws.send('headers') - assert json.loads(ws.recv())['cookie'] == 'test=ytdlp' + assert HTTPHeaderDict(json.loads(ws.recv()))['cookie'] == 'test=ytdlp' ws.close() cookiejar.clear_session_cookies() ws = ws_validate_and_send(rh, Request(self.ws_base_url)) ws.send('headers') - assert 'cookie' not in json.loads(ws.recv()) + assert 'cookie' not in HTTPHeaderDict(json.loads(ws.recv())) ws.close() def test_source_address(self, handler): @@ -341,6 +341,14 @@ def test_request_headers(self, handler): assert headers['test3'] == 'test3' ws.close() + def test_keep_header_casing(self, handler): + with handler(headers=HTTPHeaderDict({'x-TeSt1': 'test'})) as rh: + ws = ws_validate_and_send(rh, Request(self.ws_base_url, headers={'x-TeSt2': 'test'}, extensions={'keep_header_casing': True})) + ws.send('headers') + headers = json.loads(ws.recv()) + assert 'x-TeSt1' in headers + assert 'x-TeSt2' in headers + @pytest.mark.parametrize('client_cert', ( {'client_certificate': os.path.join(MTLS_CERT_DIR, 'clientwithkey.crt')}, { diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 13436f088..0f0885366 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -78,6 +78,61 @@ '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', '0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q', ), + ( + 'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + 'AAOAOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7vgpDL0QwbdV06sCIEzpWqMGkFR20CFOS21Tp-7vj_EMu-m37KtXJoOy1', + ), + ( + 'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + ), + ( + 'https://www.youtube.com/s/player/363db69b/player_ias_tce.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + ), + ( + 'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + 'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0', + ), + ( + 'https://www.youtube.com/s/player/4fcd6e4a/player_ias_tce.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + 'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0', + ), + ( + 'https://www.youtube.com/s/player/20830619/player_ias.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw', + ), + ( + 'https://www.youtube.com/s/player/20830619/player_ias_tce.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw', + ), + ( + 'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw', + ), + ( + 'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw', + ), + ( + 'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + 'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0', + ), + ( + 'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + 'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0', + ), ] _NSIG_TESTS = [ @@ -201,6 +256,66 @@ 'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js', 'YWt1qdbe8SAfkoPHW5d', 'RrRjWQOJmBiP', ), + ( + 'https://www.youtube.com/s/player/9c6dfc4a/player_ias.vflset/en_US/base.js', + 'jbu7ylIosQHyJyJV', 'uwI0ESiynAmhNg', + ), + ( + 'https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js', + 'Sy4aDGc0VpYRR9ew_', '5UPOT1VhoZxNLQ', + ), + ( + 'https://www.youtube.com/s/player/d50f54ef/player_ias_tce.vflset/en_US/base.js', + 'Ha7507LzRmH3Utygtj', 'XFTb2HoeOE5MHg', + ), + ( + 'https://www.youtube.com/s/player/074a8365/player_ias_tce.vflset/en_US/base.js', + 'Ha7507LzRmH3Utygtj', 'ufTsrE0IVYrkl8v', + ), + ( + 'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js', + 'N5uAlLqm0eg1GyHO', 'dCBQOejdq5s-ww', + ), + ( + 'https://www.youtube.com/s/player/69f581a5/tv-player-ias.vflset/tv-player-ias.js', + '-qIP447rVlTTwaZjY', 'KNcGOksBAvwqQg', + ), + ( + 'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', + 'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA', + ), + ( + 'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js', + 'eWYu5d5YeY_4LyEDc', 'XJQqf-N7Xra3gg', + ), + ( + 'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js', + 'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A', + ), + ( + 'https://www.youtube.com/s/player/4fcd6e4a/player_ias_tce.vflset/en_US/base.js', + 'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A', + ), + ( + 'https://www.youtube.com/s/player/20830619/tv-player-ias.vflset/tv-player-ias.js', + 'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4', + ), + ( + 'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js', + 'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4', + ), + ( + 'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js', + 'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4', + ), + ( + 'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js', + 'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE', + ), + ( + 'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js', + 'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE', + ), ] @@ -214,6 +329,8 @@ def test_youtube_extract_player_info(self): ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'), ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-de_DE.vflset/base.js', '64dddad9'), ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-tablet-en_US.vflset/base.js', '64dddad9'), + ('https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js', 'e7567ecf'), + ('https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', '643afba4'), # obsolete ('https://www.youtube.com/yts/jsbin/player_ias-vfle4-e03/en_US/base.js', 'vfle4-e03'), ('https://www.youtube.com/yts/jsbin/player_ias-vfl49f_g4/en_US/base.js', 'vfl49f_g4'), @@ -246,46 +363,51 @@ def t_factory(name, sig_func, url_pattern): def make_tfunc(url, sig_input, expected_sig): m = url_pattern.match(url) assert m, f'{url!r} should follow URL format' - test_id = m.group('id') + test_id = re.sub(r'[/.-]', '_', m.group('id') or m.group('compat_id')) def test_func(self): - basename = f'player-{name}-{test_id}.js' + basename = f'player-{test_id}.js' fn = os.path.join(self.TESTDATA_DIR, basename) if not os.path.exists(fn): urllib.request.urlretrieve(url, fn) with open(fn, encoding='utf-8') as testf: jscode = testf.read() - self.assertEqual(sig_func(jscode, sig_input), expected_sig) + self.assertEqual(sig_func(jscode, sig_input, url), expected_sig) test_func.__name__ = f'test_{name}_js_{test_id}' setattr(TestSignature, test_func.__name__, test_func) return make_tfunc -def signature(jscode, sig_input): - func = YoutubeIE(FakeYDL())._parse_sig_js(jscode) +def signature(jscode, sig_input, player_url): + func = YoutubeIE(FakeYDL())._parse_sig_js(jscode, player_url) src_sig = ( str(string.printable[:sig_input]) if isinstance(sig_input, int) else sig_input) return func(src_sig) -def n_sig(jscode, sig_input): +def n_sig(jscode, sig_input, player_url): ie = YoutubeIE(FakeYDL()) - funcname = ie._extract_n_function_name(jscode) + funcname = ie._extract_n_function_name(jscode, player_url=player_url) jsi = JSInterpreter(jscode) - func = jsi.extract_function_from_code(*ie._fixup_n_function_code(*jsi.extract_function_code(funcname))) + func = jsi.extract_function_from_code(*ie._fixup_n_function_code(*jsi.extract_function_code(funcname), jscode, player_url)) return func([sig_input]) make_sig_test = t_factory( - 'signature', signature, re.compile(r'.*(?:-|/player/)(?P[a-zA-Z0-9_-]+)(?:/.+\.js|(?:/watch_as3|/html5player)?\.[a-z]+)$')) + 'signature', signature, + re.compile(r'''(?x) + .+(?: + /player/(?P[a-zA-Z0-9_/.-]+)| + /html5player-(?:en_US-)?(?P[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)? + )\.js$''')) for test_spec in _SIG_TESTS: make_sig_test(*test_spec) make_nsig_test = t_factory( - 'nsig', n_sig, re.compile(r'.+/player/(?P[a-zA-Z0-9_-]+)/.+.js$')) + 'nsig', n_sig, re.compile(r'.+/player/(?P[a-zA-Z0-9_/.-]+)\.js$')) for test_spec in _NSIG_TESTS: make_nsig_test(*test_spec) diff --git a/test/testdata/plugin_packages/testpackage/yt_dlp_plugins/extractor/package.py b/test/testdata/plugin_packages/testpackage/yt_dlp_plugins/extractor/package.py index b860300d8..39020fef9 100644 --- a/test/testdata/plugin_packages/testpackage/yt_dlp_plugins/extractor/package.py +++ b/test/testdata/plugin_packages/testpackage/yt_dlp_plugins/extractor/package.py @@ -2,4 +2,5 @@ class PackagePluginIE(InfoExtractor): + _VALID_URL = 'package' pass diff --git a/test/testdata/reload_plugins/yt_dlp_plugins/extractor/normal.py b/test/testdata/reload_plugins/yt_dlp_plugins/extractor/normal.py new file mode 100644 index 000000000..6b927077f --- /dev/null +++ b/test/testdata/reload_plugins/yt_dlp_plugins/extractor/normal.py @@ -0,0 +1,10 @@ +from yt_dlp.extractor.common import InfoExtractor + + +class NormalPluginIE(InfoExtractor): + _VALID_URL = 'normal' + REPLACED = True + + +class _IgnoreUnderscorePluginIE(InfoExtractor): + pass diff --git a/test/testdata/reload_plugins/yt_dlp_plugins/postprocessor/normal.py b/test/testdata/reload_plugins/yt_dlp_plugins/postprocessor/normal.py new file mode 100644 index 000000000..5e44ba2b5 --- /dev/null +++ b/test/testdata/reload_plugins/yt_dlp_plugins/postprocessor/normal.py @@ -0,0 +1,5 @@ +from yt_dlp.postprocessor.common import PostProcessor + + +class NormalPluginPP(PostProcessor): + REPLACED = True diff --git a/test/testdata/yt_dlp_plugins/extractor/ignore.py b/test/testdata/yt_dlp_plugins/extractor/ignore.py index 816a16aa2..dca111a37 100644 --- a/test/testdata/yt_dlp_plugins/extractor/ignore.py +++ b/test/testdata/yt_dlp_plugins/extractor/ignore.py @@ -6,6 +6,7 @@ class IgnoreNotInAllPluginIE(InfoExtractor): class InAllPluginIE(InfoExtractor): + _VALID_URL = 'inallpluginie' pass diff --git a/test/testdata/yt_dlp_plugins/extractor/normal.py b/test/testdata/yt_dlp_plugins/extractor/normal.py index b09009bdc..996b2936f 100644 --- a/test/testdata/yt_dlp_plugins/extractor/normal.py +++ b/test/testdata/yt_dlp_plugins/extractor/normal.py @@ -2,8 +2,10 @@ class NormalPluginIE(InfoExtractor): - pass + _VALID_URL = 'normalpluginie' + REPLACED = False class _IgnoreUnderscorePluginIE(InfoExtractor): + _VALID_URL = 'ignoreunderscorepluginie' pass diff --git a/test/testdata/yt_dlp_plugins/extractor/override.py b/test/testdata/yt_dlp_plugins/extractor/override.py new file mode 100644 index 000000000..766dc32e1 --- /dev/null +++ b/test/testdata/yt_dlp_plugins/extractor/override.py @@ -0,0 +1,5 @@ +from yt_dlp.extractor.generic import GenericIE + + +class OverrideGenericIE(GenericIE, plugin_name='override'): + TEST_FIELD = 'override' diff --git a/test/testdata/yt_dlp_plugins/extractor/overridetwo.py b/test/testdata/yt_dlp_plugins/extractor/overridetwo.py new file mode 100644 index 000000000..826184c64 --- /dev/null +++ b/test/testdata/yt_dlp_plugins/extractor/overridetwo.py @@ -0,0 +1,5 @@ +from yt_dlp.extractor.generic import GenericIE + + +class _UnderscoreOverrideGenericIE(GenericIE, plugin_name='underscore-override'): + SECONDARY_TEST_FIELD = 'underscore-override' diff --git a/test/testdata/yt_dlp_plugins/postprocessor/normal.py b/test/testdata/yt_dlp_plugins/postprocessor/normal.py index 315b85a48..1e94d7b8b 100644 --- a/test/testdata/yt_dlp_plugins/postprocessor/normal.py +++ b/test/testdata/yt_dlp_plugins/postprocessor/normal.py @@ -2,4 +2,4 @@ class NormalPluginPP(PostProcessor): - pass + REPLACED = False diff --git a/test/testdata/zipped_plugins/yt_dlp_plugins/extractor/zipped.py b/test/testdata/zipped_plugins/yt_dlp_plugins/extractor/zipped.py index 01542e0d8..c5140bb02 100644 --- a/test/testdata/zipped_plugins/yt_dlp_plugins/extractor/zipped.py +++ b/test/testdata/zipped_plugins/yt_dlp_plugins/extractor/zipped.py @@ -2,4 +2,5 @@ class ZippedPluginIE(InfoExtractor): + _VALID_URL = 'zippedpluginie' pass diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index f8f09812c..4607e350d 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -30,9 +30,18 @@ from .cookies import CookieLoadError, LenientSimpleCookie, load_cookies from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name from .downloader.rtmp import rtmpdump_version -from .extractor import gen_extractor_classes, get_info_extractor +from .extractor import gen_extractor_classes, get_info_extractor, import_extractors from .extractor.common import UnsupportedURLIE from .extractor.openload import PhantomJSwrapper +from .globals import ( + IN_CLI, + LAZY_EXTRACTORS, + plugin_ies, + plugin_ies_overrides, + plugin_pps, + all_plugins_loaded, + plugin_dirs, +) from .minicurses import format_text from .networking import HEADRequest, Request, RequestDirector from .networking.common import _REQUEST_HANDLERS, _RH_PREFERENCES @@ -44,8 +53,7 @@ network_exceptions, ) from .networking.impersonate import ImpersonateRequestHandler -from .plugins import directories as plugin_directories -from .postprocessor import _PLUGIN_CLASSES as plugin_pps +from .plugins import directories as plugin_directories, load_all_plugins from .postprocessor import ( EmbedThumbnailPP, FFmpegFixupDuplicateMoovPP, @@ -157,7 +165,7 @@ write_json_file, write_string, ) -from .utils._utils import _UnsafeExtensionError, _YDLLogger +from .utils._utils import _UnsafeExtensionError, _YDLLogger, _ProgressState from .utils.networking import ( HTTPHeaderDict, clean_headers, @@ -266,7 +274,9 @@ class YoutubeDL: outtmpl_na_placeholder: Placeholder for unavailable meta fields. restrictfilenames: Do not allow "&" and spaces in file names trim_file_name: Limit length of filename (extension excluded) - windowsfilenames: Force the filenames to be windows compatible + windowsfilenames: True: Force filenames to be Windows compatible + False: Sanitize filenames only minimally + This option has no effect when running on Windows ignoreerrors: Do not stop on download/postprocessing errors. Can be 'only_download' to ignore only download errors. Default is 'only_download' for CLI, but False for API @@ -281,7 +291,10 @@ class YoutubeDL: lazy_playlist: Process playlist entries as they are received. matchtitle: Download only matching titles. rejecttitle: Reject downloads for matching titles. - logger: Log messages to a logging.Logger instance. + logger: A class having a `debug`, `warning` and `error` function where + each has a single string parameter, the message to be logged. + For compatibility reasons, both debug and info messages are passed to `debug`. + A debug message will have a prefix of `[debug] ` to discern it from info messages. logtostderr: Print everything to stderr instead of stdout. consoletitle: Display progress in the console window's titlebar. writedescription: Write the video description to a .description file @@ -593,7 +606,7 @@ class YoutubeDL: # NB: Keep in sync with the docstring of extractor/common.py 'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note', 'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels', - 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', + 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', 'hls_media_playlist_data', 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', 'is_dash_periods', 'request_data', 'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies', 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'extra_param_to_key_url', @@ -637,13 +650,15 @@ def __init__(self, params=None, auto_init=True): self.cache = Cache(self) self.__header_cookies = [] + # compat for API: load plugins if they have not already + if not all_plugins_loaded.value: + load_all_plugins() + stdout = sys.stderr if self.params.get('logtostderr') else sys.stdout self._out_files = Namespace( out=stdout, error=sys.stderr, screen=sys.stderr if self.params.get('quiet') else stdout, - console=None if os.name == 'nt' else next( - filter(supports_terminal_sequences, (sys.stderr, sys.stdout)), None), ) try: @@ -651,6 +666,9 @@ def __init__(self, params=None, auto_init=True): except Exception as e: self.write_debug(f'Failed to enable VT mode: {e}') + # hehe "immutable" namespace + self._out_files.console = next(filter(supports_terminal_sequences, (sys.stderr, sys.stdout)), None) + if self.params.get('no_color'): if self.params.get('color') is not None: self.params.setdefault('_warnings', []).append( @@ -954,21 +972,22 @@ def to_stderr(self, message, only_once=False): self._write_string(f'{self._bidi_workaround(message)}\n', self._out_files.error, only_once=only_once) def _send_console_code(self, code): - if os.name == 'nt' or not self._out_files.console: - return + if not supports_terminal_sequences(self._out_files.console): + return False self._write_string(code, self._out_files.console) + return True - def to_console_title(self, message): - if not self.params.get('consoletitle', False): + def to_console_title(self, message=None, progress_state=None, percent=None): + if not self.params.get('consoletitle'): return - message = remove_terminal_sequences(message) - if os.name == 'nt': - if ctypes.windll.kernel32.GetConsoleWindow(): - # c_wchar_p() might not be necessary if `message` is - # already of type unicode() - ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) - else: - self._send_console_code(f'\033]0;{message}\007') + + if message: + success = self._send_console_code(f'\033]0;{remove_terminal_sequences(message)}\007') + if not success and os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): + ctypes.windll.kernel32.SetConsoleTitleW(message) + + if isinstance(progress_state, _ProgressState): + self._send_console_code(progress_state.get_ansi_escape(percent)) def save_console_title(self): if not self.params.get('consoletitle') or self.params.get('simulate'): @@ -982,6 +1001,7 @@ def restore_console_title(self): def __enter__(self): self.save_console_title() + self.to_console_title(progress_state=_ProgressState.INDETERMINATE) return self def save_cookies(self): @@ -990,6 +1010,7 @@ def save_cookies(self): def __exit__(self, *args): self.restore_console_title() + self.to_console_title(progress_state=_ProgressState.HIDDEN) self.close() def close(self): @@ -1195,8 +1216,7 @@ def _copy_infodict(info_dict): def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False): """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict - @param sanitize Whether to sanitize the output as a filename. - For backward compatibility, a function can also be passed + @param sanitize Whether to sanitize the output as a filename """ info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set @@ -1312,14 +1332,23 @@ def get_value(mdict): na = self.params.get('outtmpl_na_placeholder', 'NA') - def filename_sanitizer(key, value, restricted=self.params.get('restrictfilenames')): + def filename_sanitizer(key, value, restricted): return sanitize_filename(str(value), restricted=restricted, is_id=( bool(re.search(r'(^|[_.])id(\.|$)', key)) if 'filename-sanitization' in self.params['compat_opts'] else NO_DEFAULT)) - sanitizer = sanitize if callable(sanitize) else filename_sanitizer - sanitize = bool(sanitize) + if callable(sanitize): + self.deprecation_warning('Passing a callable "sanitize" to YoutubeDL.prepare_outtmpl is deprecated') + elif not sanitize: + pass + elif (sys.platform != 'win32' and not self.params.get('restrictfilenames') + and self.params.get('windowsfilenames') is False): + def sanitize(key, value): + return str(value).replace('/', '\u29F8').replace('\0', '') + else: + def sanitize(key, value): + return filename_sanitizer(key, value, restricted=self.params.get('restrictfilenames')) def _dumpjson_default(obj): if isinstance(obj, (set, LazyList)): @@ -1402,13 +1431,13 @@ def create_key(outer_mobj): if sanitize: # If value is an object, sanitize might convert it to a string - # So we convert it to repr first + # So we manually convert it before sanitizing if fmt[-1] == 'r': value, fmt = repr(value), str_fmt elif fmt[-1] == 'a': value, fmt = ascii(value), str_fmt if fmt[-1] in 'csra': - value = sanitizer(last_field, value) + value = sanitize(last_field, value) key = '{}\0{}'.format(key.replace('%', '%\0'), outer_mobj.group('format')) TMPL_DICT[key] = value @@ -2111,7 +2140,7 @@ def _build_format_filter(self, filter_spec): m = operator_rex.fullmatch(filter_spec) if m: try: - comparison_value = int(m.group('value')) + comparison_value = float(m.group('value')) except ValueError: comparison_value = parse_filesize(m.group('value')) if comparison_value is None: @@ -3979,15 +4008,6 @@ def print_debug_header(self): if not self.params.get('verbose'): return - from . import _IN_CLI # Must be delayed import - - # These imports can be slow. So import them only as needed - from .extractor.extractors import _LAZY_LOADER - from .extractor.extractors import ( - _PLUGIN_CLASSES as plugin_ies, - _PLUGIN_OVERRIDES as plugin_ie_overrides, - ) - def get_encoding(stream): ret = str(getattr(stream, 'encoding', f'missing ({type(stream).__name__})')) additional_info = [] @@ -4026,17 +4046,18 @@ def get_encoding(stream): _make_label(ORIGIN, CHANNEL.partition('@')[2] or __version__, __version__), f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '', '' if source == 'unknown' else f'({source})', - '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}', + '' if IN_CLI.value else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}', delim=' ')) - if not _IN_CLI: + if not IN_CLI.value: write_debug(f'params: {self.params}') - if not _LAZY_LOADER: - if os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'): - write_debug('Lazy loading extractors is forcibly disabled') - else: - write_debug('Lazy loading extractors is disabled') + import_extractors() + lazy_extractors = LAZY_EXTRACTORS.value + if lazy_extractors is None: + write_debug('Lazy loading extractors is disabled') + elif not lazy_extractors: + write_debug('Lazy loading extractors is forcibly disabled') if self.params['compat_opts']: write_debug('Compatibility options: {}'.format(', '.join(self.params['compat_opts']))) @@ -4065,24 +4086,27 @@ def get_encoding(stream): write_debug(f'Proxy map: {self.proxies}') write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}') - if os.environ.get('YTDLP_NO_PLUGINS'): - write_debug('Plugins are forcibly disabled') - return - for plugin_type, plugins in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}.items(): - display_list = ['{}{}'.format( - klass.__name__, '' if klass.__name__ == name else f' as {name}') - for name, klass in plugins.items()] + for plugin_type, plugins in (('Extractor', plugin_ies), ('Post-Processor', plugin_pps)): + display_list = [ + klass.__name__ if klass.__name__ == name else f'{klass.__name__} as {name}' + for name, klass in plugins.value.items()] if plugin_type == 'Extractor': display_list.extend(f'{plugins[-1].IE_NAME.partition("+")[2]} ({parent.__name__})' - for parent, plugins in plugin_ie_overrides.items()) + for parent, plugins in plugin_ies_overrides.value.items()) if not display_list: continue write_debug(f'{plugin_type} Plugins: {", ".join(sorted(display_list))}') - plugin_dirs = plugin_directories() - if plugin_dirs: - write_debug(f'Plugin directories: {plugin_dirs}') + plugin_dirs_msg = 'none' + if not plugin_dirs.value: + plugin_dirs_msg = 'none (disabled)' + else: + found_plugin_directories = plugin_directories() + if found_plugin_directories: + plugin_dirs_msg = ', '.join(found_plugin_directories) + + write_debug(f'Plugin directories: {plugin_dirs_msg}') @functools.cached_property def proxies(self): @@ -4127,7 +4151,7 @@ def _get_available_impersonate_targets(self): (target, rh.RH_NAME) for rh in self._request_director.handlers.values() if isinstance(rh, ImpersonateRequestHandler) - for target in rh.supported_targets + for target in reversed(rh.supported_targets) ] def _impersonate_target_available(self, target): diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 63625115f..4612bfca3 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -19,7 +19,9 @@ from .extractor import list_extractor_classes from .extractor.adobepass import MSO_INFO from .networking.impersonate import ImpersonateTarget +from .globals import IN_CLI, plugin_dirs from .options import parseOpts +from .plugins import load_all_plugins as _load_all_plugins from .postprocessor import ( FFmpegExtractAudioPP, FFmpegMergerPP, @@ -33,7 +35,6 @@ ) from .update import Updater from .utils import ( - Config, NO_DEFAULT, POSTPROCESS_WHEN, DateRange, @@ -66,8 +67,6 @@ from .utils._utils import _UnsafeExtensionError from .YoutubeDL import YoutubeDL -_IN_CLI = False - def _exit(status=0, *args): for msg in args: @@ -261,9 +260,11 @@ def parse_retries(name, value): elif value in ('inf', 'infinite'): return float('inf') try: - return int(value) + int_value = int(value) except (TypeError, ValueError): validate(False, f'{name} retry count', value) + validate_positive(f'{name} retry count', int_value) + return int_value opts.retries = parse_retries('download', opts.retries) opts.fragment_retries = parse_retries('fragment', opts.fragment_retries) @@ -293,18 +294,20 @@ def parse_sleep_func(expr): raise ValueError(f'invalid {key} retry sleep expression {expr!r}') # Bytes - def validate_bytes(name, value): + def validate_bytes(name, value, strict_positive=False): if value is None: return None numeric_limit = parse_bytes(value) - validate(numeric_limit is not None, 'rate limit', value) + validate(numeric_limit is not None, name, value) + if strict_positive: + validate_positive(name, numeric_limit, True) return numeric_limit - opts.ratelimit = validate_bytes('rate limit', opts.ratelimit) + opts.ratelimit = validate_bytes('rate limit', opts.ratelimit, True) opts.throttledratelimit = validate_bytes('throttled rate limit', opts.throttledratelimit) opts.min_filesize = validate_bytes('min filesize', opts.min_filesize) opts.max_filesize = validate_bytes('max filesize', opts.max_filesize) - opts.buffersize = validate_bytes('buffer size', opts.buffersize) + opts.buffersize = validate_bytes('buffer size', opts.buffersize, True) opts.http_chunk_size = validate_bytes('http chunk size', opts.http_chunk_size) # Output templates @@ -429,6 +432,10 @@ def metadataparser_actions(f): } # Other options + opts.plugin_dirs = opts.plugin_dirs + if opts.plugin_dirs is None: + opts.plugin_dirs = ['default'] + if opts.playlist_items is not None: try: tuple(PlaylistEntries.parse_playlist_items(opts.playlist_items)) @@ -953,11 +960,6 @@ def _real_main(argv=None): parser, opts, all_urls, ydl_opts = parse_options(argv) - # HACK: Set the plugin dirs early on - # TODO(coletdjnz): remove when plugin globals system is implemented - if opts.plugin_dirs is not None: - Config._plugin_dirs = list(map(expand_path, opts.plugin_dirs)) - # Dump user agent if opts.dump_user_agent: ua = traverse_obj(opts.headers, 'User-Agent', casesense=False, default=std_headers['User-Agent']) @@ -972,6 +974,11 @@ def _real_main(argv=None): if opts.ffmpeg_location: FFmpegPostProcessor._ffmpeg_location.set(opts.ffmpeg_location) + # load all plugins into the global lookup + plugin_dirs.value = opts.plugin_dirs + if plugin_dirs.value: + _load_all_plugins() + with YoutubeDL(ydl_opts) as ydl: pre_process = opts.update_self or opts.rm_cachedir actual_use = all_urls or opts.load_info_filename @@ -998,8 +1005,9 @@ def _real_main(argv=None): # List of simplified targets we know are supported, # to help users know what dependencies may be required. (ImpersonateTarget('chrome'), 'curl_cffi'), - (ImpersonateTarget('edge'), 'curl_cffi'), (ImpersonateTarget('safari'), 'curl_cffi'), + (ImpersonateTarget('firefox'), 'curl_cffi>=0.10'), + (ImpersonateTarget('edge'), 'curl_cffi'), ] available_targets = ydl._get_available_impersonate_targets() @@ -1015,12 +1023,12 @@ def make_row(target, handler): for known_target, known_handler in known_targets: if not any( - known_target in target and handler == known_handler + known_target in target and known_handler.startswith(handler) for target, handler in available_targets ): - rows.append([ + rows.insert(0, [ ydl._format_out(text, ydl.Styles.SUPPRESS) - for text in make_row(known_target, f'{known_handler} (not available)') + for text in make_row(known_target, f'{known_handler} (unavailable)') ]) ydl.to_screen('[info] Available impersonate targets') @@ -1071,8 +1079,7 @@ def make_row(target, handler): def main(argv=None): - global _IN_CLI - _IN_CLI = True + IN_CLI.value = True try: _exit(*variadic(_real_main(argv))) except (CookieLoadError, DownloadError): diff --git a/yt_dlp/aes.py b/yt_dlp/aes.py index 9908434a5..065901d68 100644 --- a/yt_dlp/aes.py +++ b/yt_dlp/aes.py @@ -83,7 +83,7 @@ def aes_ecb_encrypt(data, key, iv=None): @returns {int[]} encrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) encrypted_data = [] for i in range(block_count): @@ -103,7 +103,7 @@ def aes_ecb_decrypt(data, key, iv=None): @returns {int[]} decrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) encrypted_data = [] for i in range(block_count): @@ -134,7 +134,7 @@ def aes_ctr_encrypt(data, key, iv): @returns {int[]} encrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) counter = iter_vector(iv) encrypted_data = [] @@ -158,7 +158,7 @@ def aes_cbc_decrypt(data, key, iv): @returns {int[]} decrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) decrypted_data = [] previous_cipher_block = iv @@ -183,7 +183,7 @@ def aes_cbc_encrypt(data, key, iv, *, padding_mode='pkcs7'): @returns {int[]} encrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) encrypted_data = [] previous_cipher_block = iv diff --git a/yt_dlp/downloader/__init__.py b/yt_dlp/downloader/__init__.py index 51a9f28f0..1b12bd4be 100644 --- a/yt_dlp/downloader/__init__.py +++ b/yt_dlp/downloader/__init__.py @@ -35,6 +35,7 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N from .rtsp import RtspFD from .websocket import WebSocketFragmentFD from .youtube_live_chat import YoutubeLiveChatFD +from .bunnycdn import BunnyCdnFD PROTOCOL_MAP = { 'rtmp': RtmpFD, @@ -55,6 +56,7 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N 'websocket_frag': WebSocketFragmentFD, 'youtube_live_chat': YoutubeLiveChatFD, 'youtube_live_chat_replay': YoutubeLiveChatFD, + 'bunnycdn': BunnyCdnFD, } diff --git a/yt_dlp/downloader/bunnycdn.py b/yt_dlp/downloader/bunnycdn.py new file mode 100644 index 000000000..e787f698a --- /dev/null +++ b/yt_dlp/downloader/bunnycdn.py @@ -0,0 +1,50 @@ +import hashlib +import random +import threading + +from .common import FileDownloader +from . import HlsFD +from ..networking import Request +from ..networking.exceptions import network_exceptions + + +class BunnyCdnFD(FileDownloader): + """ + Downloads from BunnyCDN with required pings + Note, this is not a part of public API, and will be removed without notice. + DO NOT USE + """ + + def real_download(self, filename, info_dict): + self.to_screen(f'[{self.FD_NAME}] Downloading from BunnyCDN') + + fd = HlsFD(self.ydl, self.params) + + stop_event = threading.Event() + ping_thread = threading.Thread(target=self.ping_thread, args=(stop_event,), kwargs=info_dict['_bunnycdn_ping_data']) + ping_thread.start() + + try: + return fd.real_download(filename, info_dict) + finally: + stop_event.set() + + def ping_thread(self, stop_event, url, headers, secret, context_id): + # Site sends ping every 4 seconds, but this throttles the download. Pinging every 2 seconds seems to work. + ping_interval = 2 + # Hard coded resolution as it doesn't seem to matter + res = 1080 + paused = 'false' + current_time = 0 + + while not stop_event.wait(ping_interval): + current_time += ping_interval + + time = current_time + round(random.random(), 6) + md5_hash = hashlib.md5(f'{secret}_{context_id}_{time}_{paused}_{res}'.encode()).hexdigest() + ping_url = f'{url}?hash={md5_hash}&time={time}&paused={paused}&resolution={res}' + + try: + self.ydl.urlopen(Request(ping_url, headers=headers)).read() + except network_exceptions as e: + self.to_screen(f'[{self.FD_NAME}] Ping failed: {e}') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index e8dcb37cc..bb9303f8a 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -31,6 +31,7 @@ timetuple_from_msec, try_call, ) +from ..utils._utils import _ProgressState class FileDownloader: @@ -333,7 +334,7 @@ def _report_progress_status(self, s, default_template): progress_dict), s.get('progress_idx') or 0) self.to_console_title(self.ydl.evaluate_outtmpl( progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s', - progress_dict)) + progress_dict), _ProgressState.from_dict(s), s.get('_percent')) def _format_progress(self, *args, **kwargs): return self.ydl._format_text( @@ -357,6 +358,7 @@ def with_fields(*tups, default=''): '_speed_str': self.format_speed(speed).strip(), '_total_bytes_str': _format_bytes('total_bytes'), '_elapsed_str': self.format_seconds(s.get('elapsed')), + '_percent': 100.0, '_percent_str': self.format_percent(100), }) self._report_progress_status(s, join_nonempty( @@ -375,13 +377,15 @@ def with_fields(*tups, default=''): return self._progress_delta_time += update_delta + progress = try_call( + lambda: 100 * s['downloaded_bytes'] / s['total_bytes'], + lambda: 100 * s['downloaded_bytes'] / s['total_bytes_estimate'], + lambda: s['downloaded_bytes'] == 0 and 0) s.update({ '_eta_str': self.format_eta(s.get('eta')).strip(), '_speed_str': self.format_speed(s.get('speed')), - '_percent_str': self.format_percent(try_call( - lambda: 100 * s['downloaded_bytes'] / s['total_bytes'], - lambda: 100 * s['downloaded_bytes'] / s['total_bytes_estimate'], - lambda: s['downloaded_bytes'] == 0 and 0)), + '_percent': progress, + '_percent_str': self.format_percent(progress), '_total_bytes_str': _format_bytes('total_bytes'), '_total_bytes_estimate_str': _format_bytes('total_bytes_estimate'), '_downloaded_bytes_str': _format_bytes('downloaded_bytes'), diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 7afee9727..77fafc36b 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -457,8 +457,6 @@ class FFmpegFD(ExternalFD): @classmethod def available(cls, path=None): - # TODO: Fix path for ffmpeg - # Fixme: This may be wrong when --ffmpeg-location is used return FFmpegPostProcessor().available def on_process_started(self, proc, stdin): diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py index 057373089..97a0380b4 100644 --- a/yt_dlp/downloader/hls.py +++ b/yt_dlp/downloader/hls.py @@ -17,6 +17,7 @@ urljoin, deprecation_warning, ) +from ..utils._utils import _request_dump_filename class HlsFD(FragmentFD): @@ -73,11 +74,23 @@ def check_results(): def real_download(self, filename, info_dict): man_url = info_dict['url'] - self.to_screen(f'[{self.FD_NAME}] Downloading m3u8 manifest') - urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url)) - man_url = urlh.url - s = urlh.read().decode('utf-8', 'ignore') + s = info_dict.get('hls_media_playlist_data') + if s: + self.to_screen(f'[{self.FD_NAME}] Using m3u8 manifest from extracted info') + else: + self.to_screen(f'[{self.FD_NAME}] Downloading m3u8 manifest') + urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url)) + man_url = urlh.url + s_bytes = urlh.read() + if self.params.get('write_pages'): + dump_filename = _request_dump_filename( + man_url, info_dict['id'], None, + trim_length=self.params.get('trim_file_name')) + self.to_screen(f'[{self.FD_NAME}] Saving request to {dump_filename}') + with open(dump_filename, 'wb') as outf: + outf.write(s_bytes) + s = s_bytes.decode('utf-8', 'ignore') can_download, message = self.can_download(s, info_dict), None if can_download: @@ -178,6 +191,7 @@ def is_ad_fragment_end(s): if external_aes_iv: external_aes_iv = binascii.unhexlify(remove_start(external_aes_iv, '0x').zfill(32)) byte_range = {} + byte_range_offset = 0 discontinuity_count = 0 frag_index = 0 ad_frag_next = False @@ -205,6 +219,11 @@ def is_ad_fragment_end(s): }) media_sequence += 1 + # If the byte_range is truthy, reset it after appending a fragment that uses it + if byte_range: + byte_range_offset = byte_range['end'] + byte_range = {} + elif line.startswith('#EXT-X-MAP'): if format_index and discontinuity_count != format_index: continue @@ -218,10 +237,12 @@ def is_ad_fragment_end(s): if extra_segment_query: frag_url = update_url_query(frag_url, extra_segment_query) + map_byte_range = {} + if map_info.get('BYTERANGE'): splitted_byte_range = map_info.get('BYTERANGE').split('@') - sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end'] - byte_range = { + sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else 0 + map_byte_range = { 'start': sub_range_start, 'end': sub_range_start + int(splitted_byte_range[0]), } @@ -230,7 +251,7 @@ def is_ad_fragment_end(s): 'frag_index': frag_index, 'url': frag_url, 'decrypt_info': decrypt_info, - 'byte_range': byte_range, + 'byte_range': map_byte_range, 'media_sequence': media_sequence, }) media_sequence += 1 @@ -258,7 +279,7 @@ def is_ad_fragment_end(s): media_sequence = int(line[22:]) elif line.startswith('#EXT-X-BYTERANGE'): splitted_byte_range = line[17:].split('@') - sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end'] + sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range_offset byte_range = { 'start': sub_range_start, 'end': sub_range_start + int(splitted_byte_range[0]), diff --git a/yt_dlp/downloader/niconico.py b/yt_dlp/downloader/niconico.py index 462c6e2d6..310144e7d 100644 --- a/yt_dlp/downloader/niconico.py +++ b/yt_dlp/downloader/niconico.py @@ -85,6 +85,7 @@ def communicate_ws(reconnect): 'quality': live_quality, 'protocol': 'hls+fmp4', 'latency': live_latency, + 'accessRightMethod': 'single_cookie', 'chasePlay': False, }, 'room': { diff --git a/yt_dlp/extractor/__init__.py b/yt_dlp/extractor/__init__.py index 6bfa4bd7b..a090e942d 100644 --- a/yt_dlp/extractor/__init__.py +++ b/yt_dlp/extractor/__init__.py @@ -1,16 +1,25 @@ from ..compat.compat_utils import passthrough_module +from ..globals import extractors as _extractors_context +from ..globals import plugin_ies as _plugin_ies_context +from ..plugins import PluginSpec, register_plugin_spec passthrough_module(__name__, '.extractors') del passthrough_module +register_plugin_spec(PluginSpec( + module_name='extractor', + suffix='IE', + destination=_extractors_context, + plugin_destination=_plugin_ies_context, +)) + def gen_extractor_classes(): """ Return a list of supported extractors. The order does matter; the first extractor matched is the one handling the URL. """ - from .extractors import _ALL_CLASSES - - return _ALL_CLASSES + import_extractors() + return list(_extractors_context.value.values()) def gen_extractors(): @@ -37,6 +46,9 @@ def list_extractors(age_limit=None): def get_info_extractor(ie_name): """Returns the info extractor class with the given ie_name""" - from . import extractors + import_extractors() + return _extractors_context.value[f'{ie_name}IE'] - return getattr(extractors, f'{ie_name}IE') + +def import_extractors(): + from . import extractors # noqa: F401 diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 967010826..047af9282 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -256,6 +256,7 @@ BilibiliCheeseIE, BilibiliCheeseSeasonIE, BilibiliCollectionListIE, + BiliBiliDynamicIE, BilibiliFavoritesListIE, BiliBiliIE, BiliBiliPlayerIE, @@ -311,6 +312,7 @@ ) from .bundesliga import BundesligaIE from .bundestag import BundestagIE +from .bunnycdn import BunnyCdnIE from .businessinsider import BusinessInsiderIE from .buzzfeed import BuzzFeedIE from .byutv import BYUtvIE @@ -334,6 +336,7 @@ from .canalalpha import CanalAlphaIE from .canalc2 import Canalc2IE from .canalplus import CanalplusIE +from .canalsurmas import CanalsurmasIE from .caracoltv import CaracolTvPlayIE from .cartoonnetwork import CartoonNetworkIE from .cbc import ( @@ -440,12 +443,6 @@ CrowdBunkerIE, ) from .crtvg import CrtvgIE -from .crunchyroll import ( - CrunchyrollArtistIE, - CrunchyrollBetaIE, - CrunchyrollBetaShowIE, - CrunchyrollMusicIE, -) from .cspan import ( CSpanCongressIE, CSpanIE, @@ -459,7 +456,10 @@ CuriosityStreamIE, CuriosityStreamSeriesIE, ) -from .cwtv import CWTVIE +from .cwtv import ( + CWTVIE, + CWTVMovieIE, +) from .cybrary import ( CybraryCourseIE, CybraryIE, @@ -496,10 +496,6 @@ from .daystar import DaystarClipIE from .dbtv import DBTVIE from .dctp import DctpTvIE -from .deezer import ( - DeezerAlbumIE, - DeezerPlaylistIE, -) from .democracynow import DemocracynowIE from .detik import DetikEmbedIE from .deuxm import ( @@ -510,6 +506,7 @@ from .dhm import DHMIE from .digitalconcerthall import DigitalConcertHallIE from .digiteka import DigitekaIE +from .digiview import DigiviewIE from .discogs import DiscogsReleasePlaylistIE from .disney import DisneyIE from .dispeak import DigitallySpeakingIE @@ -555,6 +552,7 @@ DropoutIE, DropoutSeasonIE, ) +from .drtalks import DrTalksIE from .drtuber import DrTuberIE from .drtv import ( DRTVIE, @@ -584,6 +582,10 @@ EggheadCourseIE, EggheadLessonIE, ) +from .eggs import ( + EggsArtistIE, + EggsIE, +) from .eighttracks import EightTracksIE from .eitb import EitbIE from .elementorembed import ElementorEmbedIE @@ -681,6 +683,7 @@ ) from .foxsports import FoxSportsIE from .fptplay import FptplayIE +from .francaisfacile import FrancaisFacileIE from .franceinter import FranceInterIE from .francetv import ( FranceTVIE, @@ -699,11 +702,6 @@ FrontendMastersLessonIE, ) from .fujitv import FujiTVFODPlus7IE -from .funimation import ( - FunimationIE, - FunimationPageIE, - FunimationShowIE, -) from .funk import FunkIE from .funker530 import Funker530IE from .fuyintv import FuyinTVIE @@ -842,6 +840,7 @@ from .ichinanalive import ( IchinanaLiveClipIE, IchinanaLiveIE, + IchinanaLiveVODIE, ) from .idolplus import IdolPlusIE from .ign import ( @@ -904,6 +903,7 @@ IviIE, ) from .ivideon import IvideonIE +from .ivoox import IvooxIE from .iwara import ( IwaraIE, IwaraPlaylistIE, @@ -961,7 +961,10 @@ ) from .kicker import KickerIE from .kickstarter import KickStarterIE -from .kika import KikaIE +from .kika import ( + KikaIE, + KikaPlaylistIE, +) from .kinja import KinjaEmbedIE from .kinopoisk import KinoPoiskIE from .kommunetv import KommunetvIE @@ -1054,6 +1057,7 @@ ) from .livestreamfails import LivestreamfailsIE from .lnk import LnkIE +from .loco import LocoIE from .loom import ( LoomFolderIE, LoomIE, @@ -1061,6 +1065,7 @@ from .lovehomeporn import LoveHomePornIE from .lrt import ( LRTVODIE, + LRTRadioIE, LRTStreamIE, ) from .lsm import ( @@ -1278,6 +1283,10 @@ ) from .nekohacker import NekoHackerIE from .nerdcubed import NerdCubedFeedIE +from .nest import ( + NestClipIE, + NestIE, +) from .neteasemusic import ( NetEaseMusicAlbumIE, NetEaseMusicDjRadioIE, @@ -1489,6 +1498,10 @@ ) from .parler import ParlerIE from .parlview import ParlviewIE +from .parti import ( + PartiLivestreamIE, + PartiVideoIE, +) from .patreon import ( PatreonCampaignIE, PatreonIE, @@ -1532,6 +1545,10 @@ PinterestCollectionIE, PinterestIE, ) +from .piramidetv import ( + PiramideTVChannelIE, + PiramideTVIE, +) from .pixivsketch import ( PixivSketchIE, PixivSketchUserIE, @@ -1551,6 +1568,7 @@ PluralsightIE, ) from .plutotv import PlutoTVIE +from .plvideo import PlVideoIE from .podbayfm import ( PodbayFMChannelIE, PodbayFMIE, @@ -1730,6 +1748,7 @@ RoosterTeethSeriesIE, ) from .rottentomatoes import RottenTomatoesIE +from .roya import RoyaLiveIE from .rozhlas import ( MujRozhlasIE, RozhlasIE, @@ -1764,7 +1783,6 @@ from .rtve import ( RTVEALaCartaIE, RTVEAudioIE, - RTVEInfantilIE, RTVELiveIE, RTVETelevisionIE, ) @@ -1873,6 +1891,8 @@ SkyItVideoIE, SkyItVideoLiveIE, TV8ItIE, + TV8ItLiveIE, + TV8ItPlaylistIE, ) from .skylinewebcams import SkylineWebcamsIE from .skynewsarabia import ( @@ -1886,6 +1906,7 @@ from .smotrim import SmotrimIE from .snapchat import SnapchatSpotlightIE from .snotr import SnotrIE +from .softwhiteunderbelly import SoftWhiteUnderbellyIE from .sohu import ( SohuIE, SohuVIE, @@ -1975,12 +1996,17 @@ StoryFireSeriesIE, StoryFireUserIE, ) +from .streaks import StreaksIE from .streamable import StreamableIE from .streamcz import StreamCZIE from .streetvoice import StreetVoiceIE from .stretchinternet import StretchInternetIE from .stripchat import StripchatIE from .stv import STVPlayerIE +from .subsplash import ( + SubsplashIE, + SubsplashPlaylistIE, +) from .substack import SubstackIE from .sunporno import SunPornoIE from .sverigesradio import ( @@ -2210,6 +2236,10 @@ TVPlayIE, ) from .tvplayer import TVPlayerIE +from .tvw import ( + TvwIE, + TvwTvChannelsIE, +) from .tweakers import TweakersIE from .twentymin import TwentyMinutenIE from .twentythreevideo import TwentyThreeVideoIE @@ -2333,10 +2363,6 @@ ViewLiftIE, ) from .viidea import ViideaIE -from .viki import ( - VikiChannelIE, - VikiIE, -) from .vimeo import ( VHXEmbedIE, VimeoAlbumIE, @@ -2354,10 +2380,6 @@ VimmIE, VimmRecordingIE, ) -from .vine import ( - VineIE, - VineUserIE, -) from .viously import ViouslyIE from .viqeo import ViqeoIE from .viu import ( @@ -2385,10 +2407,15 @@ VoxMediaIE, VoxMediaVolumeIE, ) +from .vrsquare import ( + VrSquareChannelIE, + VrSquareIE, + VrSquareSearchIE, + VrSquareSectionIE, +) from .vrt import ( VRTIE, DagelijkseKostIE, - KetnetIE, Radio1BeIE, VrtNUIE, ) diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py index b1343eed3..8f2fc4c80 100644 --- a/yt_dlp/extractor/abematv.py +++ b/yt_dlp/extractor/abematv.py @@ -21,6 +21,7 @@ int_or_none, time_seconds, traverse_obj, + update_url, update_url_query, ) @@ -417,18 +418,23 @@ def _real_extract(self, url): 'is_live': is_live, 'availability': availability, }) + + if thumbnail := update_url(self._og_search_thumbnail(webpage, default=''), query=None): + info['thumbnails'] = [{'url': thumbnail}] + return info class AbemaTVTitleIE(AbemaTVBaseIE): - _VALID_URL = r'https?://abema\.tv/video/title/(?P[^?/]+)' + _VALID_URL = r'https?://abema\.tv/video/title/(?P[^?/#]+)/?(?:\?(?:[^#]+&)?s=(?P[^&#]+))?' _PAGE_SIZE = 25 _TESTS = [{ - 'url': 'https://abema.tv/video/title/90-1597', + 'url': 'https://abema.tv/video/title/90-1887', 'info_dict': { - 'id': '90-1597', + 'id': '90-1887', 'title': 'シャッフルアイランド', + 'description': 'md5:61b2425308f41a5282a926edda66f178', }, 'playlist_mincount': 2, }, { @@ -436,41 +442,54 @@ class AbemaTVTitleIE(AbemaTVBaseIE): 'info_dict': { 'id': '193-132', 'title': '真心が届く~僕とスターのオフィス・ラブ!?~', + 'description': 'md5:9b59493d1f3a792bafbc7319258e7af8', }, 'playlist_mincount': 16, }, { - 'url': 'https://abema.tv/video/title/25-102', + 'url': 'https://abema.tv/video/title/25-1nzan-whrxe', 'info_dict': { - 'id': '25-102', - 'title': 'ソードアート・オンライン アリシゼーション', + 'id': '25-1nzan-whrxe', + 'title': 'ソードアート・オンライン', + 'description': 'md5:c094904052322e6978495532bdbf06e6', }, - 'playlist_mincount': 24, + 'playlist_mincount': 25, + }, { + 'url': 'https://abema.tv/video/title/26-2mzbynr-cph?s=26-2mzbynr-cph_s40', + 'info_dict': { + 'title': '〈物語〉シリーズ', + 'id': '26-2mzbynr-cph', + 'description': 'md5:e67873de1c88f360af1f0a4b84847a52', + }, + 'playlist_count': 59, }] - def _fetch_page(self, playlist_id, series_version, page): + def _fetch_page(self, playlist_id, series_version, season_id, page): + query = { + 'seriesVersion': series_version, + 'offset': str(page * self._PAGE_SIZE), + 'order': 'seq', + 'limit': str(self._PAGE_SIZE), + } + if season_id: + query['seasonId'] = season_id programs = self._call_api( f'v1/video/series/{playlist_id}/programs', playlist_id, note=f'Downloading page {page + 1}', - query={ - 'seriesVersion': series_version, - 'offset': str(page * self._PAGE_SIZE), - 'order': 'seq', - 'limit': str(self._PAGE_SIZE), - }) + query=query) yield from ( self.url_result(f'https://abema.tv/video/episode/{x}') for x in traverse_obj(programs, ('programs', ..., 'id'))) - def _entries(self, playlist_id, series_version): + def _entries(self, playlist_id, series_version, season_id): return OnDemandPagedList( - functools.partial(self._fetch_page, playlist_id, series_version), + functools.partial(self._fetch_page, playlist_id, series_version, season_id), self._PAGE_SIZE) def _real_extract(self, url): - playlist_id = self._match_id(url) + playlist_id, season_id = self._match_valid_url(url).group('id', 'season') series_info = self._call_api(f'v1/video/series/{playlist_id}', playlist_id) return self.playlist_result( - self._entries(playlist_id, series_info['version']), playlist_id=playlist_id, + self._entries(playlist_id, series_info['version'], season_id), playlist_id=playlist_id, playlist_title=series_info.get('title'), playlist_description=series_info.get('content')) diff --git a/yt_dlp/extractor/acast.py b/yt_dlp/extractor/acast.py index 8f4a2cf0f..eb467cb75 100644 --- a/yt_dlp/extractor/acast.py +++ b/yt_dlp/extractor/acast.py @@ -43,14 +43,14 @@ class ACastIE(ACastBaseIE): _VALID_URL = r'''(?x: https?:// (?: - (?:(?:embed|www)\.)?acast\.com/| + (?:(?:embed|www|shows)\.)?acast\.com/| play\.acast\.com/s/ ) - (?P[^/]+)/(?P[^/#?"]+) + (?P[^/?#]+)/(?:episodes/)?(?P[^/#?"]+) )''' _EMBED_REGEX = [rf'(?x)]+\bsrc=[\'"](?P{_VALID_URL})'] _TESTS = [{ - 'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna', + 'url': 'https://shows.acast.com/sparpodcast/episodes/2.raggarmordet-rosterurdetforflutna', 'info_dict': { 'id': '2a92b283-1a75-4ad8-8396-499c641de0d9', 'ext': 'mp3', @@ -59,7 +59,7 @@ class ACastIE(ACastBaseIE): 'timestamp': 1477346700, 'upload_date': '20161024', 'duration': 2766, - 'creator': 'Third Ear Studio', + 'creators': ['Third Ear Studio'], 'series': 'Spår', 'episode': '2. Raggarmordet - Röster ur det förflutna', 'thumbnail': 'https://assets.pippa.io/shows/616ebe1886d7b1398620b943/616ebe33c7e6e70013cae7da.jpg', @@ -74,6 +74,9 @@ class ACastIE(ACastBaseIE): }, { 'url': 'https://play.acast.com/s/rattegangspodden/s04e09styckmordetihelenelund-del2-2', 'only_matching': True, + }, { + 'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna', + 'only_matching': True, }, { 'url': 'https://play.acast.com/s/sparpodcast/2a92b283-1a75-4ad8-8396-499c641de0d9', 'only_matching': True, @@ -110,7 +113,7 @@ class ACastChannelIE(ACastBaseIE): _VALID_URL = r'''(?x) https?:// (?: - (?:www\.)?acast\.com/| + (?:(?:www|shows)\.)?acast\.com/| play\.acast\.com/s/ ) (?P[^/#?]+) @@ -120,12 +123,15 @@ class ACastChannelIE(ACastBaseIE): 'info_dict': { 'id': '4efc5294-5385-4847-98bd-519799ce5786', 'title': 'Today in Focus', - 'description': 'md5:c09ce28c91002ce4ffce71d6504abaae', + 'description': 'md5:feca253de9947634605080cd9eeea2bf', }, 'playlist_mincount': 200, }, { 'url': 'http://play.acast.com/s/ft-banking-weekly', 'only_matching': True, + }, { + 'url': 'https://shows.acast.com/sparpodcast', + 'only_matching': True, }] @classmethod diff --git a/yt_dlp/extractor/afreecatv.py b/yt_dlp/extractor/afreecatv.py index 572d1a389..aadb4d660 100644 --- a/yt_dlp/extractor/afreecatv.py +++ b/yt_dlp/extractor/afreecatv.py @@ -1,3 +1,4 @@ +import datetime as dt import functools from .common import InfoExtractor @@ -10,7 +11,7 @@ filter_dict, int_or_none, orderedSet, - unified_timestamp, + parse_iso8601, url_or_none, urlencode_postdata, urljoin, @@ -87,9 +88,9 @@ class AfreecaTVIE(AfreecaTVBaseIE): 'uploader_id': 'rlantnghks', 'uploader': '페이즈으', 'duration': 10840, - 'thumbnail': r're:https?://videoimg\.sooplive\.co/.kr/.+', + 'thumbnail': r're:https?://videoimg\.(?:sooplive\.co\.kr|afreecatv\.com)/.+', 'upload_date': '20230108', - 'timestamp': 1673218805, + 'timestamp': 1673186405, 'title': '젠지 페이즈', }, 'params': { @@ -102,7 +103,7 @@ class AfreecaTVIE(AfreecaTVBaseIE): 'id': '20170411_BE689A0E_190960999_1_2_h', 'ext': 'mp4', 'title': '혼자사는여자집', - 'thumbnail': r're:https?://(?:video|st)img\.sooplive\.co\.kr/.+', + 'thumbnail': r're:https?://(?:video|st)img\.(?:sooplive\.co\.kr|afreecatv\.com)/.+', 'uploader': '♥이슬이', 'uploader_id': 'dasl8121', 'upload_date': '20170411', @@ -119,7 +120,7 @@ class AfreecaTVIE(AfreecaTVBaseIE): 'id': '20180327_27901457_202289533_1', 'ext': 'mp4', 'title': '[생]빨개요♥ (part 1)', - 'thumbnail': r're:https?://(?:video|st)img\.sooplive\.co\.kr/.+', + 'thumbnail': r're:https?://(?:video|st)img\.(?:sooplive\.co\.kr|afreecatv\.com)/.+', 'uploader': '[SA]서아', 'uploader_id': 'bjdyrksu', 'upload_date': '20180327', @@ -187,7 +188,7 @@ def _real_extract(self, url): 'formats': formats, **traverse_obj(file_element, { 'duration': ('duration', {int_or_none(scale=1000)}), - 'timestamp': ('file_start', {unified_timestamp}), + 'timestamp': ('file_start', {parse_iso8601(delimiter=' ', timezone=dt.timedelta(hours=9))}), }), }) @@ -370,7 +371,7 @@ def _real_extract(self, url): 'title': channel_info.get('TITLE') or station_info.get('station_title'), 'uploader': channel_info.get('BJNICK') or station_info.get('station_name'), 'uploader_id': broadcaster_id, - 'timestamp': unified_timestamp(station_info.get('broad_start')), + 'timestamp': parse_iso8601(station_info.get('broad_start'), delimiter=' ', timezone=dt.timedelta(hours=9)), 'formats': formats, 'is_live': True, 'http_headers': {'Referer': url}, diff --git a/yt_dlp/extractor/agora.py b/yt_dlp/extractor/agora.py index 983558425..e040db601 100644 --- a/yt_dlp/extractor/agora.py +++ b/yt_dlp/extractor/agora.py @@ -146,7 +146,7 @@ class TokFMPodcastIE(InfoExtractor): 'url': 'https://audycje.tokfm.pl/podcast/91275,-Systemowy-rasizm-Czy-zamieszki-w-USA-po-morderstwie-w-Minneapolis-doprowadza-do-zmian-w-sluzbach-panstwowych', 'info_dict': { 'id': '91275', - 'ext': 'aac', + 'ext': 'mp3', 'title': 'md5:a9b15488009065556900169fb8061cce', 'episode': 'md5:a9b15488009065556900169fb8061cce', 'series': 'Analizy', @@ -164,23 +164,20 @@ def _real_extract(self, url): raise ExtractorError('No such podcast', expected=True) metadata = metadata[0] - formats = [] - for ext in ('aac', 'mp3'): - url_data = self._download_json( - f'https://api.podcast.radioagora.pl/api4/getSongUrl?podcast_id={media_id}&device_id={uuid.uuid4()}&ppre=false&audio={ext}', - media_id, f'Downloading podcast {ext} URL') - # prevents inserting the mp3 (default) multiple times - if 'link_ssl' in url_data and f'.{ext}' in url_data['link_ssl']: - formats.append({ - 'url': url_data['link_ssl'], - 'ext': ext, - 'vcodec': 'none', - 'acodec': ext, - }) + mp3_url = self._download_json( + 'https://api.podcast.radioagora.pl/api4/getSongUrl', + media_id, 'Downloading podcast mp3 URL', query={ + 'podcast_id': media_id, + 'device_id': str(uuid.uuid4()), + 'ppre': 'false', + 'audio': 'mp3', + })['link_ssl'] return { 'id': media_id, - 'formats': formats, + 'url': mp3_url, + 'vcodec': 'none', + 'ext': 'mp3', 'title': metadata.get('podcast_name'), 'series': metadata.get('series_name'), 'episode': metadata.get('podcast_name'), diff --git a/yt_dlp/extractor/atresplayer.py b/yt_dlp/extractor/atresplayer.py index 0fe95bec5..1258a5704 100644 --- a/yt_dlp/extractor/atresplayer.py +++ b/yt_dlp/extractor/atresplayer.py @@ -1,64 +1,105 @@ +import urllib.parse + from .common import InfoExtractor from ..networking.exceptions import HTTPError from ..utils import ( ExtractorError, int_or_none, + parse_age_limit, + url_or_none, urlencode_postdata, ) +from ..utils.traversal import traverse_obj class AtresPlayerIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/[^/]+/[^/]+/[^/]+/[^/]+/(?P.+?)_(?P[0-9a-f]{24})' + _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/(?:[^/?#]+/){4}(?P.+?)_(?P[0-9a-f]{24})' _NETRC_MACHINE = 'atresplayer' - _TESTS = [ - { - 'url': 'https://www.atresplayer.com/antena3/series/pequenas-coincidencias/temporada-1/capitulo-7-asuntos-pendientes_5d4aa2c57ed1a88fc715a615/', - 'info_dict': { - 'id': '5d4aa2c57ed1a88fc715a615', - 'ext': 'mp4', - 'title': 'Capítulo 7: Asuntos pendientes', - 'description': 'md5:7634cdcb4d50d5381bedf93efb537fbc', - 'duration': 3413, - }, - 'skip': 'This video is only available for registered users', + _TESTS = [{ + 'url': 'https://www.atresplayer.com/lasexta/programas/el-objetivo/clips/mbappe-describe-como-entrenador-a-carlo-ancelotti-sabe-cuando-tiene-que-ser-padre-jefe-amigo-entrenador_67f2dfb2fb6ab0e4c7203849/', + 'info_dict': { + 'ext': 'mp4', + 'id': '67f2dfb2fb6ab0e4c7203849', + 'display_id': 'md5:c203f8d4e425ed115ba56a1c6e4b3e6c', + 'title': 'Mbappé describe como entrenador a Carlo Ancelotti: "Sabe cuándo tiene que ser padre, jefe, amigo, entrenador..."', + 'channel': 'laSexta', + 'duration': 31, + 'thumbnail': 'https://imagenes.atresplayer.com/atp/clipping/cmsimages02/2025/04/06/B02DBE1E-D59B-4683-8404-1A9595D15269/1920x1080.jpg', + 'tags': ['Entrevista informativa', 'Actualidad', 'Debate informativo', 'Política', 'Economía', 'Sociedad', 'Cara a cara', 'Análisis', 'Más periodismo'], + 'series': 'El Objetivo', + 'season': 'Temporada 12', + 'timestamp': 1743970079, + 'upload_date': '20250406', }, - { - 'url': 'https://www.atresplayer.com/lasexta/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_5ad08edf986b2855ed47adc4/', - 'only_matching': True, + }, { + 'url': 'https://www.atresplayer.com/antena3/programas/el-hormiguero/clips/revive-la-entrevista-completa-a-miguel-bose-en-el-hormiguero_67f836baa4a5b0e4147ca59a/', + 'info_dict': { + 'ext': 'mp4', + 'id': '67f836baa4a5b0e4147ca59a', + 'display_id': 'revive-la-entrevista-completa-a-miguel-bose-en-el-hormiguero', + 'title': 'Revive la entrevista completa a Miguel Bosé en El Hormiguero', + 'description': 'md5:c6d2b591408d45a7bc2986dfb938eb72', + 'channel': 'Antena 3', + 'duration': 2556, + 'thumbnail': 'https://imagenes.atresplayer.com/atp/clipping/cmsimages02/2025/04/10/9076395F-F1FD-48BE-9F18-540DBA10EBAD/1920x1080.jpg', + 'tags': ['Entrevista', 'Variedades', 'Humor', 'Entretenimiento', 'Te sigo', 'Buen rollo', 'Cara a cara'], + 'series': 'El Hormiguero ', + 'season': 'Temporada 14', + 'timestamp': 1744320111, + 'upload_date': '20250410', }, - { - 'url': 'https://www.atresplayer.com/antena3/series/el-secreto-de-puente-viejo/el-chico-de-los-tres-lunares/capitulo-977-29-12-14_5ad51046986b2886722ccdea/', - 'only_matching': True, + }, { + 'url': 'https://www.atresplayer.com/flooxer/series/biara-proyecto-lazarus/temporada-1/capitulo-3-supervivientes_67a6038b64ceca00070f4f69/', + 'info_dict': { + 'ext': 'mp4', + 'id': '67a6038b64ceca00070f4f69', + 'display_id': 'capitulo-3-supervivientes', + 'title': 'Capítulo 3: Supervivientes', + 'description': 'md5:65b231f20302f776c2b0dd24594599a1', + 'channel': 'Flooxer', + 'duration': 1196, + 'thumbnail': 'https://imagenes.atresplayer.com/atp/clipping/cmsimages01/2025/02/14/17CF90D3-FE67-40C5-A941-7825B3E13992/1920x1080.jpg', + 'tags': ['Juvenil', 'Terror', 'Piel de gallina', 'Te sigo', 'Un break', 'Del tirón'], + 'series': 'BIARA: Proyecto Lázarus', + 'season': 'Temporada 1', + 'season_number': 1, + 'episode': 'Episode 3', + 'episode_number': 3, + 'timestamp': 1743095191, + 'upload_date': '20250327', }, - ] + }, { + 'url': 'https://www.atresplayer.com/lasexta/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_5ad08edf986b2855ed47adc4/', + 'only_matching': True, + }, { + 'url': 'https://www.atresplayer.com/antena3/series/el-secreto-de-puente-viejo/el-chico-de-los-tres-lunares/capitulo-977-29-12-14_5ad51046986b2886722ccdea/', + 'only_matching': True, + }] _API_BASE = 'https://api.atresplayer.com/' def _perform_login(self, username, password): - self._request_webpage( - self._API_BASE + 'login', None, 'Downloading login page') - try: - target_url = self._download_json( - 'https://account.atresmedia.com/api/login', None, - 'Logging in', headers={ - 'Content-Type': 'application/x-www-form-urlencoded', - }, data=urlencode_postdata({ + self._download_webpage( + 'https://account.atresplayer.com/auth/v1/login', None, + 'Logging in', 'Failed to log in', data=urlencode_postdata({ 'username': username, 'password': password, - }))['targetUrl'] + })) except ExtractorError as e: if isinstance(e.cause, HTTPError) and e.cause.status == 400: raise ExtractorError('Invalid username and/or password', expected=True) raise - self._request_webpage(target_url, None, 'Following Target URL') - def _real_extract(self, url): display_id, video_id = self._match_valid_url(url).groups() + metadata_url = self._download_json( + self._API_BASE + 'client/v1/url', video_id, 'Downloading API endpoint data', + query={'href': urllib.parse.urlparse(url).path})['href'] + metadata = self._download_json(metadata_url, video_id) + try: - episode = self._download_json( - self._API_BASE + 'client/v1/player/episode/' + video_id, video_id) + video_data = self._download_json(metadata['urlVideo'], video_id, 'Downloading video data') except ExtractorError as e: if isinstance(e.cause, HTTPError) and e.cause.status == 403: error = self._parse_json(e.cause.response.read(), None) @@ -67,37 +108,45 @@ def _real_extract(self, url): raise ExtractorError(error['error_description'], expected=True) raise - title = episode['titulo'] - formats = [] subtitles = {} - for source in episode.get('sources', []): - src = source.get('src') - if not src: - continue + for source in traverse_obj(video_data, ('sources', lambda _, v: url_or_none(v['src']))): + src_url = source['src'] src_type = source.get('type') - if src_type == 'application/vnd.apple.mpegurl': - formats, subtitles = self._extract_m3u8_formats( - src, video_id, 'mp4', 'm3u8_native', - m3u8_id='hls', fatal=False) - elif src_type == 'application/dash+xml': - formats, subtitles = self._extract_mpd_formats( - src, video_id, mpd_id='dash', fatal=False) - - heartbeat = episode.get('heartbeat') or {} - omniture = episode.get('omniture') or {} - get_meta = lambda x: heartbeat.get(x) or omniture.get(x) + if src_type in ('application/vnd.apple.mpegurl', 'application/hls+legacy', 'application/hls+hevc'): + fmts, subs = self._extract_m3u8_formats_and_subtitles( + src_url, video_id, 'mp4', m3u8_id='hls', fatal=False) + elif src_type in ('application/dash+xml', 'application/dash+hevc'): + fmts, subs = self._extract_mpd_formats_and_subtitles( + src_url, video_id, mpd_id='dash', fatal=False) + else: + continue + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) return { 'display_id': display_id, 'id': video_id, - 'title': title, - 'description': episode.get('descripcion'), - 'thumbnail': episode.get('imgPoster'), - 'duration': int_or_none(episode.get('duration')), 'formats': formats, - 'channel': get_meta('channel'), - 'season': get_meta('season'), - 'episode_number': int_or_none(get_meta('episodeNumber')), 'subtitles': subtitles, + **traverse_obj(video_data, { + 'title': ('titulo', {str}), + 'description': ('descripcion', {str}), + 'duration': ('duration', {int_or_none}), + 'thumbnail': ('imgPoster', {url_or_none}, {lambda v: f'{v}1920x1080.jpg'}), + 'age_limit': ('ageRating', {parse_age_limit}), + }), + **traverse_obj(metadata, { + 'title': ('title', {str}), + 'description': ('description', {str}), + 'duration': ('duration', {int_or_none}), + 'tags': ('tags', ..., 'title', {str}), + 'age_limit': ('ageRating', {parse_age_limit}), + 'series': ('format', 'title', {str}), + 'season': ('currentSeason', 'title', {str}), + 'season_number': ('currentSeason', 'seasonNumber', {int_or_none}), + 'episode_number': ('numberOfEpisode', {int_or_none}), + 'timestamp': ('publicationDate', {int_or_none(scale=1000)}), + 'channel': ('channel', 'title', {str}), + }), } diff --git a/yt_dlp/extractor/azmedien.py b/yt_dlp/extractor/azmedien.py index 0e3a03f03..9f6bd820c 100644 --- a/yt_dlp/extractor/azmedien.py +++ b/yt_dlp/extractor/azmedien.py @@ -1,7 +1,6 @@ -import json - from .common import InfoExtractor from .kaltura import KalturaIE +from ..utils.traversal import require, traverse_obj class AZMedienIE(InfoExtractor): @@ -9,15 +8,15 @@ class AZMedienIE(InfoExtractor): _VALID_URL = r'''(?x) https?:// (?:www\.|tv\.)? - (?P + (?: telezueri\.ch| telebaern\.tv| telem1\.ch| tvo-online\.ch )/ - [^/]+/ + [^/?#]+/ (?P - [^/]+-(?P\d+) + [^/?#]+-\d+ ) (?: \#video= @@ -47,19 +46,17 @@ class AZMedienIE(InfoExtractor): 'url': 'https://www.telebaern.tv/telebaern-news/montag-1-oktober-2018-ganze-sendung-133531189#video=0_7xjo9lf1', 'only_matching': True, }] - _API_TEMPL = 'https://www.%s/api/pub/gql/%s/NewsArticleTeaser/a4016f65fe62b81dc6664dd9f4910e4ab40383be' _PARTNER_ID = '1719221' def _real_extract(self, url): - host, display_id, article_id, entry_id = self._match_valid_url(url).groups() + display_id, entry_id = self._match_valid_url(url).groups() if not entry_id: - entry_id = self._download_json( - self._API_TEMPL % (host, host.split('.')[0]), display_id, query={ - 'variables': json.dumps({ - 'contextId': 'NewsArticle:' + article_id, - }), - })['data']['context']['mainAsset']['video']['kaltura']['kalturaId'] + webpage = self._download_webpage(url, display_id) + data = self._search_json( + r'window\.__APOLLO_STATE__\s*=', webpage, 'video data', display_id) + entry_id = traverse_obj(data, ( + lambda _, v: v['__typename'] == 'KalturaData', 'kalturaId', any, {require('kaltura id')})) return self.url_result( f'kaltura:{self._PARTNER_ID}:{entry_id}', diff --git a/yt_dlp/extractor/bandlab.py b/yt_dlp/extractor/bandlab.py index 64aa2ba70..f110b793b 100644 --- a/yt_dlp/extractor/bandlab.py +++ b/yt_dlp/extractor/bandlab.py @@ -86,7 +86,7 @@ def _parse_video(self, video_data, url=None): 'webpage_url': ( 'id', ({value(url)}, {format_field(template='https://www.bandlab.com/post/%s')}), filter, any), 'url': ('video', 'url', {url_or_none}), - 'title': ('caption', {lambda x: x.replace('\n', ' ')}, {truncate_string(left=50)}), + 'title': ('caption', {lambda x: x.replace('\n', ' ')}, {truncate_string(left=72)}), 'description': ('caption', {str}), 'thumbnail': ('video', 'picture', 'url', {url_or_none}), 'view_count': ('video', 'counters', 'plays', {int_or_none}), @@ -120,7 +120,7 @@ class BandlabIE(BandlabBaseIE): 'duration': 54.629999999999995, 'title': 'sweet black', 'upload_date': '20231210', - 'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/songs/fa082beb-b856-4730-9170-a57e4e32cc2c/', + 'thumbnail': 'https://bl-prod-images.azureedge.net/v1.0/songs/fa082beb-b856-4730-9170-a57e4e32cc2c/', 'genres': ['Lofi'], 'uploader': 'ender milze', 'comment_count': int, @@ -142,7 +142,7 @@ class BandlabIE(BandlabBaseIE): 'duration': 54.629999999999995, 'title': 'sweet black', 'upload_date': '20231210', - 'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/songs/fa082beb-b856-4730-9170-a57e4e32cc2c/', + 'thumbnail': 'https://bl-prod-images.azureedge.net/v1.0/songs/fa082beb-b856-4730-9170-a57e4e32cc2c/', 'genres': ['Lofi'], 'uploader': 'ender milze', 'comment_count': int, @@ -158,7 +158,7 @@ class BandlabIE(BandlabBaseIE): 'comment_count': int, 'genres': ['Other'], 'uploader_id': 'user8353034818103753', - 'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/songs/51b18363-da23-4b9b-a29c-2933a3e561ca/', + 'thumbnail': 'https://bl-prod-images.azureedge.net/v1.0/songs/51b18363-da23-4b9b-a29c-2933a3e561ca/', 'timestamp': 1709625771, 'track': 'PodcastMaerchen4b', 'duration': 468.14, @@ -178,7 +178,7 @@ class BandlabIE(BandlabBaseIE): 'id': '110343fc-148b-ea11-96d2-0003ffd1fc09', 'ext': 'm4a', 'timestamp': 1588273294, - 'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/users/b612e533-e4f7-4542-9f50-3fcfd8dd822c/', + 'thumbnail': 'https://bl-prod-images.azureedge.net/v1.0/users/b612e533-e4f7-4542-9f50-3fcfd8dd822c/', 'description': 'Final Revision.', 'title': 'Replay ( Instrumental)', 'uploader': 'David R Sparks', @@ -200,7 +200,7 @@ class BandlabIE(BandlabBaseIE): 'id': '5cdf9036-3857-ef11-991a-6045bd36e0d9', 'ext': 'mp4', 'duration': 44.705, - 'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/videos/67c6cef1-cef6-40d3-831e-a55bc1dcb972/', + 'thumbnail': 'https://bl-prod-images.azureedge.net/v1.0/videos/67c6cef1-cef6-40d3-831e-a55bc1dcb972/', 'comment_count': int, 'title': 'backing vocals', 'uploader_id': 'marliashya', @@ -224,7 +224,7 @@ class BandlabIE(BandlabBaseIE): 'view_count': int, 'track': 'Positronic Meltdown', 'duration': 318.55, - 'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/songs/87165bc3-5439-496e-b1f7-a9f13b541ff2/', + 'thumbnail': 'https://bl-prod-images.azureedge.net/v1.0/songs/87165bc3-5439-496e-b1f7-a9f13b541ff2/', 'description': 'Checkout my tracks at AOMX http://aomxsounds.com/', 'uploader_id': 'microfreaks', 'title': 'Positronic Meltdown', @@ -246,7 +246,7 @@ class BandlabIE(BandlabBaseIE): 'comment_count': int, 'uploader': 'Sorakime', 'uploader_id': 'sorakime', - 'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/users/572a351a-0f3a-4c6a-ac39-1a5defdeeb1c/', + 'thumbnail': 'https://bl-prod-images.azureedge.net/v1.0/users/572a351a-0f3a-4c6a-ac39-1a5defdeeb1c/', 'timestamp': 1691162128, 'upload_date': '20230804', 'media_type': 'track', diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py index 2db951a60..6508942a4 100644 --- a/yt_dlp/extractor/bilibili.py +++ b/yt_dlp/extractor/bilibili.py @@ -4,7 +4,9 @@ import itertools import json import math +import random import re +import string import time import urllib.parse import uuid @@ -1177,28 +1179,26 @@ def _extract_playlist(self, fetch_page, get_metadata, get_entries): class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE): - _VALID_URL = r'https?://space\.bilibili\.com/(?P\d+)(?P