mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2026-01-11 17:31:31 +00:00
Compare commits
72 Commits
2023.11.16
...
2023.12.30
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9f40cd2896 | ||
|
|
f10589e345 | ||
|
|
f9fb3ce86e | ||
|
|
5f009a094f | ||
|
|
225cf2b830 | ||
|
|
2d1d683a54 | ||
|
|
65de7d204c | ||
|
|
c39358a54b | ||
|
|
1f8bd8eba8 | ||
|
|
00cdda4f6f | ||
|
|
116c268438 | ||
|
|
e7d22348e7 | ||
|
|
50eaea9fd7 | ||
|
|
f45c4efcd9 | ||
|
|
13b3cb3c2b | ||
|
|
0d531c35ec | ||
|
|
bc4ab17b38 | ||
|
|
632b8ee54e | ||
|
|
c919b68f7e | ||
|
|
19741ab8a4 | ||
|
|
37755a037e | ||
|
|
196eb0fe77 | ||
|
|
db8b4edc7d | ||
|
|
1c54a98e19 | ||
|
|
00a3e47bf5 | ||
|
|
c5f01bf7d4 | ||
|
|
c91af948e4 | ||
|
|
6b5d93b0b0 | ||
|
|
298230e550 | ||
|
|
d5d1517e7d | ||
|
|
7e09c147fd | ||
|
|
e370f9ec36 | ||
|
|
b1a1ec1540 | ||
|
|
0b6f829b1d | ||
|
|
f98a3305eb | ||
|
|
04a5e06350 | ||
|
|
b03c89309e | ||
|
|
71f28097fe | ||
|
|
044886c220 | ||
|
|
993edd3f6e | ||
|
|
6a9c7a2b52 | ||
|
|
a174c453ee | ||
|
|
15f22b4880 | ||
|
|
9751a457cf | ||
|
|
5a230233d6 | ||
|
|
4903f452b6 | ||
|
|
ff2fde1b8f | ||
|
|
deeb13eae8 | ||
|
|
bb5a54e6db | ||
|
|
628fa244bb | ||
|
|
9cafb9ff17 | ||
|
|
1732eccc0a | ||
|
|
a0b19d319a | ||
|
|
cc07f5cc85 | ||
|
|
ccfd70f4c2 | ||
|
|
45d82be65f | ||
|
|
3237f8ba29 | ||
|
|
1725e943b0 | ||
|
|
9f09bdcfcb | ||
|
|
f124fa4588 | ||
|
|
585d0ed9ab | ||
|
|
1fa3f24d4b | ||
|
|
ddb2d7588b | ||
|
|
f223b1b078 | ||
|
|
6fe82491ed | ||
|
|
34df1c1f60 | ||
|
|
1d24da6c89 | ||
|
|
66a0127d45 | ||
|
|
3f90813f06 | ||
|
|
64de1a4c25 | ||
|
|
f96ab86cd8 | ||
|
|
f4b95acafc |
23
.github/workflows/build.yml
vendored
23
.github/workflows/build.yml
vendored
@@ -80,12 +80,12 @@ on:
|
||||
default: true
|
||||
type: boolean
|
||||
origin:
|
||||
description: .
|
||||
description: Origin
|
||||
required: false
|
||||
default: ''
|
||||
default: 'current repo'
|
||||
type: choice
|
||||
options:
|
||||
- ''
|
||||
- 'current repo'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
- name: Process origin
|
||||
id: process_origin
|
||||
run: |
|
||||
echo "origin=${{ inputs.origin || github.repository }}" >> "$GITHUB_OUTPUT"
|
||||
echo "origin=${{ inputs.origin == 'current repo' && github.repository || inputs.origin }}" | tee "$GITHUB_OUTPUT"
|
||||
|
||||
unix:
|
||||
needs: process
|
||||
@@ -377,8 +377,8 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v4
|
||||
with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390
|
||||
python-version: "3.7"
|
||||
with:
|
||||
python-version: "3.8"
|
||||
architecture: "x86"
|
||||
- name: Install Requirements
|
||||
run: |
|
||||
@@ -436,7 +436,16 @@ jobs:
|
||||
run: |
|
||||
cat >> _update_spec << EOF
|
||||
# This file is used for regulating self-update
|
||||
lock 2022.08.18.36 .+ Python 3.6
|
||||
lock 2022.08.18.36 .+ Python 3\.6
|
||||
lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7
|
||||
lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
|
||||
lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6
|
||||
lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7
|
||||
lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
|
||||
lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7
|
||||
lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server)
|
||||
lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7
|
||||
lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server)
|
||||
EOF
|
||||
|
||||
- name: Sign checksum files
|
||||
|
||||
40
.github/workflows/core.yml
vendored
40
.github/workflows/core.yml
vendored
@@ -1,8 +1,32 @@
|
||||
name: Core Tests
|
||||
on: [push, pull_request]
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- .github/**
|
||||
- devscripts/**
|
||||
- test/**
|
||||
- yt_dlp/**.py
|
||||
- '!yt_dlp/extractor/*.py'
|
||||
- yt_dlp/extractor/__init__.py
|
||||
- yt_dlp/extractor/common.py
|
||||
- yt_dlp/extractor/extractors.py
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/**
|
||||
- devscripts/**
|
||||
- test/**
|
||||
- yt_dlp/**.py
|
||||
- '!yt_dlp/extractor/*.py'
|
||||
- yt_dlp/extractor/__init__.py
|
||||
- yt_dlp/extractor/common.py
|
||||
- yt_dlp/extractor/extractors.py
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: core-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Core Tests
|
||||
@@ -12,30 +36,26 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
# CPython 3.11 is in quick-test
|
||||
python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.7, pypy-3.8, pypy-3.10]
|
||||
run-tests-ext: [sh]
|
||||
# CPython 3.8 is in quick-test
|
||||
python-version: ['3.9', '3.10', '3.11', '3.12', pypy-3.8, pypy-3.10]
|
||||
include:
|
||||
# atleast one of each CPython/PyPy tests must be in windows
|
||||
- os: windows-latest
|
||||
python-version: '3.7'
|
||||
run-tests-ext: bat
|
||||
python-version: '3.8'
|
||||
- os: windows-latest
|
||||
python-version: '3.12'
|
||||
run-tests-ext: bat
|
||||
- os: windows-latest
|
||||
python-version: pypy-3.9
|
||||
run-tests-ext: bat
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
- name: Install test requirements
|
||||
run: pip install pytest -r requirements.txt
|
||||
- name: Run tests
|
||||
continue-on-error: False
|
||||
run: |
|
||||
python3 -m yt_dlp -v || true # Print debug head
|
||||
./devscripts/run_tests.${{ matrix.run-tests-ext }} core
|
||||
python3 ./devscripts/run_tests.py core
|
||||
|
||||
15
.github/workflows/download.yml
vendored
15
.github/workflows/download.yml
vendored
@@ -15,10 +15,10 @@ jobs:
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install test requirements
|
||||
run: pip install pytest
|
||||
run: pip install pytest -r requirements.txt
|
||||
- name: Run tests
|
||||
continue-on-error: true
|
||||
run: ./devscripts/run_tests.sh download
|
||||
run: python3 ./devscripts/run_tests.py download
|
||||
|
||||
full:
|
||||
name: Full Download Tests
|
||||
@@ -28,24 +28,21 @@ jobs:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
python-version: ['3.7', '3.10', '3.12', pypy-3.7, pypy-3.8, pypy-3.10]
|
||||
run-tests-ext: [sh]
|
||||
python-version: ['3.10', '3.11', '3.12', pypy-3.8, pypy-3.10]
|
||||
include:
|
||||
# atleast one of each CPython/PyPy tests must be in windows
|
||||
- os: windows-latest
|
||||
python-version: '3.8'
|
||||
run-tests-ext: bat
|
||||
- os: windows-latest
|
||||
python-version: pypy-3.9
|
||||
run-tests-ext: bat
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install pytest
|
||||
run: pip install pytest
|
||||
- name: Install test requirements
|
||||
run: pip install pytest -r requirements.txt
|
||||
- name: Run tests
|
||||
continue-on-error: true
|
||||
run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} download
|
||||
run: python3 ./devscripts/run_tests.py download
|
||||
|
||||
8
.github/workflows/quick-test.yml
vendored
8
.github/workflows/quick-test.yml
vendored
@@ -10,16 +10,16 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python 3.11
|
||||
- name: Set up Python 3.8
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.11'
|
||||
python-version: '3.8'
|
||||
- name: Install test requirements
|
||||
run: pip install pytest pycryptodomex
|
||||
run: pip install pytest -r requirements.txt
|
||||
- name: Run tests
|
||||
run: |
|
||||
python3 -m yt_dlp -v || true
|
||||
./devscripts/run_tests.sh core
|
||||
python3 ./devscripts/run_tests.py core
|
||||
flake8:
|
||||
name: Linter
|
||||
if: "!contains(github.event.head_commit.message, 'ci skip all')"
|
||||
|
||||
1
.github/workflows/release-master.yml
vendored
1
.github/workflows/release-master.yml
vendored
@@ -10,7 +10,6 @@ on:
|
||||
- "pyinst.py"
|
||||
concurrency:
|
||||
group: release-master
|
||||
cancel-in-progress: true
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
24
.github/workflows/release.yml
vendored
24
.github/workflows/release.yml
vendored
@@ -64,7 +64,6 @@ jobs:
|
||||
target_tag: ${{ steps.setup_variables.outputs.target_tag }}
|
||||
pypi_project: ${{ steps.setup_variables.outputs.pypi_project }}
|
||||
pypi_suffix: ${{ steps.setup_variables.outputs.pypi_suffix }}
|
||||
pypi_token: ${{ steps.setup_variables.outputs.pypi_token }}
|
||||
head_sha: ${{ steps.get_target.outputs.head_sha }}
|
||||
|
||||
steps:
|
||||
@@ -153,7 +152,6 @@ jobs:
|
||||
${{ !!secrets[format('{0}_archive_repo_token', env.target_repo)] }} || fallback_token
|
||||
pypi_project='${{ vars[format('{0}_pypi_project', env.target_repo)] }}'
|
||||
pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.target_repo)] }}'
|
||||
${{ !secrets[format('{0}_pypi_token', env.target_repo)] }} || pypi_token='${{ env.target_repo }}_pypi_token'
|
||||
fi
|
||||
else
|
||||
target_tag="${source_tag:-${version}}"
|
||||
@@ -163,7 +161,6 @@ jobs:
|
||||
${{ !!secrets[format('{0}_archive_repo_token', env.source_repo)] }} || fallback_token
|
||||
pypi_project='${{ vars[format('{0}_pypi_project', env.source_repo)] }}'
|
||||
pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.source_repo)] }}'
|
||||
${{ !secrets[format('{0}_pypi_token', env.source_repo)] }} || pypi_token='${{ env.source_repo }}_pypi_token'
|
||||
else
|
||||
target_repo='${{ github.repository }}'
|
||||
fi
|
||||
@@ -172,13 +169,6 @@ jobs:
|
||||
if [[ "${target_repo}" == '${{ github.repository }}' ]] && ${{ !inputs.prerelease }}; then
|
||||
pypi_project='${{ vars.PYPI_PROJECT }}'
|
||||
fi
|
||||
if [[ -z "${pypi_token}" && "${pypi_project}" ]]; then
|
||||
if ${{ !secrets.PYPI_TOKEN }}; then
|
||||
pypi_token=OIDC
|
||||
else
|
||||
pypi_token=PYPI_TOKEN
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "::group::Output variables"
|
||||
cat << EOF | tee -a "$GITHUB_OUTPUT"
|
||||
@@ -189,7 +179,6 @@ jobs:
|
||||
target_tag=${target_tag}
|
||||
pypi_project=${pypi_project}
|
||||
pypi_suffix=${pypi_suffix}
|
||||
pypi_token=${pypi_token}
|
||||
EOF
|
||||
echo "::endgroup::"
|
||||
|
||||
@@ -286,18 +275,7 @@ jobs:
|
||||
python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
|
||||
python setup.py sdist bdist_wheel
|
||||
|
||||
- name: Publish to PyPI via token
|
||||
env:
|
||||
TWINE_USERNAME: __token__
|
||||
TWINE_PASSWORD: ${{ secrets[needs.prepare.outputs.pypi_token] }}
|
||||
if: |
|
||||
needs.prepare.outputs.pypi_token != 'OIDC' && env.TWINE_PASSWORD
|
||||
run: |
|
||||
twine upload dist/*
|
||||
|
||||
- name: Publish to PyPI via trusted publishing
|
||||
if: |
|
||||
needs.prepare.outputs.pypi_token == 'OIDC'
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
verbose: true
|
||||
|
||||
@@ -140,12 +140,9 @@ To run yt-dlp as a developer, you don't need to build anything either. Simply ex
|
||||
|
||||
python -m yt_dlp
|
||||
|
||||
To run the test, simply invoke your favorite test runner, or execute a test file directly; any of the following work:
|
||||
To run all the available core tests, use:
|
||||
|
||||
python -m unittest discover
|
||||
python test/test_download.py
|
||||
nosetests
|
||||
pytest
|
||||
python devscripts/run_tests.py
|
||||
|
||||
See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases.
|
||||
|
||||
@@ -187,15 +184,21 @@ After you have ensured this site is distributing its content legally, you can fo
|
||||
'url': 'https://yourextractor.com/watch/42',
|
||||
'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
|
||||
'info_dict': {
|
||||
# For videos, only the 'id' and 'ext' fields are required to RUN the test:
|
||||
'id': '42',
|
||||
'ext': 'mp4',
|
||||
'title': 'Video title goes here',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
# TODO more properties, either as:
|
||||
# * A value
|
||||
# * MD5 checksum; start the string with md5:
|
||||
# * A regular expression; start the string with re:
|
||||
# * Any Python type, e.g. int or float
|
||||
# Then if the test run fails, it will output the missing/incorrect fields.
|
||||
# Properties can be added as:
|
||||
# * A value, e.g.
|
||||
# 'title': 'Video title goes here',
|
||||
# * MD5 checksum; start the string with 'md5:', e.g.
|
||||
# 'description': 'md5:098f6bcd4621d373cade4e832627b4f6',
|
||||
# * A regular expression; start the string with 're:', e.g.
|
||||
# 'thumbnail': r're:^https?://.*\.jpg$',
|
||||
# * A count of elements in a list; start the string with 'count:', e.g.
|
||||
# 'tags': 'count:10',
|
||||
# * Any Python type, e.g.
|
||||
# 'view_count': int,
|
||||
}
|
||||
}]
|
||||
|
||||
@@ -215,14 +218,14 @@ After you have ensured this site is distributing its content legally, you can fo
|
||||
}
|
||||
```
|
||||
1. Add an import in [`yt_dlp/extractor/_extractors.py`](yt_dlp/extractor/_extractors.py). Note that the class name must end with `IE`.
|
||||
1. Run `python test/test_download.py TestDownload.test_YourExtractor` (note that `YourExtractor` doesn't end with `IE`). This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, the tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. You can also run all the tests in one go with `TestDownload.test_YourExtractor_all`
|
||||
1. Make sure you have atleast one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
|
||||
1. Run `python devscripts/run_tests.py YourExtractor`. This *may fail* at first, but you can continually re-run it until you're done. Upon failure, it will output the missing fields and/or correct values which you can copy. If you decide to add more than one test, the tests will then be named `YourExtractor`, `YourExtractor_1`, `YourExtractor_2`, etc. Note that tests with an `only_matching` key in the test's dict are not included in the count. You can also run all the tests in one go with `YourExtractor_all`
|
||||
1. Make sure you have at least one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
|
||||
1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L119-L440). Add tests and code for as many as you want.
|
||||
1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
|
||||
|
||||
$ flake8 yt_dlp/extractor/yourextractor.py
|
||||
|
||||
1. Make sure your code works under all [Python](https://www.python.org/) versions supported by yt-dlp, namely CPython and PyPy for Python 3.7 and above. Backward compatibility is not required for even older versions of Python.
|
||||
1. Make sure your code works under all [Python](https://www.python.org/) versions supported by yt-dlp, namely CPython and PyPy for Python 3.8 and above. Backward compatibility is not required for even older versions of Python.
|
||||
1. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files, [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
|
||||
|
||||
$ git add yt_dlp/extractor/_extractors.py
|
||||
|
||||
14
CONTRIBUTORS
14
CONTRIBUTORS
@@ -528,3 +528,17 @@ almx
|
||||
elivinsky
|
||||
starius
|
||||
TravisDupes
|
||||
amir16yp
|
||||
Fymyte
|
||||
Ganesh910
|
||||
hashFactory
|
||||
kclauhk
|
||||
Kyraminol
|
||||
lstrojny
|
||||
middlingphys
|
||||
NickCis
|
||||
nicodato
|
||||
prettykool
|
||||
S-Aarab
|
||||
sonmezberkay
|
||||
TSRBerry
|
||||
|
||||
87
Changelog.md
87
Changelog.md
@@ -4,6 +4,93 @@
|
||||
# To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
|
||||
-->
|
||||
|
||||
### 2023.12.30
|
||||
|
||||
#### Core changes
|
||||
- [Fix format selection parse error for CPython 3.12](https://github.com/yt-dlp/yt-dlp/commit/00cdda4f6fe18712ced13dbc64b7ea10f323e268) ([#8797](https://github.com/yt-dlp/yt-dlp/issues/8797)) by [Grub4K](https://github.com/Grub4K)
|
||||
- [Let `read_stdin` obey `--quiet`](https://github.com/yt-dlp/yt-dlp/commit/a174c453ee1e853c584ceadeac17eef2bd433dc5) by [pukkandan](https://github.com/pukkandan)
|
||||
- [Merged with youtube-dl be008e6](https://github.com/yt-dlp/yt-dlp/commit/65de7d204ce88c0225df1321060304baab85dbd8) by [bashonly](https://github.com/bashonly), [dirkf](https://github.com/dirkf), [Grub4K](https://github.com/Grub4K)
|
||||
- [Parse `release_year` from `release_date`](https://github.com/yt-dlp/yt-dlp/commit/1732eccc0a40256e076bf0435a29f0f1d8419280) ([#8524](https://github.com/yt-dlp/yt-dlp/issues/8524)) by [seproDev](https://github.com/seproDev)
|
||||
- [Release workflow and Updater cleanup](https://github.com/yt-dlp/yt-dlp/commit/632b8ee54eb2df8ac6e20746a0bd95b7ebb053aa) ([#8640](https://github.com/yt-dlp/yt-dlp/issues/8640)) by [bashonly](https://github.com/bashonly)
|
||||
- [Remove Python 3.7 support](https://github.com/yt-dlp/yt-dlp/commit/f4b95acafcd69a50040730dfdf732e797278fdcc) ([#8361](https://github.com/yt-dlp/yt-dlp/issues/8361)) by [bashonly](https://github.com/bashonly)
|
||||
- [Support `NO_COLOR` environment variable](https://github.com/yt-dlp/yt-dlp/commit/a0b19d319a6ce8b7059318fa17a34b144fde1785) ([#8385](https://github.com/yt-dlp/yt-dlp/issues/8385)) by [Grub4K](https://github.com/Grub4K), [prettykool](https://github.com/prettykool)
|
||||
- **outtmpl**: [Support multiplication](https://github.com/yt-dlp/yt-dlp/commit/993edd3f6e17e966c763bc86dc34125445cec6b6) by [pukkandan](https://github.com/pukkandan)
|
||||
- **utils**: `traverse_obj`: [Move `is_user_input` into output template](https://github.com/yt-dlp/yt-dlp/commit/0b6f829b1dfda15d3c1d7d1fbe4ea6102c26dd24) ([#8673](https://github.com/yt-dlp/yt-dlp/issues/8673)) by [Grub4K](https://github.com/Grub4K)
|
||||
- **webvtt**: [Allow spaces before newlines for CueBlock](https://github.com/yt-dlp/yt-dlp/commit/15f22b4880b6b3f71f350c64d70976ae65b9f1ca) ([#7681](https://github.com/yt-dlp/yt-dlp/issues/7681)) by [TSRBerry](https://github.com/TSRBerry) (With fixes in [298230e](https://github.com/yt-dlp/yt-dlp/commit/298230e550886b746c266724dd701d842ca2696e) by [pukkandan](https://github.com/pukkandan))
|
||||
|
||||
#### Extractor changes
|
||||
- [Add `media_type` field](https://github.com/yt-dlp/yt-dlp/commit/e370f9ec36972d06100a3db893b397bfc1b07b4d) by [trainman261](https://github.com/trainman261)
|
||||
- [Extract from `media` elements in SMIL manifests](https://github.com/yt-dlp/yt-dlp/commit/ddb2d7588bea48bae965dbfabe6df6550c9d3d43) ([#8504](https://github.com/yt-dlp/yt-dlp/issues/8504)) by [seproDev](https://github.com/seproDev)
|
||||
- **abematv**: [Fix season metadata](https://github.com/yt-dlp/yt-dlp/commit/cc07f5cc85d9e2a6cd0bedb9d961665eea0d6047) ([#8607](https://github.com/yt-dlp/yt-dlp/issues/8607)) by [middlingphys](https://github.com/middlingphys)
|
||||
- **allstar**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/3237f8ba29fe13bf95ff42b1e48b5b5109715feb) ([#8274](https://github.com/yt-dlp/yt-dlp/issues/8274)) by [S-Aarab](https://github.com/S-Aarab)
|
||||
- **altcensored**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/3f90813f0617e0d21302398010de7496c9ae36aa) ([#8291](https://github.com/yt-dlp/yt-dlp/issues/8291)) by [drzraf](https://github.com/drzraf)
|
||||
- **ard**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/5f009a094f0e8450792b097c4c8273622778052d) ([#8878](https://github.com/yt-dlp/yt-dlp/issues/8878)) by [seproDev](https://github.com/seproDev)
|
||||
- **ardbetamediathek**: [Fix series extraction](https://github.com/yt-dlp/yt-dlp/commit/1f8bd8eba82ba10ddb49ee7cc0be4540dab103d5) ([#8687](https://github.com/yt-dlp/yt-dlp/issues/8687)) by [lstrojny](https://github.com/lstrojny)
|
||||
- **bbc**
|
||||
- [Extract more formats](https://github.com/yt-dlp/yt-dlp/commit/c919b68f7e79ea5010f75f648d3c9e45405a8011) ([#8321](https://github.com/yt-dlp/yt-dlp/issues/8321)) by [barsnick](https://github.com/barsnick), [dirkf](https://github.com/dirkf)
|
||||
- [Fix JSON parsing bug](https://github.com/yt-dlp/yt-dlp/commit/19741ab8a401ec64d5e84fdbfcfb141d105e7bc8) by [bashonly](https://github.com/bashonly)
|
||||
- **bfmtv**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/4903f452b68efb62dadf22e81be8c7934fc743e7) ([#8651](https://github.com/yt-dlp/yt-dlp/issues/8651)) by [bashonly](https://github.com/bashonly)
|
||||
- **bilibili**: [Support courses and interactive videos](https://github.com/yt-dlp/yt-dlp/commit/9f09bdcfcb8e2b4b2decdc30d35d34b993bc7a94) ([#8343](https://github.com/yt-dlp/yt-dlp/issues/8343)) by [c-basalt](https://github.com/c-basalt)
|
||||
- **bitchute**: [Fix and improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/b1a1ec1540605d2ea7abdb63336ffb1c56bf6316) ([#8507](https://github.com/yt-dlp/yt-dlp/issues/8507)) by [SirElderling](https://github.com/SirElderling)
|
||||
- **box**: [Fix formats extraction](https://github.com/yt-dlp/yt-dlp/commit/5a230233d6fce06f4abd1fce0dc92b948e6f780b) ([#8649](https://github.com/yt-dlp/yt-dlp/issues/8649)) by [bashonly](https://github.com/bashonly)
|
||||
- **bundestag**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/00a3e47bf5440c96025a76e08337ff2a475ed83e) ([#8783](https://github.com/yt-dlp/yt-dlp/issues/8783)) by [Grub4K](https://github.com/Grub4K)
|
||||
- **drtv**: [Set default ext for m3u8 formats](https://github.com/yt-dlp/yt-dlp/commit/f96ab86cd837b1b5823baa87d144e15322ee9298) ([#8590](https://github.com/yt-dlp/yt-dlp/issues/8590)) by [seproDev](https://github.com/seproDev)
|
||||
- **duoplay**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/66a0127d45033c698bdbedf162cddc55d9e7b906) ([#8542](https://github.com/yt-dlp/yt-dlp/issues/8542)) by [glensc](https://github.com/glensc)
|
||||
- **eplus**: [Add login support and DRM detection](https://github.com/yt-dlp/yt-dlp/commit/d5d1517e7d838500800d193ac3234b06e89654cd) ([#8661](https://github.com/yt-dlp/yt-dlp/issues/8661)) by [pzhlkj6612](https://github.com/pzhlkj6612)
|
||||
- **facebook**
|
||||
- [Fix Memories extraction](https://github.com/yt-dlp/yt-dlp/commit/c39358a54bc6675ae0c50b81024e5a086e41656a) ([#8681](https://github.com/yt-dlp/yt-dlp/issues/8681)) by [kclauhk](https://github.com/kclauhk)
|
||||
- [Improve subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/9cafb9ff17e14475a35c9a58b5bb010c86c9db4b) ([#8296](https://github.com/yt-dlp/yt-dlp/issues/8296)) by [kclauhk](https://github.com/kclauhk)
|
||||
- **floatplane**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/628fa244bbce2ad39775a5959e99588f30cac152) ([#8639](https://github.com/yt-dlp/yt-dlp/issues/8639)) by [seproDev](https://github.com/seproDev)
|
||||
- **francetv**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/71f28097fec1c9e029f74b68a4eadc8915399840) ([#8409](https://github.com/yt-dlp/yt-dlp/issues/8409)) by [Fymyte](https://github.com/Fymyte)
|
||||
- **instagram**: [Fix stories extraction](https://github.com/yt-dlp/yt-dlp/commit/50eaea9fd7787546b53660e736325fa31c77765d) ([#8843](https://github.com/yt-dlp/yt-dlp/issues/8843)) by [bashonly](https://github.com/bashonly)
|
||||
- **joqrag**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/db8b4edc7d0bd27da462f6fe82ff6e13e3d68a04) ([#8384](https://github.com/yt-dlp/yt-dlp/issues/8384)) by [pzhlkj6612](https://github.com/pzhlkj6612)
|
||||
- **litv**: [Fix premium content extraction](https://github.com/yt-dlp/yt-dlp/commit/f45c4efcd928a173e1300a8f1ce4258e70c969b1) ([#8842](https://github.com/yt-dlp/yt-dlp/issues/8842)) by [bashonly](https://github.com/bashonly)
|
||||
- **maariv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/c5f01bf7d4b9426c87c3f8248de23934a56579e0) ([#8331](https://github.com/yt-dlp/yt-dlp/issues/8331)) by [amir16yp](https://github.com/amir16yp)
|
||||
- **mediastream**: [Fix authenticated format extraction](https://github.com/yt-dlp/yt-dlp/commit/b03c89309eb141be1a1eceeeb7475dd3b7529ad9) ([#8657](https://github.com/yt-dlp/yt-dlp/issues/8657)) by [NickCis](https://github.com/NickCis)
|
||||
- **nebula**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/45d82be65f71bb05506bd55376c6fdb36bc54142) ([#8566](https://github.com/yt-dlp/yt-dlp/issues/8566)) by [elyse0](https://github.com/elyse0), [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev)
|
||||
- **nintendo**: [Fix Nintendo Direct extraction](https://github.com/yt-dlp/yt-dlp/commit/1d24da6c899ef280d8b0a48a5e280ecd5d39cdf4) ([#8609](https://github.com/yt-dlp/yt-dlp/issues/8609)) by [Grub4K](https://github.com/Grub4K)
|
||||
- **ondemandkorea**: [Fix upgraded format extraction](https://github.com/yt-dlp/yt-dlp/commit/04a5e06350e3ef7c03f94f2f3f90dd96c6411152) ([#8677](https://github.com/yt-dlp/yt-dlp/issues/8677)) by [seproDev](https://github.com/seproDev)
|
||||
- **pr0gramm**: [Support variant formats and subtitles](https://github.com/yt-dlp/yt-dlp/commit/f98a3305eb124a0c375d03209d5c5a64fe1766c8) ([#8674](https://github.com/yt-dlp/yt-dlp/issues/8674)) by [Grub4K](https://github.com/Grub4K)
|
||||
- **rinsefm**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/c91af948e43570025e4aa887e248fd025abae394) ([#8778](https://github.com/yt-dlp/yt-dlp/issues/8778)) by [hashFactory](https://github.com/hashFactory)
|
||||
- **rudovideo**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/0d531c35eca4c2eb36e160530a7a333edbc727cc) ([#8664](https://github.com/yt-dlp/yt-dlp/issues/8664)) by [nicodato](https://github.com/nicodato)
|
||||
- **theguardian**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/1fa3f24d4b5d22176b11d78420f1f4b64a5af0a8) ([#8535](https://github.com/yt-dlp/yt-dlp/issues/8535)) by [SirElderling](https://github.com/SirElderling)
|
||||
- **theplatform**: [Extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/7e09c147fdccb44806bbf601573adc4b77210a89) ([#8635](https://github.com/yt-dlp/yt-dlp/issues/8635)) by [trainman261](https://github.com/trainman261)
|
||||
- **twitcasting**: [Detect livestreams via API and `show` page](https://github.com/yt-dlp/yt-dlp/commit/585d0ed9abcfcb957f2b2684b8ad43c3af160383) ([#8601](https://github.com/yt-dlp/yt-dlp/issues/8601)) by [bashonly](https://github.com/bashonly), [JC-Chung](https://github.com/JC-Chung)
|
||||
- **twitcastinguser**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/ff2fde1b8f922fd34bae6172602008cd67c07c93) ([#8650](https://github.com/yt-dlp/yt-dlp/issues/8650)) by [bashonly](https://github.com/bashonly)
|
||||
- **twitter**
|
||||
- [Extract stale tweets](https://github.com/yt-dlp/yt-dlp/commit/1c54a98e19d047e7c15184237b6ef8ad50af489c) ([#8724](https://github.com/yt-dlp/yt-dlp/issues/8724)) by [bashonly](https://github.com/bashonly)
|
||||
- [Prioritize m3u8 formats](https://github.com/yt-dlp/yt-dlp/commit/e7d22348e77367740da78a3db27167ecf894b7c9) ([#8826](https://github.com/yt-dlp/yt-dlp/issues/8826)) by [bashonly](https://github.com/bashonly)
|
||||
- [Work around API rate-limit](https://github.com/yt-dlp/yt-dlp/commit/116c268438ea4d3738f6fa502c169081ca8f0ee7) ([#8825](https://github.com/yt-dlp/yt-dlp/issues/8825)) by [bashonly](https://github.com/bashonly)
|
||||
- broadcast: [Extract `concurrent_view_count`](https://github.com/yt-dlp/yt-dlp/commit/6fe82491ed622b948c512cf4aab46ac3a234ae0a) ([#8600](https://github.com/yt-dlp/yt-dlp/issues/8600)) by [sonmezberkay](https://github.com/sonmezberkay)
|
||||
- **vidly**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/34df1c1f60fa652c0a6a5c712b06c10e45daf6b7) ([#8612](https://github.com/yt-dlp/yt-dlp/issues/8612)) by [seproDev](https://github.com/seproDev)
|
||||
- **vocaroo**: [Do not use deprecated `getheader`](https://github.com/yt-dlp/yt-dlp/commit/f223b1b0789f65e06619dcc9fc9e74f50d259379) ([#8606](https://github.com/yt-dlp/yt-dlp/issues/8606)) by [qbnu](https://github.com/qbnu)
|
||||
- **vvvvid**: [Set user-agent to fix extraction](https://github.com/yt-dlp/yt-dlp/commit/1725e943b0e8a8b585305660d4611e684374409c) ([#8615](https://github.com/yt-dlp/yt-dlp/issues/8615)) by [Kyraminol](https://github.com/Kyraminol)
|
||||
- **youtube**
|
||||
- [Fix `like_count` extraction](https://github.com/yt-dlp/yt-dlp/commit/6b5d93b0b0240e287389d1d43b2d5293e18aa4cc) ([#8763](https://github.com/yt-dlp/yt-dlp/issues/8763)) by [Ganesh910](https://github.com/Ganesh910)
|
||||
- [Improve detection of faulty HLS formats](https://github.com/yt-dlp/yt-dlp/commit/bb5a54e6db2422bbd155d93a0e105b6616c09467) ([#8646](https://github.com/yt-dlp/yt-dlp/issues/8646)) by [bashonly](https://github.com/bashonly)
|
||||
- [Return empty playlist when channel/tab has no videos](https://github.com/yt-dlp/yt-dlp/commit/044886c220620a7679109e92352890e18b6079e3) by [pukkandan](https://github.com/pukkandan)
|
||||
- [Support cf.piped.video](https://github.com/yt-dlp/yt-dlp/commit/6a9c7a2b52655bacfa7ab2da24fd0d14a6fff495) ([#8514](https://github.com/yt-dlp/yt-dlp/issues/8514)) by [OIRNOIR](https://github.com/OIRNOIR)
|
||||
- **zingmp3**: [Add support for radio and podcasts](https://github.com/yt-dlp/yt-dlp/commit/64de1a4c25bada90374b88d7353754fe8fbfcc51) ([#7189](https://github.com/yt-dlp/yt-dlp/issues/7189)) by [hatienl0i261299](https://github.com/hatienl0i261299)
|
||||
|
||||
#### Postprocessor changes
|
||||
- **ffmpegmetadata**: [Embed stream metadata in single format downloads](https://github.com/yt-dlp/yt-dlp/commit/deeb13eae82e60f82a2c0c5861f460399a997528) ([#8647](https://github.com/yt-dlp/yt-dlp/issues/8647)) by [bashonly](https://github.com/bashonly)
|
||||
|
||||
#### Networking changes
|
||||
- [Strip whitespace around header values](https://github.com/yt-dlp/yt-dlp/commit/196eb0fe77b78e2e5ca02c506c3837c2b1a7964c) ([#8802](https://github.com/yt-dlp/yt-dlp/issues/8802)) by [coletdjnz](https://github.com/coletdjnz)
|
||||
- **Request Handler**: websockets: [Migrate websockets to networking framework](https://github.com/yt-dlp/yt-dlp/commit/ccfd70f4c24b579c72123ca76ab50164f8f122b7) ([#7720](https://github.com/yt-dlp/yt-dlp/issues/7720)) by [coletdjnz](https://github.com/coletdjnz)
|
||||
|
||||
#### Misc. changes
|
||||
- **ci**
|
||||
- [Concurrency optimizations](https://github.com/yt-dlp/yt-dlp/commit/f124fa458826308afc86cf364c509f857686ecfd) ([#8614](https://github.com/yt-dlp/yt-dlp/issues/8614)) by [Grub4K](https://github.com/Grub4K)
|
||||
- [Run core tests only for core changes](https://github.com/yt-dlp/yt-dlp/commit/13b3cb3c2b7169a1e17d6fc62593bf744170521c) ([#8841](https://github.com/yt-dlp/yt-dlp/issues/8841)) by [Grub4K](https://github.com/Grub4K)
|
||||
- **cleanup**
|
||||
- [Fix spelling of `IE_NAME`](https://github.com/yt-dlp/yt-dlp/commit/bc4ab17b38f01000d99c5c2bedec89721fee65ec) ([#8810](https://github.com/yt-dlp/yt-dlp/issues/8810)) by [barsnick](https://github.com/barsnick)
|
||||
- [Remove dead extractors](https://github.com/yt-dlp/yt-dlp/commit/9751a457cfdb18bf99d9ee0d10e4e6a594502bbf) ([#8604](https://github.com/yt-dlp/yt-dlp/issues/8604)) by [seproDev](https://github.com/seproDev)
|
||||
- Miscellaneous: [f9fb3ce](https://github.com/yt-dlp/yt-dlp/commit/f9fb3ce86e3c6a0c3c33b45392b8d7288bceba76) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev)
|
||||
- **devscripts**: `run_tests`: [Create Python script](https://github.com/yt-dlp/yt-dlp/commit/2d1d683a541d71f3d3bb999dfe8eeb1976fb91ce) ([#8720](https://github.com/yt-dlp/yt-dlp/issues/8720)) by [Grub4K](https://github.com/Grub4K) (With fixes in [225cf2b](https://github.com/yt-dlp/yt-dlp/commit/225cf2b830a1de2c5eacd257edd2a01aed1e1114))
|
||||
- **docs**: [Update youtube-dl merge commit in `README.md`](https://github.com/yt-dlp/yt-dlp/commit/f10589e3453009bb523f55849bba144c9b91cf2a) by [bashonly](https://github.com/bashonly)
|
||||
- **test**: networking: [Update tests for OpenSSL 3.2](https://github.com/yt-dlp/yt-dlp/commit/37755a037e612bfc608c3d4722e8ef2ce6a022ee) ([#8814](https://github.com/yt-dlp/yt-dlp/issues/8814)) by [bashonly](https://github.com/bashonly)
|
||||
|
||||
### 2023.11.16
|
||||
|
||||
#### Extractor changes
|
||||
|
||||
@@ -29,6 +29,7 @@ You can also find lists of all [contributors of yt-dlp](CONTRIBUTORS) and [autho
|
||||
[](https://github.com/sponsors/coletdjnz)
|
||||
|
||||
* Improved plugin architecture
|
||||
* Rewrote the networking infrastructure, implemented support for `requests`
|
||||
* YouTube improvements including: age-gate bypass, private playlists, multiple-clients (to avoid throttling) and a lot of under-the-hood improvements
|
||||
* Added support for new websites YoutubeWebArchive, MainStreaming, PRX, nzherald, Mediaklikk, StarTV etc
|
||||
* Improved/fixed support for Patreon, panopto, gfycat, itv, pbs, SouthParkDE etc
|
||||
@@ -46,16 +47,17 @@ You can also find lists of all [contributors of yt-dlp](CONTRIBUTORS) and [autho
|
||||
|
||||
## [bashonly](https://github.com/bashonly)
|
||||
|
||||
* `--update-to`, automated release, nightly builds
|
||||
* `--cookies-from-browser` support for Firefox containers
|
||||
* Added support for new websites Genius, Kick, NBCStations, Triller, VideoKen etc
|
||||
* Improved/fixed support for Anvato, Brightcove, Instagram, ParamountPlus, Reddit, SlidesLive, TikTok, Twitter, Vimeo etc
|
||||
* `--update-to`, self-updater rewrite, automated/nightly/master releases
|
||||
* `--cookies-from-browser` support for Firefox containers, external downloader cookie handling overhaul
|
||||
* Added support for new websites like Dacast, Kick, NBCStations, Triller, VideoKen, Weverse, WrestleUniverse etc
|
||||
* Improved/fixed support for Anvato, Brightcove, Reddit, SlidesLive, TikTok, Twitter, Vimeo etc
|
||||
|
||||
|
||||
## [Grub4K](https://github.com/Grub4K)
|
||||
|
||||
[](https://ko-fi.com/Grub4K) [](https://github.com/sponsors/Grub4K)
|
||||
[](https://github.com/sponsors/Grub4K) [](https://ko-fi.com/Grub4K)
|
||||
|
||||
* `--update-to`, automated release, nightly builds
|
||||
* Rework internals like `traverse_obj`, various core refactors and bugs fixes
|
||||
* Helped fix crunchyroll, Twitter, wrestleuniverse, wistia, slideslive etc
|
||||
* `--update-to`, self-updater rewrite, automated/nightly/master releases
|
||||
* Reworked internals like `traverse_obj`, various core refactors and bugs fixes
|
||||
* Implemented proper progress reporting for parallel downloads
|
||||
* Improved/fixed/added Bundestag, crunchyroll, pr0gramm, Twitter, WrestleUniverse etc
|
||||
|
||||
16
README.md
16
README.md
@@ -76,7 +76,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
|
||||
|
||||
# NEW FEATURES
|
||||
|
||||
* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@66ab08**](https://github.com/ytdl-org/youtube-dl/commit/66ab0814c4baa2dc79c2dd5287bc0ad61a37c5b9) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
|
||||
* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@be008e6**](https://github.com/ytdl-org/youtube-dl/commit/be008e657d79832642e2158557c899249c9e31cd) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
|
||||
|
||||
* **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
|
||||
|
||||
@@ -131,7 +131,7 @@ Features marked with a **\*** have been back-ported to youtube-dl
|
||||
|
||||
Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc:
|
||||
|
||||
* yt-dlp supports only [Python 3.7+](## "Windows 7"), and *may* remove support for more versions as they [become EOL](https://devguide.python.org/versions/#python-release-cycle); while [youtube-dl still supports Python 2.6+ and 3.2+](https://github.com/ytdl-org/youtube-dl/issues/30568#issue-1118238743)
|
||||
* yt-dlp supports only [Python 3.8+](## "Windows 7"), and *may* remove support for more versions as they [become EOL](https://devguide.python.org/versions/#python-release-cycle); while [youtube-dl still supports Python 2.6+ and 3.2+](https://github.com/ytdl-org/youtube-dl/issues/30568#issue-1118238743)
|
||||
* The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
|
||||
* `avconv` is not supported as an alternative to `ffmpeg`
|
||||
* yt-dlp stores config files in slightly different locations to youtube-dl. See [CONFIGURATION](#configuration) for a list of correct locations
|
||||
@@ -159,6 +159,7 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
|
||||
* yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filter` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this
|
||||
* yt-dlp versions between 2021.11.10 and 2023.06.21 estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values
|
||||
* yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests.
|
||||
* The sub-module `swfinterp` is removed.
|
||||
|
||||
For ease of use, a few more compat options are available:
|
||||
|
||||
@@ -266,7 +267,7 @@ gpg --verify SHA2-512SUMS.sig SHA2-512SUMS
|
||||
**Note**: The manpages, shell completion (autocomplete) files etc. are available inside the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
|
||||
|
||||
## DEPENDENCIES
|
||||
Python versions 3.7+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly.
|
||||
Python versions 3.8+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly.
|
||||
|
||||
<!-- Python 3.5+ uses VC++14 and it is already embedded in the binary created
|
||||
<!x-- https://www.microsoft.com/en-us/download/details.aspx?id=26999 --x>
|
||||
@@ -299,7 +300,7 @@ While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly
|
||||
|
||||
* [**pycryptodomex**](https://github.com/Legrandin/pycryptodome)\* - For decrypting AES-128 HLS streams and various other data. Licensed under [BSD-2-Clause](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
|
||||
* [**phantomjs**](https://github.com/ariya/phantomjs) - Used in extractors where javascript needs to be run. Licensed under [BSD-3-Clause](https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD)
|
||||
* [**secretstorage**](https://github.com/mitya57/secretstorage) - For `--cookies-from-browser` to access the **Gnome** keyring while decrypting cookies of **Chromium**-based browsers on **Linux**. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
|
||||
* [**secretstorage**](https://github.com/mitya57/secretstorage)\* - For `--cookies-from-browser` to access the **Gnome** keyring while decrypting cookies of **Chromium**-based browsers on **Linux**. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
|
||||
* Any external downloader that you want to use with `--downloader`
|
||||
|
||||
### Deprecated
|
||||
@@ -334,7 +335,7 @@ On some systems, you may need to use `py` or `python` instead of `python3`.
|
||||
**Important**: Running `pyinstaller` directly **without** using `pyinst.py` is **not** officially supported. This may or may not work correctly.
|
||||
|
||||
### Platform-independent Binary (UNIX)
|
||||
You will need the build tools `python` (3.7+), `zip`, `make` (GNU), `pandoc`\* and `pytest`\*.
|
||||
You will need the build tools `python` (3.8+), `zip`, `make` (GNU), `pandoc`\* and `pytest`\*.
|
||||
|
||||
After installing these, simply run `make`.
|
||||
|
||||
@@ -1268,7 +1269,7 @@ The field names themselves (the part inside the parenthesis) can also have some
|
||||
|
||||
1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a dot `.` separator; e.g. `%(tags.0)s`, `%(subtitles.en.-1.ext)s`. You can do Python slicing with colon `:`; E.g. `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. Curly braces `{}` can be used to build dictionaries with only specific keys; e.g. `%(formats.:.{format_id,height})#j`. An empty field name `%()s` refers to the entire infodict; e.g. `%(.{id,title})s`. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
|
||||
|
||||
1. **Addition**: Addition and subtraction of numeric fields can be done using `+` and `-` respectively. E.g. `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
|
||||
1. **Arithmetic**: Simple arithmetic can be done on numeric fields using `+`, `-` and `*`. E.g. `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
|
||||
|
||||
1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. E.g. `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s`
|
||||
|
||||
@@ -1309,6 +1310,7 @@ The available fields are:
|
||||
- `upload_date` (string): Video upload date in UTC (YYYYMMDD)
|
||||
- `release_timestamp` (numeric): UNIX timestamp of the moment the video was released
|
||||
- `release_date` (string): The date (YYYYMMDD) when the video was released in UTC
|
||||
- `release_year` (numeric): Year (YYYY) when the video or album was released
|
||||
- `modified_timestamp` (numeric): UNIX timestamp of the moment the video was last modified
|
||||
- `modified_date` (string): The date (YYYYMMDD) when the video was last modified in UTC
|
||||
- `uploader_id` (string): Nickname or id of the video uploader
|
||||
@@ -1332,6 +1334,7 @@ The available fields are:
|
||||
- `was_live` (boolean): Whether this video was originally a live stream
|
||||
- `playable_in_embed` (string): Whether this video is allowed to play in embedded players on other sites
|
||||
- `availability` (string): Whether the video is "private", "premium_only", "subscriber_only", "needs_auth", "unlisted" or "public"
|
||||
- `media_type` (string): The type of media as classified by the site, e.g. "episode", "clip", "trailer"
|
||||
- `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL
|
||||
- `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL
|
||||
- `extractor` (string): Name of the extractor
|
||||
@@ -1382,7 +1385,6 @@ Available for the media that is a track or a part of a music album:
|
||||
- `album_type` (string): Type of the album
|
||||
- `album_artist` (string): List of all artists appeared on the album
|
||||
- `disc_number` (numeric): Number of the disc or other physical medium the track belongs to
|
||||
- `release_year` (numeric): Year (YYYY) when the album was released
|
||||
|
||||
Available only when using `--download-sections` and for `chapter:` prefix when using `--split-chapters` for videos with internal chapters:
|
||||
|
||||
|
||||
@@ -114,5 +114,11 @@
|
||||
"action": "add",
|
||||
"when": "f04b5bedad7b281bee9814686bba1762bae092eb",
|
||||
"short": "[priority] Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)\n\t- Disallow smuggling of arbitrary `http_headers`; extractors now only use specific headers"
|
||||
},
|
||||
{
|
||||
"action": "change",
|
||||
"when": "15f22b4880b6b3f71f350c64d70976ae65b9f1ca",
|
||||
"short": "[webvtt] Allow spaces before newlines for CueBlock (#7681)",
|
||||
"authors": ["TSRBerry"]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -40,20 +40,6 @@ class CommitGroup(enum.Enum):
|
||||
return {
|
||||
name: group
|
||||
for group, names in {
|
||||
cls.CORE: {
|
||||
'aes',
|
||||
'cache',
|
||||
'compat_utils',
|
||||
'compat',
|
||||
'cookies',
|
||||
'dependencies',
|
||||
'formats',
|
||||
'jsinterp',
|
||||
'outtmpl',
|
||||
'plugins',
|
||||
'update',
|
||||
'utils',
|
||||
},
|
||||
cls.MISC: {
|
||||
'build',
|
||||
'ci',
|
||||
@@ -404,9 +390,9 @@ class CommitRange:
|
||||
if not group:
|
||||
if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
|
||||
group = CommitGroup.EXTRACTOR
|
||||
logger.error(f'Assuming [ie] group for {commit.short!r}')
|
||||
else:
|
||||
group = CommitGroup.POSTPROCESSOR
|
||||
logger.warning(f'Failed to map {commit.short!r}, selected {group.name.lower()}')
|
||||
group = CommitGroup.CORE
|
||||
|
||||
commit_info = CommitInfo(
|
||||
details, sub_details, message.strip(),
|
||||
|
||||
@@ -9,11 +9,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import re
|
||||
|
||||
from devscripts.utils import (
|
||||
get_filename_args,
|
||||
read_file,
|
||||
write_file,
|
||||
)
|
||||
from devscripts.utils import get_filename_args, read_file, write_file
|
||||
|
||||
VERBOSE_TMPL = '''
|
||||
- type: checkboxes
|
||||
|
||||
@@ -1,17 +1,4 @@
|
||||
@setlocal
|
||||
@echo off
|
||||
cd /d %~dp0..
|
||||
|
||||
if ["%~1"]==[""] (
|
||||
set "test_set="test""
|
||||
) else if ["%~1"]==["core"] (
|
||||
set "test_set="-m not download""
|
||||
) else if ["%~1"]==["download"] (
|
||||
set "test_set="-m "download""
|
||||
) else (
|
||||
echo.Invalid test type "%~1". Use "core" ^| "download"
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
set PYTHONWARNINGS=error
|
||||
pytest %test_set%
|
||||
>&2 echo run_tests.bat is deprecated. Please use `devscripts/run_tests.py` instead
|
||||
python %~dp0run_tests.py %~1
|
||||
|
||||
71
devscripts/run_tests.py
Executable file
71
devscripts/run_tests.py
Executable file
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
fix_test_name = functools.partial(re.compile(r'IE(_all|_\d+)?$').sub, r'\1')
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Run selected yt-dlp tests')
|
||||
parser.add_argument(
|
||||
'test', help='a extractor tests, or one of "core" or "download"', nargs='*')
|
||||
parser.add_argument(
|
||||
'-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def run_tests(*tests, pattern=None, ci=False):
|
||||
run_core = 'core' in tests or (not pattern and not tests)
|
||||
run_download = 'download' in tests
|
||||
tests = list(map(fix_test_name, tests))
|
||||
|
||||
arguments = ['pytest', '-Werror', '--tb=short']
|
||||
if ci:
|
||||
arguments.append('--color=yes')
|
||||
if run_core:
|
||||
arguments.extend(['-m', 'not download'])
|
||||
elif run_download:
|
||||
arguments.extend(['-m', 'download'])
|
||||
elif pattern:
|
||||
arguments.extend(['-k', pattern])
|
||||
else:
|
||||
arguments.extend(
|
||||
f'test/test_download.py::TestDownload::test_{test}' for test in tests)
|
||||
|
||||
print(f'Running {arguments}', flush=True)
|
||||
try:
|
||||
return subprocess.call(arguments)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
arguments = [sys.executable, '-Werror', '-m', 'unittest']
|
||||
if run_core:
|
||||
print('"pytest" needs to be installed to run core tests', file=sys.stderr, flush=True)
|
||||
return 1
|
||||
elif run_download:
|
||||
arguments.append('test.test_download')
|
||||
elif pattern:
|
||||
arguments.extend(['-k', pattern])
|
||||
else:
|
||||
arguments.extend(
|
||||
f'test.test_download.TestDownload.test_{test}' for test in tests)
|
||||
|
||||
print(f'Running {arguments}', flush=True)
|
||||
return subprocess.call(arguments)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
args = parse_args()
|
||||
|
||||
os.chdir(Path(__file__).parent.parent)
|
||||
sys.exit(run_tests(*args.test, pattern=args.k, ci=bool(os.getenv('CI'))))
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
@@ -1,14 +1,4 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
test_set='test'
|
||||
elif [ "$1" = 'core' ]; then
|
||||
test_set="-m not download"
|
||||
elif [ "$1" = 'download' ]; then
|
||||
test_set="-m download"
|
||||
else
|
||||
echo 'Invalid test type "'"$1"'". Use "core" | "download"'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 -bb -Werror -m pytest "$test_set"
|
||||
>&2 echo 'run_tests.sh is deprecated. Please use `devscripts/run_tests.py` instead'
|
||||
python3 devscripts/run_tests.py "$1"
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
mutagen
|
||||
pycryptodomex
|
||||
websockets
|
||||
brotli; implementation_name=='cpython'
|
||||
brotlicffi; implementation_name!='cpython'
|
||||
certifi
|
||||
requests>=2.31.0,<3
|
||||
urllib3>=1.26.17,<3
|
||||
websockets>=12.0
|
||||
|
||||
@@ -26,7 +26,7 @@ markers =
|
||||
|
||||
[tox:tox]
|
||||
skipsdist = true
|
||||
envlist = py{36,37,38,39,310,311},pypy{36,37,38,39}
|
||||
envlist = py{38,39,310,311,312},pypy{38,39,310}
|
||||
skip_missing_interpreters = true
|
||||
|
||||
[testenv] # tox
|
||||
@@ -39,7 +39,7 @@ setenv =
|
||||
|
||||
|
||||
[isort]
|
||||
py_version = 37
|
||||
py_version = 38
|
||||
multi_line_output = VERTICAL_HANGING_INDENT
|
||||
line_length = 80
|
||||
reverse_relative = true
|
||||
|
||||
4
setup.py
4
setup.py
@@ -152,7 +152,7 @@ def main():
|
||||
url='https://github.com/yt-dlp/yt-dlp',
|
||||
packages=packages(),
|
||||
install_requires=REQUIREMENTS,
|
||||
python_requires='>=3.7',
|
||||
python_requires='>=3.8',
|
||||
project_urls={
|
||||
'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
|
||||
'Source': 'https://github.com/yt-dlp/yt-dlp',
|
||||
@@ -164,11 +164,11 @@ def main():
|
||||
'Development Status :: 5 - Production/Stable',
|
||||
'Environment :: Console',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: 3.9',
|
||||
'Programming Language :: Python :: 3.10',
|
||||
'Programming Language :: Python :: 3.11',
|
||||
'Programming Language :: Python :: 3.12',
|
||||
'Programming Language :: Python :: Implementation',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
'Programming Language :: Python :: Implementation :: PyPy',
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
# Supported sites
|
||||
- **0000studio:archive**
|
||||
- **0000studio:clip**
|
||||
- **17live**
|
||||
- **17live:clip**
|
||||
- **1News**: 1news.co.nz article videos
|
||||
@@ -9,7 +7,6 @@
|
||||
- **23video**
|
||||
- **247sports**
|
||||
- **24tv.ua**
|
||||
- **24video**
|
||||
- **3qsdn**: 3Q SDN
|
||||
- **3sat**
|
||||
- **4tube**
|
||||
@@ -50,15 +47,18 @@
|
||||
- **afreecatv**: [*afreecatv*](## "netrc machine") afreecatv.com
|
||||
- **afreecatv:live**: [*afreecatv*](## "netrc machine") afreecatv.com
|
||||
- **afreecatv:user**
|
||||
- **AirMozilla**
|
||||
- **AirTV**
|
||||
- **AitubeKZVideo**
|
||||
- **AliExpressLive**
|
||||
- **AlJazeera**
|
||||
- **Allocine**
|
||||
- **Allstar**
|
||||
- **AllstarProfile**
|
||||
- **AlphaPorno**
|
||||
- **Alsace20TV**
|
||||
- **Alsace20TVEmbed**
|
||||
- **altcensored**
|
||||
- **altcensored:channel**
|
||||
- **Alura**: [*alura*](## "netrc machine")
|
||||
- **AluraCourse**: [*aluracourse*](## "netrc machine")
|
||||
- **Amara**
|
||||
@@ -79,7 +79,7 @@
|
||||
- **ant1newsgr:embed**: ant1news.gr embedded videos
|
||||
- **antenna:watch**: antenna.gr and ant1news.gr videos
|
||||
- **Anvato**
|
||||
- **aol.com**: Yahoo screen and movies
|
||||
- **aol.com**: Yahoo screen and movies (**Currently broken**)
|
||||
- **APA**
|
||||
- **Aparat**
|
||||
- **AppleConnect**
|
||||
@@ -90,8 +90,8 @@
|
||||
- **archive.org**: archive.org video and audio
|
||||
- **ArcPublishing**
|
||||
- **ARD**
|
||||
- **ARD:mediathek**
|
||||
- **ARDBetaMediathek**
|
||||
- **ARDMediathek**
|
||||
- **ARDMediathekCollection**
|
||||
- **Arkena**
|
||||
- **arte.sky.it**
|
||||
- **ArteTV**
|
||||
@@ -100,7 +100,6 @@
|
||||
- **ArteTVPlaylist**
|
||||
- **AtresPlayer**: [*atresplayer*](## "netrc machine")
|
||||
- **AtScaleConfEvent**
|
||||
- **ATTTechChannel**
|
||||
- **ATVAt**
|
||||
- **AudiMedia**
|
||||
- **AudioBoom**
|
||||
@@ -140,12 +139,12 @@
|
||||
- **BeatBumpVideo**
|
||||
- **Beatport**
|
||||
- **Beeg**
|
||||
- **BehindKink**
|
||||
- **BehindKink**: (**Currently broken**)
|
||||
- **Bellator**
|
||||
- **BellMedia**
|
||||
- **BerufeTV**
|
||||
- **Bet**
|
||||
- **bfi:player**
|
||||
- **Bet**: (**Currently broken**)
|
||||
- **bfi:player**: (**Currently broken**)
|
||||
- **bfmtv**
|
||||
- **bfmtv:article**
|
||||
- **bfmtv:live**
|
||||
@@ -162,6 +161,8 @@
|
||||
- **BiliBiliBangumi**
|
||||
- **BiliBiliBangumiMedia**
|
||||
- **BiliBiliBangumiSeason**
|
||||
- **BilibiliCheese**
|
||||
- **BilibiliCheeseSeason**
|
||||
- **BilibiliCollectionList**
|
||||
- **BilibiliFavoritesList**
|
||||
- **BiliBiliPlayer**
|
||||
@@ -176,11 +177,8 @@
|
||||
- **BiliLive**
|
||||
- **BioBioChileTV**
|
||||
- **Biography**
|
||||
- **BIQLE**
|
||||
- **BitChute**
|
||||
- **BitChuteChannel**
|
||||
- **bitwave:replay**
|
||||
- **bitwave:stream**
|
||||
- **BlackboardCollaborate**
|
||||
- **BleacherReport**
|
||||
- **BleacherReportCMS**
|
||||
@@ -193,7 +191,7 @@
|
||||
- **Box**
|
||||
- **BoxCastVideo**
|
||||
- **Bpb**: Bundeszentrale für politische Bildung
|
||||
- **BR**: Bayerischer Rundfunk
|
||||
- **BR**: Bayerischer Rundfunk (**Currently broken**)
|
||||
- **BrainPOP**: [*brainpop*](## "netrc machine")
|
||||
- **BrainPOPELL**: [*brainpop*](## "netrc machine")
|
||||
- **BrainPOPEsp**: [*brainpop*](## "netrc machine") BrainPOP Español
|
||||
@@ -201,19 +199,18 @@
|
||||
- **BrainPOPIl**: [*brainpop*](## "netrc machine") BrainPOP Hebrew
|
||||
- **BrainPOPJr**: [*brainpop*](## "netrc machine")
|
||||
- **BravoTV**
|
||||
- **Break**
|
||||
- **BreitBart**
|
||||
- **brightcove:legacy**
|
||||
- **brightcove:new**
|
||||
- **Brilliantpala:Classes**: [*brilliantpala*](## "netrc machine") VoD on classes.brilliantpala.org
|
||||
- **Brilliantpala:Elearn**: [*brilliantpala*](## "netrc machine") VoD on elearn.brilliantpala.org
|
||||
- **BRMediathek**: Bayerischer Rundfunk Mediathek
|
||||
- **bt:article**: Bergens Tidende Articles
|
||||
- **bt:vestlendingen**: Bergens Tidende - Vestlendingen
|
||||
- **Bundesliga**
|
||||
- **Bundestag**
|
||||
- **BusinessInsider**
|
||||
- **BuzzFeed**
|
||||
- **BYUtv**
|
||||
- **BYUtv**: (**Currently broken**)
|
||||
- **CableAV**
|
||||
- **Callin**
|
||||
- **Caltrans**
|
||||
@@ -225,14 +222,11 @@
|
||||
- **CamModels**
|
||||
- **Camsoda**
|
||||
- **CamtasiaEmbed**
|
||||
- **CamWithHer**
|
||||
- **Canal1**
|
||||
- **CanalAlpha**
|
||||
- **canalc2.tv**
|
||||
- **Canalplus**: mycanal.fr and piwiplus.fr
|
||||
- **CaracolTvPlay**: [*caracoltv-play*](## "netrc machine")
|
||||
- **CarambaTV**
|
||||
- **CarambaTVPage**
|
||||
- **CartoonNetwork**
|
||||
- **cbc.ca**
|
||||
- **cbc.ca:player**
|
||||
@@ -254,16 +248,12 @@
|
||||
- **Cellebrite**
|
||||
- **CeskaTelevize**
|
||||
- **CGTN**
|
||||
- **channel9**: Channel 9
|
||||
- **CharlieRose**
|
||||
- **Chaturbate**
|
||||
- **Chilloutzone**
|
||||
- **Chingari**
|
||||
- **ChingariUser**
|
||||
- **chirbit**
|
||||
- **chirbit:profile**
|
||||
- **cielotv.it**
|
||||
- **Cinchcast**
|
||||
- **Cinemax**
|
||||
- **CinetecaMilano**
|
||||
- **Cineverse**
|
||||
@@ -276,14 +266,12 @@
|
||||
- **cliphunter**
|
||||
- **Clippit**
|
||||
- **ClipRs**
|
||||
- **Clipsyndicate**
|
||||
- **ClipYouEmbed**
|
||||
- **CloserToTruth**
|
||||
- **CloudflareStream**
|
||||
- **Cloudy**
|
||||
- **Clubic**
|
||||
- **Clubic**: (**Currently broken**)
|
||||
- **Clyp**
|
||||
- **cmt.com**
|
||||
- **cmt.com**: (**Currently broken**)
|
||||
- **CNBC**
|
||||
- **CNBCVideo**
|
||||
- **CNN**
|
||||
@@ -328,7 +316,6 @@
|
||||
- **CybraryCourse**: [*cybrary*](## "netrc machine")
|
||||
- **DacastPlaylist**
|
||||
- **DacastVOD**
|
||||
- **Daftsex**
|
||||
- **DagelijkseKost**: dagelijksekost.een.be
|
||||
- **DailyMail**
|
||||
- **dailymotion**: [*dailymotion*](## "netrc machine")
|
||||
@@ -347,13 +334,12 @@
|
||||
- **DctpTv**
|
||||
- **DeezerAlbum**
|
||||
- **DeezerPlaylist**
|
||||
- **defense.gouv.fr**
|
||||
- **democracynow**
|
||||
- **DestinationAmerica**
|
||||
- **DetikEmbed**
|
||||
- **DeuxM**
|
||||
- **DeuxMNews**
|
||||
- **DHM**: Filmarchiv - Deutsches Historisches Museum
|
||||
- **DHM**: Filmarchiv - Deutsches Historisches Museum (**Currently broken**)
|
||||
- **Digg**
|
||||
- **DigitalConcertHall**: [*digitalconcerthall*](## "netrc machine") DigitalConcertHall extractor
|
||||
- **DigitallySpeaking**
|
||||
@@ -373,7 +359,6 @@
|
||||
- **dlf:corpus**: DLF Multi-feed Archives
|
||||
- **dlive:stream**
|
||||
- **dlive:vod**
|
||||
- **Dotsub**
|
||||
- **Douyin**
|
||||
- **DouyuShow**
|
||||
- **DouyuTV**: 斗鱼直播
|
||||
@@ -392,35 +377,29 @@
|
||||
- **duboku**: www.duboku.io
|
||||
- **duboku:list**: www.duboku.io entire series
|
||||
- **Dumpert**
|
||||
- **Duoplay**
|
||||
- **dvtv**: http://video.aktualne.cz/
|
||||
- **dw**
|
||||
- **dw:article**
|
||||
- **EaglePlatform**
|
||||
- **EbaumsWorld**
|
||||
- **Ebay**
|
||||
- **EchoMsk**
|
||||
- **egghead:course**: egghead.io course
|
||||
- **egghead:lesson**: egghead.io lesson
|
||||
- **ehftv**
|
||||
- **eHow**
|
||||
- **EinsUndEinsTV**: [*1und1tv*](## "netrc machine")
|
||||
- **EinsUndEinsTVLive**: [*1und1tv*](## "netrc machine")
|
||||
- **EinsUndEinsTVRecordings**: [*1und1tv*](## "netrc machine")
|
||||
- **Einthusan**
|
||||
- **eitb.tv**
|
||||
- **ElevenSports**
|
||||
- **EllenTube**
|
||||
- **EllenTubePlaylist**
|
||||
- **EllenTubeVideo**
|
||||
- **Elonet**
|
||||
- **ElPais**: El País
|
||||
- **ElTreceTV**: El Trece TV (Argentina)
|
||||
- **Embedly**
|
||||
- **EMPFlix**
|
||||
- **Engadget**
|
||||
- **Epicon**
|
||||
- **EpiconSeries**
|
||||
- **eplus:inbound**: e+ (イープラス) overseas
|
||||
- **EpidemicSound**
|
||||
- **eplus**: [*eplus*](## "netrc machine") e+ (イープラス)
|
||||
- **Epoch**
|
||||
- **Eporner**
|
||||
- **Erocast**
|
||||
@@ -429,11 +408,9 @@
|
||||
- **ertflix**: ERTFLIX videos
|
||||
- **ertflix:codename**: ERTFLIX videos by codename
|
||||
- **ertwebtv:embed**: ert.gr webtv embedded videos
|
||||
- **Escapist**
|
||||
- **ESPN**
|
||||
- **ESPNArticle**
|
||||
- **ESPNCricInfo**
|
||||
- **EsriVideo**
|
||||
- **EttuTv**
|
||||
- **Europa**
|
||||
- **EuroParlWebstream**
|
||||
@@ -443,9 +420,7 @@
|
||||
- **EWETV**: [*ewetv*](## "netrc machine")
|
||||
- **EWETVLive**: [*ewetv*](## "netrc machine")
|
||||
- **EWETVRecordings**: [*ewetv*](## "netrc machine")
|
||||
- **ExpoTV**
|
||||
- **Expressen**
|
||||
- **ExtremeTube**
|
||||
- **EyedoTV**
|
||||
- **facebook**: [*facebook*](## "netrc machine")
|
||||
- **facebook:reel**
|
||||
@@ -465,6 +440,8 @@
|
||||
- **FiveThirtyEight**
|
||||
- **FiveTV**
|
||||
- **Flickr**
|
||||
- **Floatplane**
|
||||
- **FloatplaneChannel**
|
||||
- **Folketinget**: Folketinget (ft.dk; Danish parliament)
|
||||
- **FoodNetwork**
|
||||
- **FootyRoom**
|
||||
@@ -472,7 +449,6 @@
|
||||
- **FOX**
|
||||
- **FOX9**
|
||||
- **FOX9News**
|
||||
- **Foxgay**
|
||||
- **foxnews**: Fox News and Fox Business Video
|
||||
- **foxnews:article**
|
||||
- **FoxNewsVideo**
|
||||
@@ -496,7 +472,6 @@
|
||||
- **funimation:show**: [*funimation*](## "netrc machine")
|
||||
- **Funk**
|
||||
- **Funker530**
|
||||
- **Fusion**
|
||||
- **Fux**
|
||||
- **FuyinTV**
|
||||
- **Gab**
|
||||
@@ -522,7 +497,6 @@
|
||||
- **GeniusLyrics**
|
||||
- **Gettr**
|
||||
- **GettrStreaming**
|
||||
- **Gfycat**
|
||||
- **GiantBomb**
|
||||
- **Giga**
|
||||
- **GlattvisionTV**: [*glattvisiontv*](## "netrc machine")
|
||||
@@ -564,7 +538,6 @@
|
||||
- **HearThisAt**
|
||||
- **Heise**
|
||||
- **HellPorno**
|
||||
- **Helsinki**: helsinki.fi
|
||||
- **hetklokhuis**
|
||||
- **hgtv.com:show**
|
||||
- **HGTVDe**
|
||||
@@ -573,8 +546,6 @@
|
||||
- **HistoricFilms**
|
||||
- **history:player**
|
||||
- **history:topic**: History.com Topic
|
||||
- **hitbox**
|
||||
- **hitbox:live**
|
||||
- **HitRecord**
|
||||
- **hketv**: 香港教育局教育電視 (HKETV) Educational Television, Hong Kong Educational Bureau
|
||||
- **HollywoodReporter**
|
||||
@@ -585,8 +556,6 @@
|
||||
- **hotstar:playlist**
|
||||
- **hotstar:season**
|
||||
- **hotstar:series**
|
||||
- **Howcast**
|
||||
- **HowStuffWorks**
|
||||
- **hrfernsehen**
|
||||
- **HRTi**: [*hrti*](## "netrc machine")
|
||||
- **HRTiPlaylist**: [*hrti*](## "netrc machine")
|
||||
@@ -608,7 +577,7 @@
|
||||
- **ign.com**
|
||||
- **IGNArticle**
|
||||
- **IGNVideo**
|
||||
- **IHeartRadio**
|
||||
- **iheartradio**
|
||||
- **iheartradio:podcast**
|
||||
- **Iltalehti**
|
||||
- **imdb**: Internet Movie Database trailers
|
||||
@@ -638,7 +607,6 @@
|
||||
- **IsraelNationalNews**
|
||||
- **ITProTV**
|
||||
- **ITProTVCourse**
|
||||
- **ITTF**
|
||||
- **ITV**
|
||||
- **ITVBTCC**
|
||||
- **ivi**: ivi.ru
|
||||
@@ -658,6 +626,7 @@
|
||||
- **JioSaavnAlbum**
|
||||
- **JioSaavnSong**
|
||||
- **Joj**
|
||||
- **JoqrAg**: 超!A&G+ 文化放送 (f.k.a. AGQR) Nippon Cultural Broadcasting, Inc. (JOQR)
|
||||
- **Jove**
|
||||
- **JStream**
|
||||
- **JTBC**: jtbc.co.kr
|
||||
@@ -670,7 +639,6 @@
|
||||
- **Karaoketv**
|
||||
- **KarriereVideos**
|
||||
- **Katsomo**
|
||||
- **KeezMovies**
|
||||
- **KelbyOne**
|
||||
- **Ketnet**
|
||||
- **khanacademy**
|
||||
@@ -679,7 +647,7 @@
|
||||
- **Kicker**
|
||||
- **KickStarter**
|
||||
- **KickVOD**
|
||||
- **KinjaEmbed**
|
||||
- **kinja:embed**
|
||||
- **KinoPoisk**
|
||||
- **Kommunetv**
|
||||
- **KompasVideo**
|
||||
@@ -698,8 +666,6 @@
|
||||
- **la7.it**
|
||||
- **la7.it:pod:episode**
|
||||
- **la7.it:podcast**
|
||||
- **laola1tv**
|
||||
- **laola1tv:embed**
|
||||
- **LastFM**
|
||||
- **LastFMPlaylist**
|
||||
- **LastFMUser**
|
||||
@@ -733,7 +699,6 @@
|
||||
- **LinkedIn**: [*linkedin*](## "netrc machine")
|
||||
- **linkedin:learning**: [*linkedin*](## "netrc machine")
|
||||
- **linkedin:learning:course**: [*linkedin*](## "netrc machine")
|
||||
- **LinuxAcademy**: [*linuxacademy*](## "netrc machine")
|
||||
- **Liputan6**
|
||||
- **ListenNotes**
|
||||
- **LiTV**
|
||||
@@ -751,7 +716,7 @@
|
||||
- **Lumni**
|
||||
- **lynda**: [*lynda*](## "netrc machine") lynda.com videos
|
||||
- **lynda:course**: [*lynda*](## "netrc machine") lynda.com online courses
|
||||
- **m6**
|
||||
- **maariv.co.il**
|
||||
- **MagellanTV**
|
||||
- **MagentaMusik360**
|
||||
- **mailru**: Видео@Mail.Ru
|
||||
@@ -793,11 +758,8 @@
|
||||
- **megatvcom:embed**: megatv.com embedded videos
|
||||
- **Meipai**: 美拍
|
||||
- **MelonVOD**
|
||||
- **META**
|
||||
- **metacafe**
|
||||
- **Metacritic**
|
||||
- **mewatch**
|
||||
- **Mgoon**
|
||||
- **MiaoPai**
|
||||
- **MicrosoftEmbed**
|
||||
- **microsoftstream**: Microsoft Stream
|
||||
@@ -810,7 +772,6 @@
|
||||
- **minds:group**
|
||||
- **MinistryGrid**
|
||||
- **Minoto**
|
||||
- **miomio.tv**
|
||||
- **mirrativ**
|
||||
- **mirrativ:user**
|
||||
- **MirrorCoUK**
|
||||
@@ -825,14 +786,10 @@
|
||||
- **MLBTV**: [*mlb*](## "netrc machine")
|
||||
- **MLBVideo**
|
||||
- **MLSSoccer**
|
||||
- **Mnet**
|
||||
- **MNetTV**: [*mnettv*](## "netrc machine")
|
||||
- **MNetTVLive**: [*mnettv*](## "netrc machine")
|
||||
- **MNetTVRecordings**: [*mnettv*](## "netrc machine")
|
||||
- **MochaVideo**
|
||||
- **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
|
||||
- **Mofosex**
|
||||
- **MofosexEmbed**
|
||||
- **Mojvideo**
|
||||
- **Monstercat**
|
||||
- **MonsterSirenHypergryphMusic**
|
||||
@@ -843,13 +800,12 @@
|
||||
- **Motorsport**: motorsport.com
|
||||
- **MotorTrend**
|
||||
- **MotorTrendOnDemand**
|
||||
- **MovieClips**
|
||||
- **MovieFap**
|
||||
- **Moviepilot**
|
||||
- **MoviewPlay**
|
||||
- **Moviezine**
|
||||
- **MovingImage**
|
||||
- **MSN**
|
||||
- **MSN**: (**Currently broken**)
|
||||
- **mtg**: MTG services
|
||||
- **mtv**
|
||||
- **mtv.de**
|
||||
@@ -871,18 +827,13 @@
|
||||
- **MusicdexSong**
|
||||
- **mva**: Microsoft Virtual Academy videos
|
||||
- **mva:course**: Microsoft Virtual Academy courses
|
||||
- **Mwave**
|
||||
- **MwaveMeetGreet**
|
||||
- **Mxplayer**
|
||||
- **MxplayerShow**
|
||||
- **MyChannels**
|
||||
- **MySpace**
|
||||
- **MySpace:album**
|
||||
- **MySpass**
|
||||
- **Myvi**
|
||||
- **MyVideoGe**
|
||||
- **MyVidster**
|
||||
- **MyviEmbed**
|
||||
- **Mzaalo**
|
||||
- **n-tv.de**
|
||||
- **N1Info:article**
|
||||
@@ -894,12 +845,12 @@
|
||||
- **Naver**
|
||||
- **Naver:live**
|
||||
- **navernow**
|
||||
- **NBA**
|
||||
- **nba**
|
||||
- **nba:channel**
|
||||
- **nba:embed**
|
||||
- **nba:watch**
|
||||
- **nba:watch:collection**
|
||||
- **NBAChannel**
|
||||
- **NBAEmbed**
|
||||
- **NBAWatchEmbed**
|
||||
- **nba:watch:embed**
|
||||
- **NBC**
|
||||
- **NBCNews**
|
||||
- **nbcolympics**
|
||||
@@ -914,6 +865,7 @@
|
||||
- **NDTV**
|
||||
- **Nebula**: [*watchnebula*](## "netrc machine")
|
||||
- **nebula:channel**: [*watchnebula*](## "netrc machine")
|
||||
- **nebula:class**: [*watchnebula*](## "netrc machine")
|
||||
- **nebula:subscriptions**: [*watchnebula*](## "netrc machine")
|
||||
- **NekoHacker**
|
||||
- **NerdCubedFeed**
|
||||
@@ -935,7 +887,6 @@
|
||||
- **Newgrounds:playlist**
|
||||
- **Newgrounds:user**
|
||||
- **NewsPicks**
|
||||
- **Newstube**
|
||||
- **Newsy**
|
||||
- **NextMedia**: 蘋果日報
|
||||
- **NextMediaActionNews**: 蘋果日報 - 動新聞
|
||||
@@ -961,7 +912,6 @@
|
||||
- **nick.de**
|
||||
- **nickelodeon:br**
|
||||
- **nickelodeonru**
|
||||
- **nicknight**
|
||||
- **niconico**: [*niconico*](## "netrc machine") ニコニコ動画
|
||||
- **niconico:history**: NicoNico user history or likes. Requires cookies.
|
||||
- **niconico:live**: ニコニコ生放送
|
||||
@@ -984,9 +934,7 @@
|
||||
- **NonkTube**
|
||||
- **NoodleMagazine**
|
||||
- **Noovo**
|
||||
- **Normalboots**
|
||||
- **NOSNLArticle**
|
||||
- **NosVideo**
|
||||
- **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
|
||||
- **NovaEmbed**
|
||||
- **NovaPlay**
|
||||
@@ -1009,7 +957,7 @@
|
||||
- **NRKTVEpisodes**
|
||||
- **NRKTVSeason**
|
||||
- **NRKTVSeries**
|
||||
- **NRLTV**
|
||||
- **NRLTV**: (**Currently broken**)
|
||||
- **ntv.ru**
|
||||
- **NubilesPorn**: [*nubiles-porn*](## "netrc machine")
|
||||
- **Nuvid**
|
||||
@@ -1037,8 +985,6 @@
|
||||
- **onet.tv:channel**
|
||||
- **OnetMVP**
|
||||
- **OnionStudios**
|
||||
- **Ooyala**
|
||||
- **OoyalaExternal**
|
||||
- **Opencast**
|
||||
- **OpencastPlaylist**
|
||||
- **openrec**
|
||||
@@ -1060,7 +1006,6 @@
|
||||
- **PalcoMP3:artist**
|
||||
- **PalcoMP3:song**
|
||||
- **PalcoMP3:video**
|
||||
- **pandora.tv**: 판도라TV
|
||||
- **Panopto**
|
||||
- **PanoptoList**
|
||||
- **PanoptoPlaylist**
|
||||
@@ -1082,7 +1027,6 @@
|
||||
- **PeerTube:Playlist**
|
||||
- **peloton**: [*peloton*](## "netrc machine")
|
||||
- **peloton:live**: Peloton Live
|
||||
- **People**
|
||||
- **PerformGroup**
|
||||
- **periscope**: Periscope
|
||||
- **periscope:user**: Periscope user videos
|
||||
@@ -1104,14 +1048,11 @@
|
||||
- **PlanetMarathi**
|
||||
- **Platzi**: [*platzi*](## "netrc machine")
|
||||
- **PlatziCourse**: [*platzi*](## "netrc machine")
|
||||
- **play.fm**
|
||||
- **player.sky.it**
|
||||
- **PlayPlusTV**: [*playplustv*](## "netrc machine")
|
||||
- **PlayStuff**
|
||||
- **PlaysTV**
|
||||
- **PlaySuisse**
|
||||
- **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
|
||||
- **Playvid**
|
||||
- **PlayVids**
|
||||
- **Playwire**
|
||||
- **pluralsight**: [*pluralsight*](## "netrc machine")
|
||||
@@ -1136,11 +1077,8 @@
|
||||
- **Popcorntimes**
|
||||
- **PopcornTV**
|
||||
- **Pornbox**
|
||||
- **PornCom**
|
||||
- **PornerBros**
|
||||
- **Pornez**
|
||||
- **PornFlip**
|
||||
- **PornHd**
|
||||
- **PornHub**: [*pornhub*](## "netrc machine") PornHub and Thumbzilla
|
||||
- **PornHubPagedVideoList**: [*pornhub*](## "netrc machine")
|
||||
- **PornHubPlaylist**: [*pornhub*](## "netrc machine")
|
||||
@@ -1182,7 +1120,6 @@
|
||||
- **Radiko**
|
||||
- **RadikoRadio**
|
||||
- **radio.de**
|
||||
- **radiobremen**
|
||||
- **radiocanada**
|
||||
- **radiocanada:audiovideo**
|
||||
- **RadioComercial**
|
||||
@@ -1222,7 +1159,6 @@
|
||||
- **RCTIPlusSeries**
|
||||
- **RCTIPlusTV**
|
||||
- **RDS**: RDS.ca
|
||||
- **Recurbate**
|
||||
- **RedBull**
|
||||
- **RedBullEmbed**
|
||||
- **RedBullTV**
|
||||
@@ -1239,7 +1175,7 @@
|
||||
- **Reuters**
|
||||
- **ReverbNation**
|
||||
- **RheinMainTV**
|
||||
- **RICE**
|
||||
- **RinseFM**
|
||||
- **RMCDecouverte**
|
||||
- **RockstarGames**
|
||||
- **Rokfin**: [*rokfin*](## "netrc machine")
|
||||
@@ -1260,8 +1196,6 @@
|
||||
- **rtl.lu:tele-vod**
|
||||
- **rtl.nl**: rtl.nl and rtlxl.nl
|
||||
- **rtl2**
|
||||
- **rtl2:you**
|
||||
- **rtl2:you:series**
|
||||
- **RTLLuLive**
|
||||
- **RTLLuRadio**
|
||||
- **RTNews**
|
||||
@@ -1276,10 +1210,9 @@
|
||||
- **rtve.es:infantil**: RTVE infantil
|
||||
- **rtve.es:live**: RTVE.es live streams
|
||||
- **rtve.es:television**
|
||||
- **RTVNH**
|
||||
- **RTVS**
|
||||
- **rtvslo.si**
|
||||
- **RUHD**
|
||||
- **RudoVideo**
|
||||
- **Rule34Video**
|
||||
- **Rumble**
|
||||
- **RumbleChannel**
|
||||
@@ -1326,8 +1259,8 @@
|
||||
- **ScrippsNetworks**
|
||||
- **scrippsnetworks:watch**
|
||||
- **Scrolller**
|
||||
- **SCTE**: [*scte*](## "netrc machine")
|
||||
- **SCTECourse**: [*scte*](## "netrc machine")
|
||||
- **SCTE**: [*scte*](## "netrc machine") (**Currently broken**)
|
||||
- **SCTECourse**: [*scte*](## "netrc machine") (**Currently broken**)
|
||||
- **Seeker**
|
||||
- **SenalColombiaLive**
|
||||
- **SenateGov**
|
||||
@@ -1339,7 +1272,6 @@
|
||||
- **SeznamZpravyArticle**
|
||||
- **Shahid**: [*shahid*](## "netrc machine")
|
||||
- **ShahidShow**
|
||||
- **Shared**: shared.sx
|
||||
- **ShareVideosEmbed**
|
||||
- **ShemarooMe**
|
||||
- **ShowRoomLive**
|
||||
@@ -1391,7 +1323,6 @@
|
||||
- **SovietsClosetPlaylist**
|
||||
- **SpankBang**
|
||||
- **SpankBangPlaylist**
|
||||
- **Spankwire**
|
||||
- **Spiegel**
|
||||
- **Sport5**
|
||||
- **SportBox**
|
||||
@@ -1404,7 +1335,7 @@
|
||||
- **SpreakerShowPage**
|
||||
- **SpringboardPlatform**
|
||||
- **Sprout**
|
||||
- **sr:mediathek**: Saarländischer Rundfunk
|
||||
- **sr:mediathek**: Saarländischer Rundfunk (**Currently broken**)
|
||||
- **SRGSSR**
|
||||
- **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
|
||||
- **StacommuLive**: [*stacommu*](## "netrc machine")
|
||||
@@ -1421,7 +1352,6 @@
|
||||
- **StoryFireSeries**
|
||||
- **StoryFireUser**
|
||||
- **Streamable**
|
||||
- **streamcloud.eu**
|
||||
- **StreamCZ**
|
||||
- **StreamFF**
|
||||
- **StreetVoice**
|
||||
@@ -1437,7 +1367,6 @@
|
||||
- **SVTPlay**: SVT Play and Öppet arkiv
|
||||
- **SVTSeries**
|
||||
- **SwearnetEpisode**
|
||||
- **SWRMediathek**
|
||||
- **Syfy**
|
||||
- **SYVDK**
|
||||
- **SztvHu**
|
||||
@@ -1456,7 +1385,6 @@
|
||||
- **TeachingChannel**
|
||||
- **Teamcoco**
|
||||
- **TeamTreeHouse**: [*teamtreehouse*](## "netrc machine")
|
||||
- **TechTalks**
|
||||
- **techtv.mit.edu**
|
||||
- **TedEmbed**
|
||||
- **TedPlaylist**
|
||||
@@ -1486,6 +1414,8 @@
|
||||
- **TFO**
|
||||
- **theatercomplextown:ppv**: [*theatercomplextown*](## "netrc machine")
|
||||
- **theatercomplextown:vod**: [*theatercomplextown*](## "netrc machine")
|
||||
- **TheGuardianPodcast**
|
||||
- **TheGuardianPodcastPlaylist**
|
||||
- **TheHoleTv**
|
||||
- **TheIntercept**
|
||||
- **ThePlatform**
|
||||
@@ -1506,27 +1436,23 @@
|
||||
- **tiktok:sound**: (**Currently broken**)
|
||||
- **tiktok:tag**: (**Currently broken**)
|
||||
- **tiktok:user**: (**Currently broken**)
|
||||
- **tinypic**: tinypic.com videos
|
||||
- **TLC**
|
||||
- **TMZ**
|
||||
- **TNAFlix**
|
||||
- **TNAFlixNetworkEmbed**
|
||||
- **toggle**
|
||||
- **toggo**
|
||||
- **Tokentube**
|
||||
- **Tokentube:channel**
|
||||
- **tokfm:audition**
|
||||
- **tokfm:podcast**
|
||||
- **ToonGoggles**
|
||||
- **tou.tv**: [*toutv*](## "netrc machine")
|
||||
- **Toypics**: Toypics video
|
||||
- **ToypicsUser**: Toypics user profile
|
||||
- **Toypics**: Toypics video (**Currently broken**)
|
||||
- **ToypicsUser**: Toypics user profile (**Currently broken**)
|
||||
- **TrailerAddict**: (**Currently broken**)
|
||||
- **TravelChannel**
|
||||
- **Triller**: [*triller*](## "netrc machine")
|
||||
- **TrillerShort**
|
||||
- **TrillerUser**: [*triller*](## "netrc machine")
|
||||
- **Trilulilu**
|
||||
- **Trovo**
|
||||
- **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix
|
||||
- **TrovoChannelVod**: All VODs of a trovo.live channel; "trovovod:" prefix
|
||||
@@ -1536,7 +1462,7 @@
|
||||
- **TruNews**
|
||||
- **Truth**
|
||||
- **TruTV**
|
||||
- **Tube8**
|
||||
- **Tube8**: (**Currently broken**)
|
||||
- **TubeTuGraz**: [*tubetugraz*](## "netrc machine") tube.tugraz.at
|
||||
- **TubeTuGrazSeries**: [*tubetugraz*](## "netrc machine")
|
||||
- **TubiTv**: [*tubitv*](## "netrc machine")
|
||||
@@ -1545,7 +1471,6 @@
|
||||
- **TuneInPodcast**
|
||||
- **TuneInPodcastEpisode**
|
||||
- **TuneInStation**
|
||||
- **TunePk**
|
||||
- **Turbo**
|
||||
- **tv.dfb.de**
|
||||
- **TV2**
|
||||
@@ -1569,14 +1494,7 @@
|
||||
- **TVIPlayer**
|
||||
- **tvland.com**
|
||||
- **TVN24**
|
||||
- **TVNet**
|
||||
- **TVNoe**
|
||||
- **TVNow**
|
||||
- **TVNowAnnual**
|
||||
- **TVNowFilm**
|
||||
- **TVNowNew**
|
||||
- **TVNowSeason**
|
||||
- **TVNowShow**
|
||||
- **tvopengr:embed**: tvopen.gr embedded videos
|
||||
- **tvopengr:watch**: tvopen.gr (and ethnos.gr) videos
|
||||
- **tvp**: Telewizja Polska
|
||||
@@ -1614,7 +1532,6 @@
|
||||
- **umg:de**: Universal Music Deutschland
|
||||
- **Unistra**
|
||||
- **Unity**
|
||||
- **UnscriptedNewsVideo**
|
||||
- **uol.com.br**
|
||||
- **uplynk**
|
||||
- **uplynk:preplay**
|
||||
@@ -1629,7 +1546,6 @@
|
||||
- **Utreon**
|
||||
- **Varzesh3**
|
||||
- **Vbox7**
|
||||
- **VeeHD**
|
||||
- **Veo**
|
||||
- **Veoh**
|
||||
- **veoh:user**
|
||||
@@ -1642,7 +1558,6 @@
|
||||
- **vice**
|
||||
- **vice:article**
|
||||
- **vice:show**
|
||||
- **Vidbit**
|
||||
- **Viddler**
|
||||
- **Videa**
|
||||
- **video.arnes.si**: Arnes Video
|
||||
@@ -1664,6 +1579,7 @@
|
||||
- **VidioLive**: [*vidio*](## "netrc machine")
|
||||
- **VidioPremier**: [*vidio*](## "netrc machine")
|
||||
- **VidLii**
|
||||
- **Vidly**
|
||||
- **viewlift**
|
||||
- **viewlift:embed**
|
||||
- **Viidea**
|
||||
@@ -1683,7 +1599,6 @@
|
||||
- **Vimm:stream**
|
||||
- **ViMP**
|
||||
- **ViMP:Playlist**
|
||||
- **Vimple**: Vimple - one-click video hosting
|
||||
- **Vine**
|
||||
- **vine:user**
|
||||
- **Viqeo**
|
||||
@@ -1691,7 +1606,6 @@
|
||||
- **viu:ott**: [*viu*](## "netrc machine")
|
||||
- **viu:playlist**
|
||||
- **ViuOTTIndonesia**
|
||||
- **Vivo**: vivo.sx
|
||||
- **vk**: [*vk*](## "netrc machine") VK
|
||||
- **vk:uservideos**: [*vk*](## "netrc machine") VK - User's Videos
|
||||
- **vk:wallpost**: [*vk*](## "netrc machine")
|
||||
@@ -1699,37 +1613,27 @@
|
||||
- **VKPlayLive**
|
||||
- **vm.tiktok**
|
||||
- **Vocaroo**
|
||||
- **Vodlocker**
|
||||
- **VODPl**
|
||||
- **VODPlatform**
|
||||
- **VoiceRepublic**
|
||||
- **voicy**
|
||||
- **voicy:channel**
|
||||
- **VolejTV**
|
||||
- **Voot**: [*voot*](## "netrc machine")
|
||||
- **VootSeries**: [*voot*](## "netrc machine")
|
||||
- **Voot**: [*voot*](## "netrc machine") (**Currently broken**)
|
||||
- **VootSeries**: [*voot*](## "netrc machine") (**Currently broken**)
|
||||
- **VoxMedia**
|
||||
- **VoxMediaVolume**
|
||||
- **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
|
||||
- **vqq:series**
|
||||
- **vqq:video**
|
||||
- **Vrak**
|
||||
- **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza
|
||||
- **VrtNU**: [*vrtnu*](## "netrc machine") VRT MAX
|
||||
- **vrv**: [*vrv*](## "netrc machine")
|
||||
- **vrv:series**
|
||||
- **VShare**
|
||||
- **VTM**
|
||||
- **VTXTV**: [*vtxtv*](## "netrc machine")
|
||||
- **VTXTVLive**: [*vtxtv*](## "netrc machine")
|
||||
- **VTXTVRecordings**: [*vtxtv*](## "netrc machine")
|
||||
- **VuClip**
|
||||
- **Vupload**
|
||||
- **VVVVID**
|
||||
- **VVVVIDShow**
|
||||
- **VyboryMos**
|
||||
- **Vzaar**
|
||||
- **Wakanim**
|
||||
- **Walla**
|
||||
- **WalyTV**: [*walytv*](## "netrc machine")
|
||||
- **WalyTVLive**: [*walytv*](## "netrc machine")
|
||||
@@ -1740,9 +1644,7 @@
|
||||
- **washingtonpost**
|
||||
- **washingtonpost:article**
|
||||
- **wat.tv**
|
||||
- **WatchBox**
|
||||
- **WatchESPN**
|
||||
- **WatchIndianPorn**: Watch Indian Porn
|
||||
- **WDR**
|
||||
- **wdr:mobile**: (**Currently broken**)
|
||||
- **WDRElefant**
|
||||
@@ -1770,7 +1672,6 @@
|
||||
- **whowatch**
|
||||
- **Whyp**
|
||||
- **wikimedia.org**
|
||||
- **Willow**
|
||||
- **Wimbledon**
|
||||
- **WimTV**
|
||||
- **WinSportsVideo**
|
||||
@@ -1795,7 +1696,6 @@
|
||||
- **wykop:post**
|
||||
- **wykop:post:comment**
|
||||
- **Xanimu**
|
||||
- **XBef**
|
||||
- **XboxClips**
|
||||
- **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, WolfStream, XVideoSharing
|
||||
- **XHamster**
|
||||
@@ -1807,9 +1707,6 @@
|
||||
- **XMinus**
|
||||
- **XNXX**
|
||||
- **Xstream**
|
||||
- **XTube**
|
||||
- **XTubeUser**: XTube user profile
|
||||
- **Xuite**: 隨意窩Xuite影音
|
||||
- **XVideos**
|
||||
- **xvideos:quickies**
|
||||
- **XXXYMovies**
|
||||
@@ -1826,10 +1723,7 @@
|
||||
- **YapFiles**
|
||||
- **Yappy**
|
||||
- **YappyProfile**
|
||||
- **YesJapan**
|
||||
- **yinyuetai:video**: 音悦Tai
|
||||
- **YleAreena**
|
||||
- **Ynet**
|
||||
- **YouJizz**
|
||||
- **youku**: 优酷
|
||||
- **youku:show**
|
||||
@@ -1877,6 +1771,9 @@
|
||||
- **zingmp3:chart-home**
|
||||
- **zingmp3:chart-music-video**
|
||||
- **zingmp3:hub**
|
||||
- **zingmp3:liveradio**
|
||||
- **zingmp3:podcast**
|
||||
- **zingmp3:podcast-episode**
|
||||
- **zingmp3:user**
|
||||
- **zingmp3:week-chart**
|
||||
- **zoom**
|
||||
|
||||
@@ -19,3 +19,8 @@ def handler(request):
|
||||
pytest.skip(f'{RH_KEY} request handler is not available')
|
||||
|
||||
return functools.partial(handler, logger=FakeLogger)
|
||||
|
||||
|
||||
def validate_and_send(rh, req):
|
||||
rh.validate(req)
|
||||
return rh.send(req)
|
||||
|
||||
@@ -10,7 +10,7 @@ import types
|
||||
import yt_dlp.extractor
|
||||
from yt_dlp import YoutubeDL
|
||||
from yt_dlp.compat import compat_os_name
|
||||
from yt_dlp.utils import preferredencoding, write_string
|
||||
from yt_dlp.utils import preferredencoding, try_call, write_string
|
||||
|
||||
if 'pytest' in sys.modules:
|
||||
import pytest
|
||||
@@ -214,14 +214,19 @@ def sanitize_got_info_dict(got_dict):
|
||||
|
||||
test_info_dict = {
|
||||
key: sanitize(key, value) for key, value in got_dict.items()
|
||||
if value is not None and key not in IGNORED_FIELDS and not any(
|
||||
key.startswith(f'{prefix}_') for prefix in IGNORED_PREFIXES)
|
||||
if value is not None and key not in IGNORED_FIELDS and (
|
||||
not any(key.startswith(f'{prefix}_') for prefix in IGNORED_PREFIXES)
|
||||
or key == '_old_archive_ids')
|
||||
}
|
||||
|
||||
# display_id may be generated from id
|
||||
if test_info_dict.get('display_id') == test_info_dict.get('id'):
|
||||
test_info_dict.pop('display_id')
|
||||
|
||||
# release_year may be generated from release_date
|
||||
if try_call(lambda: test_info_dict['release_year'] == int(test_info_dict['release_date'][:4])):
|
||||
test_info_dict.pop('release_year')
|
||||
|
||||
# Check url for flat entries
|
||||
if got_dict.get('_type', 'video') != 'video' and got_dict.get('url'):
|
||||
test_info_dict['url'] = got_dict['url']
|
||||
|
||||
@@ -140,6 +140,8 @@ class TestFormatSelection(unittest.TestCase):
|
||||
test('example-with-dashes', 'example-with-dashes')
|
||||
test('all', '2', '47', '45', 'example-with-dashes', '35')
|
||||
test('mergeall', '2+47+45+example-with-dashes+35', multi=True)
|
||||
# See: https://github.com/yt-dlp/yt-dlp/pulls/8797
|
||||
test('7_a/worst', '35')
|
||||
|
||||
def test_format_selection_audio(self):
|
||||
formats = [
|
||||
@@ -728,7 +730,7 @@ class TestYoutubeDL(unittest.TestCase):
|
||||
self.assertEqual(got_dict.get(info_field), expected, info_field)
|
||||
return True
|
||||
|
||||
test('%()j', (expect_same_infodict, str))
|
||||
test('%()j', (expect_same_infodict, None))
|
||||
|
||||
# NA placeholder
|
||||
NA_TEST_OUTTMPL = '%(uploader_date)s-%(width)d-%(x|def)s-%(id)s.%(ext)s'
|
||||
@@ -797,6 +799,7 @@ class TestYoutubeDL(unittest.TestCase):
|
||||
test('%(title|%)s %(title|%%)s', '% %%')
|
||||
test('%(id+1-height+3)05d', '00158')
|
||||
test('%(width+100)05d', 'NA')
|
||||
test('%(filesize*8)d', '8192')
|
||||
test('%(formats.0) 15s', ('% 15s' % FORMATS[0], None))
|
||||
test('%(formats.0)r', (repr(FORMATS[0]), None))
|
||||
test('%(height.0)03d', '001')
|
||||
|
||||
@@ -52,6 +52,8 @@ from yt_dlp.networking.exceptions import (
|
||||
from yt_dlp.utils._utils import _YDLLogger as FakeLogger
|
||||
from yt_dlp.utils.networking import HTTPHeaderDict
|
||||
|
||||
from test.conftest import validate_and_send
|
||||
|
||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
@@ -275,11 +277,6 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
|
||||
self._headers_buffer.append(f'{keyword}: {value}\r\n'.encode())
|
||||
|
||||
|
||||
def validate_and_send(rh, req):
|
||||
rh.validate(req)
|
||||
return rh.send(req)
|
||||
|
||||
|
||||
class TestRequestHandlerBase:
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
@@ -331,7 +328,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||
https_server_thread.start()
|
||||
|
||||
with handler(verify=False) as rh:
|
||||
with pytest.raises(SSLError, match='sslv3 alert handshake failure') as exc_info:
|
||||
with pytest.raises(SSLError, match=r'ssl(?:v3|/tls) alert handshake failure') as exc_info:
|
||||
validate_and_send(rh, Request(f'https://127.0.0.1:{https_port}/headers'))
|
||||
assert not issubclass(exc_info.type, CertificateVerifyError)
|
||||
|
||||
@@ -872,8 +869,9 @@ class TestRequestsRequestHandler(TestRequestHandlerBase):
|
||||
])
|
||||
@pytest.mark.parametrize('handler', ['Requests'], indirect=True)
|
||||
def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
|
||||
from urllib3.response import HTTPResponse as Urllib3Response
|
||||
from requests.models import Response as RequestsResponse
|
||||
from urllib3.response import HTTPResponse as Urllib3Response
|
||||
|
||||
from yt_dlp.networking._requests import RequestsResponseAdapter
|
||||
requests_res = RequestsResponse()
|
||||
requests_res.raw = Urllib3Response(body=b'', status=200)
|
||||
@@ -929,13 +927,17 @@ class TestRequestHandlerValidation:
|
||||
('http', False, {}),
|
||||
('https', False, {}),
|
||||
]),
|
||||
('Websockets', [
|
||||
('ws', False, {}),
|
||||
('wss', False, {}),
|
||||
]),
|
||||
(NoCheckRH, [('http', False, {})]),
|
||||
(ValidationRH, [('http', UnsupportedRequest, {})])
|
||||
]
|
||||
|
||||
PROXY_SCHEME_TESTS = [
|
||||
# scheme, expected to fail
|
||||
('Urllib', [
|
||||
('Urllib', 'http', [
|
||||
('http', False),
|
||||
('https', UnsupportedRequest),
|
||||
('socks4', False),
|
||||
@@ -944,7 +946,7 @@ class TestRequestHandlerValidation:
|
||||
('socks5h', False),
|
||||
('socks', UnsupportedRequest),
|
||||
]),
|
||||
('Requests', [
|
||||
('Requests', 'http', [
|
||||
('http', False),
|
||||
('https', False),
|
||||
('socks4', False),
|
||||
@@ -952,8 +954,11 @@ class TestRequestHandlerValidation:
|
||||
('socks5', False),
|
||||
('socks5h', False),
|
||||
]),
|
||||
(NoCheckRH, [('http', False)]),
|
||||
(HTTPSupportedRH, [('http', UnsupportedRequest)]),
|
||||
(NoCheckRH, 'http', [('http', False)]),
|
||||
(HTTPSupportedRH, 'http', [('http', UnsupportedRequest)]),
|
||||
('Websockets', 'ws', [('http', UnsupportedRequest)]),
|
||||
(NoCheckRH, 'http', [('http', False)]),
|
||||
(HTTPSupportedRH, 'http', [('http', UnsupportedRequest)]),
|
||||
]
|
||||
|
||||
PROXY_KEY_TESTS = [
|
||||
@@ -972,7 +977,7 @@ class TestRequestHandlerValidation:
|
||||
]
|
||||
|
||||
EXTENSION_TESTS = [
|
||||
('Urllib', [
|
||||
('Urllib', 'http', [
|
||||
({'cookiejar': 'notacookiejar'}, AssertionError),
|
||||
({'cookiejar': YoutubeDLCookieJar()}, False),
|
||||
({'cookiejar': CookieJar()}, AssertionError),
|
||||
@@ -980,17 +985,21 @@ class TestRequestHandlerValidation:
|
||||
({'timeout': 'notatimeout'}, AssertionError),
|
||||
({'unsupported': 'value'}, UnsupportedRequest),
|
||||
]),
|
||||
('Requests', [
|
||||
('Requests', 'http', [
|
||||
({'cookiejar': 'notacookiejar'}, AssertionError),
|
||||
({'cookiejar': YoutubeDLCookieJar()}, False),
|
||||
({'timeout': 1}, False),
|
||||
({'timeout': 'notatimeout'}, AssertionError),
|
||||
({'unsupported': 'value'}, UnsupportedRequest),
|
||||
]),
|
||||
(NoCheckRH, [
|
||||
(NoCheckRH, 'http', [
|
||||
({'cookiejar': 'notacookiejar'}, False),
|
||||
({'somerandom': 'test'}, False), # but any extension is allowed through
|
||||
]),
|
||||
('Websockets', 'ws', [
|
||||
({'cookiejar': YoutubeDLCookieJar()}, False),
|
||||
({'timeout': 2}, False),
|
||||
]),
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize('handler,scheme,fail,handler_kwargs', [
|
||||
@@ -1016,14 +1025,14 @@ class TestRequestHandlerValidation:
|
||||
run_validation(handler, fail, Request('http://', proxies={proxy_key: 'http://example.com'}))
|
||||
run_validation(handler, fail, Request('http://'), proxies={proxy_key: 'http://example.com'})
|
||||
|
||||
@pytest.mark.parametrize('handler,scheme,fail', [
|
||||
(handler_tests[0], scheme, fail)
|
||||
@pytest.mark.parametrize('handler,req_scheme,scheme,fail', [
|
||||
(handler_tests[0], handler_tests[1], scheme, fail)
|
||||
for handler_tests in PROXY_SCHEME_TESTS
|
||||
for scheme, fail in handler_tests[1]
|
||||
for scheme, fail in handler_tests[2]
|
||||
], indirect=['handler'])
|
||||
def test_proxy_scheme(self, handler, scheme, fail):
|
||||
run_validation(handler, fail, Request('http://', proxies={'http': f'{scheme}://example.com'}))
|
||||
run_validation(handler, fail, Request('http://'), proxies={'http': f'{scheme}://example.com'})
|
||||
def test_proxy_scheme(self, handler, req_scheme, scheme, fail):
|
||||
run_validation(handler, fail, Request(f'{req_scheme}://', proxies={req_scheme: f'{scheme}://example.com'}))
|
||||
run_validation(handler, fail, Request(f'{req_scheme}://'), proxies={req_scheme: f'{scheme}://example.com'})
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Urllib', HTTPSupportedRH, 'Requests'], indirect=True)
|
||||
def test_empty_proxy(self, handler):
|
||||
@@ -1035,14 +1044,14 @@ class TestRequestHandlerValidation:
|
||||
def test_invalid_proxy_url(self, handler, proxy_url):
|
||||
run_validation(handler, UnsupportedRequest, Request('http://', proxies={'http': proxy_url}))
|
||||
|
||||
@pytest.mark.parametrize('handler,extensions,fail', [
|
||||
(handler_tests[0], extensions, fail)
|
||||
@pytest.mark.parametrize('handler,scheme,extensions,fail', [
|
||||
(handler_tests[0], handler_tests[1], extensions, fail)
|
||||
for handler_tests in EXTENSION_TESTS
|
||||
for extensions, fail in handler_tests[1]
|
||||
for extensions, fail in handler_tests[2]
|
||||
], indirect=['handler'])
|
||||
def test_extension(self, handler, extensions, fail):
|
||||
def test_extension(self, handler, scheme, extensions, fail):
|
||||
run_validation(
|
||||
handler, fail, Request('http://', extensions=extensions))
|
||||
handler, fail, Request(f'{scheme}://', extensions=extensions))
|
||||
|
||||
def test_invalid_request_type(self):
|
||||
rh = self.ValidationRH(logger=FakeLogger())
|
||||
@@ -1075,6 +1084,22 @@ class FakeRHYDL(FakeYDL):
|
||||
self._request_director = self.build_request_director([FakeRH])
|
||||
|
||||
|
||||
class AllUnsupportedRHYDL(FakeYDL):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
|
||||
class UnsupportedRH(RequestHandler):
|
||||
def _send(self, request: Request):
|
||||
pass
|
||||
|
||||
_SUPPORTED_FEATURES = ()
|
||||
_SUPPORTED_PROXY_SCHEMES = ()
|
||||
_SUPPORTED_URL_SCHEMES = ()
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
self._request_director = self.build_request_director([UnsupportedRH])
|
||||
|
||||
|
||||
class TestRequestDirector:
|
||||
|
||||
def test_handler_operations(self):
|
||||
@@ -1234,6 +1259,12 @@ class TestYoutubeDLNetworking:
|
||||
with pytest.raises(RequestError, match=r'file:// URLs are disabled by default'):
|
||||
ydl.urlopen('file://')
|
||||
|
||||
@pytest.mark.parametrize('scheme', (['ws', 'wss']))
|
||||
def test_websocket_unavailable_error(self, scheme):
|
||||
with AllUnsupportedRHYDL() as ydl:
|
||||
with pytest.raises(RequestError, match=r'This request requires WebSocket support'):
|
||||
ydl.urlopen(f'{scheme}://')
|
||||
|
||||
def test_legacy_server_connect_error(self):
|
||||
with FakeRHYDL() as ydl:
|
||||
for error in ('UNSAFE_LEGACY_RENEGOTIATION_DISABLED', 'SSLV3_ALERT_HANDSHAKE_FAILURE'):
|
||||
|
||||
@@ -210,6 +210,16 @@ class SocksHTTPTestRequestHandler(http.server.BaseHTTPRequestHandler, SocksTestR
|
||||
self.wfile.write(payload.encode())
|
||||
|
||||
|
||||
class SocksWebSocketTestRequestHandler(SocksTestRequestHandler):
|
||||
def handle(self):
|
||||
import websockets.sync.server
|
||||
protocol = websockets.ServerProtocol()
|
||||
connection = websockets.sync.server.ServerConnection(socket=self.request, protocol=protocol, close_timeout=0)
|
||||
connection.handshake()
|
||||
connection.send(json.dumps(self.socks_info))
|
||||
connection.close()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def socks_server(socks_server_class, request_handler, bind_ip=None, **socks_server_kwargs):
|
||||
server = server_thread = None
|
||||
@@ -252,8 +262,22 @@ class HTTPSocksTestProxyContext(SocksProxyTestContext):
|
||||
return json.loads(handler.send(request).read().decode())
|
||||
|
||||
|
||||
class WebSocketSocksTestProxyContext(SocksProxyTestContext):
|
||||
REQUEST_HANDLER_CLASS = SocksWebSocketTestRequestHandler
|
||||
|
||||
def socks_info_request(self, handler, target_domain=None, target_port=None, **req_kwargs):
|
||||
request = Request(f'ws://{target_domain or "127.0.0.1"}:{target_port or "40000"}', **req_kwargs)
|
||||
handler.validate(request)
|
||||
ws = handler.send(request)
|
||||
ws.send('socks_info')
|
||||
socks_info = ws.recv()
|
||||
ws.close()
|
||||
return json.loads(socks_info)
|
||||
|
||||
|
||||
CTX_MAP = {
|
||||
'http': HTTPSocksTestProxyContext,
|
||||
'ws': WebSocketSocksTestProxyContext,
|
||||
}
|
||||
|
||||
|
||||
@@ -263,7 +287,7 @@ def ctx(request):
|
||||
|
||||
|
||||
class TestSocks4Proxy:
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks4_no_auth(self, handler, ctx):
|
||||
with handler() as rh:
|
||||
with ctx.socks_server(Socks4ProxyHandler) as server_address:
|
||||
@@ -271,7 +295,7 @@ class TestSocks4Proxy:
|
||||
rh, proxies={'all': f'socks4://{server_address}'})
|
||||
assert response['version'] == 4
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks4_auth(self, handler, ctx):
|
||||
with handler() as rh:
|
||||
with ctx.socks_server(Socks4ProxyHandler, user_id='user') as server_address:
|
||||
@@ -281,7 +305,7 @@ class TestSocks4Proxy:
|
||||
rh, proxies={'all': f'socks4://user:@{server_address}'})
|
||||
assert response['version'] == 4
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks4a_ipv4_target(self, handler, ctx):
|
||||
with ctx.socks_server(Socks4ProxyHandler) as server_address:
|
||||
with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
|
||||
@@ -289,7 +313,7 @@ class TestSocks4Proxy:
|
||||
assert response['version'] == 4
|
||||
assert (response['ipv4_address'] == '127.0.0.1') != (response['domain_address'] == '127.0.0.1')
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks4a_domain_target(self, handler, ctx):
|
||||
with ctx.socks_server(Socks4ProxyHandler) as server_address:
|
||||
with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
|
||||
@@ -298,7 +322,7 @@ class TestSocks4Proxy:
|
||||
assert response['ipv4_address'] is None
|
||||
assert response['domain_address'] == 'localhost'
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_ipv4_client_source_address(self, handler, ctx):
|
||||
with ctx.socks_server(Socks4ProxyHandler) as server_address:
|
||||
source_address = f'127.0.0.{random.randint(5, 255)}'
|
||||
@@ -308,7 +332,7 @@ class TestSocks4Proxy:
|
||||
assert response['client_address'][0] == source_address
|
||||
assert response['version'] == 4
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
@pytest.mark.parametrize('reply_code', [
|
||||
Socks4CD.REQUEST_REJECTED_OR_FAILED,
|
||||
Socks4CD.REQUEST_REJECTED_CANNOT_CONNECT_TO_IDENTD,
|
||||
@@ -320,7 +344,7 @@ class TestSocks4Proxy:
|
||||
with pytest.raises(ProxyError):
|
||||
ctx.socks_info_request(rh)
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_ipv6_socks4_proxy(self, handler, ctx):
|
||||
with ctx.socks_server(Socks4ProxyHandler, bind_ip='::1') as server_address:
|
||||
with handler(proxies={'all': f'socks4://{server_address}'}) as rh:
|
||||
@@ -329,7 +353,7 @@ class TestSocks4Proxy:
|
||||
assert response['ipv4_address'] == '127.0.0.1'
|
||||
assert response['version'] == 4
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_timeout(self, handler, ctx):
|
||||
with ctx.socks_server(Socks4ProxyHandler, sleep=2) as server_address:
|
||||
with handler(proxies={'all': f'socks4://{server_address}'}, timeout=0.5) as rh:
|
||||
@@ -339,7 +363,7 @@ class TestSocks4Proxy:
|
||||
|
||||
class TestSocks5Proxy:
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks5_no_auth(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler) as server_address:
|
||||
with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
|
||||
@@ -347,7 +371,7 @@ class TestSocks5Proxy:
|
||||
assert response['auth_methods'] == [0x0]
|
||||
assert response['version'] == 5
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks5_user_pass(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler, auth=('test', 'testpass')) as server_address:
|
||||
with handler() as rh:
|
||||
@@ -360,7 +384,7 @@ class TestSocks5Proxy:
|
||||
assert response['auth_methods'] == [Socks5Auth.AUTH_NONE, Socks5Auth.AUTH_USER_PASS]
|
||||
assert response['version'] == 5
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks5_ipv4_target(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler) as server_address:
|
||||
with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
|
||||
@@ -368,7 +392,7 @@ class TestSocks5Proxy:
|
||||
assert response['ipv4_address'] == '127.0.0.1'
|
||||
assert response['version'] == 5
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks5_domain_target(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler) as server_address:
|
||||
with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
|
||||
@@ -376,7 +400,7 @@ class TestSocks5Proxy:
|
||||
assert (response['ipv4_address'] == '127.0.0.1') != (response['ipv6_address'] == '::1')
|
||||
assert response['version'] == 5
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks5h_domain_target(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler) as server_address:
|
||||
with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
|
||||
@@ -385,7 +409,7 @@ class TestSocks5Proxy:
|
||||
assert response['domain_address'] == 'localhost'
|
||||
assert response['version'] == 5
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks5h_ip_target(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler) as server_address:
|
||||
with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
|
||||
@@ -394,7 +418,7 @@ class TestSocks5Proxy:
|
||||
assert response['domain_address'] is None
|
||||
assert response['version'] == 5
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_socks5_ipv6_destination(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler) as server_address:
|
||||
with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
|
||||
@@ -402,7 +426,7 @@ class TestSocks5Proxy:
|
||||
assert response['ipv6_address'] == '::1'
|
||||
assert response['version'] == 5
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_ipv6_socks5_proxy(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler, bind_ip='::1') as server_address:
|
||||
with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
|
||||
@@ -413,7 +437,7 @@ class TestSocks5Proxy:
|
||||
|
||||
# XXX: is there any feasible way of testing IPv6 source addresses?
|
||||
# Same would go for non-proxy source_address test...
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_ipv4_client_source_address(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler) as server_address:
|
||||
source_address = f'127.0.0.{random.randint(5, 255)}'
|
||||
@@ -422,7 +446,7 @@ class TestSocks5Proxy:
|
||||
assert response['client_address'][0] == source_address
|
||||
assert response['version'] == 5
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
@pytest.mark.parametrize('reply_code', [
|
||||
Socks5Reply.GENERAL_FAILURE,
|
||||
Socks5Reply.CONNECTION_NOT_ALLOWED,
|
||||
@@ -439,7 +463,7 @@ class TestSocks5Proxy:
|
||||
with pytest.raises(ProxyError):
|
||||
ctx.socks_info_request(rh)
|
||||
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
|
||||
@pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Websockets', 'ws')], indirect=True)
|
||||
def test_timeout(self, handler, ctx):
|
||||
with ctx.socks_server(Socks5ProxyHandler, sleep=2) as server_address:
|
||||
with handler(proxies={'all': f'socks5://{server_address}'}, timeout=1) as rh:
|
||||
|
||||
@@ -9,7 +9,15 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
from test.helper import FakeYDL, report_warning
|
||||
from yt_dlp.update import Updater, UpdateInfo
|
||||
from yt_dlp.update import UpdateInfo, Updater
|
||||
|
||||
|
||||
# XXX: Keep in sync with yt_dlp.update.UPDATE_SOURCES
|
||||
TEST_UPDATE_SOURCES = {
|
||||
'stable': 'yt-dlp/yt-dlp',
|
||||
'nightly': 'yt-dlp/yt-dlp-nightly-builds',
|
||||
'master': 'yt-dlp/yt-dlp-master-builds',
|
||||
}
|
||||
|
||||
TEST_API_DATA = {
|
||||
'yt-dlp/yt-dlp/latest': {
|
||||
@@ -68,25 +76,34 @@ TEST_API_DATA = {
|
||||
},
|
||||
}
|
||||
|
||||
TEST_LOCKFILE_V1 = '''# This file is used for regulating self-update
|
||||
lock 2022.08.18.36 .+ Python 3.6
|
||||
lock 2023.11.13 .+ Python 3.7
|
||||
TEST_LOCKFILE_COMMENT = '# This file is used for regulating self-update'
|
||||
|
||||
TEST_LOCKFILE_V1 = r'''%s
|
||||
lock 2022.08.18.36 .+ Python 3\.6
|
||||
lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7
|
||||
lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
|
||||
''' % TEST_LOCKFILE_COMMENT
|
||||
|
||||
TEST_LOCKFILE_V2_TMPL = r'''%s
|
||||
lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6
|
||||
lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7
|
||||
lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
|
||||
lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7
|
||||
lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server)
|
||||
lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7
|
||||
lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server)
|
||||
'''
|
||||
|
||||
TEST_LOCKFILE_V2 = '''# This file is used for regulating self-update
|
||||
lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3.6
|
||||
lockV2 yt-dlp/yt-dlp 2023.11.13 .+ Python 3.7
|
||||
'''
|
||||
TEST_LOCKFILE_V2 = TEST_LOCKFILE_V2_TMPL % TEST_LOCKFILE_COMMENT
|
||||
|
||||
TEST_LOCKFILE_V1_V2 = '''# This file is used for regulating self-update
|
||||
lock 2022.08.18.36 .+ Python 3.6
|
||||
lock 2023.11.13 .+ Python 3.7
|
||||
lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3.6
|
||||
lockV2 yt-dlp/yt-dlp 2023.11.13 .+ Python 3.7
|
||||
TEST_LOCKFILE_ACTUAL = TEST_LOCKFILE_V2_TMPL % TEST_LOCKFILE_V1.rstrip('\n')
|
||||
|
||||
TEST_LOCKFILE_FORK = r'''%s# Test if a fork blocks updates to non-numeric tags
|
||||
lockV2 fork/yt-dlp pr0000 .+ Python 3.6
|
||||
lockV2 fork/yt-dlp pr1234 .+ Python 3.7
|
||||
lockV2 fork/yt-dlp pr1234 (?!win_x86_exe).+ Python 3\.7
|
||||
lockV2 fork/yt-dlp pr1234 win_x86_exe .+ Windows-(?:Vista|2008Server)
|
||||
lockV2 fork/yt-dlp pr9999 .+ Python 3.11
|
||||
'''
|
||||
''' % TEST_LOCKFILE_ACTUAL
|
||||
|
||||
|
||||
class FakeUpdater(Updater):
|
||||
@@ -95,9 +112,10 @@ class FakeUpdater(Updater):
|
||||
|
||||
_channel = 'stable'
|
||||
_origin = 'yt-dlp/yt-dlp'
|
||||
_update_sources = TEST_UPDATE_SOURCES
|
||||
|
||||
def _download_update_spec(self, *args, **kwargs):
|
||||
return TEST_LOCKFILE_V1_V2
|
||||
return TEST_LOCKFILE_ACTUAL
|
||||
|
||||
def _call_api(self, tag):
|
||||
tag = f'tags/{tag}' if tag != 'latest' else tag
|
||||
@@ -112,7 +130,7 @@ class TestUpdate(unittest.TestCase):
|
||||
|
||||
def test_update_spec(self):
|
||||
ydl = FakeYDL()
|
||||
updater = FakeUpdater(ydl, 'stable@latest')
|
||||
updater = FakeUpdater(ydl, 'stable')
|
||||
|
||||
def test(lockfile, identifier, input_tag, expect_tag, exact=False, repo='yt-dlp/yt-dlp'):
|
||||
updater._identifier = identifier
|
||||
@@ -124,35 +142,46 @@ class TestUpdate(unittest.TestCase):
|
||||
f'{identifier!r} requesting {repo}@{input_tag} (exact={exact}) '
|
||||
f'returned {result!r} instead of {expect_tag!r}')
|
||||
|
||||
test(TEST_LOCKFILE_V1, 'zip Python 3.11.0', '2023.11.13', '2023.11.13')
|
||||
test(TEST_LOCKFILE_V1, 'zip stable Python 3.11.0', '2023.11.13', '2023.11.13', exact=True)
|
||||
test(TEST_LOCKFILE_V1, 'zip Python 3.6.0', '2023.11.13', '2022.08.18.36')
|
||||
test(TEST_LOCKFILE_V1, 'zip stable Python 3.6.0', '2023.11.13', None, exact=True)
|
||||
test(TEST_LOCKFILE_V1, 'zip Python 3.7.0', '2023.11.13', '2023.11.13')
|
||||
test(TEST_LOCKFILE_V1, 'zip stable Python 3.7.1', '2023.11.13', '2023.11.13')
|
||||
test(TEST_LOCKFILE_V1, 'zip Python 3.7.1', '2023.12.31', '2023.11.13')
|
||||
test(TEST_LOCKFILE_V1, 'zip stable Python 3.7.1', '2023.12.31', '2023.11.13')
|
||||
for lockfile in (TEST_LOCKFILE_V1, TEST_LOCKFILE_V2, TEST_LOCKFILE_ACTUAL, TEST_LOCKFILE_FORK):
|
||||
# Normal operation
|
||||
test(lockfile, 'zip Python 3.12.0', '2023.12.31', '2023.12.31')
|
||||
test(lockfile, 'zip stable Python 3.12.0', '2023.12.31', '2023.12.31', exact=True)
|
||||
# Python 3.6 --update should update only to its lock
|
||||
test(lockfile, 'zip Python 3.6.0', '2023.11.16', '2022.08.18.36')
|
||||
# --update-to an exact version later than the lock should return None
|
||||
test(lockfile, 'zip stable Python 3.6.0', '2023.11.16', None, exact=True)
|
||||
# Python 3.7 should be able to update to its lock
|
||||
test(lockfile, 'zip Python 3.7.0', '2023.11.16', '2023.11.16')
|
||||
test(lockfile, 'zip stable Python 3.7.1', '2023.11.16', '2023.11.16', exact=True)
|
||||
# Non-win_x86_exe builds on py3.7 must be locked
|
||||
test(lockfile, 'zip Python 3.7.1', '2023.12.31', '2023.11.16')
|
||||
test(lockfile, 'zip stable Python 3.7.1', '2023.12.31', None, exact=True)
|
||||
test( # Windows Vista w/ win_x86_exe must be locked
|
||||
lockfile, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-Vista-6.0.6003-SP2',
|
||||
'2023.12.31', '2023.11.16')
|
||||
test( # Windows 2008Server w/ win_x86_exe must be locked
|
||||
lockfile, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-2008Server',
|
||||
'2023.12.31', None, exact=True)
|
||||
test( # Windows 7 w/ win_x86_exe py3.7 build should be able to update beyond lock
|
||||
lockfile, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-7-6.1.7601-SP1',
|
||||
'2023.12.31', '2023.12.31')
|
||||
test( # Windows 8.1 w/ '2008Server' in platform string should be able to update beyond lock
|
||||
lockfile, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-post2008Server-6.2.9200',
|
||||
'2023.12.31', '2023.12.31', exact=True)
|
||||
|
||||
test(TEST_LOCKFILE_V2, 'zip Python 3.11.1', '2023.11.13', '2023.11.13')
|
||||
test(TEST_LOCKFILE_V2, 'zip stable Python 3.11.1', '2023.12.31', '2023.12.31')
|
||||
test(TEST_LOCKFILE_V2, 'zip Python 3.6.1', '2023.11.13', '2022.08.18.36')
|
||||
test(TEST_LOCKFILE_V2, 'zip stable Python 3.7.2', '2023.11.13', '2023.11.13')
|
||||
test(TEST_LOCKFILE_V2, 'zip Python 3.7.2', '2023.12.31', '2023.11.13')
|
||||
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.2', '2023.11.13', '2023.11.13')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.11.2', '2023.12.31', '2023.12.31')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.2', '2023.11.13', '2022.08.18.36')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.7.3', '2023.11.13', '2023.11.13')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.3', '2023.12.31', '2023.11.13')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.3', 'pr0000', None, repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.7.4', 'pr0000', 'pr0000', repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.4', 'pr0000', None, repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.4', 'pr1234', None, repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.8.1', 'pr1234', 'pr1234', repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.5', 'pr1234', None, repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.3', 'pr9999', None, repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.12.0', 'pr9999', 'pr9999', repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.4', 'pr9999', None, repo='fork/yt-dlp')
|
||||
# Forks can block updates to non-numeric tags rather than lock
|
||||
test(TEST_LOCKFILE_FORK, 'zip Python 3.6.3', 'pr0000', None, repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_FORK, 'zip stable Python 3.7.4', 'pr0000', 'pr0000', repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_FORK, 'zip stable Python 3.7.4', 'pr1234', None, repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_FORK, 'zip Python 3.8.1', 'pr1234', 'pr1234', repo='fork/yt-dlp', exact=True)
|
||||
test(
|
||||
TEST_LOCKFILE_FORK, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-Vista-6.0.6003-SP2',
|
||||
'pr1234', None, repo='fork/yt-dlp')
|
||||
test(
|
||||
TEST_LOCKFILE_FORK, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-7-6.1.7601-SP1',
|
||||
'2023.12.31', '2023.12.31', repo='fork/yt-dlp')
|
||||
test(TEST_LOCKFILE_FORK, 'zip Python 3.11.2', 'pr9999', None, repo='fork/yt-dlp', exact=True)
|
||||
test(TEST_LOCKFILE_FORK, 'zip stable Python 3.12.0', 'pr9999', 'pr9999', repo='fork/yt-dlp')
|
||||
|
||||
def test_query_update(self):
|
||||
ydl = FakeYDL()
|
||||
|
||||
@@ -2110,6 +2110,8 @@ Line 1
|
||||
self.assertEqual(traverse_obj(_TEST_DATA, (..., {str_or_none})),
|
||||
[item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
|
||||
msg='Function in set should be a transformation')
|
||||
self.assertEqual(traverse_obj(_TEST_DATA, ('fail', {lambda _: 'const'})), 'const',
|
||||
msg='Function in set should always be called')
|
||||
if __debug__:
|
||||
with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
|
||||
traverse_obj(_TEST_DATA, set())
|
||||
@@ -2317,23 +2319,6 @@ Line 1
|
||||
self.assertEqual(traverse_obj({}, (0, slice(1)), traverse_string=True), [],
|
||||
msg='branching should result in list if `traverse_string`')
|
||||
|
||||
# Test is_user_input behavior
|
||||
_IS_USER_INPUT_DATA = {'range8': list(range(8))}
|
||||
self.assertEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3'),
|
||||
is_user_input=True), 3,
|
||||
msg='allow for string indexing if `is_user_input`')
|
||||
self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3:'),
|
||||
is_user_input=True), tuple(range(8))[3:],
|
||||
msg='allow for string slice if `is_user_input`')
|
||||
self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':4:2'),
|
||||
is_user_input=True), tuple(range(8))[:4:2],
|
||||
msg='allow step in string slice if `is_user_input`')
|
||||
self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':'),
|
||||
is_user_input=True), range(8),
|
||||
msg='`:` should be treated as `...` if `is_user_input`')
|
||||
with self.assertRaises(TypeError, msg='too many params should result in error'):
|
||||
traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':::'), is_user_input=True)
|
||||
|
||||
# Test re.Match as input obj
|
||||
mobj = re.fullmatch(r'0(12)(?P<group>3)(4)?', '0123')
|
||||
self.assertEqual(traverse_obj(mobj, ...), [x for x in mobj.groups() if x is not None],
|
||||
@@ -2387,6 +2372,11 @@ Line 1
|
||||
headers4 = HTTPHeaderDict({'ytdl-test': 'data;'})
|
||||
self.assertEqual(set(headers4.items()), {('Ytdl-Test', 'data;')})
|
||||
|
||||
# common mistake: strip whitespace from values
|
||||
# https://github.com/yt-dlp/yt-dlp/issues/8729
|
||||
headers5 = HTTPHeaderDict({'ytdl-test': ' data; '})
|
||||
self.assertEqual(set(headers5.items()), {('Ytdl-Test', 'data;')})
|
||||
|
||||
def test_extract_basic_auth(self):
|
||||
assert extract_basic_auth('http://:foo.bar') == ('http://:foo.bar', None)
|
||||
assert extract_basic_auth('http://foo.bar') == ('http://foo.bar', None)
|
||||
|
||||
380
test/test_websockets.py
Normal file
380
test/test_websockets.py
Normal file
@@ -0,0 +1,380 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import http.client
|
||||
import http.cookiejar
|
||||
import http.server
|
||||
import json
|
||||
import random
|
||||
import ssl
|
||||
import threading
|
||||
|
||||
from yt_dlp import socks
|
||||
from yt_dlp.cookies import YoutubeDLCookieJar
|
||||
from yt_dlp.dependencies import websockets
|
||||
from yt_dlp.networking import Request
|
||||
from yt_dlp.networking.exceptions import (
|
||||
CertificateVerifyError,
|
||||
HTTPError,
|
||||
ProxyError,
|
||||
RequestError,
|
||||
SSLError,
|
||||
TransportError,
|
||||
)
|
||||
from yt_dlp.utils.networking import HTTPHeaderDict
|
||||
|
||||
from test.conftest import validate_and_send
|
||||
|
||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def websocket_handler(websocket):
|
||||
for message in websocket:
|
||||
if isinstance(message, bytes):
|
||||
if message == b'bytes':
|
||||
return websocket.send('2')
|
||||
elif isinstance(message, str):
|
||||
if message == 'headers':
|
||||
return websocket.send(json.dumps(dict(websocket.request.headers)))
|
||||
elif message == 'path':
|
||||
return websocket.send(websocket.request.path)
|
||||
elif message == 'source_address':
|
||||
return websocket.send(websocket.remote_address[0])
|
||||
elif message == 'str':
|
||||
return websocket.send('1')
|
||||
return websocket.send(message)
|
||||
|
||||
|
||||
def process_request(self, request):
|
||||
if request.path.startswith('/gen_'):
|
||||
status = http.HTTPStatus(int(request.path[5:]))
|
||||
if 300 <= status.value <= 300:
|
||||
return websockets.http11.Response(
|
||||
status.value, status.phrase, websockets.datastructures.Headers([('Location', '/')]), b'')
|
||||
return self.protocol.reject(status.value, status.phrase)
|
||||
return self.protocol.accept(request)
|
||||
|
||||
|
||||
def create_websocket_server(**ws_kwargs):
|
||||
import websockets.sync.server
|
||||
wsd = websockets.sync.server.serve(websocket_handler, '127.0.0.1', 0, process_request=process_request, **ws_kwargs)
|
||||
ws_port = wsd.socket.getsockname()[1]
|
||||
ws_server_thread = threading.Thread(target=wsd.serve_forever)
|
||||
ws_server_thread.daemon = True
|
||||
ws_server_thread.start()
|
||||
return ws_server_thread, ws_port
|
||||
|
||||
|
||||
def create_ws_websocket_server():
|
||||
return create_websocket_server()
|
||||
|
||||
|
||||
def create_wss_websocket_server():
|
||||
certfn = os.path.join(TEST_DIR, 'testcert.pem')
|
||||
sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
sslctx.load_cert_chain(certfn, None)
|
||||
return create_websocket_server(ssl_context=sslctx)
|
||||
|
||||
|
||||
MTLS_CERT_DIR = os.path.join(TEST_DIR, 'testdata', 'certificate')
|
||||
|
||||
|
||||
def create_mtls_wss_websocket_server():
|
||||
certfn = os.path.join(TEST_DIR, 'testcert.pem')
|
||||
cacertfn = os.path.join(MTLS_CERT_DIR, 'ca.crt')
|
||||
|
||||
sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
sslctx.verify_mode = ssl.CERT_REQUIRED
|
||||
sslctx.load_verify_locations(cafile=cacertfn)
|
||||
sslctx.load_cert_chain(certfn, None)
|
||||
|
||||
return create_websocket_server(ssl_context=sslctx)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not websockets, reason='websockets must be installed to test websocket request handlers')
|
||||
class TestWebsSocketRequestHandlerConformance:
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
cls.ws_thread, cls.ws_port = create_ws_websocket_server()
|
||||
cls.ws_base_url = f'ws://127.0.0.1:{cls.ws_port}'
|
||||
|
||||
cls.wss_thread, cls.wss_port = create_wss_websocket_server()
|
||||
cls.wss_base_url = f'wss://127.0.0.1:{cls.wss_port}'
|
||||
|
||||
cls.bad_wss_thread, cls.bad_wss_port = create_websocket_server(ssl_context=ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER))
|
||||
cls.bad_wss_host = f'wss://127.0.0.1:{cls.bad_wss_port}'
|
||||
|
||||
cls.mtls_wss_thread, cls.mtls_wss_port = create_mtls_wss_websocket_server()
|
||||
cls.mtls_wss_base_url = f'wss://127.0.0.1:{cls.mtls_wss_port}'
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_basic_websockets(self, handler):
|
||||
with handler() as rh:
|
||||
ws = validate_and_send(rh, Request(self.ws_base_url))
|
||||
assert 'upgrade' in ws.headers
|
||||
assert ws.status == 101
|
||||
ws.send('foo')
|
||||
assert ws.recv() == 'foo'
|
||||
ws.close()
|
||||
|
||||
# https://www.rfc-editor.org/rfc/rfc6455.html#section-5.6
|
||||
@pytest.mark.parametrize('msg,opcode', [('str', 1), (b'bytes', 2)])
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_send_types(self, handler, msg, opcode):
|
||||
with handler() as rh:
|
||||
ws = validate_and_send(rh, Request(self.ws_base_url))
|
||||
ws.send(msg)
|
||||
assert int(ws.recv()) == opcode
|
||||
ws.close()
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_verify_cert(self, handler):
|
||||
with handler() as rh:
|
||||
with pytest.raises(CertificateVerifyError):
|
||||
validate_and_send(rh, Request(self.wss_base_url))
|
||||
|
||||
with handler(verify=False) as rh:
|
||||
ws = validate_and_send(rh, Request(self.wss_base_url))
|
||||
assert ws.status == 101
|
||||
ws.close()
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_ssl_error(self, handler):
|
||||
with handler(verify=False) as rh:
|
||||
with pytest.raises(SSLError, match=r'ssl(?:v3|/tls) alert handshake failure') as exc_info:
|
||||
validate_and_send(rh, Request(self.bad_wss_host))
|
||||
assert not issubclass(exc_info.type, CertificateVerifyError)
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
@pytest.mark.parametrize('path,expected', [
|
||||
# Unicode characters should be encoded with uppercase percent-encoding
|
||||
('/中文', '/%E4%B8%AD%E6%96%87'),
|
||||
# don't normalize existing percent encodings
|
||||
('/%c7%9f', '/%c7%9f'),
|
||||
])
|
||||
def test_percent_encode(self, handler, path, expected):
|
||||
with handler() as rh:
|
||||
ws = validate_and_send(rh, Request(f'{self.ws_base_url}{path}'))
|
||||
ws.send('path')
|
||||
assert ws.recv() == expected
|
||||
assert ws.status == 101
|
||||
ws.close()
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_remove_dot_segments(self, handler):
|
||||
with handler() as rh:
|
||||
# This isn't a comprehensive test,
|
||||
# but it should be enough to check whether the handler is removing dot segments
|
||||
ws = validate_and_send(rh, Request(f'{self.ws_base_url}/a/b/./../../test'))
|
||||
assert ws.status == 101
|
||||
ws.send('path')
|
||||
assert ws.recv() == '/test'
|
||||
ws.close()
|
||||
|
||||
# We are restricted to known HTTP status codes in http.HTTPStatus
|
||||
# Redirects are not supported for websockets
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
@pytest.mark.parametrize('status', (200, 204, 301, 302, 303, 400, 500, 511))
|
||||
def test_raise_http_error(self, handler, status):
|
||||
with handler() as rh:
|
||||
with pytest.raises(HTTPError) as exc_info:
|
||||
validate_and_send(rh, Request(f'{self.ws_base_url}/gen_{status}'))
|
||||
assert exc_info.value.status == status
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
@pytest.mark.parametrize('params,extensions', [
|
||||
({'timeout': 0.00001}, {}),
|
||||
({}, {'timeout': 0.00001}),
|
||||
])
|
||||
def test_timeout(self, handler, params, extensions):
|
||||
with handler(**params) as rh:
|
||||
with pytest.raises(TransportError):
|
||||
validate_and_send(rh, Request(self.ws_base_url, extensions=extensions))
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_cookies(self, handler):
|
||||
cookiejar = YoutubeDLCookieJar()
|
||||
cookiejar.set_cookie(http.cookiejar.Cookie(
|
||||
version=0, name='test', value='ytdlp', port=None, port_specified=False,
|
||||
domain='127.0.0.1', domain_specified=True, domain_initial_dot=False, path='/',
|
||||
path_specified=True, secure=False, expires=None, discard=False, comment=None,
|
||||
comment_url=None, rest={}))
|
||||
|
||||
with handler(cookiejar=cookiejar) as rh:
|
||||
ws = validate_and_send(rh, Request(self.ws_base_url))
|
||||
ws.send('headers')
|
||||
assert json.loads(ws.recv())['cookie'] == 'test=ytdlp'
|
||||
ws.close()
|
||||
|
||||
with handler() as rh:
|
||||
ws = validate_and_send(rh, Request(self.ws_base_url))
|
||||
ws.send('headers')
|
||||
assert 'cookie' not in json.loads(ws.recv())
|
||||
ws.close()
|
||||
|
||||
ws = validate_and_send(rh, Request(self.ws_base_url, extensions={'cookiejar': cookiejar}))
|
||||
ws.send('headers')
|
||||
assert json.loads(ws.recv())['cookie'] == 'test=ytdlp'
|
||||
ws.close()
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_source_address(self, handler):
|
||||
source_address = f'127.0.0.{random.randint(5, 255)}'
|
||||
with handler(source_address=source_address) as rh:
|
||||
ws = validate_and_send(rh, Request(self.ws_base_url))
|
||||
ws.send('source_address')
|
||||
assert source_address == ws.recv()
|
||||
ws.close()
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_response_url(self, handler):
|
||||
with handler() as rh:
|
||||
url = f'{self.ws_base_url}/something'
|
||||
ws = validate_and_send(rh, Request(url))
|
||||
assert ws.url == url
|
||||
ws.close()
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_request_headers(self, handler):
|
||||
with handler(headers=HTTPHeaderDict({'test1': 'test', 'test2': 'test2'})) as rh:
|
||||
# Global Headers
|
||||
ws = validate_and_send(rh, Request(self.ws_base_url))
|
||||
ws.send('headers')
|
||||
headers = HTTPHeaderDict(json.loads(ws.recv()))
|
||||
assert headers['test1'] == 'test'
|
||||
ws.close()
|
||||
|
||||
# Per request headers, merged with global
|
||||
ws = validate_and_send(rh, Request(
|
||||
self.ws_base_url, headers={'test2': 'changed', 'test3': 'test3'}))
|
||||
ws.send('headers')
|
||||
headers = HTTPHeaderDict(json.loads(ws.recv()))
|
||||
assert headers['test1'] == 'test'
|
||||
assert headers['test2'] == 'changed'
|
||||
assert headers['test3'] == 'test3'
|
||||
ws.close()
|
||||
|
||||
@pytest.mark.parametrize('client_cert', (
|
||||
{'client_certificate': os.path.join(MTLS_CERT_DIR, 'clientwithkey.crt')},
|
||||
{
|
||||
'client_certificate': os.path.join(MTLS_CERT_DIR, 'client.crt'),
|
||||
'client_certificate_key': os.path.join(MTLS_CERT_DIR, 'client.key'),
|
||||
},
|
||||
{
|
||||
'client_certificate': os.path.join(MTLS_CERT_DIR, 'clientwithencryptedkey.crt'),
|
||||
'client_certificate_password': 'foobar',
|
||||
},
|
||||
{
|
||||
'client_certificate': os.path.join(MTLS_CERT_DIR, 'client.crt'),
|
||||
'client_certificate_key': os.path.join(MTLS_CERT_DIR, 'clientencrypted.key'),
|
||||
'client_certificate_password': 'foobar',
|
||||
}
|
||||
))
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
def test_mtls(self, handler, client_cert):
|
||||
with handler(
|
||||
# Disable client-side validation of unacceptable self-signed testcert.pem
|
||||
# The test is of a check on the server side, so unaffected
|
||||
verify=False,
|
||||
client_cert=client_cert
|
||||
) as rh:
|
||||
validate_and_send(rh, Request(self.mtls_wss_base_url)).close()
|
||||
|
||||
|
||||
def create_fake_ws_connection(raised):
|
||||
import websockets.sync.client
|
||||
|
||||
class FakeWsConnection(websockets.sync.client.ClientConnection):
|
||||
def __init__(self, *args, **kwargs):
|
||||
class FakeResponse:
|
||||
body = b''
|
||||
headers = {}
|
||||
status_code = 101
|
||||
reason_phrase = 'test'
|
||||
|
||||
self.response = FakeResponse()
|
||||
|
||||
def send(self, *args, **kwargs):
|
||||
raise raised()
|
||||
|
||||
def recv(self, *args, **kwargs):
|
||||
raise raised()
|
||||
|
||||
def close(self, *args, **kwargs):
|
||||
return
|
||||
|
||||
return FakeWsConnection()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
|
||||
class TestWebsocketsRequestHandler:
|
||||
@pytest.mark.parametrize('raised,expected', [
|
||||
# https://websockets.readthedocs.io/en/stable/reference/exceptions.html
|
||||
(lambda: websockets.exceptions.InvalidURI(msg='test', uri='test://'), RequestError),
|
||||
# Requires a response object. Should be covered by HTTP error tests.
|
||||
# (lambda: websockets.exceptions.InvalidStatus(), TransportError),
|
||||
(lambda: websockets.exceptions.InvalidHandshake(), TransportError),
|
||||
# These are subclasses of InvalidHandshake
|
||||
(lambda: websockets.exceptions.InvalidHeader(name='test'), TransportError),
|
||||
(lambda: websockets.exceptions.NegotiationError(), TransportError),
|
||||
# Catch-all
|
||||
(lambda: websockets.exceptions.WebSocketException(), TransportError),
|
||||
(lambda: TimeoutError(), TransportError),
|
||||
# These may be raised by our create_connection implementation, which should also be caught
|
||||
(lambda: OSError(), TransportError),
|
||||
(lambda: ssl.SSLError(), SSLError),
|
||||
(lambda: ssl.SSLCertVerificationError(), CertificateVerifyError),
|
||||
(lambda: socks.ProxyError(), ProxyError),
|
||||
])
|
||||
def test_request_error_mapping(self, handler, monkeypatch, raised, expected):
|
||||
import websockets.sync.client
|
||||
|
||||
import yt_dlp.networking._websockets
|
||||
with handler() as rh:
|
||||
def fake_connect(*args, **kwargs):
|
||||
raise raised()
|
||||
monkeypatch.setattr(yt_dlp.networking._websockets, 'create_connection', lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(websockets.sync.client, 'connect', fake_connect)
|
||||
with pytest.raises(expected) as exc_info:
|
||||
rh.send(Request('ws://fake-url'))
|
||||
assert exc_info.type is expected
|
||||
|
||||
@pytest.mark.parametrize('raised,expected,match', [
|
||||
# https://websockets.readthedocs.io/en/stable/reference/sync/client.html#websockets.sync.client.ClientConnection.send
|
||||
(lambda: websockets.exceptions.ConnectionClosed(None, None), TransportError, None),
|
||||
(lambda: RuntimeError(), TransportError, None),
|
||||
(lambda: TimeoutError(), TransportError, None),
|
||||
(lambda: TypeError(), RequestError, None),
|
||||
(lambda: socks.ProxyError(), ProxyError, None),
|
||||
# Catch-all
|
||||
(lambda: websockets.exceptions.WebSocketException(), TransportError, None),
|
||||
])
|
||||
def test_ws_send_error_mapping(self, handler, monkeypatch, raised, expected, match):
|
||||
from yt_dlp.networking._websockets import WebsocketsResponseAdapter
|
||||
ws = WebsocketsResponseAdapter(create_fake_ws_connection(raised), url='ws://fake-url')
|
||||
with pytest.raises(expected, match=match) as exc_info:
|
||||
ws.send('test')
|
||||
assert exc_info.type is expected
|
||||
|
||||
@pytest.mark.parametrize('raised,expected,match', [
|
||||
# https://websockets.readthedocs.io/en/stable/reference/sync/client.html#websockets.sync.client.ClientConnection.recv
|
||||
(lambda: websockets.exceptions.ConnectionClosed(None, None), TransportError, None),
|
||||
(lambda: RuntimeError(), TransportError, None),
|
||||
(lambda: TimeoutError(), TransportError, None),
|
||||
(lambda: socks.ProxyError(), ProxyError, None),
|
||||
# Catch-all
|
||||
(lambda: websockets.exceptions.WebSocketException(), TransportError, None),
|
||||
])
|
||||
def test_ws_recv_error_mapping(self, handler, monkeypatch, raised, expected, match):
|
||||
from yt_dlp.networking._websockets import WebsocketsResponseAdapter
|
||||
ws = WebsocketsResponseAdapter(create_fake_ws_connection(raised), url='ws://fake-url')
|
||||
with pytest.raises(expected, match=match) as exc_info:
|
||||
ws.recv()
|
||||
assert exc_info.type is expected
|
||||
@@ -1 +1 @@
|
||||
@py -bb -Werror -Xdev "%~dp0yt_dlp\__main__.py" %*
|
||||
@py -Werror -Xdev "%~dp0yt_dlp\__main__.py" %*
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
#!/usr/bin/env sh
|
||||
exec "${PYTHON:-python3}" -bb -Werror -Xdev "$(dirname "$(realpath "$0")")/yt_dlp/__main__.py" "$@"
|
||||
exec "${PYTHON:-python3}" -Werror -Xdev "$(dirname "$(realpath "$0")")/yt_dlp/__main__.py" "$@"
|
||||
|
||||
@@ -60,7 +60,13 @@ from .postprocessor import (
|
||||
get_postprocessor,
|
||||
)
|
||||
from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
|
||||
from .update import REPOSITORY, _get_system_deprecation, _make_label, current_git_head, detect_variant
|
||||
from .update import (
|
||||
REPOSITORY,
|
||||
_get_system_deprecation,
|
||||
_make_label,
|
||||
current_git_head,
|
||||
detect_variant,
|
||||
)
|
||||
from .utils import (
|
||||
DEFAULT_OUTTMPL,
|
||||
IDENTITY,
|
||||
@@ -625,13 +631,16 @@ class YoutubeDL:
|
||||
'Overwriting params from "color" with "no_color"')
|
||||
self.params['color'] = 'no_color'
|
||||
|
||||
term_allow_color = os.environ.get('TERM', '').lower() != 'dumb'
|
||||
term_allow_color = os.getenv('TERM', '').lower() != 'dumb'
|
||||
no_color = bool(os.getenv('NO_COLOR'))
|
||||
|
||||
def process_color_policy(stream):
|
||||
stream_name = {sys.stdout: 'stdout', sys.stderr: 'stderr'}[stream]
|
||||
policy = traverse_obj(self.params, ('color', (stream_name, None), {str}), get_all=False)
|
||||
if policy in ('auto', None):
|
||||
return term_allow_color and supports_terminal_sequences(stream)
|
||||
if term_allow_color and supports_terminal_sequences(stream):
|
||||
return 'no_color' if no_color else True
|
||||
return False
|
||||
assert policy in ('always', 'never', 'no_color'), policy
|
||||
return {'always': True, 'never': False}.get(policy, policy)
|
||||
|
||||
@@ -1176,6 +1185,7 @@ class YoutubeDL:
|
||||
MATH_FUNCTIONS = {
|
||||
'+': float.__add__,
|
||||
'-': float.__sub__,
|
||||
'*': float.__mul__,
|
||||
}
|
||||
# Field is of the form key1.key2...
|
||||
# where keys (except first) can be string, int, slice or "{field, ...}"
|
||||
@@ -1197,6 +1207,15 @@ class YoutubeDL:
|
||||
(?:\|(?P<default>.*?))?
|
||||
)$''')
|
||||
|
||||
def _from_user_input(field):
|
||||
if field == ':':
|
||||
return ...
|
||||
elif ':' in field:
|
||||
return slice(*map(int_or_none, field.split(':')))
|
||||
elif int_or_none(field) is not None:
|
||||
return int(field)
|
||||
return field
|
||||
|
||||
def _traverse_infodict(fields):
|
||||
fields = [f for x in re.split(r'\.({.+?})\.?', fields)
|
||||
for f in ([x] if x.startswith('{') else x.split('.'))]
|
||||
@@ -1206,11 +1225,12 @@ class YoutubeDL:
|
||||
|
||||
for i, f in enumerate(fields):
|
||||
if not f.startswith('{'):
|
||||
fields[i] = _from_user_input(f)
|
||||
continue
|
||||
assert f.endswith('}'), f'No closing brace for {f} in {fields}'
|
||||
fields[i] = {k: k.split('.') for k in f[1:-1].split(',')}
|
||||
fields[i] = {k: list(map(_from_user_input, k.split('.'))) for k in f[1:-1].split(',')}
|
||||
|
||||
return traverse_obj(info_dict, fields, is_user_input=True, traverse_string=True)
|
||||
return traverse_obj(info_dict, fields, traverse_string=True)
|
||||
|
||||
def get_value(mdict):
|
||||
# Object traversal
|
||||
@@ -2451,9 +2471,16 @@ class YoutubeDL:
|
||||
return selector_function(ctx_copy)
|
||||
return final_selector
|
||||
|
||||
stream = io.BytesIO(format_spec.encode())
|
||||
# HACK: Python 3.12 changed the underlying parser, rendering '7_a' invalid
|
||||
# Prefix numbers with random letters to avoid it being classified as a number
|
||||
# See: https://github.com/yt-dlp/yt-dlp/pulls/8797
|
||||
# TODO: Implement parser not reliant on tokenize.tokenize
|
||||
prefix = ''.join(random.choices(string.ascii_letters, k=32))
|
||||
stream = io.BytesIO(re.sub(r'\d[_\d]*', rf'{prefix}\g<0>', format_spec).encode())
|
||||
try:
|
||||
tokens = list(_remove_unused_ops(tokenize.tokenize(stream.readline)))
|
||||
tokens = list(_remove_unused_ops(
|
||||
token._replace(string=token.string.replace(prefix, ''))
|
||||
for token in tokenize.tokenize(stream.readline)))
|
||||
except tokenize.TokenError:
|
||||
raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
|
||||
|
||||
@@ -2586,6 +2613,9 @@ class YoutubeDL:
|
||||
upload_date = datetime.datetime.fromtimestamp(info_dict[ts_key], datetime.timezone.utc)
|
||||
info_dict[date_key] = upload_date.strftime('%Y%m%d')
|
||||
|
||||
if not info_dict.get('release_year'):
|
||||
info_dict['release_year'] = traverse_obj(info_dict, ('release_date', {lambda x: int(x[:4])}))
|
||||
|
||||
live_keys = ('is_live', 'was_live')
|
||||
live_status = info_dict.get('live_status')
|
||||
if live_status is None:
|
||||
@@ -4052,6 +4082,7 @@ class YoutubeDL:
|
||||
return self._request_director.send(req)
|
||||
except NoSupportingHandlers as e:
|
||||
for ue in e.unsupported_errors:
|
||||
# FIXME: This depends on the order of errors.
|
||||
if not (ue.handler and ue.msg):
|
||||
continue
|
||||
if ue.handler.RH_KEY == 'Urllib' and 'unsupported url scheme: "file"' in ue.msg.lower():
|
||||
@@ -4061,6 +4092,15 @@ class YoutubeDL:
|
||||
if 'unsupported proxy type: "https"' in ue.msg.lower():
|
||||
raise RequestError(
|
||||
'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests')
|
||||
|
||||
elif (
|
||||
re.match(r'unsupported url scheme: "wss?"', ue.msg.lower())
|
||||
and 'websockets' not in self._request_director.handlers
|
||||
):
|
||||
raise RequestError(
|
||||
'This request requires WebSocket support. '
|
||||
'Ensure one of the following dependencies are installed: websockets',
|
||||
cause=ue) from ue
|
||||
raise
|
||||
except SSLError as e:
|
||||
if 'UNSAFE_LEGACY_RENEGOTIATION_DISABLED' in str(e):
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
try:
|
||||
import contextvars # noqa: F401
|
||||
except Exception:
|
||||
raise Exception(
|
||||
f'You are using an unsupported version of Python. Only Python versions 3.7 and above are supported by yt-dlp') # noqa: F541
|
||||
import sys
|
||||
|
||||
if sys.version_info < (3, 8):
|
||||
raise ImportError(
|
||||
f'You are using an unsupported version of Python. Only Python versions 3.8 and above are supported by yt-dlp') # noqa: F541
|
||||
|
||||
__license__ = 'Public Domain'
|
||||
|
||||
@@ -12,7 +12,6 @@ import itertools
|
||||
import optparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from .compat import compat_shlex_quote
|
||||
@@ -74,14 +73,16 @@ def _exit(status=0, *args):
|
||||
|
||||
|
||||
def get_urls(urls, batchfile, verbose):
|
||||
# Batch file verification
|
||||
"""
|
||||
@param verbose -1: quiet, 0: normal, 1: verbose
|
||||
"""
|
||||
batch_urls = []
|
||||
if batchfile is not None:
|
||||
try:
|
||||
batch_urls = read_batch_urls(
|
||||
read_stdin('URLs') if batchfile == '-'
|
||||
read_stdin(None if verbose == -1 else 'URLs') if batchfile == '-'
|
||||
else open(expand_path(batchfile), encoding='utf-8', errors='ignore'))
|
||||
if verbose:
|
||||
if verbose == 1:
|
||||
write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
|
||||
except OSError:
|
||||
_exit(f'ERROR: batch file {batchfile} could not be read')
|
||||
@@ -722,7 +723,7 @@ ParsedOptions = collections.namedtuple('ParsedOptions', ('parser', 'options', 'u
|
||||
def parse_options(argv=None):
|
||||
"""@returns ParsedOptions(parser, opts, urls, ydl_opts)"""
|
||||
parser, opts, urls = parseOpts(argv)
|
||||
urls = get_urls(urls, opts.batchfile, opts.verbose)
|
||||
urls = get_urls(urls, opts.batchfile, -1 if opts.quiet and not opts.verbose else opts.verbose)
|
||||
|
||||
set_compat_opts(opts)
|
||||
try:
|
||||
|
||||
@@ -10,17 +10,3 @@ try:
|
||||
cache # >= 3.9
|
||||
except NameError:
|
||||
cache = lru_cache(maxsize=None)
|
||||
|
||||
try:
|
||||
cached_property # >= 3.8
|
||||
except NameError:
|
||||
class cached_property:
|
||||
def __init__(self, func):
|
||||
update_wrapper(self, func)
|
||||
self.func = func
|
||||
|
||||
def __get__(self, instance, _):
|
||||
if instance is None:
|
||||
return self
|
||||
setattr(instance, self.func.__name__, self.func(instance))
|
||||
return getattr(instance, self.func.__name__)
|
||||
|
||||
@@ -6,7 +6,7 @@ from . import get_suitable_downloader
|
||||
from .common import FileDownloader
|
||||
from .external import FFmpegFD
|
||||
from ..networking import Request
|
||||
from ..utils import DownloadError, WebSocketsWrapper, str_or_none, try_get
|
||||
from ..utils import DownloadError, str_or_none, try_get
|
||||
|
||||
|
||||
class NiconicoDmcFD(FileDownloader):
|
||||
@@ -64,7 +64,6 @@ class NiconicoLiveFD(FileDownloader):
|
||||
ws_url = info_dict['url']
|
||||
ws_extractor = info_dict['ws']
|
||||
ws_origin_host = info_dict['origin']
|
||||
cookies = info_dict.get('cookies')
|
||||
live_quality = info_dict.get('live_quality', 'high')
|
||||
live_latency = info_dict.get('live_latency', 'high')
|
||||
dl = FFmpegFD(self.ydl, self.params or {})
|
||||
@@ -76,12 +75,7 @@ class NiconicoLiveFD(FileDownloader):
|
||||
|
||||
def communicate_ws(reconnect):
|
||||
if reconnect:
|
||||
ws = WebSocketsWrapper(ws_url, {
|
||||
'Cookies': str_or_none(cookies) or '',
|
||||
'Origin': f'https://{ws_origin_host}',
|
||||
'Accept': '*/*',
|
||||
'User-Agent': self.params['http_headers']['User-Agent'],
|
||||
})
|
||||
ws = self.ydl.urlopen(Request(ws_url, headers={'Origin': f'https://{ws_origin_host}'}))
|
||||
if self.ydl.params.get('verbose', False):
|
||||
self.to_screen('[debug] Sending startWatching request')
|
||||
ws.send(json.dumps({
|
||||
|
||||
@@ -77,16 +77,23 @@ from .agora import (
|
||||
WyborczaPodcastIE,
|
||||
WyborczaVideoIE,
|
||||
)
|
||||
from .airmozilla import AirMozillaIE
|
||||
from .airtv import AirTVIE
|
||||
from .aitube import AitubeKZVideoIE
|
||||
from .aljazeera import AlJazeeraIE
|
||||
from .allstar import (
|
||||
AllstarIE,
|
||||
AllstarProfileIE,
|
||||
)
|
||||
from .alphaporno import AlphaPornoIE
|
||||
from .amara import AmaraIE
|
||||
from .altcensored import (
|
||||
AltCensoredIE,
|
||||
AltCensoredChannelIE,
|
||||
)
|
||||
from .alura import (
|
||||
AluraIE,
|
||||
AluraCourseIE
|
||||
)
|
||||
from .amara import AmaraIE
|
||||
from .amcnetworks import AMCNetworksIE
|
||||
from .amazon import (
|
||||
AmazonStoreIE,
|
||||
@@ -127,8 +134,8 @@ from .arcpublishing import ArcPublishingIE
|
||||
from .arkena import ArkenaIE
|
||||
from .ard import (
|
||||
ARDBetaMediathekIE,
|
||||
ARDMediathekCollectionIE,
|
||||
ARDIE,
|
||||
ARDMediathekIE,
|
||||
)
|
||||
from .arte import (
|
||||
ArteTVIE,
|
||||
@@ -139,7 +146,6 @@ from .arte import (
|
||||
from .arnes import ArnesIE
|
||||
from .atresplayer import AtresPlayerIE
|
||||
from .atscaleconf import AtScaleConfEventIE
|
||||
from .atttechchannel import ATTTechChannelIE
|
||||
from .atvat import ATVAtIE
|
||||
from .audimedia import AudiMediaIE
|
||||
from .audioboom import AudioBoomIE
|
||||
@@ -212,6 +218,8 @@ from .bilibili import (
|
||||
BiliBiliBangumiIE,
|
||||
BiliBiliBangumiSeasonIE,
|
||||
BiliBiliBangumiMediaIE,
|
||||
BilibiliCheeseIE,
|
||||
BilibiliCheeseSeasonIE,
|
||||
BiliBiliSearchIE,
|
||||
BilibiliCategoryIE,
|
||||
BilibiliAudioIE,
|
||||
@@ -233,11 +241,6 @@ from .bitchute import (
|
||||
BitChuteIE,
|
||||
BitChuteChannelIE,
|
||||
)
|
||||
from .bitwave import (
|
||||
BitwaveReplayIE,
|
||||
BitwaveStreamIE,
|
||||
)
|
||||
from .biqle import BIQLEIE
|
||||
from .blackboardcollaborate import BlackboardCollaborateIE
|
||||
from .bleacherreport import (
|
||||
BleacherReportIE,
|
||||
@@ -252,10 +255,7 @@ from .bostonglobe import BostonGlobeIE
|
||||
from .box import BoxIE
|
||||
from .boxcast import BoxCastVideoIE
|
||||
from .bpb import BpbIE
|
||||
from .br import (
|
||||
BRIE,
|
||||
BRMediathekIE,
|
||||
)
|
||||
from .br import BRIE
|
||||
from .bravotv import BravoTVIE
|
||||
from .brainpop import (
|
||||
BrainPOPIE,
|
||||
@@ -265,7 +265,6 @@ from .brainpop import (
|
||||
BrainPOPFrIE,
|
||||
BrainPOPIlIE,
|
||||
)
|
||||
from .breakcom import BreakIE
|
||||
from .breitbart import BreitBartIE
|
||||
from .brightcove import (
|
||||
BrightcoveLegacyIE,
|
||||
@@ -277,6 +276,7 @@ from .brilliantpala import (
|
||||
)
|
||||
from .businessinsider import BusinessInsiderIE
|
||||
from .bundesliga import BundesligaIE
|
||||
from .bundestag import BundestagIE
|
||||
from .buzzfeed import BuzzFeedIE
|
||||
from .byutv import BYUtvIE
|
||||
from .c56 import C56IE
|
||||
@@ -295,16 +295,11 @@ from .camfm import (
|
||||
from .cammodels import CamModelsIE
|
||||
from .camsoda import CamsodaIE
|
||||
from .camtasia import CamtasiaEmbedIE
|
||||
from .camwithher import CamWithHerIE
|
||||
from .canal1 import Canal1IE
|
||||
from .canalalpha import CanalAlphaIE
|
||||
from .canalplus import CanalplusIE
|
||||
from .canalc2 import Canalc2IE
|
||||
from .caracoltv import CaracolTvPlayIE
|
||||
from .carambatv import (
|
||||
CarambaTVIE,
|
||||
CarambaTVPageIE,
|
||||
)
|
||||
from .cartoonnetwork import CartoonNetworkIE
|
||||
from .cbc import (
|
||||
CBCIE,
|
||||
@@ -343,7 +338,6 @@ from .cda import CDAIE
|
||||
from .cellebrite import CellebriteIE
|
||||
from .ceskatelevize import CeskaTelevizeIE
|
||||
from .cgtn import CGTNIE
|
||||
from .channel9 import Channel9IE
|
||||
from .charlierose import CharlieRoseIE
|
||||
from .chaturbate import ChaturbateIE
|
||||
from .chilloutzone import ChilloutzoneIE
|
||||
@@ -351,11 +345,6 @@ from .chingari import (
|
||||
ChingariIE,
|
||||
ChingariUserIE,
|
||||
)
|
||||
from .chirbit import (
|
||||
ChirbitIE,
|
||||
ChirbitProfileIE,
|
||||
)
|
||||
from .cinchcast import CinchcastIE
|
||||
from .cinemax import CinemaxIE
|
||||
from .cinetecamilano import CinetecaMilanoIE
|
||||
from .cineverse import (
|
||||
@@ -372,10 +361,8 @@ from .clipchamp import ClipchampIE
|
||||
from .cliphunter import CliphunterIE
|
||||
from .clippit import ClippitIE
|
||||
from .cliprs import ClipRsIE
|
||||
from .clipsyndicate import ClipsyndicateIE
|
||||
from .closertotruth import CloserToTruthIE
|
||||
from .cloudflarestream import CloudflareStreamIE
|
||||
from .cloudy import CloudyIE
|
||||
from .clubic import ClubicIE
|
||||
from .clyp import ClypIE
|
||||
from .cmt import CMTIE
|
||||
@@ -442,7 +429,6 @@ from .dacast import (
|
||||
DacastVODIE,
|
||||
DacastPlaylistIE,
|
||||
)
|
||||
from .daftsex import DaftsexIE
|
||||
from .dailymail import DailyMailIE
|
||||
from .dailymotion import (
|
||||
DailymotionIE,
|
||||
@@ -479,7 +465,6 @@ from .dlf import (
|
||||
from .dfb import DFBIE
|
||||
from .dhm import DHMIE
|
||||
from .digg import DiggIE
|
||||
from .dotsub import DotsubIE
|
||||
from .douyutv import (
|
||||
DouyuShowIE,
|
||||
DouyuTVIE,
|
||||
@@ -526,7 +511,6 @@ from .duboku import (
|
||||
DubokuPlaylistIE
|
||||
)
|
||||
from .dumpert import DumpertIE
|
||||
from .defense import DefenseGouvFrIE
|
||||
from .deuxm import (
|
||||
DeuxMIE,
|
||||
DeuxMNewsIE
|
||||
@@ -541,6 +525,7 @@ from .dropout import (
|
||||
DropoutSeasonIE,
|
||||
DropoutIE
|
||||
)
|
||||
from .duoplay import DuoplayIE
|
||||
from .dw import (
|
||||
DWIE,
|
||||
DWArticleIE,
|
||||
@@ -548,30 +533,22 @@ from .dw import (
|
||||
from .eagleplatform import EaglePlatformIE, ClipYouEmbedIE
|
||||
from .ebaumsworld import EbaumsWorldIE
|
||||
from .ebay import EbayIE
|
||||
from .echomsk import EchoMskIE
|
||||
from .egghead import (
|
||||
EggheadCourseIE,
|
||||
EggheadLessonIE,
|
||||
)
|
||||
from .ehow import EHowIE
|
||||
from .eighttracks import EightTracksIE
|
||||
from .einthusan import EinthusanIE
|
||||
from .eitb import EitbIE
|
||||
from .elevensports import ElevenSportsIE
|
||||
from .ellentube import (
|
||||
EllenTubeIE,
|
||||
EllenTubeVideoIE,
|
||||
EllenTubePlaylistIE,
|
||||
)
|
||||
from .elonet import ElonetIE
|
||||
from .elpais import ElPaisIE
|
||||
from .eltrecetv import ElTreceTVIE
|
||||
from .embedly import EmbedlyIE
|
||||
from .engadget import EngadgetIE
|
||||
from .epicon import (
|
||||
EpiconIE,
|
||||
EpiconSeriesIE,
|
||||
)
|
||||
from .epidemicsound import EpidemicSoundIE
|
||||
from .eplus import EplusIbIE
|
||||
from .epoch import EpochIE
|
||||
from .eporner import EpornerIE
|
||||
@@ -585,7 +562,6 @@ from .ertgr import (
|
||||
ERTFlixIE,
|
||||
ERTWebtvEmbedIE,
|
||||
)
|
||||
from .escapist import EscapistIE
|
||||
from .espn import (
|
||||
ESPNIE,
|
||||
WatchESPNIE,
|
||||
@@ -593,15 +569,12 @@ from .espn import (
|
||||
FiveThirtyEightIE,
|
||||
ESPNCricInfoIE,
|
||||
)
|
||||
from .esri import EsriVideoIE
|
||||
from .ettutv import EttuTvIE
|
||||
from .europa import EuropaIE, EuroParlWebstreamIE
|
||||
from .europeantour import EuropeanTourIE
|
||||
from .eurosport import EurosportIE
|
||||
from .euscreen import EUScreenIE
|
||||
from .expotv import ExpoTVIE
|
||||
from .expressen import ExpressenIE
|
||||
from .extremetube import ExtremeTubeIE
|
||||
from .eyedotv import EyedoTVIE
|
||||
from .facebook import (
|
||||
FacebookIE,
|
||||
@@ -631,6 +604,10 @@ from .filmweb import FilmwebIE
|
||||
from .firsttv import FirstTVIE
|
||||
from .fivetv import FiveTVIE
|
||||
from .flickr import FlickrIE
|
||||
from .floatplane import (
|
||||
FloatplaneIE,
|
||||
FloatplaneChannelIE,
|
||||
)
|
||||
from .folketinget import FolketingetIE
|
||||
from .footyroom import FootyRoomIE
|
||||
from .formula1 import Formula1IE
|
||||
@@ -640,16 +617,11 @@ from .fourtube import (
|
||||
PornerBrosIE,
|
||||
FuxIE,
|
||||
)
|
||||
from .fourzerostudio import (
|
||||
FourZeroStudioArchiveIE,
|
||||
FourZeroStudioClipIE,
|
||||
)
|
||||
from .fox import FOXIE
|
||||
from .fox9 import (
|
||||
FOX9IE,
|
||||
FOX9NewsIE,
|
||||
)
|
||||
from .foxgay import FoxgayIE
|
||||
from .foxnews import (
|
||||
FoxNewsIE,
|
||||
FoxNewsArticleIE,
|
||||
@@ -682,7 +654,6 @@ from .funimation import (
|
||||
)
|
||||
from .funk import FunkIE
|
||||
from .funker530 import Funker530IE
|
||||
from .fusion import FusionIE
|
||||
from .fuyintv import FuyinTVIE
|
||||
from .gab import (
|
||||
GabTVIE,
|
||||
@@ -713,7 +684,6 @@ from .gettr import (
|
||||
GettrIE,
|
||||
GettrStreamingIE,
|
||||
)
|
||||
from .gfycat import GfycatIE
|
||||
from .giantbomb import GiantBombIE
|
||||
from .giga import GigaIE
|
||||
from .glide import GlideIE
|
||||
@@ -759,12 +729,10 @@ from .hbo import HBOIE
|
||||
from .hearthisat import HearThisAtIE
|
||||
from .heise import HeiseIE
|
||||
from .hellporno import HellPornoIE
|
||||
from .helsinki import HelsinkiIE
|
||||
from .hgtv import HGTVComShowIE
|
||||
from .hketv import HKETVIE
|
||||
from .hidive import HiDiveIE
|
||||
from .historicfilms import HistoricFilmsIE
|
||||
from .hitbox import HitboxIE, HitboxLiveIE
|
||||
from .hitrecord import HitRecordIE
|
||||
from .hollywoodreporter import (
|
||||
HollywoodReporterIE,
|
||||
@@ -779,8 +747,6 @@ from .hotstar import (
|
||||
HotStarSeasonIE,
|
||||
HotStarSeriesIE,
|
||||
)
|
||||
from .howcast import HowcastIE
|
||||
from .howstuffworks import HowStuffWorksIE
|
||||
from .hrefli import HrefLiRedirectIE
|
||||
from .hrfensehen import HRFernsehenIE
|
||||
from .hrti import (
|
||||
@@ -900,6 +866,7 @@ from .jiosaavn import (
|
||||
)
|
||||
from .jove import JoveIE
|
||||
from .joj import JojIE
|
||||
from .joqrag import JoqrAgIE
|
||||
from .jstream import JStreamIE
|
||||
from .jtbc import (
|
||||
JTBCIE,
|
||||
@@ -912,7 +879,6 @@ from .kanal2 import Kanal2IE
|
||||
from .kankanews import KankaNewsIE
|
||||
from .karaoketv import KaraoketvIE
|
||||
from .karrierevideos import KarriereVideosIE
|
||||
from .keezmovies import KeezMoviesIE
|
||||
from .kelbyone import KelbyOneIE
|
||||
from .khanacademy import (
|
||||
KhanAcademyIE,
|
||||
@@ -947,12 +913,6 @@ from .la7 import (
|
||||
LA7PodcastEpisodeIE,
|
||||
LA7PodcastIE,
|
||||
)
|
||||
from .laola1tv import (
|
||||
Laola1TvEmbedIE,
|
||||
Laola1TvIE,
|
||||
EHFTVIE,
|
||||
ITTFIE,
|
||||
)
|
||||
from .lastfm import (
|
||||
LastFMIE,
|
||||
LastFMPlaylistIE,
|
||||
@@ -1007,7 +967,6 @@ from .linkedin import (
|
||||
LinkedInLearningIE,
|
||||
LinkedInLearningCourseIE,
|
||||
)
|
||||
from .linuxacademy import LinuxAcademyIE
|
||||
from .liputan6 import Liputan6IE
|
||||
from .listennotes import ListenNotesIE
|
||||
from .litv import LiTVIE
|
||||
@@ -1035,7 +994,7 @@ from .lynda import (
|
||||
LyndaIE,
|
||||
LyndaCourseIE
|
||||
)
|
||||
from .m6 import M6IE
|
||||
from .maariv import MaarivIE
|
||||
from .magellantv import MagellanTVIE
|
||||
from .magentamusik360 import MagentaMusik360IE
|
||||
from .mailru import (
|
||||
@@ -1086,10 +1045,7 @@ from .medici import MediciIE
|
||||
from .megaphone import MegaphoneIE
|
||||
from .meipai import MeipaiIE
|
||||
from .melonvod import MelonVODIE
|
||||
from .meta import METAIE
|
||||
from .metacafe import MetacafeIE
|
||||
from .metacritic import MetacriticIE
|
||||
from .mgoon import MgoonIE
|
||||
from .mgtv import MGTVIE
|
||||
from .miaopai import MiaoPaiIE
|
||||
from .microsoftstream import MicrosoftStreamIE
|
||||
@@ -1111,7 +1067,6 @@ from .minds import (
|
||||
)
|
||||
from .ministrygrid import MinistryGridIE
|
||||
from .minoto import MinotoIE
|
||||
from .miomio import MioMioIE
|
||||
from .mirrativ import (
|
||||
MirrativIE,
|
||||
MirrativUserIE,
|
||||
@@ -1135,13 +1090,7 @@ from .mlb import (
|
||||
MLBArticleIE,
|
||||
)
|
||||
from .mlssoccer import MLSSoccerIE
|
||||
from .mnet import MnetIE
|
||||
from .mocha import MochaVideoIE
|
||||
from .moevideo import MoeVideoIE
|
||||
from .mofosex import (
|
||||
MofosexIE,
|
||||
MofosexEmbedIE,
|
||||
)
|
||||
from .mojvideo import MojvideoIE
|
||||
from .monstercat import MonstercatIE
|
||||
from .morningstar import MorningstarIE
|
||||
@@ -1151,7 +1100,6 @@ from .motherless import (
|
||||
MotherlessGalleryIE,
|
||||
)
|
||||
from .motorsport import MotorsportIE
|
||||
from .movieclips import MovieClipsIE
|
||||
from .moviepilot import MoviepilotIE
|
||||
from .moview import MoviewPlayIE
|
||||
from .moviezine import MoviezineIE
|
||||
@@ -1176,18 +1124,12 @@ from .musicdex import (
|
||||
MusicdexArtistIE,
|
||||
MusicdexPlaylistIE,
|
||||
)
|
||||
from .mwave import MwaveIE, MwaveMeetGreetIE
|
||||
from .mxplayer import (
|
||||
MxplayerIE,
|
||||
MxplayerShowIE,
|
||||
)
|
||||
from .mychannels import MyChannelsIE
|
||||
from .myspace import MySpaceIE, MySpaceAlbumIE
|
||||
from .myspass import MySpassIE
|
||||
from .myvi import (
|
||||
MyviIE,
|
||||
MyviEmbedIE,
|
||||
)
|
||||
from .myvideoge import MyVideoGeIE
|
||||
from .myvidster import MyVidsterIE
|
||||
from .mzaalo import MzaaloIE
|
||||
@@ -1236,6 +1178,7 @@ from .ndr import (
|
||||
from .ndtv import NDTVIE
|
||||
from .nebula import (
|
||||
NebulaIE,
|
||||
NebulaClassIE,
|
||||
NebulaSubscriptionsIE,
|
||||
NebulaChannelIE,
|
||||
)
|
||||
@@ -1262,7 +1205,6 @@ from .newgrounds import (
|
||||
NewgroundsUserIE,
|
||||
)
|
||||
from .newspicks import NewsPicksIE
|
||||
from .newstube import NewstubeIE
|
||||
from .newsy import NewsyIE
|
||||
from .nextmedia import (
|
||||
NextMediaIE,
|
||||
@@ -1297,7 +1239,6 @@ from .nick import (
|
||||
NickIE,
|
||||
NickBrIE,
|
||||
NickDeIE,
|
||||
NickNightIE,
|
||||
NickRuIE,
|
||||
)
|
||||
from .niconico import (
|
||||
@@ -1330,8 +1271,6 @@ from .noice import NoicePodcastIE
|
||||
from .nonktube import NonkTubeIE
|
||||
from .noodlemagazine import NoodleMagazineIE
|
||||
from .noovo import NoovoIE
|
||||
from .normalboots import NormalbootsIE
|
||||
from .nosvideo import NosVideoIE
|
||||
from .nosnl import NOSNLArticleIE
|
||||
from .nova import (
|
||||
NovaEmbedIE,
|
||||
@@ -1406,10 +1345,6 @@ from .onet import (
|
||||
OnetPlIE,
|
||||
)
|
||||
from .onionstudios import OnionStudiosIE
|
||||
from .ooyala import (
|
||||
OoyalaIE,
|
||||
OoyalaExternalIE,
|
||||
)
|
||||
from .opencast import (
|
||||
OpencastIE,
|
||||
OpencastPlaylistIE,
|
||||
@@ -1438,7 +1373,6 @@ from .palcomp3 import (
|
||||
PalcoMP3ArtistIE,
|
||||
PalcoMP3VideoIE,
|
||||
)
|
||||
from .pandoratv import PandoraTVIE
|
||||
from .panopto import (
|
||||
PanoptoIE,
|
||||
PanoptoListIE,
|
||||
@@ -1466,7 +1400,6 @@ from .peloton import (
|
||||
PelotonIE,
|
||||
PelotonLiveIE
|
||||
)
|
||||
from .people import PeopleIE
|
||||
from .performgroup import PerformGroupIE
|
||||
from .periscope import (
|
||||
PeriscopeIE,
|
||||
@@ -1498,13 +1431,10 @@ from .platzi import (
|
||||
PlatziIE,
|
||||
PlatziCourseIE,
|
||||
)
|
||||
from .playfm import PlayFMIE
|
||||
from .playplustv import PlayPlusTVIE
|
||||
from .plays import PlaysTVIE
|
||||
from .playstuff import PlayStuffIE
|
||||
from .playsuisse import PlaySuisseIE
|
||||
from .playtvak import PlaytvakIE
|
||||
from .playvid import PlayvidIE
|
||||
from .playwire import PlaywireIE
|
||||
from .plutotv import PlutoTVIE
|
||||
from .pluralsight import (
|
||||
@@ -1536,9 +1466,7 @@ from .popcorntimes import PopcorntimesIE
|
||||
from .popcorntv import PopcornTVIE
|
||||
from .porn91 import Porn91IE
|
||||
from .pornbox import PornboxIE
|
||||
from .porncom import PornComIE
|
||||
from .pornflip import PornFlipIE
|
||||
from .pornhd import PornHdIE
|
||||
from .pornhub import (
|
||||
PornHubIE,
|
||||
PornHubUserIE,
|
||||
@@ -1549,7 +1477,6 @@ from .pornhub import (
|
||||
from .pornotube import PornotubeIE
|
||||
from .pornovoisines import PornoVoisinesIE
|
||||
from .pornoxo import PornoXOIE
|
||||
from .pornez import PornezIE
|
||||
from .puhutv import (
|
||||
PuhuTVIE,
|
||||
PuhuTVSerieIE,
|
||||
@@ -1593,7 +1520,6 @@ from .radiocomercial import (
|
||||
)
|
||||
from .radiode import RadioDeIE
|
||||
from .radiojavan import RadioJavanIE
|
||||
from .radiobremen import RadioBremenIE
|
||||
from .radiofrance import (
|
||||
FranceCultureIE,
|
||||
RadioFranceIE,
|
||||
@@ -1645,7 +1571,6 @@ from .rcti import (
|
||||
RCTIPlusTVIE,
|
||||
)
|
||||
from .rds import RDSIE
|
||||
from .recurbate import RecurbateIE
|
||||
from .redbee import ParliamentLiveUKIE, RTBFIE
|
||||
from .redbulltv import (
|
||||
RedBullTVIE,
|
||||
@@ -1669,7 +1594,7 @@ from .restudy import RestudyIE
|
||||
from .reuters import ReutersIE
|
||||
from .reverbnation import ReverbNationIE
|
||||
from .rheinmaintv import RheinMainTVIE
|
||||
from .rice import RICEIE
|
||||
from .rinsefm import RinseFMIE
|
||||
from .rmcdecouverte import RMCDecouverteIE
|
||||
from .rockstargames import RockstarGamesIE
|
||||
from .rokfin import (
|
||||
@@ -1693,11 +1618,7 @@ from .rtlnl import (
|
||||
RTLLuLiveIE,
|
||||
RTLLuRadioIE,
|
||||
)
|
||||
from .rtl2 import (
|
||||
RTL2IE,
|
||||
RTL2YouIE,
|
||||
RTL2YouSeriesIE,
|
||||
)
|
||||
from .rtl2 import RTL2IE
|
||||
from .rtnews import (
|
||||
RTNewsIE,
|
||||
RTDocumentryIE,
|
||||
@@ -1719,16 +1640,15 @@ from .rtve import (
|
||||
RTVEInfantilIE,
|
||||
RTVETelevisionIE,
|
||||
)
|
||||
from .rtvnh import RTVNHIE
|
||||
from .rtvs import RTVSIE
|
||||
from .rtvslo import RTVSLOIE
|
||||
from .ruhd import RUHDIE
|
||||
from .rule34video import Rule34VideoIE
|
||||
from .rumble import (
|
||||
RumbleEmbedIE,
|
||||
RumbleIE,
|
||||
RumbleChannelIE,
|
||||
)
|
||||
from .rudovideo import RudoVideoIE
|
||||
from .rutube import (
|
||||
RutubeIE,
|
||||
RutubeChannelIE,
|
||||
@@ -1804,10 +1724,6 @@ from .shahid import (
|
||||
ShahidIE,
|
||||
ShahidShowIE,
|
||||
)
|
||||
from .shared import (
|
||||
SharedIE,
|
||||
VivoIE,
|
||||
)
|
||||
from .sharevideos import ShareVideosEmbedIE
|
||||
from .sibnet import SibnetEmbedIE
|
||||
from .shemaroome import ShemarooMeIE
|
||||
@@ -1885,7 +1801,6 @@ from .spankbang import (
|
||||
SpankBangIE,
|
||||
SpankBangPlaylistIE,
|
||||
)
|
||||
from .spankwire import SpankwireIE
|
||||
from .spiegel import SpiegelIE
|
||||
from .spike import (
|
||||
BellatorIE,
|
||||
@@ -1935,7 +1850,6 @@ from .storyfire import (
|
||||
StoryFireSeriesIE,
|
||||
)
|
||||
from .streamable import StreamableIE
|
||||
from .streamcloud import StreamcloudIE
|
||||
from .streamcz import StreamCZIE
|
||||
from .streamff import StreamFFIE
|
||||
from .streetvoice import StreetVoiceIE
|
||||
@@ -1955,7 +1869,6 @@ from .svt import (
|
||||
SVTSeriesIE,
|
||||
)
|
||||
from .swearnet import SwearnetEpisodeIE
|
||||
from .swrmediathek import SWRMediathekIE
|
||||
from .syvdk import SYVDKIE
|
||||
from .syfy import SyfyIE
|
||||
from .sztvhu import SztvHuIE
|
||||
@@ -1982,7 +1895,6 @@ from .teamcoco import (
|
||||
ConanClassicIE,
|
||||
)
|
||||
from .teamtreehouse import TeamTreeHouseIE
|
||||
from .techtalks import TechTalksIE
|
||||
from .ted import (
|
||||
TedEmbedIE,
|
||||
TedPlaylistIE,
|
||||
@@ -2024,6 +1936,10 @@ from .tenplay import (
|
||||
from .testurl import TestURLIE
|
||||
from .tf1 import TF1IE
|
||||
from .tfo import TFOIE
|
||||
from .theguardian import (
|
||||
TheGuardianPodcastIE,
|
||||
TheGuardianPodcastPlaylistIE,
|
||||
)
|
||||
from .theholetv import TheHoleTvIE
|
||||
from .theintercept import TheInterceptIE
|
||||
from .theplatform import (
|
||||
@@ -2055,7 +1971,6 @@ from .tiktok import (
|
||||
TikTokLiveIE,
|
||||
DouyinIE,
|
||||
)
|
||||
from .tinypic import TinyPicIE
|
||||
from .tmz import TMZIE
|
||||
from .tnaflix import (
|
||||
TNAFlixNetworkEmbedIE,
|
||||
@@ -2070,10 +1985,6 @@ from .toggle import (
|
||||
from .toggo import (
|
||||
ToggoIE,
|
||||
)
|
||||
from .tokentube import (
|
||||
TokentubeIE,
|
||||
TokentubeChannelIE
|
||||
)
|
||||
from .tonline import TOnlineIE
|
||||
from .toongoggles import ToonGogglesIE
|
||||
from .toutv import TouTvIE
|
||||
@@ -2084,7 +1995,6 @@ from .triller import (
|
||||
TrillerUserIE,
|
||||
TrillerShortIE,
|
||||
)
|
||||
from .trilulilu import TriluliluIE
|
||||
from .trovo import (
|
||||
TrovoIE,
|
||||
TrovoVodIE,
|
||||
@@ -2109,7 +2019,6 @@ from .tunein import (
|
||||
TuneInPodcastEpisodeIE,
|
||||
TuneInShortenerIE,
|
||||
)
|
||||
from .tunepk import TunePkIE
|
||||
from .turbo import TurboIE
|
||||
from .tv2 import (
|
||||
TV2IE,
|
||||
@@ -2151,16 +2060,7 @@ from .tvigle import TvigleIE
|
||||
from .tviplayer import TVIPlayerIE
|
||||
from .tvland import TVLandIE
|
||||
from .tvn24 import TVN24IE
|
||||
from .tvnet import TVNetIE
|
||||
from .tvnoe import TVNoeIE
|
||||
from .tvnow import (
|
||||
TVNowIE,
|
||||
TVNowFilmIE,
|
||||
TVNowNewIE,
|
||||
TVNowSeasonIE,
|
||||
TVNowAnnualIE,
|
||||
TVNowShowIE,
|
||||
)
|
||||
from .tvopengr import (
|
||||
TVOpenGrWatchIE,
|
||||
TVOpenGrEmbedIE,
|
||||
@@ -2178,7 +2078,6 @@ from .tvplay import (
|
||||
)
|
||||
from .tvplayer import TVPlayerIE
|
||||
from .tweakers import TweakersIE
|
||||
from .twentyfourvideo import TwentyFourVideoIE
|
||||
from .twentymin import TwentyMinutenIE
|
||||
from .twentythreevideo import TwentyThreeVideoIE
|
||||
from .twitcasting import (
|
||||
@@ -2227,7 +2126,6 @@ from .drooble import DroobleIE
|
||||
from .umg import UMGDeIE
|
||||
from .unistra import UnistraIE
|
||||
from .unity import UnityIE
|
||||
from .unscripted import UnscriptedNewsVideoIE
|
||||
from .unsupported import KnownDRMIE, KnownPiracyIE
|
||||
from .uol import UOLIE
|
||||
from .uplynk import (
|
||||
@@ -2246,7 +2144,6 @@ from .ustudio import (
|
||||
from .utreon import UtreonIE
|
||||
from .varzesh3 import Varzesh3IE
|
||||
from .vbox7 import Vbox7IE
|
||||
from .veehd import VeeHDIE
|
||||
from .veo import VeoIE
|
||||
from .veoh import (
|
||||
VeohIE,
|
||||
@@ -2268,7 +2165,6 @@ from .vice import (
|
||||
ViceArticleIE,
|
||||
ViceShowIE,
|
||||
)
|
||||
from .vidbit import VidbitIE
|
||||
from .viddler import ViddlerIE
|
||||
from .videa import VideaIE
|
||||
from .videocampus_sachsen import (
|
||||
@@ -2296,6 +2192,7 @@ from .vidio import (
|
||||
VidioLiveIE
|
||||
)
|
||||
from .vidlii import VidLiiIE
|
||||
from .vidly import VidlyIE
|
||||
from .viewlift import (
|
||||
ViewLiftIE,
|
||||
ViewLiftEmbedIE,
|
||||
@@ -2318,7 +2215,6 @@ from .vimm import (
|
||||
VimmIE,
|
||||
VimmRecordingIE,
|
||||
)
|
||||
from .vimple import VimpleIE
|
||||
from .vine import (
|
||||
VineIE,
|
||||
VineUserIE,
|
||||
@@ -2342,10 +2238,8 @@ from .vk import (
|
||||
VKPlayLiveIE,
|
||||
)
|
||||
from .vocaroo import VocarooIE
|
||||
from .vodlocker import VodlockerIE
|
||||
from .vodpl import VODPlIE
|
||||
from .vodplatform import VODPlatformIE
|
||||
from .voicerepublic import VoiceRepublicIE
|
||||
from .voicy import (
|
||||
VoicyIE,
|
||||
VoicyChannelIE,
|
||||
@@ -2365,23 +2259,13 @@ from .vrt import (
|
||||
KetnetIE,
|
||||
DagelijkseKostIE,
|
||||
)
|
||||
from .vrak import VrakIE
|
||||
from .vrv import (
|
||||
VRVIE,
|
||||
VRVSeriesIE,
|
||||
)
|
||||
from .vshare import VShareIE
|
||||
from .vtm import VTMIE
|
||||
from .medialaan import MedialaanIE
|
||||
from .vuclip import VuClipIE
|
||||
from .vupload import VuploadIE
|
||||
from .vvvvid import (
|
||||
VVVVIDIE,
|
||||
VVVVIDShowIE,
|
||||
)
|
||||
from .vyborymos import VyboryMosIE
|
||||
from .vzaar import VzaarIE
|
||||
from .wakanim import WakanimIE
|
||||
from .walla import WallaIE
|
||||
from .washingtonpost import (
|
||||
WashingtonPostIE,
|
||||
@@ -2393,8 +2277,6 @@ from .wasdtv import (
|
||||
WASDTVClipIE,
|
||||
)
|
||||
from .wat import WatIE
|
||||
from .watchbox import WatchBoxIE
|
||||
from .watchindianporn import WatchIndianPornIE
|
||||
from .wdr import (
|
||||
WDRIE,
|
||||
WDRPageIE,
|
||||
@@ -2428,7 +2310,6 @@ from .wevidi import WeVidiIE
|
||||
from .weyyak import WeyyakIE
|
||||
from .whyp import WhypIE
|
||||
from .wikimedia import WikimediaIE
|
||||
from .willow import WillowIE
|
||||
from .wimbledon import WimbledonIE
|
||||
from .wimtv import WimTVIE
|
||||
from .whowatch import WhoWatchIE
|
||||
@@ -2462,7 +2343,6 @@ from .wykop import (
|
||||
WykopPostCommentIE,
|
||||
)
|
||||
from .xanimu import XanimuIE
|
||||
from .xbef import XBefIE
|
||||
from .xboxclips import XboxClipsIE
|
||||
from .xfileshare import XFileShareIE
|
||||
from .xhamster import (
|
||||
@@ -2478,8 +2358,6 @@ from .xinpianchang import XinpianchangIE
|
||||
from .xminus import XMinusIE
|
||||
from .xnxx import XNXXIE
|
||||
from .xstream import XstreamIE
|
||||
from .xtube import XTubeUserIE, XTubeIE
|
||||
from .xuite import XuiteIE
|
||||
from .xvideos import (
|
||||
XVideosIE,
|
||||
XVideosQuickiesIE
|
||||
@@ -2509,10 +2387,7 @@ from .yappy import (
|
||||
YappyIE,
|
||||
YappyProfileIE,
|
||||
)
|
||||
from .yesjapan import YesJapanIE
|
||||
from .yinyuetai import YinYueTaiIE
|
||||
from .yle_areena import YleAreenaIE
|
||||
from .ynet import YnetIE
|
||||
from .youjizz import YouJizzIE
|
||||
from .youku import (
|
||||
YoukuIE,
|
||||
@@ -2588,6 +2463,9 @@ from .zingmp3 import (
|
||||
ZingMp3ChartMusicVideoIE,
|
||||
ZingMp3UserIE,
|
||||
ZingMp3HubIE,
|
||||
ZingMp3LiveRadioIE,
|
||||
ZingMp3PodcastEpisodeIE,
|
||||
ZingMp3PodcastIE,
|
||||
)
|
||||
from .zoom import ZoomIE
|
||||
from .zype import ZypeIE
|
||||
|
||||
@@ -211,7 +211,8 @@ class AbemaTVIE(AbemaTVBaseIE):
|
||||
'id': '194-25_s2_p1',
|
||||
'title': '第1話 「チーズケーキ」 「モーニング再び」',
|
||||
'series': '異世界食堂2',
|
||||
'series_number': 2,
|
||||
'season': 'シーズン2',
|
||||
'season_number': 2,
|
||||
'episode': '第1話 「チーズケーキ」 「モーニング再び」',
|
||||
'episode_number': 1,
|
||||
},
|
||||
@@ -347,12 +348,12 @@ class AbemaTVIE(AbemaTVBaseIE):
|
||||
)?
|
||||
''', r'\1', og_desc)
|
||||
|
||||
# canonical URL may contain series and episode number
|
||||
# canonical URL may contain season and episode number
|
||||
mobj = re.search(r's(\d+)_p(\d+)$', canonical_url)
|
||||
if mobj:
|
||||
seri = int_or_none(mobj.group(1), default=float('inf'))
|
||||
epis = int_or_none(mobj.group(2), default=float('inf'))
|
||||
info['series_number'] = seri if seri < 100 else None
|
||||
info['season_number'] = seri if seri < 100 else None
|
||||
# some anime like Detective Conan (though not available in AbemaTV)
|
||||
# has more than 1000 episodes (1026 as of 2021/11/15)
|
||||
info['episode_number'] = epis if epis < 2000 else None
|
||||
@@ -381,7 +382,7 @@ class AbemaTVIE(AbemaTVBaseIE):
|
||||
self.report_warning('This is a premium-only stream')
|
||||
info.update(traverse_obj(api_response, {
|
||||
'series': ('series', 'title'),
|
||||
'season': ('season', 'title'),
|
||||
'season': ('season', 'name'),
|
||||
'season_number': ('season', 'sequence'),
|
||||
'episode_number': ('episode', 'number'),
|
||||
}))
|
||||
|
||||
@@ -121,11 +121,21 @@ class AENetworksIE(AENetworksBaseIE):
|
||||
'info_dict': {
|
||||
'id': '22253814',
|
||||
'ext': 'mp4',
|
||||
'title': 'Winter is Coming',
|
||||
'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
|
||||
'title': 'Winter Is Coming',
|
||||
'description': 'md5:a40e370925074260b1c8a633c632c63a',
|
||||
'timestamp': 1338306241,
|
||||
'upload_date': '20120529',
|
||||
'uploader': 'AENE-NEW',
|
||||
'duration': 2592.0,
|
||||
'thumbnail': r're:^https?://.*\.jpe?g$',
|
||||
'chapters': 'count:5',
|
||||
'tags': 'count:14',
|
||||
'categories': ['Mountain Men'],
|
||||
'episode_number': 1,
|
||||
'episode': 'Episode 1',
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'series': 'Mountain Men',
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
@@ -143,6 +153,15 @@ class AENetworksIE(AENetworksBaseIE):
|
||||
'timestamp': 1452634428,
|
||||
'upload_date': '20160112',
|
||||
'uploader': 'AENE-NEW',
|
||||
'duration': 1277.695,
|
||||
'thumbnail': r're:^https?://.*\.jpe?g$',
|
||||
'chapters': 'count:4',
|
||||
'tags': 'count:23',
|
||||
'episode': 'Episode 1',
|
||||
'episode_number': 1,
|
||||
'season': 'Season 9',
|
||||
'season_number': 9,
|
||||
'series': 'Duck Dynasty',
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
parse_duration,
|
||||
parse_iso8601,
|
||||
)
|
||||
|
||||
|
||||
class AirMozillaIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://air\.mozilla\.org/(?P<id>[0-9a-z-]+)/?'
|
||||
_TEST = {
|
||||
'url': 'https://air.mozilla.org/privacy-lab-a-meetup-for-privacy-minded-people-in-san-francisco/',
|
||||
'md5': '8d02f53ee39cf006009180e21df1f3ba',
|
||||
'info_dict': {
|
||||
'id': '6x4q2w',
|
||||
'ext': 'mp4',
|
||||
'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
|
||||
'thumbnail': r're:https?://.*/poster\.jpg',
|
||||
'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
|
||||
'timestamp': 1422487800,
|
||||
'upload_date': '20150128',
|
||||
'location': 'SFO Commons',
|
||||
'duration': 3780,
|
||||
'view_count': int,
|
||||
'categories': ['Main', 'Privacy'],
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
video_id = self._html_search_regex(r'//vid\.ly/(.*?)/embed', webpage, 'id')
|
||||
|
||||
embed_script = self._download_webpage('https://vid.ly/{0}/embed'.format(video_id), video_id)
|
||||
jwconfig = self._parse_json(self._search_regex(
|
||||
r'initCallback\((.*)\);', embed_script, 'metadata'), video_id)['config']
|
||||
|
||||
info_dict = self._parse_jwplayer_data(jwconfig, video_id)
|
||||
view_count = int_or_none(self._html_search_regex(
|
||||
r'Views since archived: ([0-9]+)',
|
||||
webpage, 'view count', fatal=False))
|
||||
timestamp = parse_iso8601(self._html_search_regex(
|
||||
r'<time datetime="(.*?)"', webpage, 'timestamp', fatal=False))
|
||||
duration = parse_duration(self._search_regex(
|
||||
r'Duration:\s*(\d+\s*hours?\s*\d+\s*minutes?)',
|
||||
webpage, 'duration', fatal=False))
|
||||
|
||||
info_dict.update({
|
||||
'id': video_id,
|
||||
'title': self._og_search_title(webpage),
|
||||
'url': self._og_search_url(webpage),
|
||||
'display_id': display_id,
|
||||
'description': self._og_search_description(webpage),
|
||||
'timestamp': timestamp,
|
||||
'location': self._html_search_regex(r'Location: (.*)', webpage, 'location', default=None),
|
||||
'duration': duration,
|
||||
'view_count': view_count,
|
||||
'categories': re.findall(r'<a href=".*?" class="channel">(.*?)</a>', webpage),
|
||||
})
|
||||
|
||||
return info_dict
|
||||
253
yt_dlp/extractor/allstar.py
Normal file
253
yt_dlp/extractor/allstar.py
Normal file
@@ -0,0 +1,253 @@
|
||||
import functools
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
OnDemandPagedList,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
parse_qs,
|
||||
urljoin,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
_FIELDS = '''
|
||||
_id
|
||||
clipImageSource
|
||||
clipImageThumb
|
||||
clipLink
|
||||
clipTitle
|
||||
createdDate
|
||||
shareId
|
||||
user { _id }
|
||||
username
|
||||
views'''
|
||||
|
||||
_EXTRA_FIELDS = '''
|
||||
clipLength
|
||||
clipSizeBytes'''
|
||||
|
||||
_QUERIES = {
|
||||
'clip': '''query ($id: String!) {
|
||||
video: getClip(clipIdentifier: $id) {
|
||||
%s %s
|
||||
}
|
||||
}''' % (_FIELDS, _EXTRA_FIELDS),
|
||||
'montage': '''query ($id: String!) {
|
||||
video: getMontage(clipIdentifier: $id) {
|
||||
%s
|
||||
}
|
||||
}''' % _FIELDS,
|
||||
'Clips': '''query ($page: Int!, $user: String!, $game: Int) {
|
||||
videos: clips(search: createdDate, page: $page, user: $user, mobile: false, game: $game) {
|
||||
data { %s %s }
|
||||
}
|
||||
}''' % (_FIELDS, _EXTRA_FIELDS),
|
||||
'Montages': '''query ($page: Int!, $user: String!) {
|
||||
videos: montages(search: createdDate, page: $page, user: $user) {
|
||||
data { %s }
|
||||
}
|
||||
}''' % _FIELDS,
|
||||
'Mobile Clips': '''query ($page: Int!, $user: String!) {
|
||||
videos: clips(search: createdDate, page: $page, user: $user, mobile: true) {
|
||||
data { %s %s }
|
||||
}
|
||||
}''' % (_FIELDS, _EXTRA_FIELDS),
|
||||
}
|
||||
|
||||
|
||||
class AllstarBaseIE(InfoExtractor):
|
||||
@staticmethod
|
||||
def _parse_video_data(video_data):
|
||||
def media_url_or_none(path):
|
||||
return urljoin('https://media.allstar.gg/', path)
|
||||
|
||||
info = traverse_obj(video_data, {
|
||||
'id': ('_id', {str}),
|
||||
'display_id': ('shareId', {str}),
|
||||
'title': ('clipTitle', {str}),
|
||||
'url': ('clipLink', {media_url_or_none}),
|
||||
'thumbnails': (('clipImageThumb', 'clipImageSource'), {'url': {media_url_or_none}}),
|
||||
'duration': ('clipLength', {int_or_none}),
|
||||
'filesize': ('clipSizeBytes', {int_or_none}),
|
||||
'timestamp': ('createdDate', {functools.partial(int_or_none, scale=1000)}),
|
||||
'uploader': ('username', {str}),
|
||||
'uploader_id': ('user', '_id', {str}),
|
||||
'view_count': ('views', {int_or_none}),
|
||||
})
|
||||
|
||||
if info.get('id') and info.get('url'):
|
||||
basename = 'clip' if '/clips/' in info['url'] else 'montage'
|
||||
info['webpage_url'] = f'https://allstar.gg/{basename}?{basename}={info["id"]}'
|
||||
|
||||
info.update({
|
||||
'extractor_key': AllstarIE.ie_key(),
|
||||
'extractor': AllstarIE.IE_NAME,
|
||||
'uploader_url': urljoin('https://allstar.gg/u/', info.get('uploader_id')),
|
||||
})
|
||||
|
||||
return info
|
||||
|
||||
def _call_api(self, query, variables, path, video_id=None, note=None):
|
||||
response = self._download_json(
|
||||
'https://a1.allstar.gg/graphql', video_id, note=note,
|
||||
headers={'content-type': 'application/json'},
|
||||
data=json.dumps({'variables': variables, 'query': query}).encode())
|
||||
|
||||
errors = traverse_obj(response, ('errors', ..., 'message', {str}))
|
||||
if errors:
|
||||
raise ExtractorError('; '.join(errors))
|
||||
|
||||
return traverse_obj(response, path)
|
||||
|
||||
|
||||
class AllstarIE(AllstarBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?P<type>(?:clip|montage))\?(?P=type)=(?P<id>[^/?#&]+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://allstar.gg/clip?clip=64482c2da9eec30008a67d1b',
|
||||
'info_dict': {
|
||||
'id': '64482c2da9eec30008a67d1b',
|
||||
'title': '4K on Inferno',
|
||||
'url': 'md5:66befb5381eef0c9456026386c25fa55',
|
||||
'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
|
||||
'uploader': 'chrk.',
|
||||
'ext': 'mp4',
|
||||
'duration': 20,
|
||||
'filesize': 21199257,
|
||||
'timestamp': 1682451501,
|
||||
'uploader_id': '62b8bdfc9021052f7905882d',
|
||||
'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
|
||||
'upload_date': '20230425',
|
||||
'view_count': int,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://allstar.gg/clip?clip=8LJLY4JKB',
|
||||
'info_dict': {
|
||||
'id': '64a1ec6b887f4c0008dc50b8',
|
||||
'display_id': '8LJLY4JKB',
|
||||
'title': 'AK-47 3K on Mirage',
|
||||
'url': 'md5:dde224fd12f035c0e2529a4ae34c4283',
|
||||
'ext': 'mp4',
|
||||
'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
|
||||
'duration': 16,
|
||||
'filesize': 30175859,
|
||||
'timestamp': 1688333419,
|
||||
'uploader': 'cherokee',
|
||||
'uploader_id': '62b8bdfc9021052f7905882d',
|
||||
'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
|
||||
'upload_date': '20230702',
|
||||
'view_count': int,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://allstar.gg/montage?montage=643e64089da7e9363e1fa66c',
|
||||
'info_dict': {
|
||||
'id': '643e64089da7e9363e1fa66c',
|
||||
'display_id': 'APQLGM2IMXW',
|
||||
'title': 'cherokee Rapid Fire Snipers Montage',
|
||||
'url': 'md5:a3ee356022115db2b27c81321d195945',
|
||||
'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
|
||||
'ext': 'mp4',
|
||||
'timestamp': 1681810448,
|
||||
'uploader': 'cherokee',
|
||||
'uploader_id': '62b8bdfc9021052f7905882d',
|
||||
'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
|
||||
'upload_date': '20230418',
|
||||
'view_count': int,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://allstar.gg/montage?montage=RILJMH6QOS',
|
||||
'info_dict': {
|
||||
'id': '64a2697372ce3703de29e868',
|
||||
'display_id': 'RILJMH6QOS',
|
||||
'title': 'cherokee Rapid Fire Snipers Montage',
|
||||
'url': 'md5:d5672e6f88579730c2310a80fdbc4030',
|
||||
'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
|
||||
'ext': 'mp4',
|
||||
'timestamp': 1688365434,
|
||||
'uploader': 'cherokee',
|
||||
'uploader_id': '62b8bdfc9021052f7905882d',
|
||||
'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
|
||||
'upload_date': '20230703',
|
||||
'view_count': int,
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
query_id, video_id = self._match_valid_url(url).group('type', 'id')
|
||||
|
||||
return self._parse_video_data(
|
||||
self._call_api(
|
||||
_QUERIES.get(query_id), {'id': video_id}, ('data', 'video'), video_id))
|
||||
|
||||
|
||||
class AllstarProfileIE(AllstarBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?:profile\?user=|u/)(?P<id>[^/?#&]+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://allstar.gg/profile?user=62b8bdfc9021052f7905882d',
|
||||
'info_dict': {
|
||||
'id': '62b8bdfc9021052f7905882d-clips',
|
||||
'title': 'cherokee - Clips',
|
||||
},
|
||||
'playlist_mincount': 15
|
||||
}, {
|
||||
'url': 'https://allstar.gg/u/cherokee?game=730&view=Clips',
|
||||
'info_dict': {
|
||||
'id': '62b8bdfc9021052f7905882d-clips-730',
|
||||
'title': 'cherokee - Clips - 730',
|
||||
},
|
||||
'playlist_mincount': 15
|
||||
}, {
|
||||
'url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d?view=Montages',
|
||||
'info_dict': {
|
||||
'id': '62b8bdfc9021052f7905882d-montages',
|
||||
'title': 'cherokee - Montages',
|
||||
},
|
||||
'playlist_mincount': 4
|
||||
}, {
|
||||
'url': 'https://allstar.gg/profile?user=cherokee&view=Mobile Clips',
|
||||
'info_dict': {
|
||||
'id': '62b8bdfc9021052f7905882d-mobile',
|
||||
'title': 'cherokee - Mobile Clips',
|
||||
},
|
||||
'playlist_mincount': 1
|
||||
}]
|
||||
|
||||
_PAGE_SIZE = 10
|
||||
|
||||
def _get_page(self, user_id, display_id, game, query, page_num):
|
||||
page_num += 1
|
||||
|
||||
for video_data in self._call_api(
|
||||
query, {
|
||||
'user': user_id,
|
||||
'page': page_num,
|
||||
'game': game,
|
||||
}, ('data', 'videos', 'data'), display_id, f'Downloading page {page_num}'):
|
||||
yield self._parse_video_data(video_data)
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
profile_data = self._download_json(
|
||||
urljoin('https://api.allstar.gg/v1/users/profile/', display_id), display_id)
|
||||
user_id = traverse_obj(profile_data, ('data', ('_id'), {str}))
|
||||
if not user_id:
|
||||
raise ExtractorError('Unable to extract the user id')
|
||||
|
||||
username = traverse_obj(profile_data, ('data', 'profile', ('username'), {str}))
|
||||
url_query = parse_qs(url)
|
||||
game = traverse_obj(url_query, ('game', 0, {int_or_none}))
|
||||
query_id = traverse_obj(url_query, ('view', 0), default='Clips')
|
||||
|
||||
if query_id not in ('Clips', 'Montages', 'Mobile Clips'):
|
||||
raise ExtractorError(f'Unsupported playlist URL type {query_id!r}')
|
||||
|
||||
return self.playlist_result(
|
||||
OnDemandPagedList(
|
||||
functools.partial(
|
||||
self._get_page, user_id, display_id, game, _QUERIES.get(query_id)), self._PAGE_SIZE),
|
||||
playlist_id=join_nonempty(user_id, query_id.lower().split()[0], game),
|
||||
playlist_title=join_nonempty((username or display_id), query_id, game, delim=' - '))
|
||||
96
yt_dlp/extractor/altcensored.py
Normal file
96
yt_dlp/extractor/altcensored.py
Normal file
@@ -0,0 +1,96 @@
|
||||
import re
|
||||
|
||||
from .archiveorg import ArchiveOrgIE
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
InAdvancePagedList,
|
||||
int_or_none,
|
||||
orderedSet,
|
||||
str_to_int,
|
||||
urljoin,
|
||||
)
|
||||
|
||||
|
||||
class AltCensoredIE(InfoExtractor):
|
||||
IE_NAME = 'altcensored'
|
||||
_VALID_URL = r'https?://(?:www\.)?altcensored\.com/(?:watch\?v=|embed/)(?P<id>[^/?#]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.altcensored.com/watch?v=k0srjLSkga8',
|
||||
'info_dict': {
|
||||
'id': 'youtube-k0srjLSkga8',
|
||||
'ext': 'webm',
|
||||
'title': "QUELLES SONT LES CONSÉQUENCES DE L'HYPERSEXUALISATION DE LA SOCIÉTÉ ?",
|
||||
'display_id': 'k0srjLSkga8.webm',
|
||||
'release_date': '20180403',
|
||||
'creator': 'Virginie Vota',
|
||||
'release_year': 2018,
|
||||
'upload_date': '20230318',
|
||||
'uploader': 'admin@altcensored.com',
|
||||
'description': 'md5:0b38a8fc04103579d5c1db10a247dc30',
|
||||
'timestamp': 1679161343,
|
||||
'track': 'k0srjLSkga8',
|
||||
'duration': 926.09,
|
||||
'thumbnail': 'https://archive.org/download/youtube-k0srjLSkga8/youtube-k0srjLSkga8.thumbs/k0srjLSkga8_000925.jpg',
|
||||
'view_count': int,
|
||||
'categories': ['News & Politics'],
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'url': f'https://archive.org/details/youtube-{video_id}',
|
||||
'ie_key': ArchiveOrgIE.ie_key(),
|
||||
'view_count': str_to_int(self._html_search_regex(
|
||||
r'YouTube Views:(?:\s| )*([\d,]+)', webpage, 'view count', default=None)),
|
||||
'categories': self._html_search_regex(
|
||||
r'<a href="/category/\d+">\s*\n?\s*([^<]+)</a>',
|
||||
webpage, 'category', default='').split() or None,
|
||||
}
|
||||
|
||||
|
||||
class AltCensoredChannelIE(InfoExtractor):
|
||||
IE_NAME = 'altcensored:channel'
|
||||
_VALID_URL = r'https?://(?:www\.)?altcensored\.com/channel/(?!page|table)(?P<id>[^/?#]+)'
|
||||
_PAGE_SIZE = 24
|
||||
_TESTS = [{
|
||||
'url': 'https://www.altcensored.com/channel/UCFPTO55xxHqFqkzRZHu4kcw',
|
||||
'info_dict': {
|
||||
'title': 'Virginie Vota',
|
||||
'id': 'UCFPTO55xxHqFqkzRZHu4kcw',
|
||||
},
|
||||
'playlist_count': 91
|
||||
}, {
|
||||
'url': 'https://altcensored.com/channel/UC9CcJ96HKMWn0LZlcxlpFTw',
|
||||
'info_dict': {
|
||||
'title': 'yukikaze775',
|
||||
'id': 'UC9CcJ96HKMWn0LZlcxlpFTw',
|
||||
},
|
||||
'playlist_count': 4
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
channel_id = self._match_id(url)
|
||||
webpage = self._download_webpage(
|
||||
url, channel_id, 'Download channel webpage', 'Unable to get channel webpage')
|
||||
title = self._html_search_meta('altcen_title', webpage, 'title', fatal=False)
|
||||
page_count = int_or_none(self._html_search_regex(
|
||||
r'<a[^>]+href="/channel/\w+/page/(\d+)">(?:\1)</a>',
|
||||
webpage, 'page count', default='1'))
|
||||
|
||||
def page_func(page_num):
|
||||
page_num += 1
|
||||
webpage = self._download_webpage(
|
||||
f'https://altcensored.com/channel/{channel_id}/page/{page_num}',
|
||||
channel_id, note=f'Downloading page {page_num}')
|
||||
|
||||
items = re.findall(r'<a[^>]+href="(/watch\?v=[^"]+)', webpage)
|
||||
return [self.url_result(urljoin('https://www.altcensored.com', path), AltCensoredIE)
|
||||
for path in orderedSet(items)]
|
||||
|
||||
return self.playlist_result(
|
||||
InAdvancePagedList(page_func, page_count, self._PAGE_SIZE),
|
||||
playlist_id=channel_id, playlist_title=title)
|
||||
@@ -10,6 +10,7 @@ from ..utils import (
|
||||
|
||||
|
||||
class AolIE(YahooIE): # XXX: Do not subclass from concrete IE
|
||||
_WORKING = False
|
||||
IE_NAME = 'aol.com'
|
||||
_VALID_URL = r'(?:aol-video:|https?://(?:www\.)?aol\.(?:com|ca|co\.uk|de|jp)/video/(?:[^/]+/)*)(?P<id>\d{9}|[0-9a-f]{24}|[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})'
|
||||
|
||||
|
||||
@@ -52,7 +52,6 @@ class ArchiveOrgIE(InfoExtractor):
|
||||
'creator': 'SRI International',
|
||||
'uploader': 'laura@archive.org',
|
||||
'thumbnail': r're:https://archive\.org/download/.*\.jpg',
|
||||
'release_year': 1968,
|
||||
'display_id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.cdr',
|
||||
'track': 'XD300-23 68HighlightsAResearchCntAugHumanIntellect',
|
||||
|
||||
@@ -134,7 +133,6 @@ class ArchiveOrgIE(InfoExtractor):
|
||||
'album': '1977-05-08 - Barton Hall - Cornell University',
|
||||
'release_date': '19770508',
|
||||
'display_id': 'gd1977-05-08d01t07.flac',
|
||||
'release_year': 1977,
|
||||
'track_number': 7,
|
||||
},
|
||||
}, {
|
||||
|
||||
@@ -1,24 +1,23 @@
|
||||
import json
|
||||
import re
|
||||
from functools import partial
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .generic import GenericIE
|
||||
from ..utils import (
|
||||
OnDemandPagedList,
|
||||
determine_ext,
|
||||
ExtractorError,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
make_archive_id,
|
||||
parse_duration,
|
||||
qualities,
|
||||
parse_iso8601,
|
||||
remove_start,
|
||||
str_or_none,
|
||||
try_get,
|
||||
unified_strdate,
|
||||
unified_timestamp,
|
||||
update_url,
|
||||
update_url_query,
|
||||
url_or_none,
|
||||
xpath_text,
|
||||
)
|
||||
from ..compat import compat_etree_fromstring
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class ARDMediathekBaseIE(InfoExtractor):
|
||||
@@ -61,45 +60,6 @@ class ARDMediathekBaseIE(InfoExtractor):
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
|
||||
def _ARD_extract_episode_info(self, title):
|
||||
"""Try to extract season/episode data from the title."""
|
||||
res = {}
|
||||
if not title:
|
||||
return res
|
||||
|
||||
for pattern in [
|
||||
# Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
|
||||
# from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
|
||||
r'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
|
||||
# E.g.: title="Fritjof aus Norwegen (2) (AD)"
|
||||
# from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
|
||||
r'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
|
||||
r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
|
||||
# E.g.: title="Folge 25/42: Symmetrie"
|
||||
# from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
|
||||
# E.g.: title="Folge 1063 - Vertrauen"
|
||||
# from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
|
||||
r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
|
||||
]:
|
||||
m = re.match(pattern, title)
|
||||
if m:
|
||||
groupdict = m.groupdict()
|
||||
res['season_number'] = int_or_none(groupdict.get('season_number'))
|
||||
res['episode_number'] = int_or_none(groupdict.get('episode_number'))
|
||||
res['episode'] = str_or_none(groupdict.get('episode'))
|
||||
# Build the episode title by removing numeric episode information:
|
||||
if groupdict.get('ep_info') and not res['episode']:
|
||||
res['episode'] = str_or_none(
|
||||
title.replace(groupdict.get('ep_info'), ''))
|
||||
if res['episode']:
|
||||
res['episode'] = res['episode'].strip()
|
||||
break
|
||||
|
||||
# As a fallback use the whole title as the episode name:
|
||||
if not res.get('episode'):
|
||||
res['episode'] = title.strip()
|
||||
return res
|
||||
|
||||
def _extract_formats(self, media_info, video_id):
|
||||
type_ = media_info.get('_type')
|
||||
media_array = media_info.get('_mediaArray', [])
|
||||
@@ -155,144 +115,12 @@ class ARDMediathekBaseIE(InfoExtractor):
|
||||
return formats
|
||||
|
||||
|
||||
class ARDMediathekIE(ARDMediathekBaseIE):
|
||||
IE_NAME = 'ARD:mediathek'
|
||||
_VALID_URL = r'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
|
||||
|
||||
_TESTS = [{
|
||||
# available till 26.07.2022
|
||||
'url': 'http://www.ardmediathek.de/tv/S%C3%9CDLICHT/Was-ist-die-Kunst-der-Zukunft-liebe-Ann/BR-Fernsehen/Video?bcastId=34633636&documentId=44726822',
|
||||
'info_dict': {
|
||||
'id': '44726822',
|
||||
'ext': 'mp4',
|
||||
'title': 'Was ist die Kunst der Zukunft, liebe Anna McCarthy?',
|
||||
'description': 'md5:4ada28b3e3b5df01647310e41f3a62f5',
|
||||
'duration': 1740,
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
'skip_download': True,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
# audio
|
||||
'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
# audio
|
||||
'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
@classmethod
|
||||
def suitable(cls, url):
|
||||
return False if ARDBetaMediathekIE.suitable(url) else super(ARDMediathekIE, cls).suitable(url)
|
||||
|
||||
def _real_extract(self, url):
|
||||
# determine video id from url
|
||||
m = self._match_valid_url(url)
|
||||
|
||||
document_id = None
|
||||
|
||||
numid = re.search(r'documentId=([0-9]+)', url)
|
||||
if numid:
|
||||
document_id = video_id = numid.group(1)
|
||||
else:
|
||||
video_id = m.group('video_id')
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
ERRORS = (
|
||||
('>Leider liegt eine Störung vor.', 'Video %s is unavailable'),
|
||||
('>Der gewünschte Beitrag ist nicht mehr verfügbar.<',
|
||||
'Video %s is no longer available'),
|
||||
)
|
||||
|
||||
for pattern, message in ERRORS:
|
||||
if pattern in webpage:
|
||||
raise ExtractorError(message % video_id, expected=True)
|
||||
|
||||
if re.search(r'[\?&]rss($|[=&])', url):
|
||||
doc = compat_etree_fromstring(webpage.encode('utf-8'))
|
||||
if doc.tag == 'rss':
|
||||
return GenericIE()._extract_rss(url, video_id, doc)
|
||||
|
||||
title = self._og_search_title(webpage, default=None) or self._html_search_regex(
|
||||
[r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
|
||||
r'<meta name="dcterms\.title" content="(.*?)"/>',
|
||||
r'<h4 class="headline">(.*?)</h4>',
|
||||
r'<title[^>]*>(.*?)</title>'],
|
||||
webpage, 'title')
|
||||
description = self._og_search_description(webpage, default=None) or self._html_search_meta(
|
||||
'dcterms.abstract', webpage, 'description', default=None)
|
||||
if description is None:
|
||||
description = self._html_search_meta(
|
||||
'description', webpage, 'meta description', default=None)
|
||||
if description is None:
|
||||
description = self._html_search_regex(
|
||||
r'<p\s+class="teasertext">(.+?)</p>',
|
||||
webpage, 'teaser text', default=None)
|
||||
|
||||
# Thumbnail is sometimes not present.
|
||||
# It is in the mobile version, but that seems to use a different URL
|
||||
# structure altogether.
|
||||
thumbnail = self._og_search_thumbnail(webpage, default=None)
|
||||
|
||||
media_streams = re.findall(r'''(?x)
|
||||
mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
|
||||
"([^"]+)"''', webpage)
|
||||
|
||||
if media_streams:
|
||||
QUALITIES = qualities(['lo', 'hi', 'hq'])
|
||||
formats = []
|
||||
for furl in set(media_streams):
|
||||
if furl.endswith('.f4m'):
|
||||
fid = 'f4m'
|
||||
else:
|
||||
fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
|
||||
fid = fid_m.group(1) if fid_m else None
|
||||
formats.append({
|
||||
'quality': QUALITIES(fid),
|
||||
'format_id': fid,
|
||||
'url': furl,
|
||||
})
|
||||
info = {
|
||||
'formats': formats,
|
||||
}
|
||||
else: # request JSON file
|
||||
if not document_id:
|
||||
video_id = self._search_regex(
|
||||
(r'/play/(?:config|media|sola)/(\d+)', r'contentId["\']\s*:\s*(\d+)'),
|
||||
webpage, 'media id', default=None)
|
||||
info = self._extract_media_info(
|
||||
'http://www.ardmediathek.de/play/media/%s' % video_id,
|
||||
webpage, video_id)
|
||||
|
||||
info.update({
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
})
|
||||
info.update(self._ARD_extract_episode_info(info['title']))
|
||||
|
||||
return info
|
||||
|
||||
|
||||
class ARDIE(InfoExtractor):
|
||||
_VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html'
|
||||
_TESTS = [{
|
||||
# available till 7.12.2023
|
||||
'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-video-424.html',
|
||||
'md5': 'a438f671e87a7eba04000336a119ccc4',
|
||||
'md5': '94812e6438488fb923c361a44469614b',
|
||||
'info_dict': {
|
||||
'id': 'maischberger-video-424',
|
||||
'display_id': 'maischberger-video-424',
|
||||
@@ -399,31 +227,35 @@ class ARDIE(InfoExtractor):
|
||||
}
|
||||
|
||||
|
||||
class ARDBetaMediathekIE(ARDMediathekBaseIE):
|
||||
class ARDBetaMediathekIE(InfoExtractor):
|
||||
IE_NAME = 'ARDMediathek'
|
||||
_VALID_URL = r'''(?x)https://
|
||||
(?:(?:beta|www)\.)?ardmediathek\.de/
|
||||
(?:(?P<client>[^/]+)/)?
|
||||
(?:player|live|video|(?P<playlist>sendung|sammlung))/
|
||||
(?:(?P<display_id>(?(playlist)[^?#]+?|[^?#]+))/)?
|
||||
(?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)
|
||||
(?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
|
||||
(?:[^/]+/)?
|
||||
(?:player|live|video)/
|
||||
(?:(?P<display_id>[^?#]+)/)?
|
||||
(?P<id>[a-zA-Z0-9]+)
|
||||
/?(?:[?#]|$)'''
|
||||
_GEO_COUNTRIES = ['DE']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.ardmediathek.de/video/filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy8xZGY0ZGJmZS00ZWQwLTRmMGItYjhhYy0wOGQ4ZmYxNjVhZDI',
|
||||
'md5': '3fd5fead7a370a819341129c8d713136',
|
||||
'url': 'https://www.ardmediathek.de/video/filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen/Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
|
||||
'md5': 'b6e8ab03f2bcc6e1f9e6cef25fcc03c4',
|
||||
'info_dict': {
|
||||
'display_id': 'filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen',
|
||||
'id': '12172961',
|
||||
'title': 'Wolfsland - Die traurigen Schwestern',
|
||||
'description': r're:^Als der Polizeiobermeister Raaben',
|
||||
'duration': 5241,
|
||||
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:efa186f7b0054957',
|
||||
'timestamp': 1670710500,
|
||||
'upload_date': '20221210',
|
||||
'display_id': 'filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen',
|
||||
'id': 'Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
|
||||
'title': 'Liebe auf vier Pfoten',
|
||||
'description': r're:^Claudia Schmitt, Anwältin in Salzburg',
|
||||
'duration': 5222,
|
||||
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:aee7cbf8f06de976?w=960&ch=ae4d0f2ee47d8b9b',
|
||||
'timestamp': 1701343800,
|
||||
'upload_date': '20231130',
|
||||
'ext': 'mp4',
|
||||
'age_limit': 12,
|
||||
'episode': 'Wolfsland - Die traurigen Schwestern',
|
||||
'series': 'Filme im MDR'
|
||||
'episode': 'Liebe auf vier Pfoten',
|
||||
'series': 'Filme im MDR',
|
||||
'age_limit': 0,
|
||||
'channel': 'MDR',
|
||||
'_old_archive_ids': ['ardbetamediathek 12939099'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
|
||||
@@ -444,7 +276,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
|
||||
'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
|
||||
'md5': '1e73ded21cb79bac065117e80c81dc88',
|
||||
'info_dict': {
|
||||
'id': '10049223',
|
||||
'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
|
||||
'ext': 'mp4',
|
||||
'title': 'tagesschau, 20:00 Uhr',
|
||||
'timestamp': 1636398000,
|
||||
@@ -454,7 +286,27 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
|
||||
'duration': 915,
|
||||
'episode': 'tagesschau, 20:00 Uhr',
|
||||
'series': 'tagesschau',
|
||||
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49',
|
||||
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49?w=960&ch=ee69108ae344f678',
|
||||
'channel': 'ARD-Aktuell',
|
||||
'_old_archive_ids': ['ardbetamediathek 10049223'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/video/7-tage/7-tage-unter-harten-jungs/hr-fernsehen/N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3',
|
||||
'md5': 'c428b9effff18ff624d4f903bda26315',
|
||||
'info_dict': {
|
||||
'id': 'N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3',
|
||||
'ext': 'mp4',
|
||||
'duration': 2700,
|
||||
'episode': '7 Tage ... unter harten Jungs',
|
||||
'description': 'md5:0f215470dcd2b02f59f4bd10c963f072',
|
||||
'upload_date': '20231005',
|
||||
'timestamp': 1696491171,
|
||||
'display_id': '7-tage/7-tage-unter-harten-jungs/hr-fernsehen',
|
||||
'series': '7 Tage ...',
|
||||
'channel': 'HR',
|
||||
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:f6e6d5ffac41925c?w=960&ch=fa32ba69bc87989a',
|
||||
'title': '7 Tage ... unter harten Jungs',
|
||||
'_old_archive_ids': ['ardbetamediathek 94834686'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
|
||||
@@ -471,203 +323,230 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _extract_episode_info(self, title):
|
||||
patterns = [
|
||||
# Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
|
||||
# from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
|
||||
r'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
|
||||
# E.g.: title="Fritjof aus Norwegen (2) (AD)"
|
||||
# from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
|
||||
r'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
|
||||
r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
|
||||
# E.g.: title="Folge 25/42: Symmetrie"
|
||||
# from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
|
||||
# E.g.: title="Folge 1063 - Vertrauen"
|
||||
# from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
|
||||
r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
|
||||
# As a fallback use the full title
|
||||
r'(?P<title>.*)',
|
||||
]
|
||||
|
||||
return traverse_obj(patterns, (..., {partial(re.match, string=title)}, {
|
||||
'season_number': ('season_number', {int_or_none}),
|
||||
'episode_number': ('episode_number', {int_or_none}),
|
||||
'episode': ((
|
||||
('episode', {str_or_none}),
|
||||
('ep_info', {lambda x: title.replace(x, '')}),
|
||||
('title', {str}),
|
||||
), {str.strip}),
|
||||
}), get_all=False)
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
|
||||
|
||||
page_data = self._download_json(
|
||||
f'https://api.ardmediathek.de/page-gateway/pages/ard/item/{video_id}', video_id, query={
|
||||
'embedded': 'false',
|
||||
'mcV6': 'true',
|
||||
})
|
||||
|
||||
player_data = traverse_obj(
|
||||
page_data, ('widgets', lambda _, v: v['type'] in ('player_ondemand', 'player_live'), {dict}), get_all=False)
|
||||
is_live = player_data.get('type') == 'player_live'
|
||||
media_data = traverse_obj(player_data, ('mediaCollection', 'embedded', {dict}))
|
||||
|
||||
if player_data.get('blockedByFsk'):
|
||||
self.raise_no_formats('This video is only available after 22:00', expected=True)
|
||||
|
||||
formats = []
|
||||
subtitles = {}
|
||||
for stream in traverse_obj(media_data, ('streams', ..., {dict})):
|
||||
kind = stream.get('kind')
|
||||
# Prioritize main stream over sign language and others
|
||||
preference = 1 if kind == 'main' else None
|
||||
for media in traverse_obj(stream, ('media', lambda _, v: url_or_none(v['url']))):
|
||||
media_url = media['url']
|
||||
|
||||
audio_kind = traverse_obj(media, (
|
||||
'audios', 0, 'kind', {str}), default='').replace('standard', '')
|
||||
lang_code = traverse_obj(media, ('audios', 0, 'languageCode', {str})) or 'deu'
|
||||
lang = join_nonempty(lang_code, audio_kind)
|
||||
language_preference = 10 if lang == 'deu' else -10
|
||||
|
||||
if determine_ext(media_url) == 'm3u8':
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(
|
||||
media_url, video_id, m3u8_id=f'hls-{kind}', preference=preference, fatal=False, live=is_live)
|
||||
for f in fmts:
|
||||
f['language'] = lang
|
||||
f['language_preference'] = language_preference
|
||||
formats.extend(fmts)
|
||||
self._merge_subtitles(subs, target=subtitles)
|
||||
else:
|
||||
formats.append({
|
||||
'url': media_url,
|
||||
'format_id': f'http-{kind}',
|
||||
'preference': preference,
|
||||
'language': lang,
|
||||
'language_preference': language_preference,
|
||||
**traverse_obj(media, {
|
||||
'format_note': ('forcedLabel', {str}),
|
||||
'width': ('maxHResolutionPx', {int_or_none}),
|
||||
'height': ('maxVResolutionPx', {int_or_none}),
|
||||
'vcodec': ('videoCodec', {str}),
|
||||
}),
|
||||
})
|
||||
|
||||
for sub in traverse_obj(media_data, ('subtitles', ..., {dict})):
|
||||
for sources in traverse_obj(sub, ('sources', lambda _, v: url_or_none(v['url']))):
|
||||
subtitles.setdefault(sub.get('languageCode') or 'deu', []).append({
|
||||
'url': sources['url'],
|
||||
'ext': {'webvtt': 'vtt', 'ebutt': 'ttml'}.get(sources.get('kind')),
|
||||
})
|
||||
|
||||
age_limit = traverse_obj(page_data, ('fskRating', {lambda x: remove_start(x, 'FSK')}, {int_or_none}))
|
||||
old_id = traverse_obj(page_data, ('tracking', 'atiCustomVars', 'contentId'))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'is_live': is_live,
|
||||
'age_limit': age_limit,
|
||||
**traverse_obj(media_data, ('meta', {
|
||||
'title': 'title',
|
||||
'description': 'synopsis',
|
||||
'timestamp': ('broadcastedOnDateTime', {parse_iso8601}),
|
||||
'series': 'seriesTitle',
|
||||
'thumbnail': ('images', 0, 'url', {url_or_none}),
|
||||
'duration': ('durationSeconds', {int_or_none}),
|
||||
'channel': 'clipSourceName',
|
||||
})),
|
||||
**self._extract_episode_info(page_data.get('title')),
|
||||
'_old_archive_ids': [make_archive_id(ARDBetaMediathekIE, old_id)],
|
||||
}
|
||||
|
||||
|
||||
class ARDMediathekCollectionIE(InfoExtractor):
|
||||
_VALID_URL = r'''(?x)https://
|
||||
(?:(?:beta|www)\.)?ardmediathek\.de/
|
||||
(?:[^/?#]+/)?
|
||||
(?P<playlist>sendung|serie|sammlung)/
|
||||
(?:(?P<display_id>[^?#]+?)/)?
|
||||
(?P<id>[a-zA-Z0-9]+)
|
||||
(?:/(?P<season>\d+)(?:/(?P<version>OV|AD))?)?/?(?:[?#]|$)'''
|
||||
_GEO_COUNTRIES = ['DE']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.ardmediathek.de/serie/quiz/staffel-1-originalversion/Y3JpZDovL3dkci5kZS9vbmUvcXVpeg/1/OV',
|
||||
'info_dict': {
|
||||
'id': 'Y3JpZDovL3dkci5kZS9vbmUvcXVpeg_1_OV',
|
||||
'display_id': 'quiz/staffel-1-originalversion',
|
||||
'title': 'Staffel 1 Originalversion',
|
||||
},
|
||||
'playlist_count': 3,
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/serie/babylon-berlin/staffel-4-mit-audiodeskription/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu/4/AD',
|
||||
'info_dict': {
|
||||
'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu_4_AD',
|
||||
'display_id': 'babylon-berlin/staffel-4-mit-audiodeskription',
|
||||
'title': 'Staffel 4 mit Audiodeskription',
|
||||
},
|
||||
'playlist_count': 12,
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/serie/babylon-berlin/staffel-1/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu/1/',
|
||||
'info_dict': {
|
||||
'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu_1',
|
||||
'display_id': 'babylon-berlin/staffel-1',
|
||||
'title': 'Staffel 1',
|
||||
},
|
||||
'playlist_count': 8,
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/sendung/tatort/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydA',
|
||||
'info_dict': {
|
||||
'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydA',
|
||||
'display_id': 'tatort',
|
||||
'title': 'Tatort',
|
||||
},
|
||||
'playlist_mincount': 500,
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2',
|
||||
'info_dict': {
|
||||
'id': '5eOHzt8XB2sqeFXbIoJlg2',
|
||||
'display_id': 'die-kirche-bleibt-im-dorf',
|
||||
'title': 'Die Kirche bleibt im Dorf',
|
||||
'description': 'Die Kirche bleibt im Dorf',
|
||||
},
|
||||
'playlist_count': 4,
|
||||
}, {
|
||||
# playlist of type 'sendung'
|
||||
'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
# playlist of type 'serie'
|
||||
'url': 'https://www.ardmediathek.de/serie/nachtstreife/staffel-1/Y3JpZDovL3N3ci5kZS9zZGIvc3RJZC8xMjQy/1',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
# playlist of type 'sammlung'
|
||||
'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3dkci5kZS9CZWl0cmFnLWQ2NDJjYWEzLTMwZWYtNGI4NS1iMTI2LTU1N2UxYTcxOGIzOQ/tatort-duo-koeln-leipzig-ihr-kinderlein-kommet',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, pageNumber):
|
||||
""" Query the ARD server for playlist information
|
||||
and returns the data in "raw" format """
|
||||
if mode == 'sendung':
|
||||
graphQL = json.dumps({
|
||||
'query': '''{
|
||||
showPage(
|
||||
client: "%s"
|
||||
showId: "%s"
|
||||
pageNumber: %d
|
||||
) {
|
||||
pagination {
|
||||
pageSize
|
||||
totalElements
|
||||
}
|
||||
teasers { # Array
|
||||
mediumTitle
|
||||
links { target { id href title } }
|
||||
type
|
||||
}
|
||||
}}''' % (client, playlist_id, pageNumber),
|
||||
}).encode()
|
||||
else: # mode == 'sammlung'
|
||||
graphQL = json.dumps({
|
||||
'query': '''{
|
||||
morePage(
|
||||
client: "%s"
|
||||
compilationId: "%s"
|
||||
pageNumber: %d
|
||||
) {
|
||||
widget {
|
||||
pagination {
|
||||
pageSize
|
||||
totalElements
|
||||
}
|
||||
teasers { # Array
|
||||
mediumTitle
|
||||
links { target { id href title } }
|
||||
type
|
||||
}
|
||||
}
|
||||
}}''' % (client, playlist_id, pageNumber),
|
||||
}).encode()
|
||||
# Ressources for ARD graphQL debugging:
|
||||
# https://api-test.ardmediathek.de/public-gateway
|
||||
show_page = self._download_json(
|
||||
'https://api.ardmediathek.de/public-gateway',
|
||||
'[Playlist] %s' % display_id,
|
||||
data=graphQL,
|
||||
headers={'Content-Type': 'application/json'})['data']
|
||||
# align the structure of the returned data:
|
||||
if mode == 'sendung':
|
||||
show_page = show_page['showPage']
|
||||
else: # mode == 'sammlung'
|
||||
show_page = show_page['morePage']['widget']
|
||||
return show_page
|
||||
|
||||
def _ARD_extract_playlist(self, url, playlist_id, display_id, client, mode):
|
||||
""" Collects all playlist entries and returns them as info dict.
|
||||
Supports playlists of mode 'sendung' and 'sammlung', and also nested
|
||||
playlists. """
|
||||
entries = []
|
||||
pageNumber = 0
|
||||
while True: # iterate by pageNumber
|
||||
show_page = self._ARD_load_playlist_snipped(
|
||||
playlist_id, display_id, client, mode, pageNumber)
|
||||
for teaser in show_page['teasers']: # process playlist items
|
||||
if '/compilation/' in teaser['links']['target']['href']:
|
||||
# alternativ cond.: teaser['type'] == "compilation"
|
||||
# => This is an nested compilation, e.g. like:
|
||||
# https://www.ardmediathek.de/ard/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2/
|
||||
link_mode = 'sammlung'
|
||||
else:
|
||||
link_mode = 'video'
|
||||
|
||||
item_url = 'https://www.ardmediathek.de/%s/%s/%s/%s/%s' % (
|
||||
client, link_mode, display_id,
|
||||
# perform HTLM quoting of episode title similar to ARD:
|
||||
re.sub('^-|-$', '', # remove '-' from begin/end
|
||||
re.sub('[^a-zA-Z0-9]+', '-', # replace special chars by -
|
||||
teaser['links']['target']['title'].lower()
|
||||
.replace('ä', 'ae').replace('ö', 'oe')
|
||||
.replace('ü', 'ue').replace('ß', 'ss'))),
|
||||
teaser['links']['target']['id'])
|
||||
entries.append(self.url_result(
|
||||
item_url,
|
||||
ie=ARDBetaMediathekIE.ie_key()))
|
||||
|
||||
if (show_page['pagination']['pageSize'] * (pageNumber + 1)
|
||||
>= show_page['pagination']['totalElements']):
|
||||
# we've processed enough pages to get all playlist entries
|
||||
break
|
||||
pageNumber = pageNumber + 1
|
||||
|
||||
return self.playlist_result(entries, playlist_id, playlist_title=display_id)
|
||||
_PAGE_SIZE = 100
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id, display_id, playlist_type, client, season_number = self._match_valid_url(url).group(
|
||||
'id', 'display_id', 'playlist', 'client', 'season')
|
||||
display_id, client = display_id or video_id, client or 'ard'
|
||||
playlist_id, display_id, playlist_type, season_number, version = self._match_valid_url(url).group(
|
||||
'id', 'display_id', 'playlist', 'season', 'version')
|
||||
|
||||
if playlist_type:
|
||||
# TODO: Extract only specified season
|
||||
return self._ARD_extract_playlist(url, video_id, display_id, client, playlist_type)
|
||||
def call_api(page_num):
|
||||
api_path = 'compilations/ard' if playlist_type == 'sammlung' else 'widgets/ard/asset'
|
||||
return self._download_json(
|
||||
f'https://api.ardmediathek.de/page-gateway/{api_path}/{playlist_id}', playlist_id,
|
||||
f'Downloading playlist page {page_num}', query={
|
||||
'pageNumber': page_num,
|
||||
'pageSize': self._PAGE_SIZE,
|
||||
**({
|
||||
'seasoned': 'true',
|
||||
'seasonNumber': season_number,
|
||||
'withOriginalversion': 'true' if version == 'OV' else 'false',
|
||||
'withAudiodescription': 'true' if version == 'AD' else 'false',
|
||||
} if season_number else {}),
|
||||
})
|
||||
|
||||
player_page = self._download_json(
|
||||
'https://api.ardmediathek.de/public-gateway',
|
||||
display_id, data=json.dumps({
|
||||
'query': '''{
|
||||
playerPage(client:"%s", clipId: "%s") {
|
||||
blockedByFsk
|
||||
broadcastedOn
|
||||
maturityContentRating
|
||||
mediaCollection {
|
||||
_duration
|
||||
_geoblocked
|
||||
_isLive
|
||||
_mediaArray {
|
||||
_mediaStreamArray {
|
||||
_quality
|
||||
_server
|
||||
_stream
|
||||
}
|
||||
}
|
||||
_previewImage
|
||||
_subtitleUrl
|
||||
_type
|
||||
}
|
||||
show {
|
||||
title
|
||||
}
|
||||
image {
|
||||
src
|
||||
}
|
||||
synopsis
|
||||
title
|
||||
tracking {
|
||||
atiCustomVars {
|
||||
contentId
|
||||
}
|
||||
}
|
||||
}
|
||||
}''' % (client, video_id),
|
||||
}).encode(), headers={
|
||||
'Content-Type': 'application/json'
|
||||
})['data']['playerPage']
|
||||
title = player_page['title']
|
||||
content_id = str_or_none(try_get(
|
||||
player_page, lambda x: x['tracking']['atiCustomVars']['contentId']))
|
||||
media_collection = player_page.get('mediaCollection') or {}
|
||||
if not media_collection and content_id:
|
||||
media_collection = self._download_json(
|
||||
'https://www.ardmediathek.de/play/media/' + content_id,
|
||||
content_id, fatal=False) or {}
|
||||
info = self._parse_media_info(
|
||||
media_collection, content_id or video_id,
|
||||
player_page.get('blockedByFsk'))
|
||||
age_limit = None
|
||||
description = player_page.get('synopsis')
|
||||
maturity_content_rating = player_page.get('maturityContentRating')
|
||||
if maturity_content_rating:
|
||||
age_limit = int_or_none(maturity_content_rating.lstrip('FSK'))
|
||||
if not age_limit and description:
|
||||
age_limit = int_or_none(self._search_regex(
|
||||
r'\(FSK\s*(\d+)\)\s*$', description, 'age limit', default=None))
|
||||
info.update({
|
||||
'age_limit': age_limit,
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'timestamp': unified_timestamp(player_page.get('broadcastedOn')),
|
||||
'series': try_get(player_page, lambda x: x['show']['title']),
|
||||
'thumbnail': (media_collection.get('_previewImage')
|
||||
or try_get(player_page, lambda x: update_url(x['image']['src'], query=None, fragment=None))
|
||||
or self.get_thumbnail_from_html(display_id, url)),
|
||||
})
|
||||
info.update(self._ARD_extract_episode_info(info['title']))
|
||||
return info
|
||||
def fetch_page(page_num):
|
||||
for item in traverse_obj(call_api(page_num), ('teasers', ..., {dict})):
|
||||
item_id = traverse_obj(item, ('links', 'target', ('urlId', 'id')), 'id', get_all=False)
|
||||
if not item_id or item_id == playlist_id:
|
||||
continue
|
||||
item_mode = 'sammlung' if item.get('type') == 'compilation' else 'video'
|
||||
yield self.url_result(
|
||||
f'https://www.ardmediathek.de/{item_mode}/{item_id}',
|
||||
ie=(ARDMediathekCollectionIE if item_mode == 'sammlung' else ARDBetaMediathekIE),
|
||||
**traverse_obj(item, {
|
||||
'id': ('id', {str}),
|
||||
'title': ('longTitle', {str}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'timestamp': ('broadcastedOn', {parse_iso8601}),
|
||||
}))
|
||||
|
||||
def get_thumbnail_from_html(self, display_id, url):
|
||||
webpage = self._download_webpage(url, display_id, fatal=False) or ''
|
||||
return (
|
||||
self._og_search_thumbnail(webpage, default=None)
|
||||
or self._html_search_meta('thumbnailUrl', webpage, default=None))
|
||||
page_data = call_api(0)
|
||||
full_id = join_nonempty(playlist_id, season_number, version, delim='_')
|
||||
|
||||
return self.playlist_result(
|
||||
OnDemandPagedList(fetch_page, self._PAGE_SIZE), full_id, display_id=display_id,
|
||||
title=page_data.get('title'), description=page_data.get('synopsis'))
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import unified_strdate
|
||||
|
||||
|
||||
class ATTTechChannelIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://techchannel\.att\.com/play-video\.cfm/([^/]+/)*(?P<id>.+)'
|
||||
_TEST = {
|
||||
'url': 'http://techchannel.att.com/play-video.cfm/2014/1/27/ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
|
||||
'info_dict': {
|
||||
'id': '11316',
|
||||
'display_id': 'ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
|
||||
'ext': 'flv',
|
||||
'title': 'AT&T Archives : The UNIX System: Making Computers Easier to Use',
|
||||
'description': 'A 1982 film about UNIX is the foundation for software in use around Bell Labs and AT&T.',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'upload_date': '20140127',
|
||||
},
|
||||
'params': {
|
||||
# rtmp download
|
||||
'skip_download': True,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
video_url = self._search_regex(
|
||||
r"url\s*:\s*'(rtmp://[^']+)'",
|
||||
webpage, 'video URL')
|
||||
|
||||
video_id = self._search_regex(
|
||||
r'mediaid\s*=\s*(\d+)',
|
||||
webpage, 'video id', fatal=False)
|
||||
|
||||
title = self._og_search_title(webpage)
|
||||
description = self._og_search_description(webpage)
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
upload_date = unified_strdate(self._search_regex(
|
||||
r'[Rr]elease\s+date:\s*(\d{1,2}/\d{1,2}/\d{4})',
|
||||
webpage, 'upload date', fatal=False), False)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'url': video_url,
|
||||
'ext': 'flv',
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
'upload_date': upload_date,
|
||||
}
|
||||
@@ -152,7 +152,7 @@ class BanByeChannelIE(BanByeBaseIE):
|
||||
'sort': 'new',
|
||||
'limit': self._PAGE_SIZE,
|
||||
'offset': page_num * self._PAGE_SIZE,
|
||||
}, note=f'Downloading page {page_num+1}')
|
||||
}, note=f'Downloading page {page_num + 1}')
|
||||
return [
|
||||
self.url_result(f"{self._VIDEO_BASE}/{video['_id']}", BanByeIE)
|
||||
for video in data['items']
|
||||
|
||||
@@ -317,16 +317,25 @@ class BBCCoUkIE(InfoExtractor):
|
||||
|
||||
def _download_media_selector(self, programme_id):
|
||||
last_exception = None
|
||||
formats, subtitles = [], {}
|
||||
for media_set in self._MEDIA_SETS:
|
||||
try:
|
||||
return self._download_media_selector_url(
|
||||
fmts, subs = self._download_media_selector_url(
|
||||
self._MEDIA_SELECTOR_URL_TEMPL % (media_set, programme_id), programme_id)
|
||||
formats.extend(fmts)
|
||||
if subs:
|
||||
self._merge_subtitles(subs, target=subtitles)
|
||||
except BBCCoUkIE.MediaSelectionError as e:
|
||||
if e.id in ('notukerror', 'geolocation', 'selectionunavailable'):
|
||||
last_exception = e
|
||||
continue
|
||||
self._raise_extractor_error(e)
|
||||
self._raise_extractor_error(last_exception)
|
||||
if last_exception:
|
||||
if formats or subtitles:
|
||||
self.report_warning(f'{self.IE_NAME} returned error: {last_exception.id}')
|
||||
else:
|
||||
self._raise_extractor_error(last_exception)
|
||||
return formats, subtitles
|
||||
|
||||
def _download_media_selector_url(self, url, programme_id=None):
|
||||
media_selection = self._download_json(
|
||||
@@ -1188,7 +1197,7 @@ class BBCIE(BBCCoUkIE): # XXX: Do not subclass from concrete IE
|
||||
if initial_data is None:
|
||||
initial_data = self._search_regex(
|
||||
r'window\.__INITIAL_DATA__\s*=\s*({.+?})\s*;', webpage,
|
||||
'preload state', default={})
|
||||
'preload state', default='{}')
|
||||
else:
|
||||
initial_data = self._parse_json(initial_data or '"{}"', playlist_id, fatal=False)
|
||||
initial_data = self._parse_json(initial_data, playlist_id, fatal=False)
|
||||
|
||||
@@ -3,6 +3,7 @@ from ..utils import url_basename
|
||||
|
||||
|
||||
class BehindKinkIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'https?://(?:www\.)?behindkink\.com/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<id>[^/#?_]+)'
|
||||
_TEST = {
|
||||
'url': 'http://www.behindkink.com/2014/12/05/what-are-you-passionate-about-marley-blaze/',
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
from .mtv import MTVServicesInfoExtractor
|
||||
from ..utils import unified_strdate
|
||||
|
||||
# TODO Remove - Reason: Outdated Site
|
||||
|
||||
|
||||
class BetIE(MTVServicesInfoExtractor):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'
|
||||
_TESTS = [
|
||||
{
|
||||
|
||||
@@ -5,6 +5,7 @@ from ..utils import extract_attributes
|
||||
|
||||
|
||||
class BFIPlayerIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
IE_NAME = 'bfi:player'
|
||||
_VALID_URL = r'https?://player\.bfi\.org\.uk/[^/]+/film/watch-(?P<id>[\w-]+)-online'
|
||||
_TEST = {
|
||||
|
||||
@@ -7,7 +7,7 @@ from ..utils import extract_attributes
|
||||
class BFMTVBaseIE(InfoExtractor):
|
||||
_VALID_URL_BASE = r'https?://(?:www\.|rmc\.)?bfmtv\.com/'
|
||||
_VALID_URL_TMPL = _VALID_URL_BASE + r'(?:[^/]+/)*[^/?&#]+_%s[A-Z]-(?P<id>\d{12})\.html'
|
||||
_VIDEO_BLOCK_REGEX = r'(<div[^>]+class="video_block"[^>]*>)'
|
||||
_VIDEO_BLOCK_REGEX = r'(<div[^>]+class="video_block[^"]*"[^>]*>)'
|
||||
BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
|
||||
|
||||
def _brightcove_url_result(self, video_id, video_block):
|
||||
@@ -55,8 +55,11 @@ class BFMTVLiveIE(BFMTVIE): # XXX: Do not subclass from concrete IE
|
||||
'ext': 'mp4',
|
||||
'title': r're:^le direct BFMTV WEB \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
|
||||
'uploader_id': '876450610001',
|
||||
'upload_date': '20171018',
|
||||
'timestamp': 1508329950,
|
||||
'upload_date': '20220926',
|
||||
'timestamp': 1664207191,
|
||||
'live_status': 'is_live',
|
||||
'thumbnail': r're:https://.+/image\.jpg',
|
||||
'tags': [],
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
|
||||
@@ -2,6 +2,7 @@ import base64
|
||||
import functools
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
@@ -16,9 +17,11 @@ from ..utils import (
|
||||
InAdvancePagedList,
|
||||
OnDemandPagedList,
|
||||
bool_or_none,
|
||||
clean_html,
|
||||
filter_dict,
|
||||
float_or_none,
|
||||
format_field,
|
||||
get_element_by_class,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
make_archive_id,
|
||||
@@ -88,6 +91,12 @@ class BilibiliBaseIE(InfoExtractor):
|
||||
|
||||
return formats
|
||||
|
||||
def _download_playinfo(self, video_id, cid):
|
||||
return self._download_json(
|
||||
'https://api.bilibili.com/x/player/playurl', video_id,
|
||||
query={'bvid': video_id, 'cid': cid, 'fnval': 4048},
|
||||
note=f'Downloading video formats for cid {cid}')['data']
|
||||
|
||||
def json2srt(self, json_data):
|
||||
srt_data = ''
|
||||
for idx, line in enumerate(json_data.get('body') or []):
|
||||
@@ -96,7 +105,7 @@ class BilibiliBaseIE(InfoExtractor):
|
||||
f'{line["content"]}\n\n')
|
||||
return srt_data
|
||||
|
||||
def _get_subtitles(self, video_id, aid, cid):
|
||||
def _get_subtitles(self, video_id, cid, aid=None):
|
||||
subtitles = {
|
||||
'danmaku': [{
|
||||
'ext': 'xml',
|
||||
@@ -104,8 +113,15 @@ class BilibiliBaseIE(InfoExtractor):
|
||||
}]
|
||||
}
|
||||
|
||||
video_info_json = self._download_json(f'https://api.bilibili.com/x/player/v2?aid={aid}&cid={cid}', video_id)
|
||||
for s in traverse_obj(video_info_json, ('data', 'subtitle', 'subtitles', ...)):
|
||||
subtitle_info = traverse_obj(self._download_json(
|
||||
'https://api.bilibili.com/x/player/v2', video_id,
|
||||
query={'aid': aid, 'cid': cid} if aid else {'bvid': video_id, 'cid': cid},
|
||||
note=f'Extracting subtitle info {cid}'), ('data', 'subtitle'))
|
||||
subs_list = traverse_obj(subtitle_info, ('subtitles', lambda _, v: v['subtitle_url'] and v['lan']))
|
||||
if not subs_list and traverse_obj(subtitle_info, 'allow_submit'):
|
||||
if not self._get_cookies('https://api.bilibili.com').get('SESSDATA'): # no login session cookie
|
||||
self.report_warning(f'CC subtitles (if any) are only visible when logged in. {self._login_hint()}', only_once=True)
|
||||
for s in subs_list:
|
||||
subtitles.setdefault(s['lan'], []).append({
|
||||
'ext': 'srt',
|
||||
'data': self.json2srt(self._download_json(s['subtitle_url'], video_id))
|
||||
@@ -155,7 +171,54 @@ class BilibiliBaseIE(InfoExtractor):
|
||||
for entry in traverse_obj(season_info, (
|
||||
'result', 'main_section', 'episodes',
|
||||
lambda _, v: url_or_none(v['share_url']) and v['id'])):
|
||||
yield self.url_result(entry['share_url'], BiliBiliBangumiIE, f'ep{entry["id"]}')
|
||||
yield self.url_result(entry['share_url'], BiliBiliBangumiIE, str_or_none(entry.get('id')))
|
||||
|
||||
def _get_divisions(self, video_id, graph_version, edges, edge_id, cid_edges=None):
|
||||
cid_edges = cid_edges or {}
|
||||
division_data = self._download_json(
|
||||
'https://api.bilibili.com/x/stein/edgeinfo_v2', video_id,
|
||||
query={'graph_version': graph_version, 'edge_id': edge_id, 'bvid': video_id},
|
||||
note=f'Extracting divisions from edge {edge_id}')
|
||||
edges.setdefault(edge_id, {}).update(
|
||||
traverse_obj(division_data, ('data', 'story_list', lambda _, v: v['edge_id'] == edge_id, {
|
||||
'title': ('title', {str}),
|
||||
'cid': ('cid', {int_or_none}),
|
||||
}), get_all=False))
|
||||
|
||||
edges[edge_id].update(traverse_obj(division_data, ('data', {
|
||||
'title': ('title', {str}),
|
||||
'choices': ('edges', 'questions', ..., 'choices', ..., {
|
||||
'edge_id': ('id', {int_or_none}),
|
||||
'cid': ('cid', {int_or_none}),
|
||||
'text': ('option', {str}),
|
||||
}),
|
||||
})))
|
||||
# use dict to combine edges that use the same video section (same cid)
|
||||
cid_edges.setdefault(edges[edge_id]['cid'], {})[edge_id] = edges[edge_id]
|
||||
for choice in traverse_obj(edges, (edge_id, 'choices', ...)):
|
||||
if choice['edge_id'] not in edges:
|
||||
edges[choice['edge_id']] = {'cid': choice['cid']}
|
||||
self._get_divisions(video_id, graph_version, edges, choice['edge_id'], cid_edges=cid_edges)
|
||||
return cid_edges
|
||||
|
||||
def _get_interactive_entries(self, video_id, cid, metainfo):
|
||||
graph_version = traverse_obj(
|
||||
self._download_json(
|
||||
'https://api.bilibili.com/x/player/wbi/v2', video_id,
|
||||
'Extracting graph version', query={'bvid': video_id, 'cid': cid}),
|
||||
('data', 'interaction', 'graph_version', {int_or_none}))
|
||||
cid_edges = self._get_divisions(video_id, graph_version, {1: {'cid': cid}}, 1)
|
||||
for cid, edges in cid_edges.items():
|
||||
play_info = self._download_playinfo(video_id, cid)
|
||||
yield {
|
||||
**metainfo,
|
||||
'id': f'{video_id}_{cid}',
|
||||
'title': f'{metainfo.get("title")} - {list(edges.values())[0].get("title")}',
|
||||
'formats': self.extract_formats(play_info),
|
||||
'description': f'{json.dumps(edges, ensure_ascii=False)}\n{metainfo.get("description", "")}',
|
||||
'duration': float_or_none(play_info.get('timelength'), scale=1000),
|
||||
'subtitles': self.extract_subtitles(video_id, cid),
|
||||
}
|
||||
|
||||
|
||||
class BiliBiliIE(BilibiliBaseIE):
|
||||
@@ -180,7 +243,7 @@ class BiliBiliIE(BilibiliBaseIE):
|
||||
'view_count': int,
|
||||
},
|
||||
}, {
|
||||
# old av URL version
|
||||
'note': 'old av URL version',
|
||||
'url': 'http://www.bilibili.com/video/av1074402/',
|
||||
'info_dict': {
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg)$',
|
||||
@@ -212,7 +275,7 @@ class BiliBiliIE(BilibiliBaseIE):
|
||||
'id': 'BV1bK411W797_p1',
|
||||
'ext': 'mp4',
|
||||
'title': '物语中的人物是如何吐槽自己的OP的 p01 Staple Stable/战场原+羽川',
|
||||
'tags': 'count:11',
|
||||
'tags': 'count:10',
|
||||
'timestamp': 1589601697,
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
|
||||
'uploader': '打牌还是打桩',
|
||||
@@ -232,7 +295,7 @@ class BiliBiliIE(BilibiliBaseIE):
|
||||
'id': 'BV1bK411W797_p1',
|
||||
'ext': 'mp4',
|
||||
'title': '物语中的人物是如何吐槽自己的OP的 p01 Staple Stable/战场原+羽川',
|
||||
'tags': 'count:11',
|
||||
'tags': 'count:10',
|
||||
'timestamp': 1589601697,
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
|
||||
'uploader': '打牌还是打桩',
|
||||
@@ -343,18 +406,120 @@ class BiliBiliIE(BilibiliBaseIE):
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
|
||||
},
|
||||
'params': {'skip_download': True},
|
||||
}, {
|
||||
'note': 'interactive/split-path video',
|
||||
'url': 'https://www.bilibili.com/video/BV1af4y1H7ga/',
|
||||
'info_dict': {
|
||||
'id': 'BV1af4y1H7ga',
|
||||
'title': '【互动游戏】花了大半年时间做的自我介绍~请查收!!',
|
||||
'timestamp': 1630500414,
|
||||
'upload_date': '20210901',
|
||||
'description': 'md5:01113e39ab06e28042d74ac356a08786',
|
||||
'tags': list,
|
||||
'uploader': '钉宫妮妮Ninico',
|
||||
'duration': 1503,
|
||||
'uploader_id': '8881297',
|
||||
'comment_count': int,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
|
||||
},
|
||||
'playlist_count': 33,
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': 'BV1af4y1H7ga_400950101',
|
||||
'ext': 'mp4',
|
||||
'title': '【互动游戏】花了大半年时间做的自我介绍~请查收!! - 听见猫猫叫~',
|
||||
'timestamp': 1630500414,
|
||||
'upload_date': '20210901',
|
||||
'description': 'md5:db66ac7a2813a94b8291dbce990cc5b2',
|
||||
'tags': list,
|
||||
'uploader': '钉宫妮妮Ninico',
|
||||
'duration': 11.605,
|
||||
'uploader_id': '8881297',
|
||||
'comment_count': int,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
|
||||
},
|
||||
}],
|
||||
}, {
|
||||
'note': '301 redirect to bangumi link',
|
||||
'url': 'https://www.bilibili.com/video/BV1TE411f7f1',
|
||||
'info_dict': {
|
||||
'id': '288525',
|
||||
'title': '李永乐老师 钱学森弹道和乘波体飞行器是什么?',
|
||||
'ext': 'mp4',
|
||||
'series': '我和我的祖国',
|
||||
'series_id': '4780',
|
||||
'season': '幕后纪实',
|
||||
'season_id': '28609',
|
||||
'season_number': 1,
|
||||
'episode': '钱学森弹道和乘波体飞行器是什么?',
|
||||
'episode_id': '288525',
|
||||
'episode_number': 105,
|
||||
'duration': 1183.957,
|
||||
'timestamp': 1571648124,
|
||||
'upload_date': '20191021',
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.bilibili.com/video/BV1jL41167ZG/',
|
||||
'info_dict': {
|
||||
'id': 'BV1jL41167ZG',
|
||||
'title': '一场大火引发的离奇死亡!古典推理经典短篇集《不可能犯罪诊断书》!',
|
||||
'ext': 'mp4',
|
||||
},
|
||||
'skip': 'supporter-only video',
|
||||
}, {
|
||||
'url': 'https://www.bilibili.com/video/BV1Ks411f7aQ/',
|
||||
'info_dict': {
|
||||
'id': 'BV1Ks411f7aQ',
|
||||
'title': '【BD1080P】狼与香辛料I【华盟】',
|
||||
'ext': 'mp4',
|
||||
},
|
||||
'skip': 'login required',
|
||||
}, {
|
||||
'url': 'https://www.bilibili.com/video/BV1GJ411x7h7/',
|
||||
'info_dict': {
|
||||
'id': 'BV1GJ411x7h7',
|
||||
'title': '【官方 MV】Never Gonna Give You Up - Rick Astley',
|
||||
'ext': 'mp4',
|
||||
},
|
||||
'skip': 'geo-restricted',
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
webpage, urlh = self._download_webpage_handle(url, video_id)
|
||||
if not self._match_valid_url(urlh.url):
|
||||
return self.url_result(urlh.url)
|
||||
|
||||
initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', video_id)
|
||||
|
||||
is_festival = 'videoData' not in initial_state
|
||||
if is_festival:
|
||||
video_data = initial_state['videoInfo']
|
||||
else:
|
||||
play_info = self._search_json(r'window\.__playinfo__\s*=', webpage, 'play info', video_id)['data']
|
||||
play_info_obj = self._search_json(
|
||||
r'window\.__playinfo__\s*=', webpage, 'play info', video_id, fatal=False)
|
||||
if not play_info_obj:
|
||||
if traverse_obj(initial_state, ('error', 'trueCode')) == -403:
|
||||
self.raise_login_required()
|
||||
if traverse_obj(initial_state, ('error', 'trueCode')) == -404:
|
||||
raise ExtractorError(
|
||||
'This video may be deleted or geo-restricted. '
|
||||
'You might want to try a VPN or a proxy server (with --proxy)', expected=True)
|
||||
play_info = traverse_obj(play_info_obj, ('data', {dict}))
|
||||
if not play_info:
|
||||
if traverse_obj(play_info_obj, 'code') == 87007:
|
||||
toast = get_element_by_class('tips-toast', webpage) or ''
|
||||
msg = clean_html(
|
||||
f'{get_element_by_class("belongs-to", toast) or ""},'
|
||||
+ (get_element_by_class('level', toast) or ''))
|
||||
raise ExtractorError(
|
||||
f'This is a supporter-only video: {msg}. {self._login_hint()}', expected=True)
|
||||
raise ExtractorError('Failed to extract play info')
|
||||
video_data = initial_state['videoData']
|
||||
|
||||
video_id, title = video_data['bvid'], video_data.get('title')
|
||||
@@ -385,10 +550,7 @@ class BiliBiliIE(BilibiliBaseIE):
|
||||
|
||||
festival_info = {}
|
||||
if is_festival:
|
||||
play_info = self._download_json(
|
||||
'https://api.bilibili.com/x/player/playurl', video_id,
|
||||
query={'bvid': video_id, 'cid': cid, 'fnval': 4048},
|
||||
note='Extracting festival video formats')['data']
|
||||
play_info = self._download_playinfo(video_id, cid)
|
||||
|
||||
festival_info = traverse_obj(initial_state, {
|
||||
'uploader': ('videoInfo', 'upName'),
|
||||
@@ -397,7 +559,7 @@ class BiliBiliIE(BilibiliBaseIE):
|
||||
'thumbnail': ('sectionEpisodes', lambda _, v: v['bvid'] == video_id, 'cover'),
|
||||
}, get_all=False)
|
||||
|
||||
return {
|
||||
metainfo = {
|
||||
**traverse_obj(initial_state, {
|
||||
'uploader': ('upData', 'name'),
|
||||
'uploader_id': ('upData', 'mid', {str_or_none}),
|
||||
@@ -413,28 +575,59 @@ class BiliBiliIE(BilibiliBaseIE):
|
||||
'comment_count': ('stat', 'reply', {int_or_none}),
|
||||
}, get_all=False),
|
||||
'id': f'{video_id}{format_field(part_id, None, "_p%d")}',
|
||||
'formats': self.extract_formats(play_info),
|
||||
'_old_archive_ids': [make_archive_id(self, old_video_id)] if old_video_id else None,
|
||||
'title': title,
|
||||
'duration': float_or_none(play_info.get('timelength'), scale=1000),
|
||||
'chapters': self._get_chapters(aid, cid),
|
||||
'subtitles': self.extract_subtitles(video_id, aid, cid),
|
||||
'__post_extractor': self.extract_comments(aid),
|
||||
'http_headers': {'Referer': url},
|
||||
}
|
||||
|
||||
is_interactive = traverse_obj(video_data, ('rights', 'is_stein_gate'))
|
||||
if is_interactive:
|
||||
return self.playlist_result(
|
||||
self._get_interactive_entries(video_id, cid, metainfo), **metainfo, **{
|
||||
'duration': traverse_obj(initial_state, ('videoData', 'duration', {int_or_none})),
|
||||
'__post_extractor': self.extract_comments(aid),
|
||||
})
|
||||
else:
|
||||
return {
|
||||
**metainfo,
|
||||
'duration': float_or_none(play_info.get('timelength'), scale=1000),
|
||||
'chapters': self._get_chapters(aid, cid),
|
||||
'subtitles': self.extract_subtitles(video_id, cid),
|
||||
'formats': self.extract_formats(play_info),
|
||||
'__post_extractor': self.extract_comments(aid),
|
||||
}
|
||||
|
||||
|
||||
class BiliBiliBangumiIE(BilibiliBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?bilibili\.com/bangumi/play/(?P<id>ep\d+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?bilibili\.com/bangumi/play/ep(?P<id>\d+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.bilibili.com/bangumi/play/ep21495/',
|
||||
'info_dict': {
|
||||
'id': '21495',
|
||||
'ext': 'mp4',
|
||||
'series': '悠久之翼',
|
||||
'series_id': '774',
|
||||
'season': '第二季',
|
||||
'season_id': '1182',
|
||||
'season_number': 2,
|
||||
'episode': 'forever/ef',
|
||||
'episode_id': '21495',
|
||||
'episode_number': 12,
|
||||
'title': '12 forever/ef',
|
||||
'duration': 1420.791,
|
||||
'timestamp': 1320412200,
|
||||
'upload_date': '20111104',
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.bilibili.com/bangumi/play/ep267851',
|
||||
'info_dict': {
|
||||
'id': '267851',
|
||||
'ext': 'mp4',
|
||||
'series': '鬼灭之刃',
|
||||
'series_id': '4358',
|
||||
'season': '鬼灭之刃',
|
||||
'season': '立志篇',
|
||||
'season_id': '26801',
|
||||
'season_number': 1,
|
||||
'episode': '残酷',
|
||||
@@ -446,13 +639,32 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
|
||||
'upload_date': '20190406',
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
|
||||
},
|
||||
'skip': 'According to the copyright owner\'s request, you may only watch the video after you are premium member.'
|
||||
'skip': 'Geo-restricted',
|
||||
}, {
|
||||
'note': 'a making-of which falls outside main section',
|
||||
'url': 'https://www.bilibili.com/bangumi/play/ep345120',
|
||||
'info_dict': {
|
||||
'id': '345120',
|
||||
'ext': 'mp4',
|
||||
'series': '鬼灭之刃',
|
||||
'series_id': '4358',
|
||||
'season': '立志篇',
|
||||
'season_id': '26801',
|
||||
'season_number': 1,
|
||||
'episode': '炭治郎篇',
|
||||
'episode_id': '345120',
|
||||
'episode_number': 27,
|
||||
'title': '#1 炭治郎篇',
|
||||
'duration': 1922.129,
|
||||
'timestamp': 1602853860,
|
||||
'upload_date': '20201016',
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
episode_id = video_id[2:]
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
episode_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, episode_id)
|
||||
|
||||
if '您所在的地区无法观看本片' in webpage:
|
||||
raise GeoRestrictedError('This video is restricted')
|
||||
@@ -461,7 +673,7 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
|
||||
|
||||
headers = {'Referer': url, **self.geo_verification_headers()}
|
||||
play_info = self._download_json(
|
||||
'https://api.bilibili.com/pgc/player/web/v2/playurl', video_id,
|
||||
'https://api.bilibili.com/pgc/player/web/v2/playurl', episode_id,
|
||||
'Extracting episode', query={'fnval': '4048', 'ep_id': episode_id},
|
||||
headers=headers)
|
||||
premium_only = play_info.get('code') == -10403
|
||||
@@ -472,40 +684,43 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
|
||||
self.raise_login_required('This video is for premium members only')
|
||||
|
||||
bangumi_info = self._download_json(
|
||||
'https://api.bilibili.com/pgc/view/web/season', video_id, 'Get episode details',
|
||||
'https://api.bilibili.com/pgc/view/web/season', episode_id, 'Get episode details',
|
||||
query={'ep_id': episode_id}, headers=headers)['result']
|
||||
|
||||
episode_number, episode_info = next((
|
||||
(idx, ep) for idx, ep in enumerate(traverse_obj(
|
||||
bangumi_info, ('episodes', ..., {dict})), 1)
|
||||
bangumi_info, (('episodes', ('section', ..., 'episodes')), ..., {dict})), 1)
|
||||
if str_or_none(ep.get('id')) == episode_id), (1, {}))
|
||||
|
||||
season_id = bangumi_info.get('season_id')
|
||||
season_number = season_id and next((
|
||||
idx + 1 for idx, e in enumerate(
|
||||
season_number, season_title = season_id and next((
|
||||
(idx + 1, e.get('season_title')) for idx, e in enumerate(
|
||||
traverse_obj(bangumi_info, ('seasons', ...)))
|
||||
if e.get('season_id') == season_id
|
||||
), None)
|
||||
), (None, None))
|
||||
|
||||
aid = episode_info.get('aid')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'id': episode_id,
|
||||
'formats': formats,
|
||||
**traverse_obj(bangumi_info, {
|
||||
'series': ('series', 'series_title', {str}),
|
||||
'series_id': ('series', 'series_id', {str_or_none}),
|
||||
'thumbnail': ('square_cover', {url_or_none}),
|
||||
}),
|
||||
'title': join_nonempty('title', 'long_title', delim=' ', from_dict=episode_info),
|
||||
'episode': episode_info.get('long_title'),
|
||||
**traverse_obj(episode_info, {
|
||||
'episode': ('long_title', {str}),
|
||||
'episode_number': ('title', {int_or_none}, {lambda x: x or episode_number}),
|
||||
'timestamp': ('pub_time', {int_or_none}),
|
||||
'title': {lambda v: v and join_nonempty('title', 'long_title', delim=' ', from_dict=v)},
|
||||
}),
|
||||
'episode_id': episode_id,
|
||||
'episode_number': int_or_none(episode_info.get('title')) or episode_number,
|
||||
'season': str_or_none(season_title),
|
||||
'season_id': str_or_none(season_id),
|
||||
'season_number': season_number,
|
||||
'timestamp': int_or_none(episode_info.get('pub_time')),
|
||||
'duration': float_or_none(play_info.get('timelength'), scale=1000),
|
||||
'subtitles': self.extract_subtitles(video_id, aid, episode_info.get('cid')),
|
||||
'subtitles': self.extract_subtitles(episode_id, episode_info.get('cid'), aid=aid),
|
||||
'__post_extractor': self.extract_comments(aid),
|
||||
'http_headers': headers,
|
||||
}
|
||||
@@ -517,17 +732,53 @@ class BiliBiliBangumiMediaIE(BilibiliBaseIE):
|
||||
'url': 'https://www.bilibili.com/bangumi/media/md24097891',
|
||||
'info_dict': {
|
||||
'id': '24097891',
|
||||
'title': 'CAROLE & TUESDAY',
|
||||
'description': 'md5:42417ad33d1eaa1c93bfd2dd1626b829',
|
||||
},
|
||||
'playlist_mincount': 25,
|
||||
}, {
|
||||
'url': 'https://www.bilibili.com/bangumi/media/md1565/',
|
||||
'info_dict': {
|
||||
'id': '1565',
|
||||
'title': '攻壳机动队 S.A.C. 2nd GIG',
|
||||
'description': 'md5:46cac00bafd645b97f4d6df616fc576d',
|
||||
},
|
||||
'playlist_count': 26,
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '68540',
|
||||
'ext': 'mp4',
|
||||
'series': '攻壳机动队',
|
||||
'series_id': '1077',
|
||||
'season': '第二季',
|
||||
'season_id': '1565',
|
||||
'season_number': 2,
|
||||
'episode': '再启动 REEMBODY',
|
||||
'episode_id': '68540',
|
||||
'episode_number': 1,
|
||||
'title': '1 再启动 REEMBODY',
|
||||
'duration': 1525.777,
|
||||
'timestamp': 1425074413,
|
||||
'upload_date': '20150227',
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
|
||||
},
|
||||
}],
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
media_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, media_id)
|
||||
ss_id = self._search_json(
|
||||
r'window\.__INITIAL_STATE__\s*=', webpage, 'initial_state', media_id)['mediaInfo']['season_id']
|
||||
|
||||
return self.playlist_result(self._get_episodes_from_season(ss_id, url), media_id)
|
||||
initial_state = self._search_json(
|
||||
r'window\.__INITIAL_STATE__\s*=', webpage, 'initial_state', media_id)
|
||||
ss_id = initial_state['mediaInfo']['season_id']
|
||||
|
||||
return self.playlist_result(
|
||||
self._get_episodes_from_season(ss_id, url), media_id,
|
||||
**traverse_obj(initial_state, ('mediaInfo', {
|
||||
'title': ('title', {str}),
|
||||
'description': ('evaluate', {str}),
|
||||
})))
|
||||
|
||||
|
||||
class BiliBiliBangumiSeasonIE(BilibiliBaseIE):
|
||||
@@ -535,15 +786,183 @@ class BiliBiliBangumiSeasonIE(BilibiliBaseIE):
|
||||
_TESTS = [{
|
||||
'url': 'https://www.bilibili.com/bangumi/play/ss26801',
|
||||
'info_dict': {
|
||||
'id': '26801'
|
||||
'id': '26801',
|
||||
'title': '鬼灭之刃',
|
||||
'description': 'md5:e2cc9848b6f69be6db79fc2a82d9661b',
|
||||
},
|
||||
'playlist_mincount': 26
|
||||
}, {
|
||||
'url': 'https://www.bilibili.com/bangumi/play/ss2251',
|
||||
'info_dict': {
|
||||
'id': '2251',
|
||||
'title': '玲音',
|
||||
'description': 'md5:1fd40e3df4c08d4d9d89a6a34844bdc4',
|
||||
},
|
||||
'playlist_count': 13,
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '50188',
|
||||
'ext': 'mp4',
|
||||
'series': '玲音',
|
||||
'series_id': '1526',
|
||||
'season': 'TV',
|
||||
'season_id': '2251',
|
||||
'season_number': 1,
|
||||
'episode': 'WEIRD',
|
||||
'episode_id': '50188',
|
||||
'episode_number': 1,
|
||||
'title': '1 WEIRD',
|
||||
'duration': 1436.992,
|
||||
'timestamp': 1343185080,
|
||||
'upload_date': '20120725',
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
|
||||
},
|
||||
}],
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
ss_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, ss_id)
|
||||
metainfo = traverse_obj(
|
||||
self._search_json(r'<script[^>]+type="application/ld\+json"[^>]*>', webpage, 'info', ss_id),
|
||||
('itemListElement', ..., {
|
||||
'title': ('name', {str}),
|
||||
'description': ('description', {str}),
|
||||
}), get_all=False)
|
||||
|
||||
return self.playlist_result(self._get_episodes_from_season(ss_id, url), ss_id)
|
||||
return self.playlist_result(self._get_episodes_from_season(ss_id, url), ss_id, **metainfo)
|
||||
|
||||
|
||||
class BilibiliCheeseBaseIE(BilibiliBaseIE):
|
||||
_HEADERS = {'Referer': 'https://www.bilibili.com/'}
|
||||
|
||||
def _extract_episode(self, season_info, ep_id):
|
||||
episode_info = traverse_obj(season_info, (
|
||||
'episodes', lambda _, v: v['id'] == int(ep_id)), get_all=False)
|
||||
aid, cid = episode_info['aid'], episode_info['cid']
|
||||
|
||||
if traverse_obj(episode_info, 'ep_status') == -1:
|
||||
raise ExtractorError('This course episode is not yet available.', expected=True)
|
||||
if not traverse_obj(episode_info, 'playable'):
|
||||
self.raise_login_required('You need to purchase the course to download this episode')
|
||||
|
||||
play_info = self._download_json(
|
||||
'https://api.bilibili.com/pugv/player/web/playurl', ep_id,
|
||||
query={'avid': aid, 'cid': cid, 'ep_id': ep_id, 'fnval': 16, 'fourk': 1},
|
||||
headers=self._HEADERS, note='Downloading playinfo')['data']
|
||||
|
||||
return {
|
||||
'id': str_or_none(ep_id),
|
||||
'episode_id': str_or_none(ep_id),
|
||||
'formats': self.extract_formats(play_info),
|
||||
'extractor_key': BilibiliCheeseIE.ie_key(),
|
||||
'extractor': BilibiliCheeseIE.IE_NAME,
|
||||
'webpage_url': f'https://www.bilibili.com/cheese/play/ep{ep_id}',
|
||||
**traverse_obj(episode_info, {
|
||||
'episode': ('title', {str}),
|
||||
'title': {lambda v: v and join_nonempty('index', 'title', delim=' - ', from_dict=v)},
|
||||
'alt_title': ('subtitle', {str}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'episode_number': ('index', {int_or_none}),
|
||||
'thumbnail': ('cover', {url_or_none}),
|
||||
'timestamp': ('release_date', {int_or_none}),
|
||||
'view_count': ('play', {int_or_none}),
|
||||
}),
|
||||
**traverse_obj(season_info, {
|
||||
'uploader': ('up_info', 'uname', {str}),
|
||||
'uploader_id': ('up_info', 'mid', {str_or_none}),
|
||||
}),
|
||||
'subtitles': self.extract_subtitles(ep_id, cid, aid=aid),
|
||||
'__post_extractor': self.extract_comments(aid),
|
||||
'http_headers': self._HEADERS,
|
||||
}
|
||||
|
||||
def _download_season_info(self, query_key, video_id):
|
||||
return self._download_json(
|
||||
f'https://api.bilibili.com/pugv/view/web/season?{query_key}={video_id}', video_id,
|
||||
headers=self._HEADERS, note='Downloading season info')['data']
|
||||
|
||||
|
||||
class BilibiliCheeseIE(BilibiliCheeseBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?bilibili\.com/cheese/play/ep(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.bilibili.com/cheese/play/ep229832',
|
||||
'info_dict': {
|
||||
'id': '229832',
|
||||
'ext': 'mp4',
|
||||
'title': '1 - 课程先导片',
|
||||
'alt_title': '视频课 · 3分41秒',
|
||||
'uploader': '马督工',
|
||||
'uploader_id': '316568752',
|
||||
'episode': '课程先导片',
|
||||
'episode_id': '229832',
|
||||
'episode_number': 1,
|
||||
'duration': 221,
|
||||
'timestamp': 1695549606,
|
||||
'upload_date': '20230924',
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
|
||||
'view_count': int,
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
ep_id = self._match_id(url)
|
||||
return self._extract_episode(self._download_season_info('ep_id', ep_id), ep_id)
|
||||
|
||||
|
||||
class BilibiliCheeseSeasonIE(BilibiliCheeseBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?bilibili\.com/cheese/play/ss(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.bilibili.com/cheese/play/ss5918',
|
||||
'info_dict': {
|
||||
'id': '5918',
|
||||
'title': '【限时五折】新闻系学不到:马督工教你做自媒体',
|
||||
'description': '帮普通人建立世界模型,降低人与人的沟通门槛',
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '229832',
|
||||
'ext': 'mp4',
|
||||
'title': '1 - 课程先导片',
|
||||
'alt_title': '视频课 · 3分41秒',
|
||||
'uploader': '马督工',
|
||||
'uploader_id': '316568752',
|
||||
'episode': '课程先导片',
|
||||
'episode_id': '229832',
|
||||
'episode_number': 1,
|
||||
'duration': 221,
|
||||
'timestamp': 1695549606,
|
||||
'upload_date': '20230924',
|
||||
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
|
||||
'view_count': int,
|
||||
}
|
||||
}],
|
||||
'params': {'playlist_items': '1'},
|
||||
}, {
|
||||
'url': 'https://www.bilibili.com/cheese/play/ss5918',
|
||||
'info_dict': {
|
||||
'id': '5918',
|
||||
'title': '【限时五折】新闻系学不到:马督工教你做自媒体',
|
||||
'description': '帮普通人建立世界模型,降低人与人的沟通门槛',
|
||||
},
|
||||
'playlist_mincount': 5,
|
||||
'skip': 'paid video in list',
|
||||
}]
|
||||
|
||||
def _get_cheese_entries(self, season_info):
|
||||
for ep_id in traverse_obj(season_info, ('episodes', lambda _, v: v['episode_can_view'], 'id')):
|
||||
yield self._extract_episode(season_info, ep_id)
|
||||
|
||||
def _real_extract(self, url):
|
||||
season_id = self._match_id(url)
|
||||
season_info = self._download_season_info('season_id', season_id)
|
||||
|
||||
return self.playlist_result(
|
||||
self._get_cheese_entries(season_info), season_id,
|
||||
**traverse_obj(season_info, {
|
||||
'title': ('title', {str}),
|
||||
'description': ('subtitle', {str}),
|
||||
}))
|
||||
|
||||
|
||||
class BilibiliSpaceBaseIE(InfoExtractor):
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from .vk import VKIE
|
||||
from ..compat import compat_b64decode
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
traverse_obj,
|
||||
unified_timestamp,
|
||||
)
|
||||
|
||||
|
||||
class BIQLEIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?biqle\.(?:com|org|ru)/watch/(?P<id>-?\d+_\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://biqle.ru/watch/-2000421746_85421746',
|
||||
'md5': 'ae6ef4f04d19ac84e4658046d02c151c',
|
||||
'info_dict': {
|
||||
'id': '-2000421746_85421746',
|
||||
'ext': 'mp4',
|
||||
'title': 'Forsaken By Hope Studio Clip',
|
||||
'description': 'Forsaken By Hope Studio Clip — Смотреть онлайн',
|
||||
'upload_date': '19700101',
|
||||
'thumbnail': r're:https://[^/]+/impf/7vN3ACwSTgChP96OdOfzFjUCzFR6ZglDQgWsIw/KPaACiVJJxM\.jpg\?size=800x450&quality=96&keep_aspect_ratio=1&background=000000&sign=b48ea459c4d33dbcba5e26d63574b1cb&type=video_thumb',
|
||||
'timestamp': 0,
|
||||
},
|
||||
}, {
|
||||
'url': 'http://biqle.org/watch/-44781847_168547604',
|
||||
'md5': '7f24e72af1db0edf7c1aaba513174f97',
|
||||
'info_dict': {
|
||||
'id': '-44781847_168547604',
|
||||
'ext': 'mp4',
|
||||
'title': 'Ребенок в шоке от автоматической мойки',
|
||||
'description': 'Ребенок в шоке от автоматической мойки — Смотреть онлайн',
|
||||
'timestamp': 1396633454,
|
||||
'upload_date': '20140404',
|
||||
'thumbnail': r're:https://[^/]+/c535507/u190034692/video/l_b84df002\.jpg',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
title = self._html_search_meta('name', webpage, 'Title', fatal=False)
|
||||
timestamp = unified_timestamp(self._html_search_meta('uploadDate', webpage, 'Upload Date', default=None))
|
||||
description = self._html_search_meta('description', webpage, 'Description', default=None)
|
||||
|
||||
global_embed_url = self._search_regex(
|
||||
r'<script[^<]+?window.globEmbedUrl\s*=\s*\'((?:https?:)?//(?:daxab\.com|dxb\.to|[^/]+/player)/[^\']+)\'',
|
||||
webpage, 'global Embed url')
|
||||
hash = self._search_regex(
|
||||
r'<script id="data-embed-video[^<]+?hash: "([^"]+)"[^<]*</script>', webpage, 'Hash')
|
||||
|
||||
embed_url = global_embed_url + hash
|
||||
|
||||
if VKIE.suitable(embed_url):
|
||||
return self.url_result(embed_url, VKIE.ie_key(), video_id)
|
||||
|
||||
embed_page = self._download_webpage(
|
||||
embed_url, video_id, 'Downloading embed webpage', headers={'Referer': url})
|
||||
|
||||
glob_params = self._parse_json(self._search_regex(
|
||||
r'<script id="globParams">[^<]*window.globParams = ([^;]+);[^<]+</script>',
|
||||
embed_page, 'Global Parameters'), video_id, transform_source=js_to_json)
|
||||
host_name = compat_b64decode(glob_params['server'][::-1]).decode()
|
||||
|
||||
item = self._download_json(
|
||||
f'https://{host_name}/method/video.get/{video_id}', video_id,
|
||||
headers={'Referer': url}, query={
|
||||
'token': glob_params['video']['access_token'],
|
||||
'videos': video_id,
|
||||
'ckey': glob_params['c_key'],
|
||||
'credentials': glob_params['video']['credentials'],
|
||||
})['response']['items'][0]
|
||||
|
||||
formats = []
|
||||
for f_id, f_url in item.get('files', {}).items():
|
||||
if f_id == 'external':
|
||||
return self.url_result(f_url)
|
||||
ext, height = f_id.split('_')
|
||||
height_extra_key = traverse_obj(glob_params, ('video', 'partial', 'quality', height))
|
||||
if height_extra_key:
|
||||
formats.append({
|
||||
'format_id': f'{height}p',
|
||||
'url': f'https://{host_name}/{f_url[8:]}&videos={video_id}&extra_key={height_extra_key}',
|
||||
'height': int_or_none(height),
|
||||
'ext': ext,
|
||||
})
|
||||
|
||||
thumbnails = []
|
||||
for k, v in item.items():
|
||||
if k.startswith('photo_') and v:
|
||||
width = k.replace('photo_', '')
|
||||
thumbnails.append({
|
||||
'id': width,
|
||||
'url': v,
|
||||
'width': int_or_none(width),
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'comment_count': int_or_none(item.get('comments')),
|
||||
'description': description,
|
||||
'duration': int_or_none(item.get('duration')),
|
||||
'thumbnails': thumbnails,
|
||||
'timestamp': timestamp,
|
||||
'view_count': int_or_none(item.get('views')),
|
||||
}
|
||||
@@ -7,8 +7,10 @@ from ..utils import (
|
||||
ExtractorError,
|
||||
OnDemandPagedList,
|
||||
clean_html,
|
||||
extract_attributes,
|
||||
get_element_by_class,
|
||||
get_element_by_id,
|
||||
get_element_html_by_class,
|
||||
get_elements_html_by_class,
|
||||
int_or_none,
|
||||
orderedSet,
|
||||
@@ -17,6 +19,7 @@ from ..utils import (
|
||||
traverse_obj,
|
||||
unified_strdate,
|
||||
urlencode_postdata,
|
||||
urljoin,
|
||||
)
|
||||
|
||||
|
||||
@@ -34,6 +37,25 @@ class BitChuteIE(InfoExtractor):
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'uploader': 'BitChute',
|
||||
'upload_date': '20170103',
|
||||
'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
|
||||
'channel': 'BitChute',
|
||||
'channel_url': 'https://www.bitchute.com/channel/bitchute/'
|
||||
},
|
||||
}, {
|
||||
# test case: video with different channel and uploader
|
||||
'url': 'https://www.bitchute.com/video/Yti_j9A-UZ4/',
|
||||
'md5': 'f10e6a8e787766235946d0868703f1d0',
|
||||
'info_dict': {
|
||||
'id': 'Yti_j9A-UZ4',
|
||||
'ext': 'mp4',
|
||||
'title': 'Israel at War | Full Measure',
|
||||
'description': 'md5:38cf7bc6f42da1a877835539111c69ef',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'uploader': 'sharylattkisson',
|
||||
'upload_date': '20231106',
|
||||
'uploader_url': 'https://www.bitchute.com/profile/9K0kUWA9zmd9/',
|
||||
'channel': 'Full Measure with Sharyl Attkisson',
|
||||
'channel_url': 'https://www.bitchute.com/channel/sharylattkisson/'
|
||||
},
|
||||
}, {
|
||||
# video not downloadable in browser, but we can recover it
|
||||
@@ -48,6 +70,9 @@ class BitChuteIE(InfoExtractor):
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'uploader': 'BitChute',
|
||||
'upload_date': '20181113',
|
||||
'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
|
||||
'channel': 'BitChute',
|
||||
'channel_url': 'https://www.bitchute.com/channel/bitchute/'
|
||||
},
|
||||
'params': {'check_formats': None},
|
||||
}, {
|
||||
@@ -99,6 +124,11 @@ class BitChuteIE(InfoExtractor):
|
||||
reason = clean_html(get_element_by_id('page-detail', webpage)) or page_title
|
||||
self.raise_geo_restricted(reason)
|
||||
|
||||
@staticmethod
|
||||
def _make_url(html):
|
||||
path = extract_attributes(get_element_html_by_class('spa', html) or '').get('href')
|
||||
return urljoin('https://www.bitchute.com', path)
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(
|
||||
@@ -121,12 +151,19 @@ class BitChuteIE(InfoExtractor):
|
||||
'Video is unavailable. Please make sure this video is playable in the browser '
|
||||
'before reporting this issue.', expected=True, video_id=video_id)
|
||||
|
||||
details = get_element_by_class('details', webpage) or ''
|
||||
uploader_html = get_element_html_by_class('creator', details) or ''
|
||||
channel_html = get_element_html_by_class('name', details) or ''
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
|
||||
'description': self._og_search_description(webpage, default=None),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'uploader': clean_html(get_element_by_class('owner', webpage)),
|
||||
'uploader': clean_html(uploader_html),
|
||||
'uploader_url': self._make_url(uploader_html),
|
||||
'channel': clean_html(channel_html),
|
||||
'channel_url': self._make_url(channel_html),
|
||||
'upload_date': unified_strdate(self._search_regex(
|
||||
r'at \d+:\d+ UTC on (.+?)\.', publish_date, 'upload date', fatal=False)),
|
||||
'formats': formats,
|
||||
@@ -154,6 +191,9 @@ class BitChuteChannelIE(InfoExtractor):
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'uploader': 'BitChute',
|
||||
'upload_date': '20170103',
|
||||
'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
|
||||
'channel': 'BitChute',
|
||||
'channel_url': 'https://www.bitchute.com/channel/bitchute/',
|
||||
'duration': 16,
|
||||
'view_count': int,
|
||||
},
|
||||
@@ -169,7 +209,7 @@ class BitChuteChannelIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': 'wV9Imujxasw9',
|
||||
'title': 'Bruce MacDonald and "The Light of Darkness"',
|
||||
'description': 'md5:04913227d2714af1d36d804aa2ab6b1e',
|
||||
'description': 'md5:747724ef404eebdfc04277714f81863e',
|
||||
}
|
||||
}]
|
||||
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class BitwaveReplayIE(InfoExtractor):
|
||||
IE_NAME = 'bitwave:replay'
|
||||
_VALID_URL = r'https?://(?:www\.)?bitwave\.tv/(?P<user>\w+)/replay/(?P<id>\w+)/?$'
|
||||
_TEST = {
|
||||
'url': 'https://bitwave.tv/RhythmicCarnage/replay/z4P6eq5L7WDrM85UCrVr',
|
||||
'only_matching': True
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
replay_id = self._match_id(url)
|
||||
replay = self._download_json(
|
||||
'https://api.bitwave.tv/v1/replays/' + replay_id,
|
||||
replay_id
|
||||
)
|
||||
|
||||
return {
|
||||
'id': replay_id,
|
||||
'title': replay['data']['title'],
|
||||
'uploader': replay['data']['name'],
|
||||
'uploader_id': replay['data']['name'],
|
||||
'url': replay['data']['url'],
|
||||
'thumbnails': [
|
||||
{'url': x} for x in replay['data']['thumbnails']
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class BitwaveStreamIE(InfoExtractor):
|
||||
IE_NAME = 'bitwave:stream'
|
||||
_VALID_URL = r'https?://(?:www\.)?bitwave\.tv/(?P<id>\w+)/?$'
|
||||
_TEST = {
|
||||
'url': 'https://bitwave.tv/doomtube',
|
||||
'only_matching': True
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
username = self._match_id(url)
|
||||
channel = self._download_json(
|
||||
'https://api.bitwave.tv/v1/channels/' + username,
|
||||
username)
|
||||
|
||||
formats = self._extract_m3u8_formats(
|
||||
channel['data']['url'], username,
|
||||
'mp4')
|
||||
|
||||
return {
|
||||
'id': username,
|
||||
'title': channel['data']['title'],
|
||||
'uploader': username,
|
||||
'uploader_id': username,
|
||||
'formats': formats,
|
||||
'thumbnail': channel['data']['thumbnail'],
|
||||
'is_live': True,
|
||||
'view_count': channel['data']['viewCount']
|
||||
}
|
||||
@@ -22,7 +22,7 @@ class BleacherReportIE(InfoExtractor):
|
||||
'upload_date': '20150615',
|
||||
'uploader': 'Team Stream Now ',
|
||||
},
|
||||
'add_ie': ['Ooyala'],
|
||||
'skip': 'Video removed',
|
||||
}, {
|
||||
'url': 'http://bleacherreport.com/articles/2586817-aussie-golfers-get-fright-of-their-lives-after-being-chased-by-angry-kangaroo',
|
||||
'md5': '6a5cd403418c7b01719248ca97fb0692',
|
||||
@@ -70,8 +70,6 @@ class BleacherReportIE(InfoExtractor):
|
||||
video_type = video['type']
|
||||
if video_type in ('cms.bleacherreport.com', 'vid.bleacherreport.com'):
|
||||
info['url'] = 'http://bleacherreport.com/video_embed?id=%s' % video['id']
|
||||
elif video_type == 'ooyala.com':
|
||||
info['url'] = 'ooyala:%s' % video['id']
|
||||
elif video_type == 'youtube.com':
|
||||
info['url'] = video['id']
|
||||
elif video_type == 'vine.co':
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
import json
|
||||
import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
parse_iso8601,
|
||||
# try_get,
|
||||
update_url_query,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class BoxIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:[^.]+\.)?app\.box\.com/s/(?P<shared_name>[^/]+)/file/(?P<id>\d+)'
|
||||
_VALID_URL = r'https?://(?:[^.]+\.)?app\.box\.com/s/(?P<shared_name>[^/?#]+)/file/(?P<id>\d+)'
|
||||
_TEST = {
|
||||
'url': 'https://mlssoccer.app.box.com/s/0evd2o3e08l60lr4ygukepvnkord1o1x/file/510727257538',
|
||||
'md5': '1f81b2fd3960f38a40a3b8823e5fcd43',
|
||||
@@ -18,11 +19,12 @@ class BoxIE(InfoExtractor):
|
||||
'id': '510727257538',
|
||||
'ext': 'mp4',
|
||||
'title': 'Garber St. Louis will be 28th MLS team +scarving.mp4',
|
||||
'uploader': 'MLS Video',
|
||||
'uploader': '',
|
||||
'timestamp': 1566320259,
|
||||
'upload_date': '20190820',
|
||||
'uploader_id': '235196876',
|
||||
}
|
||||
},
|
||||
'params': {'skip_download': 'dash fragment too small'},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
@@ -58,26 +60,15 @@ class BoxIE(InfoExtractor):
|
||||
|
||||
formats = []
|
||||
|
||||
# for entry in (try_get(f, lambda x: x['representations']['entries'], list) or []):
|
||||
# entry_url_template = try_get(
|
||||
# entry, lambda x: x['content']['url_template'])
|
||||
# if not entry_url_template:
|
||||
# continue
|
||||
# representation = entry.get('representation')
|
||||
# if representation == 'dash':
|
||||
# TODO: append query to every fragment URL
|
||||
# formats.extend(self._extract_mpd_formats(
|
||||
# entry_url_template.replace('{+asset_path}', 'manifest.mpd'),
|
||||
# file_id, query=query))
|
||||
|
||||
authenticated_download_url = f.get('authenticated_download_url')
|
||||
if authenticated_download_url and f.get('is_download_available'):
|
||||
formats.append({
|
||||
'ext': f.get('extension') or determine_ext(title),
|
||||
'filesize': f.get('size'),
|
||||
'format_id': 'download',
|
||||
'url': update_url_query(authenticated_download_url, query),
|
||||
})
|
||||
for url_tmpl in traverse_obj(f, (
|
||||
'representations', 'entries', lambda _, v: v['representation'] == 'dash',
|
||||
'content', 'url_template', {url_or_none}
|
||||
)):
|
||||
manifest_url = update_url_query(url_tmpl.replace('{+asset_path}', 'manifest.mpd'), query)
|
||||
fmts = self._extract_mpd_formats(manifest_url, file_id)
|
||||
for fmt in fmts:
|
||||
fmt['extra_param_to_segment_url'] = urllib.parse.urlparse(manifest_url).query
|
||||
formats.extend(fmts)
|
||||
|
||||
creator = f.get('created_by') or {}
|
||||
|
||||
|
||||
@@ -1,18 +1,15 @@
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
ExtractorError,
|
||||
int_or_none,
|
||||
parse_duration,
|
||||
parse_iso8601,
|
||||
xpath_element,
|
||||
xpath_text,
|
||||
)
|
||||
|
||||
|
||||
class BRIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
IE_DESC = 'Bayerischer Rundfunk'
|
||||
_VALID_URL = r'(?P<base_url>https?://(?:www\.)?br(?:-klassik)?\.de)/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'
|
||||
|
||||
@@ -167,142 +164,3 @@ class BRIE(InfoExtractor):
|
||||
} for variant in variants.findall('variant') if xpath_text(variant, 'url')]
|
||||
thumbnails.sort(key=lambda x: x['width'] * x['height'], reverse=True)
|
||||
return thumbnails
|
||||
|
||||
|
||||
class BRMediathekIE(InfoExtractor):
|
||||
IE_DESC = 'Bayerischer Rundfunk Mediathek'
|
||||
_VALID_URL = r'https?://(?:www\.)?br\.de/mediathek//?video/(?:[^/?&#]+?-)?(?P<id>av:[0-9a-f]{24})'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.br.de/mediathek/video/gesundheit-die-sendung-vom-28112017-av:5a1e6a6e8fce6d001871cc8e',
|
||||
'md5': 'fdc3d485835966d1622587d08ba632ec',
|
||||
'info_dict': {
|
||||
'id': 'av:5a1e6a6e8fce6d001871cc8e',
|
||||
'ext': 'mp4',
|
||||
'title': 'Die Sendung vom 28.11.2017',
|
||||
'description': 'md5:6000cdca5912ab2277e5b7339f201ccc',
|
||||
'timestamp': 1511942766,
|
||||
'upload_date': '20171129',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://www.br.de/mediathek//video/av:61b0db581aed360007558c12',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
clip_id = self._match_id(url)
|
||||
|
||||
clip = self._download_json(
|
||||
'https://proxy-base.master.mango.express/graphql',
|
||||
clip_id, data=json.dumps({
|
||||
"query": """{
|
||||
viewer {
|
||||
clip(id: "%s") {
|
||||
title
|
||||
description
|
||||
duration
|
||||
createdAt
|
||||
ageRestriction
|
||||
videoFiles {
|
||||
edges {
|
||||
node {
|
||||
publicLocation
|
||||
fileSize
|
||||
videoProfile {
|
||||
width
|
||||
height
|
||||
bitrate
|
||||
encoding
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
captionFiles {
|
||||
edges {
|
||||
node {
|
||||
publicLocation
|
||||
}
|
||||
}
|
||||
}
|
||||
teaserImages {
|
||||
edges {
|
||||
node {
|
||||
imageFiles {
|
||||
edges {
|
||||
node {
|
||||
publicLocation
|
||||
width
|
||||
height
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}""" % clip_id}).encode(), headers={
|
||||
'Content-Type': 'application/json',
|
||||
})['data']['viewer']['clip']
|
||||
title = clip['title']
|
||||
|
||||
formats = []
|
||||
for edge in clip.get('videoFiles', {}).get('edges', []):
|
||||
node = edge.get('node', {})
|
||||
n_url = node.get('publicLocation')
|
||||
if not n_url:
|
||||
continue
|
||||
ext = determine_ext(n_url)
|
||||
if ext == 'm3u8':
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
n_url, clip_id, 'mp4', 'm3u8_native',
|
||||
m3u8_id='hls', fatal=False))
|
||||
else:
|
||||
video_profile = node.get('videoProfile', {})
|
||||
tbr = int_or_none(video_profile.get('bitrate'))
|
||||
format_id = 'http'
|
||||
if tbr:
|
||||
format_id += '-%d' % tbr
|
||||
formats.append({
|
||||
'format_id': format_id,
|
||||
'url': n_url,
|
||||
'width': int_or_none(video_profile.get('width')),
|
||||
'height': int_or_none(video_profile.get('height')),
|
||||
'tbr': tbr,
|
||||
'filesize': int_or_none(node.get('fileSize')),
|
||||
})
|
||||
|
||||
subtitles = {}
|
||||
for edge in clip.get('captionFiles', {}).get('edges', []):
|
||||
node = edge.get('node', {})
|
||||
n_url = node.get('publicLocation')
|
||||
if not n_url:
|
||||
continue
|
||||
subtitles.setdefault('de', []).append({
|
||||
'url': n_url,
|
||||
})
|
||||
|
||||
thumbnails = []
|
||||
for edge in clip.get('teaserImages', {}).get('edges', []):
|
||||
for image_edge in edge.get('node', {}).get('imageFiles', {}).get('edges', []):
|
||||
node = image_edge.get('node', {})
|
||||
n_url = node.get('publicLocation')
|
||||
if not n_url:
|
||||
continue
|
||||
thumbnails.append({
|
||||
'url': n_url,
|
||||
'width': int_or_none(node.get('width')),
|
||||
'height': int_or_none(node.get('height')),
|
||||
})
|
||||
|
||||
return {
|
||||
'id': clip_id,
|
||||
'title': title,
|
||||
'description': clip.get('description'),
|
||||
'duration': int_or_none(clip.get('duration')),
|
||||
'timestamp': parse_iso8601(clip.get('createdAt')),
|
||||
'age_limit': int_or_none(clip.get('ageRestriction')),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'thumbnails': thumbnails,
|
||||
}
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from .youtube import YoutubeIE
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class BreakIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?break\.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
|
||||
_TESTS = [{
|
||||
'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
|
||||
'info_dict': {
|
||||
'id': '2468056',
|
||||
'ext': 'mp4',
|
||||
'title': 'When Girls Act Like D-Bags',
|
||||
'age_limit': 13,
|
||||
},
|
||||
}, {
|
||||
# youtube embed
|
||||
'url': 'http://www.break.com/video/someone-forgot-boat-brakes-work',
|
||||
'info_dict': {
|
||||
'id': 'RrrDLdeL2HQ',
|
||||
'ext': 'mp4',
|
||||
'title': 'Whale Watching Boat Crashing Into San Diego Dock',
|
||||
'description': 'md5:afc1b2772f0a8468be51dd80eb021069',
|
||||
'upload_date': '20160331',
|
||||
'uploader': 'Steve Holden',
|
||||
'uploader_id': 'sdholden07',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
}
|
||||
}, {
|
||||
'url': 'http://www.break.com/video/ugc/baby-flex-2773063',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id, video_id = self._match_valid_url(url).groups()
|
||||
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
youtube_url = YoutubeIE._extract_url(webpage)
|
||||
if youtube_url:
|
||||
return self.url_result(youtube_url, ie=YoutubeIE.ie_key())
|
||||
|
||||
content = self._parse_json(
|
||||
self._search_regex(
|
||||
r'(?s)content["\']\s*:\s*(\[.+?\])\s*[,\n]', webpage,
|
||||
'content'),
|
||||
display_id)
|
||||
|
||||
formats = []
|
||||
for video in content:
|
||||
video_url = url_or_none(video.get('url'))
|
||||
if not video_url:
|
||||
continue
|
||||
bitrate = int_or_none(self._search_regex(
|
||||
r'(\d+)_kbps', video_url, 'tbr', default=None))
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
'format_id': 'http-%d' % bitrate if bitrate else 'http',
|
||||
'tbr': bitrate,
|
||||
})
|
||||
|
||||
title = self._search_regex(
|
||||
(r'title["\']\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
|
||||
r'<h1[^>]*>(?P<value>[^<]+)'), webpage, 'title', group='value')
|
||||
|
||||
def get(key, name):
|
||||
return int_or_none(self._search_regex(
|
||||
r'%s["\']\s*:\s*["\'](\d+)' % key, webpage, name,
|
||||
default=None))
|
||||
|
||||
age_limit = get('ratings', 'age limit')
|
||||
video_id = video_id or get('pid', 'video id') or display_id
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'age_limit': age_limit,
|
||||
'formats': formats,
|
||||
}
|
||||
123
yt_dlp/extractor/bundestag.py
Normal file
123
yt_dlp/extractor/bundestag.py
Normal file
@@ -0,0 +1,123 @@
|
||||
import re
|
||||
from functools import partial
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..networking.exceptions import HTTPError
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
bug_reports_message,
|
||||
clean_html,
|
||||
format_field,
|
||||
get_element_text_and_html_by_tag,
|
||||
int_or_none,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class BundestagIE(InfoExtractor):
|
||||
_VALID_URL = [
|
||||
r'https?://dbtg\.tv/[cf]vid/(?P<id>\d+)',
|
||||
r'https?://www\.bundestag\.de/mediathek/?\?(?:[^#]+&)?videoid=(?P<id>\d+)',
|
||||
]
|
||||
_TESTS = [{
|
||||
'url': 'https://dbtg.tv/cvid/7605304',
|
||||
'info_dict': {
|
||||
'id': '7605304',
|
||||
'ext': 'mp4',
|
||||
'title': '145. Sitzung vom 15.12.2023, TOP 24 Barrierefreiheit',
|
||||
'description': 'md5:321a9dc6bdad201264c0045efc371561',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.bundestag.de/mediathek?videoid=7602120&url=L21lZGlhdGhla292ZXJsYXk=&mod=mediathek',
|
||||
'info_dict': {
|
||||
'id': '7602120',
|
||||
'ext': 'mp4',
|
||||
'title': '130. Sitzung vom 18.10.2023, TOP 1 Befragung der Bundesregierung',
|
||||
'description': 'Befragung der Bundesregierung',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.bundestag.de/mediathek?videoid=7604941#url=L21lZGlhdGhla292ZXJsYXk/dmlkZW9pZD03NjA0OTQx&mod=mediathek',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'http://dbtg.tv/fvid/3594346',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_OVERLAY_URL = 'https://www.bundestag.de/mediathekoverlay'
|
||||
_INSTANCE_FORMAT = 'https://cldf-wzw-od.r53.cdn.tv1.eu/13014bundestagod/_definst_/13014bundestag/ondemand/3777parlamentsfernsehen/archiv/app144277506/145293313/{0}/{0}_playlist.smil/playlist.m3u8'
|
||||
|
||||
_SHARE_URL = 'https://webtv.bundestag.de/player/macros/_x_s-144277506/shareData.json?contentId='
|
||||
_SHARE_AUDIO_REGEX = r'/\d+_(?P<codec>\w+)_(?P<bitrate>\d+)kb_(?P<channels>\w+)_\w+_\d+\.(?P<ext>\w+)'
|
||||
_SHARE_VIDEO_REGEX = r'/\d+_(?P<codec>\w+)_(?P<width>\w+)_(?P<height>\w+)_(?P<bitrate>\d+)kb_\w+_\w+_\d+\.(?P<ext>\w+)'
|
||||
|
||||
def _bt_extract_share_formats(self, video_id):
|
||||
share_data = self._download_json(
|
||||
f'{self._SHARE_URL}{video_id}', video_id, note='Downloading share format JSON')
|
||||
if traverse_obj(share_data, ('status', 'code', {int})) != 1:
|
||||
self.report_warning(format_field(
|
||||
share_data, [('status', 'message', {str})],
|
||||
'Share API response: %s', default='Unknown Share API Error')
|
||||
+ bug_reports_message())
|
||||
return
|
||||
|
||||
for name, url in share_data.items():
|
||||
if not isinstance(name, str) or not url_or_none(url):
|
||||
continue
|
||||
|
||||
elif name.startswith('audio'):
|
||||
match = re.search(self._SHARE_AUDIO_REGEX, url)
|
||||
yield {
|
||||
'format_id': name,
|
||||
'url': url,
|
||||
'vcodec': 'none',
|
||||
**traverse_obj(match, {
|
||||
'acodec': 'codec',
|
||||
'audio_channels': ('channels', {{'mono': 1, 'stereo': 2}.get}),
|
||||
'abr': ('bitrate', {int_or_none}),
|
||||
'ext': 'ext',
|
||||
}),
|
||||
}
|
||||
|
||||
elif name.startswith('download'):
|
||||
match = re.search(self._SHARE_VIDEO_REGEX, url)
|
||||
yield {
|
||||
'format_id': name,
|
||||
'url': url,
|
||||
**traverse_obj(match, {
|
||||
'vcodec': 'codec',
|
||||
'tbr': ('bitrate', {int_or_none}),
|
||||
'width': ('width', {int_or_none}),
|
||||
'height': ('height', {int_or_none}),
|
||||
'ext': 'ext',
|
||||
}),
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
formats = []
|
||||
result = {'id': video_id, 'formats': formats}
|
||||
|
||||
try:
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
self._INSTANCE_FORMAT.format(video_id), video_id, m3u8_id='instance'))
|
||||
except ExtractorError as error:
|
||||
if isinstance(error.cause, HTTPError) and error.cause.status == 404:
|
||||
raise ExtractorError('Could not find video id', expected=True)
|
||||
self.report_warning(f'Error extracting hls formats: {error}', video_id)
|
||||
formats.extend(self._bt_extract_share_formats(video_id))
|
||||
if not formats:
|
||||
self.raise_no_formats('Could not find suitable formats', video_id=video_id)
|
||||
|
||||
result.update(traverse_obj(self._download_webpage(
|
||||
self._OVERLAY_URL, video_id,
|
||||
query={'videoid': video_id, 'view': 'main'},
|
||||
note='Downloading metadata overlay', fatal=False,
|
||||
), {
|
||||
'title': (
|
||||
{partial(get_element_text_and_html_by_tag, 'h3')}, 0,
|
||||
{partial(re.sub, r'<span[^>]*>[^<]+</span>', '')}, {clean_html}),
|
||||
'description': ({partial(get_element_text_and_html_by_tag, 'p')}, 0, {clean_html}),
|
||||
}))
|
||||
|
||||
return result
|
||||
@@ -8,9 +8,9 @@ from ..utils import (
|
||||
|
||||
|
||||
class BYUtvIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'https?://(?:www\.)?byutv\.org/(?:watch|player)/(?!event/)(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
|
||||
_TESTS = [{
|
||||
# ooyalaVOD
|
||||
'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
|
||||
'info_dict': {
|
||||
'id': 'ZvanRocTpW-G5_yZFeltTAMv6jxOU9KH',
|
||||
@@ -24,7 +24,6 @@ class BYUtvIE(InfoExtractor):
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
'add_ie': ['Ooyala'],
|
||||
}, {
|
||||
# dvr
|
||||
'url': 'https://www.byutv.org/player/8f1dab9b-b243-47c8-b525-3e2d021a3451/byu-softball-pacific-vs-byu-41219---game-2',
|
||||
@@ -63,19 +62,6 @@ class BYUtvIE(InfoExtractor):
|
||||
'x-byutv-platformkey': 'xsaaw9c7y5',
|
||||
})
|
||||
|
||||
ep = video.get('ooyalaVOD')
|
||||
if ep:
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'ie_key': 'Ooyala',
|
||||
'url': 'ooyala:%s' % ep['providerId'],
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'title': ep.get('title'),
|
||||
'description': ep.get('description'),
|
||||
'thumbnail': ep.get('imageThumbnail'),
|
||||
}
|
||||
|
||||
info = {}
|
||||
formats = []
|
||||
subtitles = {}
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
parse_duration,
|
||||
unified_strdate,
|
||||
)
|
||||
|
||||
|
||||
class CamWithHerIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?camwithher\.tv/view_video\.php\?.*\bviewkey=(?P<id>\w+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://camwithher.tv/view_video.php?viewkey=6e9a24e2c0e842e1f177&page=&viewtype=&category=',
|
||||
'info_dict': {
|
||||
'id': '5644',
|
||||
'ext': 'flv',
|
||||
'title': 'Periscope Tease',
|
||||
'description': 'In the clouds teasing on periscope to my favorite song',
|
||||
'duration': 240,
|
||||
'view_count': int,
|
||||
'comment_count': int,
|
||||
'uploader': 'MileenaK',
|
||||
'upload_date': '20160322',
|
||||
'age_limit': 18,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
}
|
||||
}, {
|
||||
'url': 'http://camwithher.tv/view_video.php?viewkey=6dfd8b7c97531a459937',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'http://camwithher.tv/view_video.php?page=&viewkey=6e9a24e2c0e842e1f177&viewtype=&category=',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'http://camwithher.tv/view_video.php?viewkey=b6c3b5bea9515d1a1fc4&page=&viewtype=&category=mv',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
flv_id = self._html_search_regex(
|
||||
r'<a[^>]+href=["\']/download/\?v=(\d+)', webpage, 'video id')
|
||||
|
||||
# Video URL construction algorithm is reverse-engineered from cwhplayer.swf
|
||||
rtmp_url = 'rtmp://camwithher.tv/clipshare/%s' % (
|
||||
('mp4:%s.mp4' % flv_id) if int(flv_id) > 2010 else flv_id)
|
||||
|
||||
title = self._html_search_regex(
|
||||
r'<div[^>]+style="float:left"[^>]*>\s*<h2>(.+?)</h2>', webpage, 'title')
|
||||
description = self._html_search_regex(
|
||||
r'>Description:</span>(.+?)</div>', webpage, 'description', default=None)
|
||||
|
||||
runtime = self._search_regex(
|
||||
r'Runtime\s*:\s*(.+?) \|', webpage, 'duration', default=None)
|
||||
if runtime:
|
||||
runtime = re.sub(r'[\s-]', '', runtime)
|
||||
duration = parse_duration(runtime)
|
||||
view_count = int_or_none(self._search_regex(
|
||||
r'Views\s*:\s*(\d+)', webpage, 'view count', default=None))
|
||||
comment_count = int_or_none(self._search_regex(
|
||||
r'Comments\s*:\s*(\d+)', webpage, 'comment count', default=None))
|
||||
|
||||
uploader = self._search_regex(
|
||||
r'Added by\s*:\s*<a[^>]+>([^<]+)</a>', webpage, 'uploader', default=None)
|
||||
upload_date = unified_strdate(self._search_regex(
|
||||
r'Added on\s*:\s*([\d-]+)', webpage, 'upload date', default=None))
|
||||
|
||||
return {
|
||||
'id': flv_id,
|
||||
'url': rtmp_url,
|
||||
'ext': 'flv',
|
||||
'no_resume': True,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'duration': duration,
|
||||
'view_count': view_count,
|
||||
'comment_count': comment_count,
|
||||
'uploader': uploader,
|
||||
'upload_date': upload_date,
|
||||
'age_limit': 18
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_str
|
||||
from ..utils import (
|
||||
format_field,
|
||||
float_or_none,
|
||||
int_or_none,
|
||||
try_get,
|
||||
)
|
||||
|
||||
from .videomore import VideomoreIE
|
||||
|
||||
|
||||
class CarambaTVIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:carambatv:|https?://video1\.carambatv\.ru/v/)(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://video1.carambatv.ru/v/191910501',
|
||||
'md5': '2f4a81b7cfd5ab866ee2d7270cb34a2a',
|
||||
'info_dict': {
|
||||
'id': '191910501',
|
||||
'ext': 'mp4',
|
||||
'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
|
||||
'thumbnail': r're:^https?://.*\.jpg',
|
||||
'duration': 2678.31,
|
||||
},
|
||||
}, {
|
||||
'url': 'carambatv:191910501',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
video = self._download_json(
|
||||
'http://video1.carambatv.ru/v/%s/videoinfo.js' % video_id,
|
||||
video_id)
|
||||
|
||||
title = video['title']
|
||||
|
||||
base_url = video.get('video') or 'http://video1.carambatv.ru/v/%s/' % video_id
|
||||
|
||||
formats = [{
|
||||
'url': base_url + f['fn'],
|
||||
'height': int_or_none(f.get('height')),
|
||||
'format_id': format_field(f, 'height', '%sp'),
|
||||
} for f in video['qualities'] if f.get('fn')]
|
||||
|
||||
thumbnail = video.get('splash')
|
||||
duration = float_or_none(try_get(
|
||||
video, lambda x: x['annotations'][0]['end_time'], compat_str))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail,
|
||||
'duration': duration,
|
||||
'formats': formats,
|
||||
}
|
||||
|
||||
|
||||
class CarambaTVPageIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://carambatv\.ru/(?:[^/]+/)+(?P<id>[^/?#&]+)'
|
||||
_TEST = {
|
||||
'url': 'http://carambatv.ru/movie/bad-comedian/razborka-v-manile/',
|
||||
'md5': 'a49fb0ec2ad66503eeb46aac237d3c86',
|
||||
'info_dict': {
|
||||
'id': '475222',
|
||||
'ext': 'flv',
|
||||
'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
|
||||
'thumbnail': r're:^https?://.*\.jpg',
|
||||
# duration reported by videomore is incorrect
|
||||
'duration': int,
|
||||
},
|
||||
'add_ie': [VideomoreIE.ie_key()],
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
videomore_url = VideomoreIE._extract_url(webpage)
|
||||
if not videomore_url:
|
||||
videomore_id = self._search_regex(
|
||||
r'getVMCode\s*\(\s*["\']?(\d+)', webpage, 'videomore id',
|
||||
default=None)
|
||||
if videomore_id:
|
||||
videomore_url = 'videomore:%s' % videomore_id
|
||||
if videomore_url:
|
||||
title = self._og_search_title(webpage)
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'url': videomore_url,
|
||||
'ie_key': VideomoreIE.ie_key(),
|
||||
'title': title,
|
||||
}
|
||||
|
||||
video_url = self._og_search_property('video:iframe', webpage, default=None)
|
||||
|
||||
if not video_url:
|
||||
video_id = self._search_regex(
|
||||
r'(?:video_id|crmb_vuid)\s*[:=]\s*["\']?(\d+)',
|
||||
webpage, 'video id')
|
||||
video_url = 'carambatv:%s' % video_id
|
||||
|
||||
return self.url_result(video_url, CarambaTVIE.ie_key())
|
||||
@@ -180,6 +180,13 @@ class CBCPlayerIE(InfoExtractor):
|
||||
'thumbnail': 'http://thumbnails.cbc.ca/maven_legacy/thumbnails/sonali-karnick-220.jpg',
|
||||
'chapters': [],
|
||||
'duration': 494.811,
|
||||
'categories': ['AudioMobile/All in a Weekend Montreal'],
|
||||
'tags': 'count:8',
|
||||
'location': 'Quebec',
|
||||
'series': 'All in a Weekend Montreal',
|
||||
'season': 'Season 2015',
|
||||
'season_number': 2015,
|
||||
'media_type': 'Excerpt',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://www.cbc.ca/player/play/2164402062',
|
||||
@@ -195,25 +202,37 @@ class CBCPlayerIE(InfoExtractor):
|
||||
'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/277/67/cancer_852x480_2164412612.jpg',
|
||||
'chapters': [],
|
||||
'duration': 186.867,
|
||||
'series': 'CBC News: Windsor at 6:00',
|
||||
'categories': ['News/Canada/Windsor'],
|
||||
'location': 'Windsor',
|
||||
'tags': ['cancer'],
|
||||
'creator': 'Allison Johnson',
|
||||
'media_type': 'Excerpt',
|
||||
},
|
||||
}, {
|
||||
# Has subtitles
|
||||
# These broadcasts expire after ~1 month, can find new test URL here:
|
||||
# https://www.cbc.ca/player/news/TV%20Shows/The%20National/Latest%20Broadcast
|
||||
'url': 'http://www.cbc.ca/player/play/2249992771553',
|
||||
'md5': '2f2fb675dd4f0f8a5bb7588d1b13bacd',
|
||||
'url': 'http://www.cbc.ca/player/play/2284799043667',
|
||||
'md5': '9b49f0839e88b6ec0b01d840cf3d42b5',
|
||||
'info_dict': {
|
||||
'id': '2249992771553',
|
||||
'id': '2284799043667',
|
||||
'ext': 'mp4',
|
||||
'title': 'The National | Women’s soccer pay, Florida seawater, Swift quake',
|
||||
'description': 'md5:adba28011a56cfa47a080ff198dad27a',
|
||||
'timestamp': 1690596000,
|
||||
'duration': 2716.333,
|
||||
'title': 'The National | Hockey coach charged, Green grants, Safer drugs',
|
||||
'description': 'md5:84ef46321c94bcf7d0159bb565d26bfa',
|
||||
'timestamp': 1700272800,
|
||||
'duration': 2718.833,
|
||||
'subtitles': {'eng': [{'ext': 'vtt', 'protocol': 'm3u8_native'}]},
|
||||
'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/481/326/thumbnail.jpeg',
|
||||
'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/907/171/thumbnail.jpeg',
|
||||
'uploader': 'CBCC-NEW',
|
||||
'chapters': 'count:5',
|
||||
'upload_date': '20230729',
|
||||
'upload_date': '20231118',
|
||||
'categories': 'count:4',
|
||||
'series': 'The National - Full Show',
|
||||
'tags': 'count:1',
|
||||
'creator': 'News',
|
||||
'location': 'Canada',
|
||||
'media_type': 'Full Program',
|
||||
},
|
||||
}]
|
||||
|
||||
|
||||
@@ -1,252 +0,0 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
qualities,
|
||||
unescapeHTML,
|
||||
)
|
||||
|
||||
|
||||
class Channel9IE(InfoExtractor):
|
||||
IE_DESC = 'Channel 9'
|
||||
IE_NAME = 'channel9'
|
||||
_VALID_URL = r'https?://(?:www\.)?(?:channel9\.msdn\.com|s\.ch9\.ms)/(?P<contentpath>.+?)(?P<rss>/RSS)?/?(?:[?#&]|$)'
|
||||
_EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>https?://channel9\.msdn\.com/(?:[^/]+/)+)player\b']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://channel9.msdn.com/Events/TechEd/Australia/2013/KOS002',
|
||||
'md5': '32083d4eaf1946db6d454313f44510ca',
|
||||
'info_dict': {
|
||||
'id': '6c413323-383a-49dc-88f9-a22800cab024',
|
||||
'ext': 'wmv',
|
||||
'title': 'Developer Kick-Off Session: Stuff We Love',
|
||||
'description': 'md5:b80bf9355a503c193aff7ec6cd5a7731',
|
||||
'duration': 4576,
|
||||
'thumbnail': r're:https?://.*\.jpg',
|
||||
'timestamp': 1377717420,
|
||||
'upload_date': '20130828',
|
||||
'session_code': 'KOS002',
|
||||
'session_room': 'Arena 1A',
|
||||
'session_speakers': 'count:5',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://channel9.msdn.com/posts/Self-service-BI-with-Power-BI-nuclear-testing',
|
||||
'md5': 'dcf983ee6acd2088e7188c3cf79b46bc',
|
||||
'info_dict': {
|
||||
'id': 'fe8e435f-bb93-4e01-8e97-a28c01887024',
|
||||
'ext': 'wmv',
|
||||
'title': 'Self-service BI with Power BI - nuclear testing',
|
||||
'description': 'md5:2d17fec927fc91e9e17783b3ecc88f54',
|
||||
'duration': 1540,
|
||||
'thumbnail': r're:https?://.*\.jpg',
|
||||
'timestamp': 1386381991,
|
||||
'upload_date': '20131207',
|
||||
'authors': ['Mike Wilmot'],
|
||||
},
|
||||
}, {
|
||||
# low quality mp4 is best
|
||||
'url': 'https://channel9.msdn.com/Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
|
||||
'info_dict': {
|
||||
'id': '33ad69d2-6a4e-4172-83a1-a523013dec76',
|
||||
'ext': 'mp4',
|
||||
'title': 'Ranges for the Standard Library',
|
||||
'description': 'md5:9895e0a9fd80822d2f01c454b8f4a372',
|
||||
'duration': 5646,
|
||||
'thumbnail': r're:https?://.*\.jpg',
|
||||
'upload_date': '20150930',
|
||||
'timestamp': 1443640735,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://channel9.msdn.com/Events/DEVintersection/DEVintersection-2016/RSS',
|
||||
'info_dict': {
|
||||
'id': 'Events/DEVintersection/DEVintersection-2016',
|
||||
'title': 'DEVintersection 2016 Orlando Sessions',
|
||||
},
|
||||
'playlist_mincount': 14,
|
||||
}, {
|
||||
'url': 'https://channel9.msdn.com/Niners/Splendid22/Queue/76acff796e8f411184b008028e0d492b/RSS',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://channel9.msdn.com/Events/Speakers/scott-hanselman/RSS?UrlSafeName=scott-hanselman',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_RSS_URL = 'http://channel9.msdn.com/%s/RSS'
|
||||
|
||||
def _extract_list(self, video_id, rss_url=None):
|
||||
if not rss_url:
|
||||
rss_url = self._RSS_URL % video_id
|
||||
rss = self._download_xml(rss_url, video_id, 'Downloading RSS')
|
||||
entries = [self.url_result(session_url.text, 'Channel9')
|
||||
for session_url in rss.findall('./channel/item/link')]
|
||||
title_text = rss.find('./channel/title').text
|
||||
return self.playlist_result(entries, video_id, title_text)
|
||||
|
||||
def _real_extract(self, url):
|
||||
content_path, rss = self._match_valid_url(url).groups()
|
||||
|
||||
if rss:
|
||||
return self._extract_list(content_path, url)
|
||||
|
||||
webpage = self._download_webpage(
|
||||
url, content_path, 'Downloading web page')
|
||||
|
||||
episode_data = self._search_regex(
|
||||
r"data-episode='([^']+)'", webpage, 'episode data', default=None)
|
||||
if episode_data:
|
||||
episode_data = self._parse_json(unescapeHTML(
|
||||
episode_data), content_path)
|
||||
content_id = episode_data['contentId']
|
||||
is_session = '/Sessions(' in episode_data['api']
|
||||
content_url = 'https://channel9.msdn.com/odata' + episode_data['api'] + '?$select=Captions,CommentCount,MediaLengthInSeconds,PublishedDate,Rating,RatingCount,Title,VideoMP4High,VideoMP4Low,VideoMP4Medium,VideoPlayerPreviewImage,VideoWMV,VideoWMVHQ,Views,'
|
||||
if is_session:
|
||||
content_url += 'Code,Description,Room,Slides,Speakers,ZipFile&$expand=Speakers'
|
||||
else:
|
||||
content_url += 'Authors,Body&$expand=Authors'
|
||||
content_data = self._download_json(content_url, content_id)
|
||||
title = content_data['Title']
|
||||
|
||||
QUALITIES = (
|
||||
'mp3',
|
||||
'wmv', 'mp4',
|
||||
'wmv-low', 'mp4-low',
|
||||
'wmv-mid', 'mp4-mid',
|
||||
'wmv-high', 'mp4-high',
|
||||
)
|
||||
|
||||
quality_key = qualities(QUALITIES)
|
||||
|
||||
def quality(quality_id, format_url):
|
||||
return (len(QUALITIES) if '_Source.' in format_url
|
||||
else quality_key(quality_id))
|
||||
|
||||
formats = []
|
||||
urls = set()
|
||||
|
||||
SITE_QUALITIES = {
|
||||
'MP3': 'mp3',
|
||||
'MP4': 'mp4',
|
||||
'Low Quality WMV': 'wmv-low',
|
||||
'Low Quality MP4': 'mp4-low',
|
||||
'Mid Quality WMV': 'wmv-mid',
|
||||
'Mid Quality MP4': 'mp4-mid',
|
||||
'High Quality WMV': 'wmv-high',
|
||||
'High Quality MP4': 'mp4-high',
|
||||
}
|
||||
|
||||
formats_select = self._search_regex(
|
||||
r'(?s)<select[^>]+name=["\']format[^>]+>(.+?)</select', webpage,
|
||||
'formats select', default=None)
|
||||
if formats_select:
|
||||
for mobj in re.finditer(
|
||||
r'<option\b[^>]+\bvalue=(["\'])(?P<url>(?:(?!\1).)+)\1[^>]*>\s*(?P<format>[^<]+?)\s*<',
|
||||
formats_select):
|
||||
format_url = mobj.group('url')
|
||||
if format_url in urls:
|
||||
continue
|
||||
urls.add(format_url)
|
||||
format_id = mobj.group('format')
|
||||
quality_id = SITE_QUALITIES.get(format_id, format_id)
|
||||
formats.append({
|
||||
'url': format_url,
|
||||
'format_id': quality_id,
|
||||
'quality': quality(quality_id, format_url),
|
||||
'vcodec': 'none' if quality_id == 'mp3' else None,
|
||||
})
|
||||
|
||||
API_QUALITIES = {
|
||||
'VideoMP4Low': 'mp4-low',
|
||||
'VideoWMV': 'wmv-mid',
|
||||
'VideoMP4Medium': 'mp4-mid',
|
||||
'VideoMP4High': 'mp4-high',
|
||||
'VideoWMVHQ': 'wmv-hq',
|
||||
}
|
||||
|
||||
for format_id, q in API_QUALITIES.items():
|
||||
q_url = content_data.get(format_id)
|
||||
if not q_url or q_url in urls:
|
||||
continue
|
||||
urls.add(q_url)
|
||||
formats.append({
|
||||
'url': q_url,
|
||||
'format_id': q,
|
||||
'quality': quality(q, q_url),
|
||||
})
|
||||
|
||||
slides = content_data.get('Slides')
|
||||
zip_file = content_data.get('ZipFile')
|
||||
|
||||
if not formats and not slides and not zip_file:
|
||||
self.raise_no_formats(
|
||||
'None of recording, slides or zip are available for %s' % content_path)
|
||||
|
||||
subtitles = {}
|
||||
for caption in content_data.get('Captions', []):
|
||||
caption_url = caption.get('Url')
|
||||
if not caption_url:
|
||||
continue
|
||||
subtitles.setdefault(caption.get('Language', 'en'), []).append({
|
||||
'url': caption_url,
|
||||
'ext': 'vtt',
|
||||
})
|
||||
|
||||
common = {
|
||||
'id': content_id,
|
||||
'title': title,
|
||||
'description': clean_html(content_data.get('Description') or content_data.get('Body')),
|
||||
'thumbnail': content_data.get('VideoPlayerPreviewImage'),
|
||||
'duration': int_or_none(content_data.get('MediaLengthInSeconds')),
|
||||
'timestamp': parse_iso8601(content_data.get('PublishedDate')),
|
||||
'avg_rating': int_or_none(content_data.get('Rating')),
|
||||
'rating_count': int_or_none(content_data.get('RatingCount')),
|
||||
'view_count': int_or_none(content_data.get('Views')),
|
||||
'comment_count': int_or_none(content_data.get('CommentCount')),
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
if is_session:
|
||||
speakers = []
|
||||
for s in content_data.get('Speakers', []):
|
||||
speaker_name = s.get('FullName')
|
||||
if not speaker_name:
|
||||
continue
|
||||
speakers.append(speaker_name)
|
||||
|
||||
common.update({
|
||||
'session_code': content_data.get('Code'),
|
||||
'session_room': content_data.get('Room'),
|
||||
'session_speakers': speakers,
|
||||
})
|
||||
else:
|
||||
authors = []
|
||||
for a in content_data.get('Authors', []):
|
||||
author_name = a.get('DisplayName')
|
||||
if not author_name:
|
||||
continue
|
||||
authors.append(author_name)
|
||||
common['authors'] = authors
|
||||
|
||||
contents = []
|
||||
|
||||
if slides:
|
||||
d = common.copy()
|
||||
d.update({'title': title + '-Slides', 'url': slides})
|
||||
contents.append(d)
|
||||
|
||||
if zip_file:
|
||||
d = common.copy()
|
||||
d.update({'title': title + '-Zip', 'url': zip_file})
|
||||
contents.append(d)
|
||||
|
||||
if formats:
|
||||
d = common.copy()
|
||||
d.update({'title': title, 'formats': formats})
|
||||
contents.append(d)
|
||||
return self.playlist_result(contents)
|
||||
else:
|
||||
return self._extract_list(content_path)
|
||||
@@ -1,88 +0,0 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_b64decode
|
||||
from ..utils import parse_duration
|
||||
|
||||
|
||||
class ChirbitIE(InfoExtractor):
|
||||
IE_NAME = 'chirbit'
|
||||
_VALID_URL = r'https?://(?:www\.)?chirb\.it/(?:(?:wp|pl)/|fb_chirbit_player\.swf\?key=)?(?P<id>[\da-zA-Z]+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://chirb.it/be2abG',
|
||||
'info_dict': {
|
||||
'id': 'be2abG',
|
||||
'ext': 'mp3',
|
||||
'title': 'md5:f542ea253f5255240be4da375c6a5d7e',
|
||||
'description': 'md5:f24a4e22a71763e32da5fed59e47c770',
|
||||
'duration': 306,
|
||||
'uploader': 'Gerryaudio',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://chirb.it/fb_chirbit_player.swf?key=PrIPv5',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://chirb.it/wp/MN58c2',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
audio_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(
|
||||
'http://chirb.it/%s' % audio_id, audio_id)
|
||||
|
||||
data_fd = self._search_regex(
|
||||
r'data-fd=(["\'])(?P<url>(?:(?!\1).)+)\1',
|
||||
webpage, 'data fd', group='url')
|
||||
|
||||
# Reverse engineered from https://chirb.it/js/chirbit.player.js (look
|
||||
# for soundURL)
|
||||
audio_url = compat_b64decode(data_fd[::-1]).decode('utf-8')
|
||||
|
||||
title = self._search_regex(
|
||||
r'class=["\']chirbit-title["\'][^>]*>([^<]+)', webpage, 'title')
|
||||
description = self._search_regex(
|
||||
r'<h3>Description</h3>\s*<pre[^>]*>([^<]+)</pre>',
|
||||
webpage, 'description', default=None)
|
||||
duration = parse_duration(self._search_regex(
|
||||
r'class=["\']c-length["\'][^>]*>([^<]+)',
|
||||
webpage, 'duration', fatal=False))
|
||||
uploader = self._search_regex(
|
||||
r'id=["\']chirbit-username["\'][^>]*>([^<]+)',
|
||||
webpage, 'uploader', fatal=False)
|
||||
|
||||
return {
|
||||
'id': audio_id,
|
||||
'url': audio_url,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'duration': duration,
|
||||
'uploader': uploader,
|
||||
}
|
||||
|
||||
|
||||
class ChirbitProfileIE(InfoExtractor):
|
||||
IE_NAME = 'chirbit:profile'
|
||||
_VALID_URL = r'https?://(?:www\.)?chirbit\.com/(?:rss/)?(?P<id>[^/]+)'
|
||||
_TEST = {
|
||||
'url': 'http://chirbit.com/ScarletBeauty',
|
||||
'info_dict': {
|
||||
'id': 'ScarletBeauty',
|
||||
},
|
||||
'playlist_mincount': 3,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
profile_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, profile_id)
|
||||
|
||||
entries = [
|
||||
self.url_result(self._proto_relative_url('//chirb.it/' + video_id))
|
||||
for _, video_id in re.findall(r'<input[^>]+id=([\'"])copy-btn-(?P<id>[0-9a-zA-Z]+)\1', webpage)]
|
||||
|
||||
return self.playlist_result(entries, profile_id)
|
||||
@@ -1,56 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
unified_strdate,
|
||||
xpath_text,
|
||||
)
|
||||
|
||||
|
||||
class CinchcastIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://player\.cinchcast\.com/.*?(?:assetId|show_id)=(?P<id>[0-9]+)'
|
||||
_EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://player.cinchcast.com/?show_id=5258197&platformId=1&assetType=single',
|
||||
'info_dict': {
|
||||
'id': '5258197',
|
||||
'ext': 'mp3',
|
||||
'title': 'Train Your Brain to Up Your Game with Coach Mandy',
|
||||
'upload_date': '20130816',
|
||||
},
|
||||
}, {
|
||||
# Actual test is run in generic, look for undergroundwellness
|
||||
'url': 'http://player.cinchcast.com/?platformId=1&assetType=single&assetId=7141703',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
doc = self._download_xml(
|
||||
'http://www.blogtalkradio.com/playerasset/mrss?assetType=single&assetId=%s' % video_id,
|
||||
video_id)
|
||||
|
||||
item = doc.find('.//item')
|
||||
title = xpath_text(item, './title', fatal=True)
|
||||
date_str = xpath_text(
|
||||
item, './{http://developer.longtailvideo.com/trac/}date')
|
||||
upload_date = unified_strdate(date_str, day_first=False)
|
||||
# duration is present but wrong
|
||||
formats = [{
|
||||
'format_id': 'main',
|
||||
'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url'],
|
||||
}]
|
||||
backup_url = xpath_text(
|
||||
item, './{http://developer.longtailvideo.com/trac/}backupContent')
|
||||
if backup_url:
|
||||
formats.append({
|
||||
'preference': 2, # seems to be more reliable
|
||||
'format_id': 'backup',
|
||||
'url': backup_url,
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'upload_date': upload_date,
|
||||
'formats': formats,
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
find_xpath_attr,
|
||||
fix_xml_ampersands
|
||||
)
|
||||
|
||||
|
||||
class ClipsyndicateIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe',
|
||||
'md5': '4d7d549451bad625e0ff3d7bd56d776c',
|
||||
'info_dict': {
|
||||
'id': '4629301',
|
||||
'ext': 'mp4',
|
||||
'title': 'Brick Briscoe',
|
||||
'duration': 612,
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://chic.clipsyndicate.com/video/play/5844117/shark_attack',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
js_player = self._download_webpage(
|
||||
'http://eplayer.clipsyndicate.com/embed/player.js?va_id=%s' % video_id,
|
||||
video_id, 'Downlaoding player')
|
||||
# it includes a required token
|
||||
flvars = self._search_regex(r'flvars: "(.*?)"', js_player, 'flvars')
|
||||
|
||||
pdoc = self._download_xml(
|
||||
'http://eplayer.clipsyndicate.com/osmf/playlist?%s' % flvars,
|
||||
video_id, 'Downloading video info',
|
||||
transform_source=fix_xml_ampersands)
|
||||
|
||||
track_doc = pdoc.find('trackList/track')
|
||||
|
||||
def find_param(name):
|
||||
node = find_xpath_attr(track_doc, './/param', 'name', name)
|
||||
if node is not None:
|
||||
return node.attrib['value']
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': find_param('title'),
|
||||
'url': track_doc.find('location').text,
|
||||
'thumbnail': find_param('thumbnail'),
|
||||
'duration': int(find_param('duration')),
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
str_to_int,
|
||||
unified_strdate,
|
||||
)
|
||||
|
||||
|
||||
class CloudyIE(InfoExtractor):
|
||||
_IE_DESC = 'cloudy.ec'
|
||||
_VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.cloudy.ec/v/af511e2527aac',
|
||||
'md5': '29832b05028ead1b58be86bf319397ca',
|
||||
'info_dict': {
|
||||
'id': 'af511e2527aac',
|
||||
'ext': 'mp4',
|
||||
'title': 'Funny Cats and Animals Compilation june 2013',
|
||||
'upload_date': '20130913',
|
||||
'view_count': int,
|
||||
}
|
||||
}, {
|
||||
'url': 'http://www.cloudy.ec/embed.php?autoplay=1&id=af511e2527aac',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(
|
||||
'https://www.cloudy.ec/embed.php', video_id, query={
|
||||
'id': video_id,
|
||||
'playerPage': 1,
|
||||
'autoplay': 1,
|
||||
})
|
||||
|
||||
info = self._parse_html5_media_entries(url, webpage, video_id)[0]
|
||||
|
||||
webpage = self._download_webpage(
|
||||
'https://www.cloudy.ec/v/%s' % video_id, video_id, fatal=False)
|
||||
|
||||
if webpage:
|
||||
info.update({
|
||||
'title': self._search_regex(
|
||||
r'<h\d[^>]*>([^<]+)<', webpage, 'title'),
|
||||
'upload_date': unified_strdate(self._search_regex(
|
||||
r'>Published at (\d{4}-\d{1,2}-\d{1,2})', webpage,
|
||||
'upload date', fatal=False)),
|
||||
'view_count': str_to_int(self._search_regex(
|
||||
r'([\d,.]+) views<', webpage, 'view count', fatal=False)),
|
||||
})
|
||||
|
||||
if not info.get('title'):
|
||||
info['title'] = video_id
|
||||
|
||||
info['id'] = video_id
|
||||
|
||||
return info
|
||||
@@ -6,6 +6,7 @@ from ..utils import (
|
||||
|
||||
|
||||
class ClubicIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'https?://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
|
||||
|
||||
_TESTS = [{
|
||||
|
||||
@@ -4,6 +4,7 @@ from .mtv import MTVIE
|
||||
|
||||
|
||||
class CMTIE(MTVIE): # XXX: Do not subclass from concrete IE
|
||||
_WORKING = False
|
||||
IE_NAME = 'cmt.com'
|
||||
_VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|(?:full-)?episodes|video-clips)/(?P<id>[^/]+)'
|
||||
|
||||
|
||||
@@ -286,6 +286,9 @@ class InfoExtractor:
|
||||
If it is not clear whether to use timestamp or this, use the former
|
||||
release_date: The date (YYYYMMDD) when the video was released in UTC.
|
||||
If not explicitly set, calculated from release_timestamp
|
||||
release_year: Year (YYYY) as integer when the video or album was released.
|
||||
To be used if no exact release date is known.
|
||||
If not explicitly set, calculated from release_date.
|
||||
modified_timestamp: UNIX timestamp of the moment the video was last modified.
|
||||
modified_date: The date (YYYYMMDD) when the video was last modified in UTC.
|
||||
If not explicitly set, calculated from modified_timestamp
|
||||
@@ -379,6 +382,7 @@ class InfoExtractor:
|
||||
'private', 'premium_only', 'subscriber_only', 'needs_auth',
|
||||
'unlisted' or 'public'. Use 'InfoExtractor._availability'
|
||||
to set it
|
||||
media_type: The type of media as classified by the site, e.g. "episode", "clip", "trailer"
|
||||
_old_archive_ids: A list of old archive ids needed for backward compatibility
|
||||
_format_sort_fields: A list of fields to use for sorting formats
|
||||
__post_extractor: A function to be called just before the metadata is
|
||||
@@ -427,7 +431,6 @@ class InfoExtractor:
|
||||
and compilations).
|
||||
disc_number: Number of the disc or other physical medium the track belongs to,
|
||||
as an integer.
|
||||
release_year: Year (YYYY) when the album was released.
|
||||
composer: Composer of the piece
|
||||
|
||||
The following fields should only be set for clips that should be cut from the original video:
|
||||
@@ -2341,7 +2344,9 @@ class InfoExtractor:
|
||||
imgs_count = 0
|
||||
|
||||
srcs = set()
|
||||
media = smil.findall(self._xpath_ns('.//video', namespace)) + smil.findall(self._xpath_ns('.//audio', namespace))
|
||||
media = itertools.chain.from_iterable(
|
||||
smil.findall(self._xpath_ns(arg, namespace))
|
||||
for arg in ['.//video', './/audio', './/media'])
|
||||
for medium in media:
|
||||
src = medium.get('src')
|
||||
if not src or src in srcs:
|
||||
|
||||
@@ -46,6 +46,10 @@ class CWTVIE(InfoExtractor):
|
||||
'timestamp': 1444107300,
|
||||
'age_limit': 14,
|
||||
'uploader': 'CWTV',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g$',
|
||||
'chapters': 'count:4',
|
||||
'episode': 'Episode 20',
|
||||
'season': 'Season 11',
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_b64decode
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
parse_count,
|
||||
parse_duration,
|
||||
traverse_obj,
|
||||
try_get,
|
||||
unified_timestamp,
|
||||
)
|
||||
|
||||
|
||||
class DaftsexIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?daft\.sex/watch/(?P<id>-?\d+_\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://daft.sex/watch/-35370899_456246186',
|
||||
'md5': '64c04ef7b4c7b04b308f3b0c78efe7cd',
|
||||
'info_dict': {
|
||||
'id': '-35370899_456246186',
|
||||
'ext': 'mp4',
|
||||
'title': 'just relaxing',
|
||||
'description': 'just relaxing – Watch video Watch video in high quality',
|
||||
'upload_date': '20201113',
|
||||
'timestamp': 1605261911,
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'age_limit': 18,
|
||||
'duration': 15.0,
|
||||
'view_count': int
|
||||
},
|
||||
}, {
|
||||
'url': 'https://daft.sex/watch/-156601359_456242791',
|
||||
'info_dict': {
|
||||
'id': '-156601359_456242791',
|
||||
'ext': 'mp4',
|
||||
'title': 'Skye Blue - Dinner And A Show',
|
||||
'description': 'Skye Blue - Dinner And A Show - Watch video Watch video in high quality',
|
||||
'upload_date': '20200916',
|
||||
'timestamp': 1600250735,
|
||||
'thumbnail': 'https://psv153-1.crazycloud.ru/videos/-156601359/456242791/thumb.jpg?extra=i3D32KaBbBFf9TqDRMAVmQ',
|
||||
},
|
||||
'skip': 'deleted / private'
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
title = self._html_search_meta('name', webpage, 'title')
|
||||
timestamp = unified_timestamp(self._html_search_meta('uploadDate', webpage, 'Upload Date', default=None))
|
||||
description = self._html_search_meta('description', webpage, 'Description', default=None)
|
||||
|
||||
duration = parse_duration(self._search_regex(
|
||||
r'Duration: ((?:[0-9]{2}:){0,2}[0-9]{2})',
|
||||
webpage, 'duration', fatal=False))
|
||||
views = parse_count(self._search_regex(
|
||||
r'Views: ([0-9 ]+)',
|
||||
webpage, 'views', fatal=False))
|
||||
|
||||
player_hash = self._search_regex(
|
||||
r'DaxabPlayer\.Init\({[\s\S]*hash:\s*"([0-9a-zA-Z_\-]+)"[\s\S]*}',
|
||||
webpage, 'player hash')
|
||||
player_color = self._search_regex(
|
||||
r'DaxabPlayer\.Init\({[\s\S]*color:\s*"([0-9a-z]+)"[\s\S]*}',
|
||||
webpage, 'player color', fatal=False) or ''
|
||||
|
||||
embed_page = self._download_webpage(
|
||||
'https://dxb.to/player/%s?color=%s' % (player_hash, player_color),
|
||||
video_id, headers={'Referer': url})
|
||||
video_params = self._parse_json(
|
||||
self._search_regex(
|
||||
r'window\.globParams\s*=\s*({[\S\s]+})\s*;\s*<\/script>',
|
||||
embed_page, 'video parameters'),
|
||||
video_id, transform_source=js_to_json)
|
||||
|
||||
server_domain = 'https://%s' % compat_b64decode(video_params['server'][::-1]).decode('utf-8')
|
||||
|
||||
cdn_files = traverse_obj(video_params, ('video', 'cdn_files')) or {}
|
||||
if cdn_files:
|
||||
formats = []
|
||||
for format_id, format_data in cdn_files.items():
|
||||
ext, height = format_id.split('_')
|
||||
formats.append({
|
||||
'format_id': format_id,
|
||||
'url': f'{server_domain}/videos/{video_id.replace("_", "/")}/{height}.mp4?extra={format_data.split(".")[-1]}',
|
||||
'height': int_or_none(height),
|
||||
'ext': ext,
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'description': description,
|
||||
'duration': duration,
|
||||
'thumbnail': try_get(video_params, lambda vi: 'https:' + compat_b64decode(vi['video']['thumb']).decode('utf-8')),
|
||||
'timestamp': timestamp,
|
||||
'view_count': views,
|
||||
'age_limit': 18,
|
||||
}
|
||||
|
||||
items = self._download_json(
|
||||
f'{server_domain}/method/video.get/{video_id}', video_id,
|
||||
headers={'Referer': url}, query={
|
||||
'token': video_params['video']['access_token'],
|
||||
'videos': video_id,
|
||||
'ckey': video_params['c_key'],
|
||||
'credentials': video_params['video']['credentials'],
|
||||
})['response']['items']
|
||||
|
||||
if not items:
|
||||
raise ExtractorError('Video is not available', video_id=video_id, expected=True)
|
||||
|
||||
item = items[0]
|
||||
formats = []
|
||||
for f_id, f_url in item.get('files', {}).items():
|
||||
if f_id == 'external':
|
||||
return self.url_result(f_url)
|
||||
ext, height = f_id.split('_')
|
||||
height_extra_key = traverse_obj(video_params, ('video', 'partial', 'quality', height))
|
||||
if height_extra_key:
|
||||
formats.append({
|
||||
'format_id': f'{height}p',
|
||||
'url': f'{server_domain}/{f_url[8:]}&videos={video_id}&extra_key={height_extra_key}',
|
||||
'height': int_or_none(height),
|
||||
'ext': ext,
|
||||
})
|
||||
|
||||
thumbnails = []
|
||||
for k, v in item.items():
|
||||
if k.startswith('photo_') and v:
|
||||
width = k.replace('photo_', '')
|
||||
thumbnails.append({
|
||||
'id': width,
|
||||
'url': v,
|
||||
'width': int_or_none(width),
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'comment_count': int_or_none(item.get('comments')),
|
||||
'description': description,
|
||||
'duration': duration,
|
||||
'thumbnails': thumbnails,
|
||||
'timestamp': timestamp,
|
||||
'view_count': views,
|
||||
'age_limit': 18,
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class DefenseGouvFrIE(InfoExtractor):
|
||||
IE_NAME = 'defense.gouv.fr'
|
||||
_VALID_URL = r'https?://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'
|
||||
|
||||
_TEST = {
|
||||
'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
|
||||
'md5': '75bba6124da7e63d2d60b5244ec9430c',
|
||||
'info_dict': {
|
||||
'id': '11213',
|
||||
'ext': 'mp4',
|
||||
'title': 'attaque-chimique-syrienne-du-21-aout-2013-1'
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
title = self._match_id(url)
|
||||
webpage = self._download_webpage(url, title)
|
||||
|
||||
video_id = self._search_regex(
|
||||
r"flashvars.pvg_id=\"(\d+)\";",
|
||||
webpage, 'ID')
|
||||
|
||||
json_url = (
|
||||
'http://static.videos.gouv.fr/brightcovehub/export/json/%s' %
|
||||
video_id)
|
||||
info = self._download_json(json_url, title, 'Downloading JSON config')
|
||||
video_url = info['renditions'][0]['url']
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'ext': 'mp4',
|
||||
'url': video_url,
|
||||
'title': title,
|
||||
}
|
||||
@@ -3,6 +3,7 @@ from ..utils import parse_duration
|
||||
|
||||
|
||||
class DHMIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
IE_DESC = 'Filmarchiv - Deutsches Historisches Museum'
|
||||
_VALID_URL = r'https?://(?:www\.)?dhm\.de/filmarchiv/(?:[^/]+/)+(?P<id>[^/]+)'
|
||||
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
float_or_none,
|
||||
int_or_none,
|
||||
)
|
||||
|
||||
|
||||
class DotsubIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://dotsub.com/view/9c63db2a-fa95-4838-8e6e-13deafe47f09',
|
||||
'md5': '21c7ff600f545358134fea762a6d42b6',
|
||||
'info_dict': {
|
||||
'id': '9c63db2a-fa95-4838-8e6e-13deafe47f09',
|
||||
'ext': 'flv',
|
||||
'title': 'MOTIVATION - "It\'s Possible" Best Inspirational Video Ever',
|
||||
'description': 'md5:41af1e273edbbdfe4e216a78b9d34ac6',
|
||||
'thumbnail': 're:^https?://dotsub.com/media/9c63db2a-fa95-4838-8e6e-13deafe47f09/p',
|
||||
'duration': 198,
|
||||
'uploader': 'liuxt',
|
||||
'timestamp': 1385778501.104,
|
||||
'upload_date': '20131130',
|
||||
'view_count': int,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://dotsub.com/view/747bcf58-bd59-45b7-8c8c-ac312d084ee6',
|
||||
'md5': '2bb4a83896434d5c26be868c609429a3',
|
||||
'info_dict': {
|
||||
'id': '168006778',
|
||||
'ext': 'mp4',
|
||||
'title': 'Apartments and flats in Raipur the white symphony',
|
||||
'description': 'md5:784d0639e6b7d1bc29530878508e38fe',
|
||||
'thumbnail': 're:^https?://dotsub.com/media/747bcf58-bd59-45b7-8c8c-ac312d084ee6/p',
|
||||
'duration': 290,
|
||||
'timestamp': 1476767794.2809999,
|
||||
'upload_date': '20161018',
|
||||
'uploader': 'parthivi001',
|
||||
'uploader_id': 'user52596202',
|
||||
'view_count': int,
|
||||
},
|
||||
'add_ie': ['Vimeo'],
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
info = self._download_json(
|
||||
'https://dotsub.com/api/media/%s/metadata' % video_id, video_id)
|
||||
video_url = info.get('mediaURI')
|
||||
|
||||
if not video_url:
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_url = self._search_regex(
|
||||
[r'<source[^>]+src="([^"]+)"', r'"file"\s*:\s*\'([^\']+)'],
|
||||
webpage, 'video url', default=None)
|
||||
info_dict = {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': 'flv',
|
||||
}
|
||||
|
||||
if not video_url:
|
||||
setup_data = self._parse_json(self._html_search_regex(
|
||||
r'(?s)data-setup=([\'"])(?P<content>(?!\1).+?)\1',
|
||||
webpage, 'setup data', group='content'), video_id)
|
||||
info_dict = {
|
||||
'_type': 'url_transparent',
|
||||
'url': setup_data['src'],
|
||||
}
|
||||
|
||||
info_dict.update({
|
||||
'title': info['title'],
|
||||
'description': info.get('description'),
|
||||
'thumbnail': info.get('screenshotURI'),
|
||||
'duration': int_or_none(info.get('duration'), 1000),
|
||||
'uploader': info.get('user'),
|
||||
'timestamp': float_or_none(info.get('dateCreated'), 1000),
|
||||
'view_count': int_or_none(info.get('numberOfViews')),
|
||||
})
|
||||
|
||||
return info_dict
|
||||
@@ -209,7 +209,7 @@ class DRTVIE(InfoExtractor):
|
||||
elif access_service == 'StandardVideo':
|
||||
preference = 1
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(
|
||||
stream.get('url'), video_id, preference=preference, m3u8_id=format_id, fatal=False)
|
||||
stream.get('url'), video_id, ext='mp4', preference=preference, m3u8_id=format_id, fatal=False)
|
||||
formats.extend(fmts)
|
||||
|
||||
api_subtitles = traverse_obj(stream, ('subtitles', lambda _, v: url_or_none(v['link']), {dict}))
|
||||
|
||||
104
yt_dlp/extractor/duoplay.py
Normal file
104
yt_dlp/extractor/duoplay.py
Normal file
@@ -0,0 +1,104 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
extract_attributes,
|
||||
get_element_text_and_html_by_tag,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
str_or_none,
|
||||
try_call,
|
||||
unified_timestamp,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class DuoplayIE(InfoExtractor):
|
||||
_VALID_URL = r'https://duoplay\.ee/(?P<id>\d+)/[\w-]+/?(?:\?(?:[^#]+&)?ep=(?P<ep>\d+))?'
|
||||
_TESTS = [{
|
||||
'note': 'Siberi võmm S02E12',
|
||||
'url': 'https://duoplay.ee/4312/siberi-vomm?ep=24',
|
||||
'md5': '1ff59d535310ac9c5cf5f287d8f91b2d',
|
||||
'info_dict': {
|
||||
'id': '4312_24',
|
||||
'ext': 'mp4',
|
||||
'title': 'Operatsioon "Öö"',
|
||||
'thumbnail': r're:https://.+\.jpg(?:\?c=\d+)?$',
|
||||
'description': 'md5:8ef98f38569d6b8b78f3d350ccc6ade8',
|
||||
'upload_date': '20170523',
|
||||
'timestamp': 1495567800,
|
||||
'series': 'Siberi võmm',
|
||||
'series_id': '4312',
|
||||
'season': 'Season 2',
|
||||
'season_number': 2,
|
||||
'episode': 'Operatsioon "Öö"',
|
||||
'episode_number': 12,
|
||||
'episode_id': 24,
|
||||
},
|
||||
}, {
|
||||
'note': 'Empty title',
|
||||
'url': 'https://duoplay.ee/17/uhikarotid?ep=14',
|
||||
'md5': '6aca68be71112314738dd17cced7f8bf',
|
||||
'info_dict': {
|
||||
'id': '17_14',
|
||||
'ext': 'mp4',
|
||||
'title': 'Ühikarotid',
|
||||
'thumbnail': r're:https://.+\.jpg(?:\?c=\d+)?$',
|
||||
'description': 'md5:4719b418e058c209def41d48b601276e',
|
||||
'upload_date': '20100916',
|
||||
'timestamp': 1284661800,
|
||||
'series': 'Ühikarotid',
|
||||
'series_id': '17',
|
||||
'season': 'Season 2',
|
||||
'season_number': 2,
|
||||
'episode_id': 14,
|
||||
'release_year': 2010,
|
||||
},
|
||||
}, {
|
||||
'note': 'Movie without expiry',
|
||||
'url': 'https://duoplay.ee/5501/pilvede-all.-neljas-ode',
|
||||
'md5': '7abf63d773a49ef7c39f2c127842b8fd',
|
||||
'info_dict': {
|
||||
'id': '5501',
|
||||
'ext': 'mp4',
|
||||
'title': 'Pilvede all. Neljas õde',
|
||||
'thumbnail': r're:https://.+\.jpg(?:\?c=\d+)?$',
|
||||
'description': 'md5:d86a70f8f31e82c369d4d4f4c79b1279',
|
||||
'cast': 'count:9',
|
||||
'upload_date': '20221214',
|
||||
'timestamp': 1671054000,
|
||||
'release_year': 2018,
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
telecast_id, episode = self._match_valid_url(url).group('id', 'ep')
|
||||
video_id = join_nonempty(telecast_id, episode, delim='_')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_player = try_call(lambda: extract_attributes(
|
||||
get_element_text_and_html_by_tag('video-player', webpage)[1]))
|
||||
if not video_player or not video_player.get('manifest-url'):
|
||||
raise ExtractorError('No video found', expected=True)
|
||||
|
||||
episode_attr = self._parse_json(video_player.get(':episode') or '', video_id, fatal=False) or {}
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': self._extract_m3u8_formats(video_player['manifest-url'], video_id, 'mp4'),
|
||||
**traverse_obj(episode_attr, {
|
||||
'title': 'title',
|
||||
'description': 'synopsis',
|
||||
'thumbnail': ('images', 'original'),
|
||||
'timestamp': ('airtime', {lambda x: unified_timestamp(x + ' +0200')}),
|
||||
'cast': ('cast', {lambda x: x.split(', ')}),
|
||||
'release_year': ('year', {int_or_none}),
|
||||
}),
|
||||
**(traverse_obj(episode_attr, {
|
||||
'title': (None, ('subtitle', ('episode_nr', {lambda x: f'Episode {x}' if x else None}))),
|
||||
'series': 'title',
|
||||
'series_id': ('telecast_id', {str_or_none}),
|
||||
'season_number': ('season_id', {int_or_none}),
|
||||
'episode': 'subtitle',
|
||||
'episode_number': ('episode_nr', {int_or_none}),
|
||||
'episode_id': ('episode_id', {int_or_none}),
|
||||
}, get_all=False) if episode_attr.get('category') != 'movies' else {}),
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class EchoMskIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?echo\.msk\.ru/sounds/(?P<id>\d+)'
|
||||
_TEST = {
|
||||
'url': 'http://www.echo.msk.ru/sounds/1464134.html',
|
||||
'md5': '2e44b3b78daff5b458e4dbc37f191f7c',
|
||||
'info_dict': {
|
||||
'id': '1464134',
|
||||
'ext': 'mp3',
|
||||
'title': 'Особое мнение - 29 декабря 2014, 19:08',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
audio_url = self._search_regex(
|
||||
r'<a rel="mp3" href="([^"]+)">', webpage, 'audio URL')
|
||||
|
||||
title = self._html_search_regex(
|
||||
r'<a href="/programs/[^"]+" target="_blank">([^<]+)</a>',
|
||||
webpage, 'title')
|
||||
|
||||
air_date = self._html_search_regex(
|
||||
r'(?s)<div class="date">(.+?)</div>',
|
||||
webpage, 'date', fatal=False, default=None)
|
||||
|
||||
if air_date:
|
||||
air_date = re.sub(r'(\s)\1+', r'\1', air_date)
|
||||
if air_date:
|
||||
title = '%s - %s' % (title, air_date)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': audio_url,
|
||||
'title': title,
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_urllib_parse_unquote
|
||||
|
||||
|
||||
class EHowIE(InfoExtractor):
|
||||
IE_NAME = 'eHow'
|
||||
_VALID_URL = r'https?://(?:www\.)?ehow\.com/[^/_?]*_(?P<id>[0-9]+)'
|
||||
_TEST = {
|
||||
'url': 'http://www.ehow.com/video_12245069_hardwood-flooring-basics.html',
|
||||
'md5': '9809b4e3f115ae2088440bcb4efbf371',
|
||||
'info_dict': {
|
||||
'id': '12245069',
|
||||
'ext': 'flv',
|
||||
'title': 'Hardwood Flooring Basics',
|
||||
'description': 'Hardwood flooring may be time consuming, but its ultimately a pretty straightforward concept. Learn about hardwood flooring basics with help from a hardware flooring business owner in this free video...',
|
||||
'uploader': 'Erick Nathan',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_url = self._search_regex(
|
||||
r'(?:file|source)=(http[^\'"&]*)', webpage, 'video URL')
|
||||
final_url = compat_urllib_parse_unquote(video_url)
|
||||
uploader = self._html_search_meta('uploader', webpage)
|
||||
title = self._og_search_title(webpage).replace(' | eHow', '')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': final_url,
|
||||
'title': title,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'description': self._og_search_description(webpage),
|
||||
'uploader': uploader,
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
parse_iso8601,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class ElevenSportsIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?elevensports\.com/view/event/(?P<id>\w+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://elevensports.com/view/event/clf46yr3kenn80jgrqsjmwefk',
|
||||
'md5': 'c0958d9ff90e4503a75544358758921d',
|
||||
'info_dict': {
|
||||
'id': 'clf46yr3kenn80jgrqsjmwefk',
|
||||
'title': 'Cleveland SC vs Lionsbridge FC',
|
||||
'ext': 'mp4',
|
||||
'description': 'md5:03b5238d6549f4ea1fddadf69b5e0b58',
|
||||
'upload_date': '20230323',
|
||||
'timestamp': 1679612400,
|
||||
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'}
|
||||
}, {
|
||||
'url': 'https://elevensports.com/view/event/clhpyd53b06160jez74qhgkmf',
|
||||
'md5': 'c0958d9ff90e4503a75544358758921d',
|
||||
'info_dict': {
|
||||
'id': 'clhpyd53b06160jez74qhgkmf',
|
||||
'title': 'AJNLF vs ARRAF',
|
||||
'ext': 'mp4',
|
||||
'description': 'md5:c8c5e75c78f37c6d15cd6c475e43a8c1',
|
||||
'upload_date': '20230521',
|
||||
'timestamp': 1684684800,
|
||||
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
event_id = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['event']['mclsEventId']
|
||||
event_data = self._download_json(
|
||||
f'https://mcls-api.mycujoo.tv/bff/events/v1beta1/{event_id}', video_id,
|
||||
headers={'Authorization': 'Bearer FBVKACGN37JQC5SFA0OVK8KKSIOP153G'})
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
event_data['streams'][0]['full_url'], video_id, 'mp4', m3u8_id='hls')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(event_data, {
|
||||
'title': ('title', {str}),
|
||||
'description': ('description', {str}),
|
||||
'timestamp': ('start_time', {parse_iso8601}),
|
||||
'thumbnail': ('thumbnail_url', {url_or_none}),
|
||||
}),
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
extract_attributes,
|
||||
float_or_none,
|
||||
int_or_none,
|
||||
try_get,
|
||||
)
|
||||
|
||||
|
||||
class EllenTubeBaseIE(InfoExtractor):
|
||||
def _extract_data_config(self, webpage, video_id):
|
||||
details = self._search_regex(
|
||||
r'(<[^>]+\bdata-component=(["\'])[Dd]etails.+?></div>)', webpage,
|
||||
'details')
|
||||
return self._parse_json(
|
||||
extract_attributes(details)['data-config'], video_id)
|
||||
|
||||
def _extract_video(self, data, video_id):
|
||||
title = data['title']
|
||||
|
||||
formats = []
|
||||
duration = None
|
||||
for entry in data.get('media'):
|
||||
if entry.get('id') == 'm3u8':
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
entry['url'], video_id, 'mp4',
|
||||
entry_protocol='m3u8_native', m3u8_id='hls')
|
||||
duration = int_or_none(entry.get('duration'))
|
||||
break
|
||||
|
||||
def get_insight(kind):
|
||||
return int_or_none(try_get(
|
||||
data, lambda x: x['insight']['%ss' % kind]))
|
||||
|
||||
return {
|
||||
'extractor_key': EllenTubeIE.ie_key(),
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': data.get('description'),
|
||||
'duration': duration,
|
||||
'thumbnail': data.get('thumbnail'),
|
||||
'timestamp': float_or_none(data.get('publishTime'), scale=1000),
|
||||
'view_count': get_insight('view'),
|
||||
'like_count': get_insight('like'),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
|
||||
|
||||
class EllenTubeIE(EllenTubeBaseIE):
|
||||
_VALID_URL = r'''(?x)
|
||||
(?:
|
||||
ellentube:|
|
||||
https://api-prod\.ellentube\.com/ellenapi/api/item/
|
||||
)
|
||||
(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})
|
||||
'''
|
||||
_TESTS = [{
|
||||
'url': 'https://api-prod.ellentube.com/ellenapi/api/item/0822171c-3829-43bf-b99f-d77358ae75e3',
|
||||
'md5': '2fabc277131bddafdd120e0fc0f974c9',
|
||||
'info_dict': {
|
||||
'id': '0822171c-3829-43bf-b99f-d77358ae75e3',
|
||||
'ext': 'mp4',
|
||||
'title': 'Ellen Meets Las Vegas Survivors Jesus Campos and Stephen Schuck',
|
||||
'description': 'md5:76e3355e2242a78ad9e3858e5616923f',
|
||||
'thumbnail': r're:^https?://.+?',
|
||||
'duration': 514,
|
||||
'timestamp': 1508505120,
|
||||
'upload_date': '20171020',
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
}
|
||||
}, {
|
||||
'url': 'ellentube:734a3353-f697-4e79-9ca9-bfc3002dc1e0',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
data = self._download_json(
|
||||
'https://api-prod.ellentube.com/ellenapi/api/item/%s' % video_id,
|
||||
video_id)
|
||||
return self._extract_video(data, video_id)
|
||||
|
||||
|
||||
class EllenTubeVideoIE(EllenTubeBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?ellentube\.com/video/(?P<id>.+?)\.html'
|
||||
_TEST = {
|
||||
'url': 'https://www.ellentube.com/video/ellen-meets-las-vegas-survivors-jesus-campos-and-stephen-schuck.html',
|
||||
'only_matching': True,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
video_id = self._extract_data_config(webpage, display_id)['id']
|
||||
return self.url_result(
|
||||
'ellentube:%s' % video_id, ie=EllenTubeIE.ie_key(),
|
||||
video_id=video_id)
|
||||
|
||||
|
||||
class EllenTubePlaylistIE(EllenTubeBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?ellentube\.com/(?:episode|studios)/(?P<id>.+?)\.html'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.ellentube.com/episode/dax-shepard-jordan-fisher-haim.html',
|
||||
'info_dict': {
|
||||
'id': 'dax-shepard-jordan-fisher-haim',
|
||||
'title': "Dax Shepard, 'DWTS' Team Jordan Fisher & Lindsay Arnold, HAIM",
|
||||
'description': 'md5:bfc982194dabb3f4e325e43aa6b2e21c',
|
||||
},
|
||||
'playlist_count': 6,
|
||||
}, {
|
||||
'url': 'https://www.ellentube.com/studios/macey-goes-rving0.html',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
data = self._extract_data_config(webpage, display_id)['data']
|
||||
feed = self._download_json(
|
||||
'https://api-prod.ellentube.com/ellenapi/api/feed/?%s'
|
||||
% data['filter'], display_id)
|
||||
entries = [
|
||||
self._extract_video(elem, elem['id'])
|
||||
for elem in feed if elem.get('type') == 'VIDEO' and elem.get('id')]
|
||||
return self.playlist_result(
|
||||
entries, display_id, data.get('title'),
|
||||
clean_html(data.get('description')))
|
||||
@@ -1,15 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class EngadgetIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?engadget\.com/video/(?P<id>[^/?#]+)'
|
||||
|
||||
_TESTS = [{
|
||||
# video with vidible ID
|
||||
'url': 'https://www.engadget.com/video/57a28462134aa15a39f0421a/',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
return self.url_result('aol-video:%s' % video_id)
|
||||
107
yt_dlp/extractor/epidemicsound.py
Normal file
107
yt_dlp/extractor/epidemicsound.py
Normal file
@@ -0,0 +1,107 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
float_or_none,
|
||||
int_or_none,
|
||||
orderedSet,
|
||||
parse_iso8601,
|
||||
parse_qs,
|
||||
parse_resolution,
|
||||
str_or_none,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class EpidemicSoundIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?epidemicsound\.com/track/(?P<id>[0-9a-zA-Z]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.epidemicsound.com/track/yFfQVRpSPz/',
|
||||
'md5': 'd98ff2ddb49e8acab9716541cbc9dfac',
|
||||
'info_dict': {
|
||||
'id': '45014',
|
||||
'display_id': 'yFfQVRpSPz',
|
||||
'ext': 'mp3',
|
||||
'title': 'Door Knock Door 1',
|
||||
'alt_title': 'Door Knock Door 1',
|
||||
'tags': ['foley', 'door', 'knock', 'glass', 'window', 'glass door knock'],
|
||||
'categories': ['Misc. Door'],
|
||||
'duration': 1,
|
||||
'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/default-sfx/3000x3000.jpg',
|
||||
'timestamp': 1415320353,
|
||||
'upload_date': '20141107',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.epidemicsound.com/track/mj8GTTwsZd/',
|
||||
'md5': 'c82b745890f9baf18dc2f8d568ee3830',
|
||||
'info_dict': {
|
||||
'id': '148700',
|
||||
'display_id': 'mj8GTTwsZd',
|
||||
'ext': 'mp3',
|
||||
'title': 'Noplace',
|
||||
'tags': ['liquid drum n bass', 'energetic'],
|
||||
'categories': ['drum and bass'],
|
||||
'duration': 237,
|
||||
'timestamp': 1694426482,
|
||||
'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/11138/3000x3000.jpg',
|
||||
'upload_date': '20230911',
|
||||
'release_timestamp': 1700535606,
|
||||
'release_date': '20231121',
|
||||
},
|
||||
}]
|
||||
|
||||
@staticmethod
|
||||
def _epidemic_parse_thumbnail(url: str):
|
||||
if not url_or_none(url):
|
||||
return None
|
||||
|
||||
return {
|
||||
'url': url,
|
||||
**(traverse_obj(url, ({parse_qs}, {
|
||||
'width': ('width', 0, {int_or_none}),
|
||||
'height': ('height', 0, {int_or_none}),
|
||||
})) or parse_resolution(url)),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _epidemic_fmt_or_none(f):
|
||||
if not f.get('format'):
|
||||
f['format'] = f.get('format_id')
|
||||
elif not f.get('format_id'):
|
||||
f['format_id'] = f['format']
|
||||
if not f['url'] or not f['format']:
|
||||
return None
|
||||
if f.get('format_note'):
|
||||
f['format_note'] = f'track ID {f["format_note"]}'
|
||||
if f['format'] != 'full':
|
||||
f['preference'] = -2
|
||||
return f
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
json_data = self._download_json(f'https://www.epidemicsound.com/json/track/{video_id}', video_id)
|
||||
|
||||
thumbnails = traverse_obj(json_data, [('imageUrl', 'cover')])
|
||||
thumb_base_url = traverse_obj(json_data, ('coverArt', 'baseUrl', {url_or_none}))
|
||||
if thumb_base_url:
|
||||
thumbnails.extend(traverse_obj(json_data, (
|
||||
'coverArt', 'sizes', ..., {thumb_base_url.__add__})))
|
||||
|
||||
return traverse_obj(json_data, {
|
||||
'id': ('id', {str_or_none}),
|
||||
'display_id': ('publicSlug', {str}),
|
||||
'title': ('title', {str}),
|
||||
'alt_title': ('oldTitle', {str}),
|
||||
'duration': ('length', {float_or_none}),
|
||||
'timestamp': ('added', {parse_iso8601}),
|
||||
'release_timestamp': ('releaseDate', {parse_iso8601}),
|
||||
'categories': ('genres', ..., 'tag', {str}),
|
||||
'tags': ('metadataTags', ..., {str}),
|
||||
'age_limit': ('isExplicit', {lambda b: 18 if b else None}),
|
||||
'thumbnails': ({lambda _: thumbnails}, {orderedSet}, ..., {self._epidemic_parse_thumbnail}),
|
||||
'formats': ('stems', {dict.items}, ..., {
|
||||
'format': (0, {str_or_none}),
|
||||
'format_note': (1, 's3TrackId', {str_or_none}),
|
||||
'format_id': (1, 'stemType', {str}),
|
||||
'url': (1, 'lqMp3Url', {url_or_none}),
|
||||
}, {self._epidemic_fmt_or_none}),
|
||||
})
|
||||
@@ -1,15 +1,20 @@
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
try_call,
|
||||
unified_timestamp,
|
||||
urlencode_postdata,
|
||||
)
|
||||
|
||||
|
||||
class EplusIbIE(InfoExtractor):
|
||||
IE_NAME = 'eplus:inbound'
|
||||
IE_DESC = 'e+ (イープラス) overseas'
|
||||
_VALID_URL = r'https?://live\.eplus\.jp/ex/player\?ib=(?P<id>(?:\w|%2B|%2F){86}%3D%3D)'
|
||||
_NETRC_MACHINE = 'eplus'
|
||||
IE_NAME = 'eplus'
|
||||
IE_DESC = 'e+ (イープラス)'
|
||||
_VALID_URL = [r'https?://live\.eplus\.jp/ex/player\?ib=(?P<id>(?:\w|%2B|%2F){86}%3D%3D)',
|
||||
r'https?://live\.eplus\.jp/(?P<id>sample|\d+)']
|
||||
_TESTS = [{
|
||||
'url': 'https://live.eplus.jp/ex/player?ib=YEFxb3Vyc2Dombnjg7blkrLlrablnJLjgrnjgq%2Fjg7zjg6vjgqLjgqTjg4njg6vlkIzlpb3kvJpgTGllbGxhIQ%3D%3D',
|
||||
'info_dict': {
|
||||
@@ -29,14 +34,97 @@ class EplusIbIE(InfoExtractor):
|
||||
'No video formats found!',
|
||||
'Requested format is not available',
|
||||
],
|
||||
}, {
|
||||
'url': 'https://live.eplus.jp/sample',
|
||||
'info_dict': {
|
||||
'id': 'stream1ng20210719-test-005',
|
||||
'title': 'Online streaming test for DRM',
|
||||
'live_status': 'was_live',
|
||||
'release_date': '20210719',
|
||||
'release_timestamp': 1626703200,
|
||||
'description': None,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
'ignore_no_formats_error': True,
|
||||
},
|
||||
'expected_warnings': [
|
||||
'Could not find the playlist URL. This event may not be accessible',
|
||||
'No video formats found!',
|
||||
'Requested format is not available',
|
||||
'This video is DRM protected',
|
||||
],
|
||||
}, {
|
||||
'url': 'https://live.eplus.jp/2053935',
|
||||
'info_dict': {
|
||||
'id': '331320-0001-001',
|
||||
'title': '丘みどり2020配信LIVE Vol.2 ~秋麗~ 【Streaming+(配信チケット)】',
|
||||
'live_status': 'was_live',
|
||||
'release_date': '20200920',
|
||||
'release_timestamp': 1600596000,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
'ignore_no_formats_error': True,
|
||||
},
|
||||
'expected_warnings': [
|
||||
'Could not find the playlist URL. This event may not be accessible',
|
||||
'No video formats found!',
|
||||
'Requested format is not available',
|
||||
],
|
||||
}]
|
||||
|
||||
_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0'
|
||||
|
||||
def _login(self, username, password, urlh):
|
||||
if not self._get_cookies('https://live.eplus.jp/').get('ci_session'):
|
||||
raise ExtractorError('Unable to get ci_session cookie')
|
||||
|
||||
cltft_token = urlh.headers.get('X-CLTFT-Token')
|
||||
if not cltft_token:
|
||||
raise ExtractorError('Unable to get X-CLTFT-Token')
|
||||
self._set_cookie('live.eplus.jp', 'X-CLTFT-Token', cltft_token)
|
||||
|
||||
login_json = self._download_json(
|
||||
'https://live.eplus.jp/member/api/v1/FTAuth/idpw', None,
|
||||
note='Sending pre-login info', errnote='Unable to send pre-login info', headers={
|
||||
'Content-Type': 'application/json; charset=UTF-8',
|
||||
'Referer': urlh.url,
|
||||
'X-Cltft-Token': cltft_token,
|
||||
'Accept': '*/*',
|
||||
}, data=json.dumps({
|
||||
'loginId': username,
|
||||
'loginPassword': password,
|
||||
}).encode())
|
||||
if not login_json.get('isSuccess'):
|
||||
raise ExtractorError('Login failed: Invalid id or password', expected=True)
|
||||
|
||||
self._request_webpage(
|
||||
urlh.url, None, note='Logging in', errnote='Unable to log in',
|
||||
data=urlencode_postdata({
|
||||
'loginId': username,
|
||||
'loginPassword': password,
|
||||
'Token.Default': cltft_token,
|
||||
'op': 'nextPage',
|
||||
}), headers={'Referer': urlh.url})
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
webpage, urlh = self._download_webpage_handle(
|
||||
url, video_id, headers={'User-Agent': self._USER_AGENT})
|
||||
if urlh.url.startswith('https://live.eplus.jp/member/auth'):
|
||||
username, password = self._get_login_info()
|
||||
if not username:
|
||||
self.raise_login_required()
|
||||
self._login(username, password, urlh)
|
||||
webpage = self._download_webpage(
|
||||
url, video_id, headers={'User-Agent': self._USER_AGENT})
|
||||
|
||||
data_json = self._search_json(r'<script>\s*var app\s*=', webpage, 'data json', video_id)
|
||||
|
||||
if data_json.get('drm_mode') == 'ON':
|
||||
self.report_drm(video_id)
|
||||
|
||||
delivery_status = data_json.get('delivery_status')
|
||||
archive_mode = data_json.get('archive_mode')
|
||||
release_timestamp = try_call(lambda: unified_timestamp(data_json['event_datetime']) - 32400)
|
||||
@@ -64,7 +152,7 @@ class EplusIbIE(InfoExtractor):
|
||||
formats = []
|
||||
|
||||
m3u8_playlist_urls = self._search_json(
|
||||
r'var listChannels\s*=', webpage, 'hls URLs', video_id, contains_pattern=r'\[.+\]', default=[])
|
||||
r'var\s+listChannels\s*=', webpage, 'hls URLs', video_id, contains_pattern=r'\[.+\]', default=[])
|
||||
if not m3u8_playlist_urls:
|
||||
if live_status == 'is_upcoming':
|
||||
self.raise_no_formats(
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
clean_html,
|
||||
int_or_none,
|
||||
float_or_none,
|
||||
)
|
||||
|
||||
|
||||
def _decrypt_config(key, string):
|
||||
a = ''
|
||||
i = ''
|
||||
r = ''
|
||||
|
||||
while len(a) < (len(string) / 2):
|
||||
a += key
|
||||
|
||||
a = a[0:int(len(string) / 2)]
|
||||
|
||||
t = 0
|
||||
while t < len(string):
|
||||
i += chr(int(string[t] + string[t + 1], 16))
|
||||
t += 2
|
||||
|
||||
icko = [s for s in i]
|
||||
|
||||
for t, c in enumerate(a):
|
||||
r += chr(ord(c) ^ ord(icko[t]))
|
||||
|
||||
return r
|
||||
|
||||
|
||||
class EscapistIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://?(?:(?:www|v1)\.)?escapistmagazine\.com/videos/view/[^/]+/(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
|
||||
'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
|
||||
'info_dict': {
|
||||
'id': '6618',
|
||||
'ext': 'mp4',
|
||||
'description': "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
|
||||
'title': "Breaking Down Baldur's Gate",
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'duration': 264,
|
||||
'uploader': 'The Escapist',
|
||||
}
|
||||
}, {
|
||||
'url': 'http://www.escapistmagazine.com/videos/view/zero-punctuation/10044-Evolve-One-vs-Multiplayer',
|
||||
'md5': '9e8c437b0dbb0387d3bd3255ca77f6bf',
|
||||
'info_dict': {
|
||||
'id': '10044',
|
||||
'ext': 'mp4',
|
||||
'description': 'This week, Zero Punctuation reviews Evolve.',
|
||||
'title': 'Evolve - One vs Multiplayer',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'duration': 304,
|
||||
'uploader': 'The Escapist',
|
||||
}
|
||||
}, {
|
||||
'url': 'http://escapistmagazine.com/videos/view/the-escapist-presents/6618',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://v1.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
ims_video = self._parse_json(
|
||||
self._search_regex(
|
||||
r'imsVideo\.play\(({.+?})\);', webpage, 'imsVideo'),
|
||||
video_id)
|
||||
video_id = ims_video['videoID']
|
||||
key = ims_video['hash']
|
||||
|
||||
config = self._download_webpage(
|
||||
'http://www.escapistmagazine.com/videos/vidconfig.php',
|
||||
video_id, 'Downloading video config', headers={
|
||||
'Referer': url,
|
||||
}, query={
|
||||
'videoID': video_id,
|
||||
'hash': key,
|
||||
})
|
||||
|
||||
data = self._parse_json(_decrypt_config(key, config), video_id)
|
||||
|
||||
video_data = data['videoData']
|
||||
|
||||
title = clean_html(video_data['title'])
|
||||
|
||||
formats = [{
|
||||
'url': video['src'],
|
||||
'format_id': '%s-%sp' % (determine_ext(video['src']), video['res']),
|
||||
'height': int_or_none(video.get('res')),
|
||||
} for video in data['files']['videos']]
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'title': title,
|
||||
'thumbnail': self._og_search_thumbnail(webpage) or data.get('poster'),
|
||||
'description': self._og_search_description(webpage),
|
||||
'duration': float_or_none(video_data.get('duration'), 1000),
|
||||
'uploader': video_data.get('publisher'),
|
||||
'series': video_data.get('show'),
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_urlparse
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
parse_filesize,
|
||||
unified_strdate,
|
||||
)
|
||||
|
||||
|
||||
class EsriVideoIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://video\.esri\.com/watch/(?P<id>[0-9]+)'
|
||||
_TEST = {
|
||||
'url': 'https://video.esri.com/watch/1124/arcgis-online-_dash_-developing-applications',
|
||||
'md5': 'd4aaf1408b221f1b38227a9bbaeb95bc',
|
||||
'info_dict': {
|
||||
'id': '1124',
|
||||
'ext': 'mp4',
|
||||
'title': 'ArcGIS Online - Developing Applications',
|
||||
'description': 'Jeremy Bartley demonstrates how to develop applications with ArcGIS Online.',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'duration': 185,
|
||||
'upload_date': '20120419',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
formats = []
|
||||
for width, height, content in re.findall(
|
||||
r'(?s)<li><strong>(\d+)x(\d+):</strong>(.+?)</li>', webpage):
|
||||
for video_url, ext, filesize in re.findall(
|
||||
r'<a[^>]+href="([^"]+)">([^<]+) \(([^<]+)\)</a>', content):
|
||||
formats.append({
|
||||
'url': compat_urlparse.urljoin(url, video_url),
|
||||
'ext': ext.lower(),
|
||||
'format_id': '%s-%s' % (ext.lower(), height),
|
||||
'width': int(width),
|
||||
'height': int(height),
|
||||
'filesize_approx': parse_filesize(filesize),
|
||||
})
|
||||
|
||||
title = self._html_search_meta('title', webpage, 'title')
|
||||
description = self._html_search_meta(
|
||||
'description', webpage, 'description', fatal=False)
|
||||
|
||||
thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail', fatal=False)
|
||||
if thumbnail:
|
||||
thumbnail = re.sub(r'_[st]\.jpg$', '_x.jpg', thumbnail)
|
||||
|
||||
duration = int_or_none(self._search_regex(
|
||||
[r'var\s+videoSeconds\s*=\s*(\d+)', r"'duration'\s*:\s*(\d+)"],
|
||||
webpage, 'duration', fatal=False))
|
||||
|
||||
upload_date = unified_strdate(self._html_search_meta(
|
||||
'last-modified', webpage, 'upload date', fatal=False))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
'duration': duration,
|
||||
'upload_date': upload_date,
|
||||
'formats': formats
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
unified_strdate,
|
||||
)
|
||||
|
||||
|
||||
class ExpoTVIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?expotv\.com/videos/[^?#]*/(?P<id>[0-9]+)($|[?#])'
|
||||
_TEST = {
|
||||
'url': 'http://www.expotv.com/videos/reviews/3/40/NYX-Butter-lipstick/667916',
|
||||
'md5': 'fe1d728c3a813ff78f595bc8b7a707a8',
|
||||
'info_dict': {
|
||||
'id': '667916',
|
||||
'ext': 'mp4',
|
||||
'title': 'NYX Butter Lipstick Little Susie',
|
||||
'description': 'Goes on like butter, but looks better!',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'uploader': 'Stephanie S.',
|
||||
'upload_date': '20150520',
|
||||
'view_count': int,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
player_key = self._search_regex(
|
||||
r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
|
||||
config = self._download_json(
|
||||
'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
|
||||
video_id, 'Downloading video configuration')
|
||||
|
||||
formats = []
|
||||
for fcfg in config['sources']:
|
||||
media_url = fcfg.get('file')
|
||||
if not media_url:
|
||||
continue
|
||||
if fcfg.get('type') == 'm3u8':
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
media_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls'))
|
||||
else:
|
||||
formats.append({
|
||||
'url': media_url,
|
||||
'height': int_or_none(fcfg.get('height')),
|
||||
'format_id': fcfg.get('label'),
|
||||
'ext': self._search_regex(
|
||||
r'filename=.*\.([a-z0-9_A-Z]+)&', media_url,
|
||||
'file extension', default=None) or fcfg.get('type'),
|
||||
})
|
||||
|
||||
title = self._og_search_title(webpage)
|
||||
description = self._og_search_description(webpage)
|
||||
thumbnail = config.get('image')
|
||||
view_count = int_or_none(self._search_regex(
|
||||
r'<h5>Plays: ([0-9]+)</h5>', webpage, 'view counts'))
|
||||
uploader = self._search_regex(
|
||||
r'<div class="reviewer">\s*<img alt="([^"]+)"', webpage, 'uploader',
|
||||
fatal=False)
|
||||
upload_date = unified_strdate(self._search_regex(
|
||||
r'<h5>Reviewed on ([0-9/.]+)</h5>', webpage, 'upload date',
|
||||
fatal=False), day_first=False)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'view_count': view_count,
|
||||
'thumbnail': thumbnail,
|
||||
'uploader': uploader,
|
||||
'upload_date': upload_date,
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
from ..utils import str_to_int
|
||||
from .keezmovies import KeezMoviesIE
|
||||
|
||||
|
||||
class ExtremeTubeIE(KeezMoviesIE): # XXX: Do not subclass from concrete IE
|
||||
_VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
|
||||
'md5': '92feaafa4b58e82f261e5419f39c60cb',
|
||||
'info_dict': {
|
||||
'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431',
|
||||
'ext': 'mp4',
|
||||
'title': 'Music Video 14 british euro brit european cumshots swallow',
|
||||
'uploader': 'anonim',
|
||||
'view_count': int,
|
||||
'age_limit': 18,
|
||||
}
|
||||
}, {
|
||||
'url': 'http://www.extremetube.com/gay/video/abcde-1234',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'http://www.extremetube.com/video/latina-slut-fucked-by-fat-black-dick',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'http://www.extremetube.com/video/652431',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
webpage, info = self._extract_info(url)
|
||||
|
||||
if not info['title']:
|
||||
info['title'] = self._search_regex(
|
||||
r'<h1[^>]+title="([^"]+)"[^>]*>', webpage, 'title')
|
||||
|
||||
uploader = self._html_search_regex(
|
||||
r'Uploaded by:\s*</[^>]+>\s*<a[^>]+>(.+?)</a>',
|
||||
webpage, 'uploader', fatal=False)
|
||||
view_count = str_to_int(self._search_regex(
|
||||
r'Views:\s*</[^>]+>\s*<[^>]+>([\d,\.]+)</',
|
||||
webpage, 'view count', fatal=False))
|
||||
|
||||
info.update({
|
||||
'uploader': uploader,
|
||||
'view_count': view_count,
|
||||
})
|
||||
|
||||
return info
|
||||
@@ -16,6 +16,7 @@ from ..utils import (
|
||||
determine_ext,
|
||||
error_to_compat_str,
|
||||
float_or_none,
|
||||
format_field,
|
||||
get_element_by_id,
|
||||
get_first,
|
||||
int_or_none,
|
||||
@@ -51,7 +52,7 @@ class FacebookIE(InfoExtractor):
|
||||
)\?(?:.*?)(?:v|video_id|story_fbid)=|
|
||||
[^/]+/videos/(?:[^/]+/)?|
|
||||
[^/]+/posts/|
|
||||
groups/[^/]+/permalink/|
|
||||
groups/[^/]+/(?:permalink|posts)/|
|
||||
watchparty/
|
||||
)|
|
||||
facebook:
|
||||
@@ -231,6 +232,21 @@ class FacebookIE(InfoExtractor):
|
||||
'uploader_id': '100013949973717',
|
||||
},
|
||||
'skip': 'Requires logging in',
|
||||
}, {
|
||||
# data.node.comet_sections.content.story.attachments[].throwbackStyles.attachment_target_renderer.attachment.target.attachments[].styles.attachment.media
|
||||
'url': 'https://www.facebook.com/groups/1645456212344334/posts/3737828833107051/',
|
||||
'info_dict': {
|
||||
'id': '1569199726448814',
|
||||
'ext': 'mp4',
|
||||
'title': 'Pence MUST GO!',
|
||||
'description': 'Vickie Gentry shared a memory.',
|
||||
'timestamp': 1511548260,
|
||||
'upload_date': '20171124',
|
||||
'uploader': 'Vickie Gentry',
|
||||
'uploader_id': 'pfbid0FuZhHCeWDAxWxEbr3yKPFaRstXvRxgsp9uCPG6GjD4J2AitB35NUAuJ4Q75KcjiDl',
|
||||
'thumbnail': r're:^https?://.*',
|
||||
'duration': 148.435,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.facebook.com/video.php?v=10204634152394104',
|
||||
'only_matching': True,
|
||||
@@ -420,6 +436,29 @@ class FacebookIE(InfoExtractor):
|
||||
r'data-sjs>({.*?ScheduledServerJS.*?})</script>', webpage)]
|
||||
post = traverse_obj(post_data, (
|
||||
..., 'require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., '__bbox', 'result', 'data'), expected_type=dict) or []
|
||||
|
||||
automatic_captions, subtitles = {}, {}
|
||||
subs_data = traverse_obj(post, (..., 'video', ..., 'attachments', ..., lambda k, v: (
|
||||
k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')))
|
||||
is_video_broadcast = get_first(subs_data, 'is_video_broadcast', expected_type=bool)
|
||||
captions = get_first(subs_data, 'video_available_captions_locales', 'captions_url')
|
||||
if url_or_none(captions): # if subs_data only had a 'captions_url'
|
||||
locale = self._html_search_meta(['og:locale', 'twitter:locale'], webpage, 'locale', default='en_US')
|
||||
subtitles[locale] = [{'url': captions}]
|
||||
# or else subs_data had 'video_available_captions_locales', a list of dicts
|
||||
for caption in traverse_obj(captions, (
|
||||
{lambda x: sorted(x, key=lambda c: c['locale'])}, lambda _, v: v['captions_url'])
|
||||
):
|
||||
lang = caption.get('localized_language') or ''
|
||||
subs = {
|
||||
'url': caption['captions_url'],
|
||||
'name': format_field(caption, 'localized_country', f'{lang} (%s)', default=lang),
|
||||
}
|
||||
if caption.get('localized_creation_method') or is_video_broadcast:
|
||||
automatic_captions.setdefault(caption['locale'], []).append(subs)
|
||||
else:
|
||||
subtitles.setdefault(caption['locale'], []).append(subs)
|
||||
|
||||
media = traverse_obj(post, (..., 'attachments', ..., lambda k, v: (
|
||||
k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')), expected_type=dict)
|
||||
title = get_first(media, ('title', 'text'))
|
||||
@@ -463,6 +502,8 @@ class FacebookIE(InfoExtractor):
|
||||
webpage, 'view count', default=None)),
|
||||
'concurrent_view_count': get_first(post, (
|
||||
('video', (..., ..., 'attachments', ..., 'media')), 'liveViewerCount', {int_or_none})),
|
||||
'automatic_captions': automatic_captions,
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
|
||||
info_json_ld = self._search_json_ld(webpage, video_id, default={})
|
||||
@@ -586,9 +627,11 @@ class FacebookIE(InfoExtractor):
|
||||
nodes = variadic(traverse_obj(data, 'nodes', 'node') or [])
|
||||
attachments = traverse_obj(nodes, (
|
||||
..., 'comet_sections', 'content', 'story', (None, 'attached_story'), 'attachments',
|
||||
..., ('styles', 'style_type_renderer'), 'attachment'), expected_type=dict) or []
|
||||
..., ('styles', 'style_type_renderer', ('throwbackStyles', 'attachment_target_renderer')),
|
||||
'attachment', {dict}))
|
||||
for attachment in attachments:
|
||||
ns = try_get(attachment, lambda x: x['all_subattachments']['nodes'], list) or []
|
||||
ns = traverse_obj(attachment, ('all_subattachments', 'nodes', ..., {dict}),
|
||||
('target', 'attachments', ..., 'styles', 'attachment', {dict}))
|
||||
for n in ns:
|
||||
parse_attachment(n)
|
||||
parse_attachment(attachment)
|
||||
@@ -611,7 +654,7 @@ class FacebookIE(InfoExtractor):
|
||||
if len(entries) > 1:
|
||||
return self.playlist_result(entries, video_id)
|
||||
|
||||
video_info = entries[0]
|
||||
video_info = entries[0] if entries else {'id': video_id}
|
||||
webpage_info = extract_metadata(webpage)
|
||||
# honor precise duration in video info
|
||||
if video_info.get('duration'):
|
||||
|
||||
@@ -2,11 +2,9 @@ import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_parse_qs
|
||||
from ..dependencies import websockets
|
||||
from ..networking import Request
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
WebSocketsWrapper,
|
||||
js_to_json,
|
||||
traverse_obj,
|
||||
update_url_query,
|
||||
@@ -167,8 +165,6 @@ class FC2LiveIE(InfoExtractor):
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
if not websockets:
|
||||
raise ExtractorError('websockets library is not available. Please install it.', expected=True)
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage('https://live.fc2.com/%s/' % video_id, video_id)
|
||||
|
||||
@@ -199,13 +195,9 @@ class FC2LiveIE(InfoExtractor):
|
||||
ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']})
|
||||
playlist_data = None
|
||||
|
||||
self.to_screen('%s: Fetching HLS playlist info via WebSocket' % video_id)
|
||||
ws = WebSocketsWrapper(ws_url, {
|
||||
'Cookie': str(self._get_cookies('https://live.fc2.com/'))[12:],
|
||||
ws = self._request_webpage(Request(ws_url, headers={
|
||||
'Origin': 'https://live.fc2.com',
|
||||
'Accept': '*/*',
|
||||
'User-Agent': self.get_param('http_headers')['User-Agent'],
|
||||
})
|
||||
}), video_id, note='Fetching HLS playlist info via WebSocket')
|
||||
|
||||
self.write_debug('Sending HLS server request')
|
||||
|
||||
|
||||
268
yt_dlp/extractor/floatplane.py
Normal file
268
yt_dlp/extractor/floatplane.py
Normal file
@@ -0,0 +1,268 @@
|
||||
import functools
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
OnDemandPagedList,
|
||||
clean_html,
|
||||
determine_ext,
|
||||
format_field,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
parse_codecs,
|
||||
parse_iso8601,
|
||||
urljoin,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class FloatplaneIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/post/(?P<id>\w+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.floatplane.com/post/2Yf3UedF7C',
|
||||
'info_dict': {
|
||||
'id': 'yuleLogLTT',
|
||||
'ext': 'mp4',
|
||||
'display_id': '2Yf3UedF7C',
|
||||
'title': '8K Yule Log Fireplace with Crackling Fire Sounds - 10 Hours',
|
||||
'description': 'md5:adf2970e0de1c5e3df447818bb0309f6',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g$',
|
||||
'duration': 36035,
|
||||
'comment_count': int,
|
||||
'like_count': int,
|
||||
'dislike_count': int,
|
||||
'release_date': '20191206',
|
||||
'release_timestamp': 1575657000,
|
||||
'uploader': 'LinusTechTips',
|
||||
'uploader_id': '59f94c0bdd241b70349eb72b',
|
||||
'uploader_url': 'https://www.floatplane.com/channel/linustechtips/home',
|
||||
'channel': 'Linus Tech Tips',
|
||||
'channel_id': '63fe42c309e691e4e36de93d',
|
||||
'channel_url': 'https://www.floatplane.com/channel/linustechtips/home/main',
|
||||
'availability': 'subscriber_only',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://www.floatplane.com/post/j2jqG3JmgJ',
|
||||
'info_dict': {
|
||||
'id': 'j2jqG3JmgJ',
|
||||
'title': 'TJM: Does Anyone Care About Avatar: The Way of Water?',
|
||||
'description': 'md5:00bf17dc5733e4031e99b7fd6489f274',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g$',
|
||||
'comment_count': int,
|
||||
'like_count': int,
|
||||
'dislike_count': int,
|
||||
'release_timestamp': 1671915900,
|
||||
'release_date': '20221224',
|
||||
'uploader': 'LinusTechTips',
|
||||
'uploader_id': '59f94c0bdd241b70349eb72b',
|
||||
'uploader_url': 'https://www.floatplane.com/channel/linustechtips/home',
|
||||
'channel': "They're Just Movies",
|
||||
'channel_id': '64135f82fc76ab7f9fbdc876',
|
||||
'channel_url': 'https://www.floatplane.com/channel/linustechtips/home/tajm',
|
||||
'availability': 'subscriber_only',
|
||||
},
|
||||
'playlist_count': 2,
|
||||
}, {
|
||||
'url': 'https://www.floatplane.com/post/3tK2tInhoN',
|
||||
'info_dict': {
|
||||
'id': '3tK2tInhoN',
|
||||
'title': 'Extras - How Linus Communicates with Editors (Compensator 4)',
|
||||
'description': 'md5:83cd40aae1ce124df33769600c80ca5b',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g$',
|
||||
'comment_count': int,
|
||||
'like_count': int,
|
||||
'dislike_count': int,
|
||||
'release_timestamp': 1700529120,
|
||||
'release_date': '20231121',
|
||||
'uploader': 'LinusTechTips',
|
||||
'uploader_id': '59f94c0bdd241b70349eb72b',
|
||||
'uploader_url': 'https://www.floatplane.com/channel/linustechtips/home',
|
||||
'channel': 'FP Exclusives',
|
||||
'channel_id': '6413623f5b12cca228a28e78',
|
||||
'channel_url': 'https://www.floatplane.com/channel/linustechtips/home/fpexclusive',
|
||||
'availability': 'subscriber_only',
|
||||
},
|
||||
'playlist_count': 2,
|
||||
}, {
|
||||
'url': 'https://beta.floatplane.com/post/d870PEFXS1',
|
||||
'info_dict': {
|
||||
'id': 'bg9SuYKEww',
|
||||
'ext': 'mp4',
|
||||
'display_id': 'd870PEFXS1',
|
||||
'title': 'LCS Drama, TLOU 2 Remaster, Destiny 2 Player Count Drops, + More!',
|
||||
'description': 'md5:80d612dcabf41b17487afcbe303ec57d',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g$',
|
||||
'release_timestamp': 1700622000,
|
||||
'release_date': '20231122',
|
||||
'duration': 513,
|
||||
'like_count': int,
|
||||
'dislike_count': int,
|
||||
'comment_count': int,
|
||||
'uploader': 'LinusTechTips',
|
||||
'uploader_id': '59f94c0bdd241b70349eb72b',
|
||||
'uploader_url': 'https://www.floatplane.com/channel/linustechtips/home',
|
||||
'channel': 'GameLinked',
|
||||
'channel_id': '649dbade3540dbc3945eeda7',
|
||||
'channel_url': 'https://www.floatplane.com/channel/linustechtips/home/gamelinked',
|
||||
'availability': 'subscriber_only',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}]
|
||||
|
||||
def _real_initialize(self):
|
||||
if not self._get_cookies('https://www.floatplane.com').get('sails.sid'):
|
||||
self.raise_login_required()
|
||||
|
||||
def _real_extract(self, url):
|
||||
post_id = self._match_id(url)
|
||||
|
||||
post_data = self._download_json(
|
||||
'https://www.floatplane.com/api/v3/content/post', post_id, query={'id': post_id},
|
||||
note='Downloading post data', errnote='Unable to download post data')
|
||||
|
||||
if not any(traverse_obj(post_data, ('metadata', ('hasVideo', 'hasAudio')))):
|
||||
raise ExtractorError('Post does not contain a video or audio track', expected=True)
|
||||
|
||||
items = []
|
||||
for media in traverse_obj(post_data, (('videoAttachments', 'audioAttachments'), ...)):
|
||||
media_id = media['id']
|
||||
media_typ = media.get('type') or 'video'
|
||||
|
||||
metadata = self._download_json(
|
||||
f'https://www.floatplane.com/api/v3/content/{media_typ}', media_id, query={'id': media_id},
|
||||
note=f'Downloading {media_typ} metadata')
|
||||
|
||||
stream = self._download_json(
|
||||
'https://www.floatplane.com/api/v2/cdn/delivery', media_id, query={
|
||||
'type': 'vod' if media_typ == 'video' else 'aod',
|
||||
'guid': metadata['guid']
|
||||
}, note=f'Downloading {media_typ} stream data')
|
||||
|
||||
path_template = traverse_obj(stream, ('resource', 'uri', {str}))
|
||||
|
||||
def format_path(params):
|
||||
path = path_template
|
||||
for i, val in (params or {}).items():
|
||||
path = path.replace(f'{{qualityLevelParams.{i}}}', val)
|
||||
return path
|
||||
|
||||
formats = []
|
||||
for quality in traverse_obj(stream, ('resource', 'data', 'qualityLevels', ...)):
|
||||
url = urljoin(stream['cdn'], format_path(traverse_obj(
|
||||
stream, ('resource', 'data', 'qualityLevelParams', quality['name']))))
|
||||
formats.append({
|
||||
**traverse_obj(quality, {
|
||||
'format_id': 'name',
|
||||
'format_note': 'label',
|
||||
'width': ('width', {int}),
|
||||
'height': ('height', {int}),
|
||||
}),
|
||||
**parse_codecs(quality.get('codecs')),
|
||||
'url': url,
|
||||
'ext': determine_ext(url.partition('/chunk.m3u8')[0], 'mp4'),
|
||||
})
|
||||
|
||||
items.append({
|
||||
'id': media_id,
|
||||
**traverse_obj(metadata, {
|
||||
'title': 'title',
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'thumbnail': ('thumbnail', 'path'),
|
||||
}),
|
||||
'formats': formats,
|
||||
})
|
||||
|
||||
uploader_url = format_field(
|
||||
post_data, [('creator', 'urlname')], 'https://www.floatplane.com/channel/%s/home') or None
|
||||
channel_url = urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname')))
|
||||
|
||||
post_info = {
|
||||
'id': post_id,
|
||||
'display_id': post_id,
|
||||
**traverse_obj(post_data, {
|
||||
'title': 'title',
|
||||
'description': ('text', {clean_html}),
|
||||
'uploader': ('creator', 'title'),
|
||||
'uploader_id': ('creator', 'id'),
|
||||
'channel': ('channel', 'title'),
|
||||
'channel_id': ('channel', 'id'),
|
||||
'like_count': ('likes', {int_or_none}),
|
||||
'dislike_count': ('dislikes', {int_or_none}),
|
||||
'comment_count': ('comments', {int_or_none}),
|
||||
'release_timestamp': ('releaseDate', {parse_iso8601}),
|
||||
'thumbnail': ('thumbnail', 'path'),
|
||||
}),
|
||||
'uploader_url': uploader_url,
|
||||
'channel_url': channel_url,
|
||||
'availability': self._availability(needs_subscription=True),
|
||||
}
|
||||
|
||||
if len(items) > 1:
|
||||
return self.playlist_result(items, **post_info)
|
||||
|
||||
post_info.update(items[0])
|
||||
return post_info
|
||||
|
||||
|
||||
class FloatplaneChannelIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/channel/(?P<id>[\w-]+)/home(?:/(?P<channel>[\w-]+))?'
|
||||
_PAGE_SIZE = 20
|
||||
_TESTS = [{
|
||||
'url': 'https://www.floatplane.com/channel/linustechtips/home/ltxexpo',
|
||||
'info_dict': {
|
||||
'id': 'linustechtips/ltxexpo',
|
||||
'title': 'LTX Expo',
|
||||
'description': 'md5:9819002f9ebe7fd7c75a3a1d38a59149',
|
||||
},
|
||||
'playlist_mincount': 51,
|
||||
}, {
|
||||
'url': 'https://www.floatplane.com/channel/ShankMods/home',
|
||||
'info_dict': {
|
||||
'id': 'ShankMods',
|
||||
'title': 'Shank Mods',
|
||||
'description': 'md5:6dff1bb07cad8e5448e04daad9be1b30',
|
||||
},
|
||||
'playlist_mincount': 14,
|
||||
}, {
|
||||
'url': 'https://beta.floatplane.com/channel/bitwit_ultra/home',
|
||||
'info_dict': {
|
||||
'id': 'bitwit_ultra',
|
||||
'title': 'Bitwit Ultra',
|
||||
'description': 'md5:1452f280bb45962976d4789200f676dd',
|
||||
},
|
||||
'playlist_mincount': 200,
|
||||
}]
|
||||
|
||||
def _fetch_page(self, display_id, creator_id, channel_id, page):
|
||||
query = {
|
||||
'id': creator_id,
|
||||
'limit': self._PAGE_SIZE,
|
||||
'fetchAfter': page * self._PAGE_SIZE,
|
||||
}
|
||||
if channel_id:
|
||||
query['channel'] = channel_id
|
||||
page_data = self._download_json(
|
||||
'https://www.floatplane.com/api/v3/content/creator', display_id,
|
||||
query=query, note=f'Downloading page {page + 1}')
|
||||
for post in page_data or []:
|
||||
yield self.url_result(
|
||||
f'https://www.floatplane.com/post/{post["id"]}',
|
||||
FloatplaneIE, id=post['id'], title=post.get('title'),
|
||||
release_timestamp=parse_iso8601(post.get('releaseDate')))
|
||||
|
||||
def _real_extract(self, url):
|
||||
creator, channel = self._match_valid_url(url).group('id', 'channel')
|
||||
display_id = join_nonempty(creator, channel, delim='/')
|
||||
|
||||
creator_data = self._download_json(
|
||||
'https://www.floatplane.com/api/v3/creator/named',
|
||||
display_id, query={'creatorURL[0]': creator})[0]
|
||||
|
||||
channel_data = traverse_obj(
|
||||
creator_data, ('channels', lambda _, v: v['urlname'] == channel), get_all=False) or {}
|
||||
|
||||
return self.playlist_result(OnDemandPagedList(functools.partial(
|
||||
self._fetch_page, display_id, creator_data['id'], channel_data.get('id')), self._PAGE_SIZE),
|
||||
display_id, title=channel_data.get('title') or creator_data.get('title'),
|
||||
description=channel_data.get('about') or creator_data.get('about'))
|
||||
@@ -1,106 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import traverse_obj, unified_timestamp
|
||||
|
||||
|
||||
class FourZeroStudioArchiveIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://0000\.studio/(?P<uploader_id>[^/]+)/broadcasts/(?P<id>[^/]+)/archive'
|
||||
IE_NAME = '0000studio:archive'
|
||||
_TESTS = [{
|
||||
'url': 'https://0000.studio/mumeijiten/broadcasts/1290f433-fce0-4909-a24a-5f7df09665dc/archive',
|
||||
'info_dict': {
|
||||
'id': '1290f433-fce0-4909-a24a-5f7df09665dc',
|
||||
'title': 'noteで『canape』様へのファンレターを執筆します。(数秘術その2)',
|
||||
'timestamp': 1653802534,
|
||||
'release_timestamp': 1653796604,
|
||||
'thumbnails': 'count:1',
|
||||
'comments': 'count:7',
|
||||
'uploader': '『中崎雄心』の執務室。',
|
||||
'uploader_id': 'mumeijiten',
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id, uploader_id = self._match_valid_url(url).group('id', 'uploader_id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
nuxt_data = self._search_nuxt_data(webpage, video_id, traverse=None)
|
||||
|
||||
pcb = traverse_obj(nuxt_data, ('ssrRefs', lambda _, v: v['__typename'] == 'PublicCreatorBroadcast'), get_all=False)
|
||||
uploader_internal_id = traverse_obj(nuxt_data, (
|
||||
'ssrRefs', lambda _, v: v['__typename'] == 'PublicUser', 'id'), get_all=False)
|
||||
|
||||
formats, subs = self._extract_m3u8_formats_and_subtitles(pcb['archiveUrl'], video_id, ext='mp4')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': pcb.get('title'),
|
||||
'age_limit': 18 if pcb.get('isAdult') else None,
|
||||
'timestamp': unified_timestamp(pcb.get('finishTime')),
|
||||
'release_timestamp': unified_timestamp(pcb.get('createdAt')),
|
||||
'thumbnails': [{
|
||||
'url': pcb['thumbnailUrl'],
|
||||
'ext': 'png',
|
||||
}] if pcb.get('thumbnailUrl') else None,
|
||||
'formats': formats,
|
||||
'subtitles': subs,
|
||||
'comments': [{
|
||||
'author': c.get('username'),
|
||||
'author_id': c.get('postedUserId'),
|
||||
'author_thumbnail': c.get('userThumbnailUrl'),
|
||||
'id': c.get('id'),
|
||||
'text': c.get('body'),
|
||||
'timestamp': unified_timestamp(c.get('createdAt')),
|
||||
'like_count': c.get('likeCount'),
|
||||
'is_favorited': c.get('isLikedByOwner'),
|
||||
'author_is_uploader': c.get('postedUserId') == uploader_internal_id,
|
||||
} for c in traverse_obj(nuxt_data, (
|
||||
'ssrRefs', ..., lambda _, v: v['__typename'] == 'PublicCreatorBroadcastComment')) or []],
|
||||
'uploader_id': uploader_id,
|
||||
'uploader': traverse_obj(nuxt_data, (
|
||||
'ssrRefs', lambda _, v: v['__typename'] == 'PublicUser', 'username'), get_all=False),
|
||||
}
|
||||
|
||||
|
||||
class FourZeroStudioClipIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://0000\.studio/(?P<uploader_id>[^/]+)/archive-clip/(?P<id>[^/]+)'
|
||||
IE_NAME = '0000studio:clip'
|
||||
_TESTS = [{
|
||||
'url': 'https://0000.studio/soeji/archive-clip/e46b0278-24cd-40a8-92e1-b8fc2b21f34f',
|
||||
'info_dict': {
|
||||
'id': 'e46b0278-24cd-40a8-92e1-b8fc2b21f34f',
|
||||
'title': 'わたベーさんからイラスト差し入れいただきました。ありがとうございました!',
|
||||
'timestamp': 1652109105,
|
||||
'like_count': 1,
|
||||
'uploader': 'ソエジマケイタ',
|
||||
'uploader_id': 'soeji',
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id, uploader_id = self._match_valid_url(url).group('id', 'uploader_id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
nuxt_data = self._search_nuxt_data(webpage, video_id, traverse=None)
|
||||
|
||||
clip_info = traverse_obj(nuxt_data, ('ssrRefs', lambda _, v: v['__typename'] == 'PublicCreatorArchivedClip'), get_all=False)
|
||||
|
||||
info = next((
|
||||
m for m in self._parse_html5_media_entries(url, webpage, video_id)
|
||||
if 'mp4' in traverse_obj(m, ('formats', ..., 'ext'))
|
||||
), None)
|
||||
if not info:
|
||||
self.report_warning('Failed to find a desired media element. Falling back to using NUXT data.')
|
||||
info = {
|
||||
'formats': [{
|
||||
'ext': 'mp4',
|
||||
'url': url,
|
||||
} for url in clip_info.get('mediaFiles') or [] if url],
|
||||
}
|
||||
return {
|
||||
**info,
|
||||
'id': video_id,
|
||||
'title': clip_info.get('clipComment'),
|
||||
'timestamp': unified_timestamp(clip_info.get('createdAt')),
|
||||
'like_count': clip_info.get('likeCount'),
|
||||
'uploader_id': uploader_id,
|
||||
'uploader': traverse_obj(nuxt_data, (
|
||||
'ssrRefs', lambda _, v: v['__typename'] == 'PublicUser', 'username'), get_all=False),
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
import itertools
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
get_element_by_id,
|
||||
int_or_none,
|
||||
remove_end,
|
||||
)
|
||||
|
||||
|
||||
class FoxgayIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
|
||||
_TEST = {
|
||||
'url': 'http://foxgay.com/videos/fuck-turkish-style-2582.shtml',
|
||||
'md5': '344558ccfea74d33b7adbce22e577f54',
|
||||
'info_dict': {
|
||||
'id': '2582',
|
||||
'ext': 'mp4',
|
||||
'title': 'Fuck Turkish-style',
|
||||
'description': 'md5:6ae2d9486921891efe89231ace13ffdf',
|
||||
'age_limit': 18,
|
||||
'thumbnail': r're:https?://.*\.jpg$',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
title = remove_end(self._html_extract_title(webpage), ' - Foxgay.com')
|
||||
description = get_element_by_id('inf_tit', webpage)
|
||||
|
||||
# The default user-agent with foxgay cookies leads to pages without videos
|
||||
self.cookiejar.clear('.foxgay.com')
|
||||
# Find the URL for the iFrame which contains the actual video.
|
||||
iframe_url = self._html_search_regex(
|
||||
r'<iframe[^>]+src=([\'"])(?P<url>[^\'"]+)\1', webpage,
|
||||
'video frame', group='url')
|
||||
iframe = self._download_webpage(
|
||||
iframe_url, video_id, headers={'User-Agent': 'curl/7.50.1'},
|
||||
note='Downloading video frame')
|
||||
video_data = self._parse_json(self._search_regex(
|
||||
r'video_data\s*=\s*([^;]+);', iframe, 'video data'), video_id)
|
||||
|
||||
formats = [{
|
||||
'url': source,
|
||||
'height': int_or_none(resolution),
|
||||
} for source, resolution in zip(
|
||||
video_data['sources'], video_data.get('resolutions', itertools.repeat(None)))]
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'description': description,
|
||||
'thumbnail': video_data.get('act_vid', {}).get('thumb'),
|
||||
'age_limit': 18,
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
from .common import InfoExtractor
|
||||
from .dailymotion import DailymotionIE
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
ExtractorError,
|
||||
determine_ext,
|
||||
format_field,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
parse_iso8601,
|
||||
parse_qs,
|
||||
)
|
||||
from .dailymotion import DailymotionIE
|
||||
|
||||
|
||||
class FranceTVBaseInfoExtractor(InfoExtractor):
|
||||
@@ -82,6 +84,8 @@ class FranceTVIE(InfoExtractor):
|
||||
videos = []
|
||||
title = None
|
||||
subtitle = None
|
||||
episode_number = None
|
||||
season_number = None
|
||||
image = None
|
||||
duration = None
|
||||
timestamp = None
|
||||
@@ -112,7 +116,9 @@ class FranceTVIE(InfoExtractor):
|
||||
if meta:
|
||||
if title is None:
|
||||
title = meta.get('title')
|
||||
# XXX: what is meta['pre_title']?
|
||||
# meta['pre_title'] contains season and episode number for series in format "S<ID> E<ID>"
|
||||
season_number, episode_number = self._search_regex(
|
||||
r'S(\d+)\s*E(\d+)', meta.get('pre_title'), 'episode info', group=(1, 2), default=(None, None))
|
||||
if subtitle is None:
|
||||
subtitle = meta.get('additional_title')
|
||||
if image is None:
|
||||
@@ -191,19 +197,19 @@ class FranceTVIE(InfoExtractor):
|
||||
} for sheet in spritesheets]
|
||||
})
|
||||
|
||||
if subtitle:
|
||||
title += ' - %s' % subtitle
|
||||
title = title.strip()
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'title': join_nonempty(title, subtitle, delim=' - ').strip(),
|
||||
'thumbnail': image,
|
||||
'duration': duration,
|
||||
'timestamp': timestamp,
|
||||
'is_live': is_live,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'episode': subtitle if episode_number else None,
|
||||
'series': title if episode_number else None,
|
||||
'episode_number': int_or_none(episode_number),
|
||||
'season_number': int_or_none(season_number),
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
@@ -230,14 +236,31 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
|
||||
'id': 'ec217ecc-0733-48cf-ac06-af1347b849d1',
|
||||
'ext': 'mp4',
|
||||
'title': '13h15, le dimanche... - Les mystères de Jésus',
|
||||
'description': 'md5:75efe8d4c0a8205e5904498ffe1e1a42',
|
||||
'timestamp': 1502623500,
|
||||
'duration': 2580,
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'upload_date': '20170813',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
'add_ie': [FranceTVIE.ie_key()],
|
||||
}, {
|
||||
'url': 'https://www.france.tv/enfants/six-huit-ans/foot2rue/saison-1/3066387-duel-au-vieux-port.html',
|
||||
'info_dict': {
|
||||
'id': 'a9050959-eedd-4b4a-9b0d-de6eeaa73e44',
|
||||
'ext': 'mp4',
|
||||
'title': 'Foot2Rue - Duel au vieux port',
|
||||
'episode': 'Duel au vieux port',
|
||||
'series': 'Foot2Rue',
|
||||
'episode_number': 1,
|
||||
'season_number': 1,
|
||||
'timestamp': 1642761360,
|
||||
'upload_date': '20220121',
|
||||
'season': 'Season 1',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'duration': 1441,
|
||||
},
|
||||
}, {
|
||||
# france3
|
||||
'url': 'https://www.france.tv/france-3/des-chiffres-et-des-lettres/139063-emission-du-mardi-9-mai-2017.html',
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
int_or_none,
|
||||
mimetype2ext,
|
||||
parse_iso8601,
|
||||
)
|
||||
|
||||
|
||||
class FusionIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?fusion\.(?:net|tv)/(?:video/|show/.+?\bvideo=)(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://fusion.tv/video/201781/u-s-and-panamanian-forces-work-together-to-stop-a-vessel-smuggling-drugs/',
|
||||
'info_dict': {
|
||||
'id': '3145868',
|
||||
'ext': 'mp4',
|
||||
'title': 'U.S. and Panamanian forces work together to stop a vessel smuggling drugs',
|
||||
'description': 'md5:0cc84a9943c064c0f46b128b41b1b0d7',
|
||||
'duration': 140.0,
|
||||
'timestamp': 1442589635,
|
||||
'uploader': 'UNIVISON',
|
||||
'upload_date': '20150918',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
'add_ie': ['Anvato'],
|
||||
}, {
|
||||
'url': 'http://fusion.tv/video/201781',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://fusion.tv/show/food-exposed-with-nelufar-hedayat/?ancla=full-episodes&video=588644',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
video = self._download_json(
|
||||
'https://platform.fusion.net/wp-json/fusiondotnet/v1/video/' + video_id, video_id)
|
||||
|
||||
info = {
|
||||
'id': video_id,
|
||||
'title': video['title'],
|
||||
'description': video.get('excerpt'),
|
||||
'timestamp': parse_iso8601(video.get('published')),
|
||||
'series': video.get('show'),
|
||||
}
|
||||
|
||||
formats = []
|
||||
src = video.get('src') or {}
|
||||
for f_id, f in src.items():
|
||||
for q_id, q in f.items():
|
||||
q_url = q.get('url')
|
||||
if not q_url:
|
||||
continue
|
||||
ext = determine_ext(q_url, mimetype2ext(q.get('type')))
|
||||
if ext == 'smil':
|
||||
formats.extend(self._extract_smil_formats(q_url, video_id, fatal=False))
|
||||
elif f_id == 'm3u8-variant' or (ext == 'm3u8' and q_id == 'Variant'):
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
q_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
|
||||
else:
|
||||
formats.append({
|
||||
'format_id': '-'.join([f_id, q_id]),
|
||||
'url': q_url,
|
||||
'width': int_or_none(q.get('width')),
|
||||
'height': int_or_none(q.get('height')),
|
||||
'tbr': int_or_none(self._search_regex(r'_(\d+)\.m(?:p4|3u8)', q_url, 'bitrate')),
|
||||
'ext': 'mp4' if ext == 'm3u8' else ext,
|
||||
'protocol': 'm3u8_native' if ext == 'm3u8' else 'https',
|
||||
})
|
||||
if formats:
|
||||
info['formats'] = formats
|
||||
else:
|
||||
info.update({
|
||||
'_type': 'url',
|
||||
'url': 'anvato:uni:' + video['video_ids']['anvato'],
|
||||
'ie_key': 'Anvato',
|
||||
})
|
||||
|
||||
return info
|
||||
@@ -35,8 +35,8 @@ from ..utils import (
|
||||
unified_timestamp,
|
||||
unsmuggle_url,
|
||||
update_url_query,
|
||||
urlhandle_detect_ext,
|
||||
url_or_none,
|
||||
urlhandle_detect_ext,
|
||||
urljoin,
|
||||
variadic,
|
||||
xpath_attr,
|
||||
@@ -374,46 +374,6 @@ class GenericIE(InfoExtractor):
|
||||
},
|
||||
'skip': 'There is a limit of 200 free downloads / month for the test song',
|
||||
},
|
||||
# ooyala video
|
||||
{
|
||||
'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
|
||||
'md5': '166dd577b433b4d4ebfee10b0824d8ff',
|
||||
'info_dict': {
|
||||
'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
|
||||
'ext': 'mp4',
|
||||
'title': '2cc213299525360.mov', # that's what we get
|
||||
'duration': 238.231,
|
||||
},
|
||||
'add_ie': ['Ooyala'],
|
||||
},
|
||||
{
|
||||
# ooyala video embedded with http://player.ooyala.com/iframe.js
|
||||
'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
|
||||
'info_dict': {
|
||||
'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
|
||||
'ext': 'mp4',
|
||||
'title': '"Steve Jobs: Man in the Machine" trailer',
|
||||
'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
|
||||
'duration': 135.427,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
'skip': 'movie expired',
|
||||
},
|
||||
# ooyala video embedded with http://player.ooyala.com/static/v4/production/latest/core.min.js
|
||||
{
|
||||
'url': 'http://wnep.com/2017/07/22/steampunk-fest-comes-to-honesdale/',
|
||||
'info_dict': {
|
||||
'id': 'lwYWYxYzE6V5uJMjNGyKtwwiw9ZJD7t2',
|
||||
'ext': 'mp4',
|
||||
'title': 'Steampunk Fest Comes to Honesdale',
|
||||
'duration': 43.276,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
}
|
||||
},
|
||||
# embed.ly video
|
||||
{
|
||||
'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
|
||||
@@ -506,7 +466,8 @@ class GenericIE(InfoExtractor):
|
||||
'title': 'Ужастики, русский трейлер (2015)',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'duration': 153,
|
||||
}
|
||||
},
|
||||
'skip': 'Site dead',
|
||||
},
|
||||
# XHamster embed
|
||||
{
|
||||
@@ -778,14 +739,16 @@ class GenericIE(InfoExtractor):
|
||||
'playlist_mincount': 1,
|
||||
'add_ie': ['Youtube'],
|
||||
},
|
||||
# Cinchcast embed
|
||||
# Libsyn embed
|
||||
{
|
||||
'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
|
||||
'info_dict': {
|
||||
'id': '7141703',
|
||||
'id': '3793998',
|
||||
'ext': 'mp3',
|
||||
'upload_date': '20141126',
|
||||
'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
|
||||
'title': 'Underground Wellness Radio - Jack Tips: 5 Steps to Permanent Gut Healing',
|
||||
'thumbnail': 'https://assets.libsyn.com/secure/item/3793998/?height=90&width=90',
|
||||
'duration': 3989.0,
|
||||
}
|
||||
},
|
||||
# Cinerama player
|
||||
@@ -1567,16 +1530,6 @@ class GenericIE(InfoExtractor):
|
||||
'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
|
||||
},
|
||||
},
|
||||
{
|
||||
# vzaar embed
|
||||
'url': 'http://help.vzaar.com/article/165-embedding-video',
|
||||
'md5': '7e3919d9d2620b89e3e00bec7fe8c9d4',
|
||||
'info_dict': {
|
||||
'id': '8707641',
|
||||
'ext': 'mp4',
|
||||
'title': 'Building A Business Online: Principal Chairs Q & A',
|
||||
},
|
||||
},
|
||||
{
|
||||
# multiple HTML5 videos on one page
|
||||
'url': 'https://www.paragon-software.com/home/rk-free/keyscenarios.html',
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
float_or_none,
|
||||
qualities,
|
||||
ExtractorError,
|
||||
)
|
||||
|
||||
|
||||
class GfycatIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:(?:www|giant|thumbs)\.)?gfycat\.com/(?i:ru/|ifr/|gifs/detail/)?(?P<id>[^-/?#\."\']+)'
|
||||
_EMBED_REGEX = [rf'<(?:iframe|source)[^>]+\bsrc=["\'](?P<url>{_VALID_URL})']
|
||||
_TESTS = [{
|
||||
'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
|
||||
'info_dict': {
|
||||
'id': 'DeadlyDecisiveGermanpinscher',
|
||||
'ext': 'mp4',
|
||||
'title': 'Ghost in the Shell',
|
||||
'timestamp': 1410656006,
|
||||
'upload_date': '20140914',
|
||||
'uploader': 'anonymous',
|
||||
'duration': 10.4,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'categories': list,
|
||||
'age_limit': 0,
|
||||
'uploader_id': 'anonymous',
|
||||
'description': '',
|
||||
}
|
||||
}, {
|
||||
'url': 'http://gfycat.com/ifr/JauntyTimelyAmazontreeboa',
|
||||
'info_dict': {
|
||||
'id': 'JauntyTimelyAmazontreeboa',
|
||||
'ext': 'mp4',
|
||||
'title': 'JauntyTimelyAmazontreeboa',
|
||||
'timestamp': 1411720126,
|
||||
'upload_date': '20140926',
|
||||
'uploader': 'anonymous',
|
||||
'duration': 3.52,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'categories': list,
|
||||
'age_limit': 0,
|
||||
'uploader_id': 'anonymous',
|
||||
'description': '',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://gfycat.com/alienatedsolidgreathornedowl',
|
||||
'info_dict': {
|
||||
'id': 'alienatedsolidgreathornedowl',
|
||||
'ext': 'mp4',
|
||||
'upload_date': '20211226',
|
||||
'uploader_id': 'reactions',
|
||||
'timestamp': 1640536930,
|
||||
'like_count': int,
|
||||
'description': '',
|
||||
'title': 'Ingrid Michaelson, Zooey Deschanel - Merry Christmas Happy New Year',
|
||||
'categories': list,
|
||||
'age_limit': 0,
|
||||
'duration': 2.9583333333333335,
|
||||
'uploader': 'Reaction GIFs',
|
||||
'view_count': int,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://gfycat.com/ru/RemarkableDrearyAmurstarfish',
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'https://gfycat.com/gifs/detail/UnconsciousLankyIvorygull',
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'https://gfycat.com/acceptablehappygoluckyharborporpoise-baseball',
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'https://thumbs.gfycat.com/acceptablehappygoluckyharborporpoise-size_restricted.gif',
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'https://giant.gfycat.com/acceptablehappygoluckyharborporpoise.mp4',
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'http://gfycat.com/IFR/JauntyTimelyAmazontreeboa',
|
||||
'only_matching': True
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
gfy = self._download_json(
|
||||
'https://api.gfycat.com/v1/gfycats/%s' % video_id,
|
||||
video_id, 'Downloading video info')
|
||||
if 'error' in gfy:
|
||||
raise ExtractorError('Gfycat said: ' + gfy['error'], expected=True)
|
||||
gfy = gfy['gfyItem']
|
||||
|
||||
title = gfy.get('title') or gfy['gfyName']
|
||||
description = gfy.get('description')
|
||||
timestamp = int_or_none(gfy.get('createDate'))
|
||||
uploader = gfy.get('userName') or gfy.get('username')
|
||||
view_count = int_or_none(gfy.get('views'))
|
||||
like_count = int_or_none(gfy.get('likes'))
|
||||
dislike_count = int_or_none(gfy.get('dislikes'))
|
||||
age_limit = 18 if gfy.get('nsfw') == '1' else 0
|
||||
|
||||
width = int_or_none(gfy.get('width'))
|
||||
height = int_or_none(gfy.get('height'))
|
||||
fps = int_or_none(gfy.get('frameRate'))
|
||||
num_frames = int_or_none(gfy.get('numFrames'))
|
||||
|
||||
duration = float_or_none(num_frames, fps) if num_frames and fps else None
|
||||
|
||||
categories = gfy.get('tags') or gfy.get('extraLemmas') or []
|
||||
|
||||
FORMATS = ('gif', 'webm', 'mp4')
|
||||
quality = qualities(FORMATS)
|
||||
|
||||
formats = []
|
||||
for format_id in FORMATS:
|
||||
video_url = gfy.get('%sUrl' % format_id)
|
||||
if not video_url:
|
||||
continue
|
||||
filesize = int_or_none(gfy.get('%sSize' % format_id))
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
'format_id': format_id,
|
||||
'width': width,
|
||||
'height': height,
|
||||
'fps': fps,
|
||||
'filesize': filesize,
|
||||
'quality': quality(format_id),
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'timestamp': timestamp,
|
||||
'uploader': gfy.get('userDisplayName') or uploader,
|
||||
'uploader_id': uploader,
|
||||
'duration': duration,
|
||||
'view_count': view_count,
|
||||
'like_count': like_count,
|
||||
'dislike_count': dislike_count,
|
||||
'categories': categories,
|
||||
'age_limit': age_limit,
|
||||
'formats': formats,
|
||||
}
|
||||
@@ -31,7 +31,6 @@ class GrouponIE(InfoExtractor):
|
||||
}
|
||||
|
||||
_PROVIDERS = {
|
||||
'ooyala': ('ooyala:%s', 'Ooyala'),
|
||||
'youtube': ('%s', 'Youtube'),
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user