Merge branch 'yt-dlp:master' into pr/live-sections

2025-12-13 19:55:25 +00:00 · 2024-10-26 09:35:14 -05:00
parent 9dd8574b68 6abef74232
commit e05694e550
62 changed files with 1802 additions and 948 deletions
--- a/test/test_plugins.py
+++ b/test/test_plugins.py
@@ -10,6 +10,7 @@ TEST_DATA_DIR = Path(os.path.dirname(os.path.abspath(__file__)), 'testdata')
 sys.path.append(str(TEST_DATA_DIR))
 importlib.invalidate_caches()

+from yt_dlp.utils import Config
 from yt_dlp.plugins import PACKAGE_NAME, directories, load_plugins


@@ -68,6 +69,24 @@ class TestPlugins(unittest.TestCase):
            os.remove(zip_path)
            importlib.invalidate_caches()  # reset the import caches

+    def test_plugin_dirs(self):
+        # Internal plugin dirs hack for CLI --plugin-dirs
+        # To be replaced with proper system later
+        custom_plugin_dir = TEST_DATA_DIR / 'plugin_packages'
+        Config._plugin_dirs = [str(custom_plugin_dir)]
+        importlib.invalidate_caches()  # reset the import caches
+
+        try:
+            package = importlib.import_module(f'{PACKAGE_NAME}.extractor')
+            self.assertIn(custom_plugin_dir / 'testpackage' / PACKAGE_NAME / 'extractor', map(Path, package.__path__))
+
+            plugins_ie = load_plugins('extractor', 'IE')
+            self.assertIn('PackagePluginIE', plugins_ie.keys())
+
+        finally:
+            Config._plugin_dirs = []
+            importlib.invalidate_caches()  # reset the import caches
+

 if __name__ == '__main__':
    unittest.main()
--- a/test/test_traversal.py
+++ b/test/test_traversal.py
@@ -4,8 +4,18 @@ import xml.etree.ElementTree

 import pytest

-from yt_dlp.utils import dict_get, int_or_none, str_or_none
-from yt_dlp.utils.traversal import traverse_obj
+from yt_dlp.utils import (
+    ExtractorError,
+    determine_ext,
+    dict_get,
+    int_or_none,
+    str_or_none,
+)
+from yt_dlp.utils.traversal import (
+    traverse_obj,
+    require,
+    subs_list_to_dict,
+)

 _TEST_DATA = {
    100: 100,
@@ -420,6 +430,71 @@ class TestTraversal:
        assert traverse_obj(morsel, [(None,), any]) == morsel, \
            'Morsel should not be implicitly changed to dict on usage'

+    def test_traversal_filter(self):
+        data = [None, False, True, 0, 1, 0.0, 1.1, '', 'str', {}, {0: 0}, [], [1]]
+
+        assert traverse_obj(data, [..., filter]) == [True, 1, 1.1, 'str', {0: 0}, [1]], \
+            '`filter` should filter falsy values'
+
+
+class TestTraversalHelpers:
+    def test_traversal_require(self):
+        with pytest.raises(ExtractorError):
+            traverse_obj(_TEST_DATA, ['None', {require('value')}])
+        assert traverse_obj(_TEST_DATA, ['str', {require('value')}]) == 'str', \
+            '`require` should pass through non `None` values'
+
+    def test_subs_list_to_dict(self):
+        assert traverse_obj([
+            {'name': 'de', 'url': 'https://example.com/subs/de.vtt'},
+            {'name': 'en', 'url': 'https://example.com/subs/en1.ass'},
+            {'name': 'en', 'url': 'https://example.com/subs/en2.ass'},
+        ], [..., {
+            'id': 'name',
+            'url': 'url',
+        }, all, {subs_list_to_dict}]) == {
+            'de': [{'url': 'https://example.com/subs/de.vtt'}],
+            'en': [
+                {'url': 'https://example.com/subs/en1.ass'},
+                {'url': 'https://example.com/subs/en2.ass'},
+            ],
+        }, 'function should build subtitle dict from list of subtitles'
+        assert traverse_obj([
+            {'name': 'de', 'url': 'https://example.com/subs/de.ass'},
+            {'name': 'de'},
+            {'name': 'en', 'content': 'content'},
+            {'url': 'https://example.com/subs/en'},
+        ], [..., {
+            'id': 'name',
+            'data': 'content',
+            'url': 'url',
+        }, all, {subs_list_to_dict}]) == {
+            'de': [{'url': 'https://example.com/subs/de.ass'}],
+            'en': [{'data': 'content'}],
+        }, 'subs with mandatory items missing should be filtered'
+        assert traverse_obj([
+            {'url': 'https://example.com/subs/de.ass', 'name': 'de'},
+            {'url': 'https://example.com/subs/en', 'name': 'en'},
+        ], [..., {
+            'id': 'name',
+            'ext': ['url', {lambda x: determine_ext(x, default_ext=None)}],
+            'url': 'url',
+        }, all, {subs_list_to_dict(ext='ext')}]) == {
+            'de': [{'url': 'https://example.com/subs/de.ass', 'ext': 'ass'}],
+            'en': [{'url': 'https://example.com/subs/en', 'ext': 'ext'}],
+        }, '`ext` should set default ext but leave existing value untouched'
+        assert traverse_obj([
+            {'name': 'en', 'url': 'https://example.com/subs/en2', 'prio': True},
+            {'name': 'en', 'url': 'https://example.com/subs/en1', 'prio': False},
+        ], [..., {
+            'id': 'name',
+            'quality': ['prio', {int}],
+            'url': 'url',
+        }, all, {subs_list_to_dict(ext='ext')}]) == {'en': [
+            {'url': 'https://example.com/subs/en1', 'ext': 'ext'},
+            {'url': 'https://example.com/subs/en2', 'ext': 'ext'},
+        ]}, '`quality` key should sort subtitle list accordingly'
+

 class TestDictGet:
    def test_dict_get(self):
--- a/test/test_update.py
+++ b/test/test_update.py
@@ -82,16 +82,32 @@ TEST_LOCKFILE_V1 = rf'''{TEST_LOCKFILE_COMMENT}
 lock 2022.08.18.36 .+ Python 3\.6
 lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7
 lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
+lock 2024.10.22 py2exe .+
+lock 2024.10.22 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b
+lock 2024.10.22 (?!\w+_exe).+ Python 3\.8
+lock 2024.10.22 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2)
 '''

 TEST_LOCKFILE_V2_TMPL = r'''%s
 lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6
 lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7
 lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
+lockV2 yt-dlp/yt-dlp 2024.10.22 py2exe .+
+lockV2 yt-dlp/yt-dlp 2024.10.22 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b
+lockV2 yt-dlp/yt-dlp 2024.10.22 (?!\w+_exe).+ Python 3\.8
+lockV2 yt-dlp/yt-dlp 2024.10.22 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2)
 lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7
 lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server)
+lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 py2exe .+
+lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b
+lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 (?!\w+_exe).+ Python 3\.8
+lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2)
 lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7
 lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server)
+lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.045052 py2exe .+
+lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b
+lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 (?!\w+_exe).+ Python 3\.8
+lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2)
 '''

 TEST_LOCKFILE_V2 = TEST_LOCKFILE_V2_TMPL % TEST_LOCKFILE_COMMENT
@@ -145,43 +161,76 @@ class TestUpdate(unittest.TestCase):
        for lockfile in (TEST_LOCKFILE_V1, TEST_LOCKFILE_V2, TEST_LOCKFILE_ACTUAL, TEST_LOCKFILE_FORK):
            # Normal operation
            test(lockfile, 'zip Python 3.12.0', '2023.12.31', '2023.12.31')
-            test(lockfile, 'zip stable Python 3.12.0', '2023.12.31', '2023.12.31', exact=True)
-            # Python 3.6 --update should update only to its lock
+            test(lockfile, 'zip Python 3.12.0', '2023.12.31', '2023.12.31', exact=True)
+            # py2exe should never update beyond 2024.10.22
+            test(lockfile, 'py2exe Python 3.8', '2025.01.01', '2024.10.22')
+            test(lockfile, 'py2exe Python 3.8', '2025.01.01', None, exact=True)
+            # Python 3.6 --update should update only to the py3.6 lock
            test(lockfile, 'zip Python 3.6.0', '2023.11.16', '2022.08.18.36')
-            # --update-to an exact version later than the lock should return None
-            test(lockfile, 'zip stable Python 3.6.0', '2023.11.16', None, exact=True)
-            # Python 3.7 should be able to update to its lock
+            # Python 3.6 --update-to an exact version later than the py3.6 lock should return None
+            test(lockfile, 'zip Python 3.6.0', '2023.11.16', None, exact=True)
+            # Python 3.7 should be able to update to the py3.7 lock
            test(lockfile, 'zip Python 3.7.0', '2023.11.16', '2023.11.16')
-            test(lockfile, 'zip stable Python 3.7.1', '2023.11.16', '2023.11.16', exact=True)
-            # Non-win_x86_exe builds on py3.7 must be locked
+            test(lockfile, 'zip Python 3.7.1', '2023.11.16', '2023.11.16', exact=True)
+            # Non-win_x86_exe builds on py3.7 must be locked at py3.7 lock
            test(lockfile, 'zip Python 3.7.1', '2023.12.31', '2023.11.16')
-            test(lockfile, 'zip stable Python 3.7.1', '2023.12.31', None, exact=True)
-            test(  # Windows Vista w/ win_x86_exe must be locked
-                lockfile, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-Vista-6.0.6003-SP2',
+            test(lockfile, 'zip Python 3.7.1', '2023.12.31', None, exact=True)
+            # Python 3.8 should only update to the py3.8 lock
+            test(lockfile, 'zip Python 3.8.10', '2025.01.01', '2024.10.22')
+            test(lockfile, 'zip Python 3.8.110', '2025.01.01', None, exact=True)
+            test(  # Windows Vista w/ win_x86_exe must be locked at Vista lock
+                lockfile, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-Vista-6.0.6003-SP2',
                '2023.12.31', '2023.11.16')
-            test(  # Windows 2008Server w/ win_x86_exe must be locked
+            test(  # Windows 2008Server w/ win_x86_exe must be locked at Vista lock
                lockfile, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-2008Server',
                '2023.12.31', None, exact=True)
-            test(  # Windows 7 w/ win_x86_exe py3.7 build should be able to update beyond lock
-                lockfile, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-7-6.1.7601-SP1',
-                '2023.12.31', '2023.12.31')
-            test(  # Windows 8.1 w/ '2008Server' in platform string should be able to update beyond lock
+            test(  # Windows 7 w/ win_x86_exe py3.7 build should be able to update beyond py3.7 lock
+                lockfile, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-7-6.1.7601-SP1',
+                '2023.12.31', '2023.12.31', exact=True)
+            test(  # Windows 7 win_x86_exe should only update to Win7 lock
+                lockfile, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-7-6.1.7601-SP1',
+                '2025.01.01', '2024.10.22')
+            test(  # Windows 2008ServerR2 win_exe should only update to Win7 lock
+                lockfile, 'win_exe Python 3.8.10 (CPython x86 32bit) - Windows-2008ServerR2',
+                '2025.12.31', '2024.10.22')
+            test(  # Windows 8.1 w/ '2008Server' in platform string should be able to update beyond py3.7 lock
                lockfile, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-post2008Server-6.2.9200',
                '2023.12.31', '2023.12.31', exact=True)
+            test(  # win_exe built w/Python 3.8 on Windows>=8 should be able to update beyond py3.8 lock
+                lockfile, 'win_exe Python 3.8.10 (CPython AMD64 64bit) - Windows-10-10.0.20348-SP0',
+                '2025.01.01', '2025.01.01', exact=True)
+            test(  # linux_armv7l_exe w/glibc2.7 should only update to glibc<2.31 lock
+                lockfile, 'linux_armv7l_exe Python 3.8.0 (CPython armv7l 32bit) - Linux-6.5.0-1025-azure-armv7l-with-glibc2.7',
+                '2025.01.01', '2024.10.22')
+            test(  # linux_armv7l_exe w/Python 3.8 and glibc>=2.31 should be able to update beyond py3.8 and glibc<2.31 locks
+                lockfile, 'linux_armv7l_exe Python 3.8.0 (CPython armv7l 32bit) - Linux-6.5.0-1025-azure-armv7l-with-glibc2.31',
+                '2025.01.01', '2025.01.01')
+            test(  # linux_armv7l_exe w/glibc2.30 should only update to glibc<2.31 lock
+                lockfile, 'linux_armv7l_exe Python 3.8.0 (CPython armv7l 64bit) - Linux-6.5.0-1025-azure-aarch64-with-glibc2.30 (OpenSSL',
+                '2025.01.01', '2024.10.22')
+            test(  # linux_aarch64_exe w/glibc2.17 should only update to glibc<2.31 lock
+                lockfile, 'linux_aarch64_exe Python 3.8.0 (CPython aarch64 64bit) - Linux-6.5.0-1025-azure-aarch64-with-glibc2.17',
+                '2025.01.01', '2024.10.22')
+            test(  # linux_aarch64_exe w/glibc2.40 and glibc>=2.31 should be able to update beyond py3.8 and glibc<2.31 locks
+                lockfile, 'linux_aarch64_exe Python 3.8.0 (CPython aarch64 64bit) - Linux-6.5.0-1025-azure-aarch64-with-glibc2.40',
+                '2025.01.01', '2025.01.01')
+            test(  # linux_aarch64_exe w/glibc2.3 should only update to glibc<2.31 lock
+                lockfile, 'linux_aarch64_exe Python 3.8.0 (CPython aarch64 64bit) - Linux-6.5.0-1025-azure-aarch64-with-glibc2.3 (OpenSSL',
+                '2025.01.01', '2024.10.22')

        # Forks can block updates to non-numeric tags rather than lock
        test(TEST_LOCKFILE_FORK, 'zip Python 3.6.3', 'pr0000', None, repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_FORK, 'zip stable Python 3.7.4', 'pr0000', 'pr0000', repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_FORK, 'zip stable Python 3.7.4', 'pr1234', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_FORK, 'zip Python 3.7.4', 'pr0000', 'pr0000', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_FORK, 'zip Python 3.7.4', 'pr1234', None, repo='fork/yt-dlp')
        test(TEST_LOCKFILE_FORK, 'zip Python 3.8.1', 'pr1234', 'pr1234', repo='fork/yt-dlp', exact=True)
        test(
-            TEST_LOCKFILE_FORK, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-Vista-6.0.6003-SP2',
+            TEST_LOCKFILE_FORK, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-Vista-6.0.6003-SP2',
            'pr1234', None, repo='fork/yt-dlp')
        test(
-            TEST_LOCKFILE_FORK, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-7-6.1.7601-SP1',
+            TEST_LOCKFILE_FORK, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-7-6.1.7601-SP1',
            '2023.12.31', '2023.12.31', repo='fork/yt-dlp')
        test(TEST_LOCKFILE_FORK, 'zip Python 3.11.2', 'pr9999', None, repo='fork/yt-dlp', exact=True)
-        test(TEST_LOCKFILE_FORK, 'zip stable Python 3.12.0', 'pr9999', 'pr9999', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_FORK, 'zip Python 3.12.0', 'pr9999', 'pr9999', repo='fork/yt-dlp')

    def test_query_update(self):
        ydl = FakeYDL()
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -221,9 +221,10 @@ class TestUtil(unittest.TestCase):
        self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')

    def test_sanitize_path(self):
-        if sys.platform != 'win32':
-            return
+        with unittest.mock.patch('sys.platform', 'win32'):
+            self._test_sanitize_path()

+    def _test_sanitize_path(self):
        self.assertEqual(sanitize_path('abc'), 'abc')
        self.assertEqual(sanitize_path('abc/def'), 'abc\\def')
        self.assertEqual(sanitize_path('abc\\def'), 'abc\\def')
@@ -256,6 +257,11 @@ class TestUtil(unittest.TestCase):
        self.assertEqual(sanitize_path('./abc'), 'abc')
        self.assertEqual(sanitize_path('./../abc'), '..\\abc')

+        self.assertEqual(sanitize_path('\\abc'), '\\abc')
+        self.assertEqual(sanitize_path('C:abc'), 'C:abc')
+        self.assertEqual(sanitize_path('C:abc\\..\\'), 'C:..')
+        self.assertEqual(sanitize_path('C:\\abc:%(title)s.%(ext)s'), 'C:\\abc#%(title)s.%(ext)s')
+
    def test_sanitize_url(self):
        self.assertEqual(sanitize_url('//foo.bar'), 'http://foo.bar')
        self.assertEqual(sanitize_url('httpss://foo.bar'), 'https://foo.bar')
--- a/test/testdata/plugin_packages/testpackage/yt_dlp_plugins/extractor/package.py
+++ b/test/testdata/plugin_packages/testpackage/yt_dlp_plugins/extractor/package.py
@@ -0,0 +1,5 @@
+from yt_dlp.extractor.common import InfoExtractor
+
+
+class PackagePluginIE(InfoExtractor):
+    pass