[core] Load plugins on demand (#11305)

- Adds `--no-plugin-dirs` to disable plugin loading
- `--plugin-dirs` now supports post-processors

Authored by: coletdjnz, Grub4K, pukkandan

yt_dlp/extractor/__init__.py

@@ -1,16 +1,25 @@
 from ..compat.compat_utils import passthrough_module
+from ..globals import extractors as _extractors_context
+from ..globals import plugin_ies as _plugin_ies_context
+from ..plugins import PluginSpec, register_plugin_spec
 
 passthrough_module(__name__, '.extractors')
 del passthrough_module
 
+register_plugin_spec(PluginSpec(
+    module_name='extractor',
+    suffix='IE',
+    destination=_extractors_context,
+    plugin_destination=_plugin_ies_context,
+))
+
 
 def gen_extractor_classes():
     """ Return a list of supported extractors.
     The order does matter; the first extractor matched is the one handling the URL.
     """
-    from .extractors import _ALL_CLASSES
-
-    return _ALL_CLASSES
+    import_extractors()
+    return list(_extractors_context.value.values())
 
 
 def gen_extractors():
@@ -37,6 +46,9 @@ def list_extractors(age_limit=None):
 
 def get_info_extractor(ie_name):
     """Returns the info extractor class with the given ie_name"""
-    from . import extractors
+    import_extractors()
+    return _extractors_context.value[f'{ie_name}IE']
 
-    return getattr(extractors, f'{ie_name}IE')
+
+def import_extractors():
+    from . import extractors  # noqa: F401

yt_dlp/extractor/common.py

@@ -29,6 +29,7 @@ from ..compat import (
 from ..cookies import LenientSimpleCookie
 from ..downloader.f4m import get_base_url, remove_encrypted_media
 from ..downloader.hls import HlsFD
+from ..globals import plugin_ies_overrides
 from ..networking import HEADRequest, Request
 from ..networking.exceptions import (
     HTTPError,
@@ -3954,14 +3955,18 @@ class InfoExtractor:
     def __init_subclass__(cls, *, plugin_name=None, **kwargs):
         if plugin_name:
             mro = inspect.getmro(cls)
-            super_class = cls.__wrapped__ = mro[mro.index(cls) + 1]
-            cls.PLUGIN_NAME, cls.ie_key = plugin_name, super_class.ie_key
-            cls.IE_NAME = f'{super_class.IE_NAME}+{plugin_name}'
+            next_mro_class = super_class = mro[mro.index(cls) + 1]
+
             while getattr(super_class, '__wrapped__', None):
                 super_class = super_class.__wrapped__
-            setattr(sys.modules[super_class.__module__], super_class.__name__, cls)
-            _PLUGIN_OVERRIDES[super_class].append(cls)
 
+            if not any(override.PLUGIN_NAME == plugin_name for override in plugin_ies_overrides.value[super_class]):
+                cls.__wrapped__ = next_mro_class
+                cls.PLUGIN_NAME, cls.ie_key = plugin_name, next_mro_class.ie_key
+                cls.IE_NAME = f'{next_mro_class.IE_NAME}+{plugin_name}'
+
+                setattr(sys.modules[super_class.__module__], super_class.__name__, cls)
+                plugin_ies_overrides.value[super_class].append(cls)
         return super().__init_subclass__(**kwargs)
 
 
@@ -4017,6 +4022,3 @@ class UnsupportedURLIE(InfoExtractor):
 
     def _real_extract(self, url):
         raise UnsupportedError(url)
-
-
-_PLUGIN_OVERRIDES = collections.defaultdict(list)

yt_dlp/extractor/extractors.py

@@ -1,28 +1,35 @@
-import contextlib
+import inspect
 import os
 
-from ..plugins import load_plugins
+from ..globals import LAZY_EXTRACTORS
+from ..globals import extractors as _extractors_context
 
-# NB: Must be before other imports so that plugins can be correctly injected
-_PLUGIN_CLASSES = load_plugins('extractor', 'IE')
-
-_LAZY_LOADER = False
+_CLASS_LOOKUP = None
 if not os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'):
-    with contextlib.suppress(ImportError):
-        from .lazy_extractors import *  # noqa: F403
-        from .lazy_extractors import _ALL_CLASSES
-        _LAZY_LOADER = True
+    try:
+        from .lazy_extractors import _CLASS_LOOKUP
+        LAZY_EXTRACTORS.value = True
+    except ImportError:
+        LAZY_EXTRACTORS.value = False
 
-if not _LAZY_LOADER:
-    from ._extractors import *  # noqa: F403
-    _ALL_CLASSES = [  # noqa: F811
-        klass
-        for name, klass in globals().items()
+if not _CLASS_LOOKUP:
+    from . import _extractors
+
+    _CLASS_LOOKUP = {
+        name: value
+        for name, value in inspect.getmembers(_extractors)
         if name.endswith('IE') and name != 'GenericIE'
-    ]
-    _ALL_CLASSES.append(GenericIE)  # noqa: F405
+    }
+    _CLASS_LOOKUP['GenericIE'] = _extractors.GenericIE
 
-globals().update(_PLUGIN_CLASSES)
-_ALL_CLASSES[:0] = _PLUGIN_CLASSES.values()
+# We want to append to the main lookup
+_current = _extractors_context.value
+for name, ie in _CLASS_LOOKUP.items():
+    _current.setdefault(name, ie)
 
-from .common import _PLUGIN_OVERRIDES  # noqa: F401
+
+def __getattr__(name):
+    value = _CLASS_LOOKUP.get(name)
+    if not value:
+        raise AttributeError(f'module {__name__} has no attribute {name}')
+    return value