Improve plugin architecture (#5553)

to make plugins easier to develop and use:
* Plugins are now loaded as namespace packages.
* Plugins can be loaded in any distribution of yt-dlp (binary, pip, source, etc.).
* Plugin packages can be installed and managed via pip, or dropped into any of the documented locations.
* Users do not need to edit any code files to install plugins.
* Backwards-compatible with previous plugin architecture.

As a side-effect, yt-dlp will now search in a few more locations for config files.

Closes https://github.com/yt-dlp/yt-dlp/issues/1389

Authored by: flashdagger, coletdjnz, pukkandan, Grub4K
Co-authored-by: Marcel <flashdagger@googlemail.com>
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
Co-authored-by: Simon Sawicki <accounts@grub4k.xyz>

yt_dlp/YoutubeDL.py

@@ -32,6 +32,7 @@ from .extractor import gen_extractor_classes, get_info_extractor
 from .extractor.common import UnsupportedURLIE
 from .extractor.openload import PhantomJSwrapper
 from .minicurses import format_text
+from .plugins import directories as plugin_directories
 from .postprocessor import _PLUGIN_CLASSES as plugin_postprocessors
 from .postprocessor import (
     EmbedThumbnailPP,
@@ -3773,10 +3774,6 @@ class YoutubeDL:
                 write_debug('Lazy loading extractors is forcibly disabled')
             else:
                 write_debug('Lazy loading extractors is disabled')
-        if plugin_extractors or plugin_postprocessors:
-            write_debug('Plugins: %s' % [
-                '%s%s' % (klass.__name__, '' if klass.__name__ == name else f' as {name}')
-                for name, klass in itertools.chain(plugin_extractors.items(), plugin_postprocessors.items())])
         if self.params['compat_opts']:
             write_debug('Compatibility options: %s' % ', '.join(self.params['compat_opts']))
 
@@ -3810,6 +3807,16 @@ class YoutubeDL:
                 proxy_map.update(handler.proxies)
         write_debug(f'Proxy map: {proxy_map}')
 
+        for plugin_type, plugins in {'Extractor': plugin_extractors, 'Post-Processor': plugin_postprocessors}.items():
+            if not plugins:
+                continue
+            write_debug(f'{plugin_type} Plugins: %s' % (', '.join(sorted(('%s%s' % (
+                klass.__name__, '' if klass.__name__ == name else f' as {name}')
+                for name, klass in plugins.items())))))
+        plugin_dirs = plugin_directories()
+        if plugin_dirs:
+            write_debug(f'Plugin directories: {plugin_dirs}')
+
         # Not implemented
         if False and self.params.get('call_home'):
             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode()

yt_dlp/extractor/extractors.py

@@ -1,10 +1,10 @@
 import contextlib
 import os
 
-from ..utils import load_plugins
+from ..plugins import load_plugins
 
 # NB: Must be before other imports so that plugins can be correctly injected
-_PLUGIN_CLASSES = load_plugins('extractor', 'IE', {})
+_PLUGIN_CLASSES = load_plugins('extractor', 'IE')
 
 _LAZY_LOADER = False
 if not os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'):

yt_dlp/options.py

@@ -29,6 +29,8 @@ from .utils import (
     expand_path,
     format_field,
     get_executable_path,
+    get_system_config_dirs,
+    get_user_config_dirs,
     join_nonempty,
     orderedSet_from_options,
     remove_end,
@@ -42,62 +44,67 @@ def parseOpts(overrideArguments=None, ignore_config_files='if_override'):
     if ignore_config_files == 'if_override':
         ignore_config_files = overrideArguments is not None
 
-    def _readUserConf(package_name, default=[]):
-        # .config
+    def _load_from_config_dirs(config_dirs):
+        for config_dir in config_dirs:
+            conf_file_path = os.path.join(config_dir, 'config')
+            conf = Config.read_file(conf_file_path, default=None)
+            if conf is None:
+                conf_file_path += '.txt'
+                conf = Config.read_file(conf_file_path, default=None)
+            if conf is not None:
+                return conf, conf_file_path
+        return None, None
+
+    def _read_user_conf(package_name, default=None):
+        # .config/package_name.conf
         xdg_config_home = os.getenv('XDG_CONFIG_HOME') or compat_expanduser('~/.config')
-        userConfFile = os.path.join(xdg_config_home, package_name, 'config')
-        if not os.path.isfile(userConfFile):
-            userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name)
-        userConf = Config.read_file(userConfFile, default=None)
-        if userConf is not None:
-            return userConf, userConfFile
+        user_conf_file = os.path.join(xdg_config_home, '%s.conf' % package_name)
+        user_conf = Config.read_file(user_conf_file, default=None)
+        if user_conf is not None:
+            return user_conf, user_conf_file
 
-        # appdata
-        appdata_dir = os.getenv('appdata')
-        if appdata_dir:
-            userConfFile = os.path.join(appdata_dir, package_name, 'config')
-            userConf = Config.read_file(userConfFile, default=None)
-            if userConf is None:
-                userConfFile += '.txt'
-                userConf = Config.read_file(userConfFile, default=None)
-        if userConf is not None:
-            return userConf, userConfFile
+        # home (~/package_name.conf or ~/package_name.conf.txt)
+        user_conf_file = os.path.join(compat_expanduser('~'), '%s.conf' % package_name)
+        user_conf = Config.read_file(user_conf_file, default=None)
+        if user_conf is None:
+            user_conf_file += '.txt'
+            user_conf = Config.read_file(user_conf_file, default=None)
+        if user_conf is not None:
+            return user_conf, user_conf_file
 
-        # home
-        userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name)
-        userConf = Config.read_file(userConfFile, default=None)
-        if userConf is None:
-            userConfFile += '.txt'
-            userConf = Config.read_file(userConfFile, default=None)
-        if userConf is not None:
-            return userConf, userConfFile
+        # Package config directories (e.g. ~/.config/package_name/package_name.txt)
+        user_conf, user_conf_file = _load_from_config_dirs(get_user_config_dirs(package_name))
+        if user_conf is not None:
+            return user_conf, user_conf_file
+        return default if default is not None else [], None
 
-        return default, None
+    def _read_system_conf(package_name, default=None):
+        system_conf, system_conf_file = _load_from_config_dirs(get_system_config_dirs(package_name))
+        if system_conf is not None:
+            return system_conf, system_conf_file
+        return default if default is not None else [], None
 
-    def add_config(label, path, user=False):
+    def add_config(label, path=None, func=None):
         """ Adds config and returns whether to continue """
         if root.parse_known_args()[0].ignoreconfig:
             return False
-        # Multiple package names can be given here
-        # E.g. ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for
-        # the configuration file of any of these three packages
-        for package in ('yt-dlp',):
-            if user:
-                args, current_path = _readUserConf(package, default=None)
-            else:
-                current_path = os.path.join(path, '%s.conf' % package)
-                args = Config.read_file(current_path, default=None)
-            if args is not None:
-                root.append_config(args, current_path, label=label)
-                return True
+        elif func:
+            assert path is None
+            args, current_path = func('yt-dlp')
+        else:
+            current_path = os.path.join(path, 'yt-dlp.conf')
+            args = Config.read_file(current_path, default=None)
+        if args is not None:
+            root.append_config(args, current_path, label=label)
+            return True
         return True
 
     def load_configs():
         yield not ignore_config_files
         yield add_config('Portable', get_executable_path())
         yield add_config('Home', expand_path(root.parse_known_args()[0].paths.get('home', '')).strip())
-        yield add_config('User', None, user=True)
-        yield add_config('System', '/etc')
+        yield add_config('User', func=_read_user_conf)
+        yield add_config('System', func=_read_system_conf)
 
     opts = optparse.Values({'verbose': True, 'print_help': False})
     try:

yt_dlp/plugins.py

@@ -0,0 +1,171 @@
+import contextlib
+import importlib
+import importlib.abc
+import importlib.machinery
+import importlib.util
+import inspect
+import itertools
+import os
+import pkgutil
+import sys
+import traceback
+import zipimport
+from pathlib import Path
+from zipfile import ZipFile
+
+from .compat import functools  # isort: split
+from .compat import compat_expanduser
+from .utils import (
+    get_executable_path,
+    get_system_config_dirs,
+    get_user_config_dirs,
+    write_string,
+)
+
+PACKAGE_NAME = 'yt_dlp_plugins'
+COMPAT_PACKAGE_NAME = 'ytdlp_plugins'
+
+
+class PluginLoader(importlib.abc.Loader):
+    """Dummy loader for virtual namespace packages"""
+
+    def exec_module(self, module):
+        return None
+
+
+@functools.cache
+def dirs_in_zip(archive):
+    with ZipFile(archive) as zip:
+        return set(itertools.chain.from_iterable(
+            Path(file).parents for file in zip.namelist()))
+
+
+class PluginFinder(importlib.abc.MetaPathFinder):
+    """
+    This class provides one or multiple namespace packages.
+    It searches in sys.path and yt-dlp config folders for
+    the existing subdirectories from which the modules can be imported
+    """
+
+    def __init__(self, *packages):
+        self._zip_content_cache = {}
+        self.packages = set(itertools.chain.from_iterable(
+            itertools.accumulate(name.split('.'), lambda a, b: '.'.join((a, b)))
+            for name in packages))
+
+    def search_locations(self, fullname):
+        candidate_locations = []
+
+        def _get_package_paths(*root_paths, containing_folder='plugins'):
+            for config_dir in map(Path, root_paths):
+                plugin_dir = config_dir / containing_folder
+                if not plugin_dir.is_dir():
+                    continue
+                yield from plugin_dir.iterdir()
+
+        # Load from yt-dlp config folders
+        candidate_locations.extend(_get_package_paths(
+            *get_user_config_dirs('yt-dlp'), *get_system_config_dirs('yt-dlp'),
+            containing_folder='plugins'))
+
+        # Load from yt-dlp-plugins folders
+        candidate_locations.extend(_get_package_paths(
+            get_executable_path(),
+            compat_expanduser('~'),
+            '/etc',
+            os.getenv('XDG_CONFIG_HOME') or compat_expanduser('~/.config'),
+            containing_folder='yt-dlp-plugins'))
+
+        candidate_locations.extend(map(Path, sys.path))  # PYTHONPATH
+
+        parts = Path(*fullname.split('.'))
+        locations = set()
+        for path in dict.fromkeys(candidate_locations):
+            candidate = path / parts
+            if candidate.is_dir():
+                locations.add(str(candidate))
+            elif path.name and any(path.with_suffix(suffix).is_file() for suffix in {'.zip', '.egg', '.whl'}):
+                with contextlib.suppress(FileNotFoundError):
+                    if parts in dirs_in_zip(path):
+                        locations.add(str(candidate))
+        return locations
+
+    def find_spec(self, fullname, path=None, target=None):
+        if fullname not in self.packages:
+            return None
+
+        search_locations = self.search_locations(fullname)
+        if not search_locations:
+            return None
+
+        spec = importlib.machinery.ModuleSpec(fullname, PluginLoader(), is_package=True)
+        spec.submodule_search_locations = search_locations
+        return spec
+
+    def invalidate_caches(self):
+        dirs_in_zip.cache_clear()
+        for package in self.packages:
+            if package in sys.modules:
+                del sys.modules[package]
+
+
+def directories():
+    spec = importlib.util.find_spec(PACKAGE_NAME)
+    return spec.submodule_search_locations if spec else []
+
+
+def iter_modules(subpackage):
+    fullname = f'{PACKAGE_NAME}.{subpackage}'
+    with contextlib.suppress(ModuleNotFoundError):
+        pkg = importlib.import_module(fullname)
+        yield from pkgutil.iter_modules(path=pkg.__path__, prefix=f'{fullname}.')
+
+
+def load_module(module, module_name, suffix):
+    return inspect.getmembers(module, lambda obj: (
+        inspect.isclass(obj)
+        and obj.__name__.endswith(suffix)
+        and obj.__module__.startswith(module_name)
+        and not obj.__name__.startswith('_')
+        and obj.__name__ in getattr(module, '__all__', [obj.__name__])))
+
+
+def load_plugins(name, suffix):
+    classes = {}
+
+    for finder, module_name, _ in iter_modules(name):
+        if any(x.startswith('_') for x in module_name.split('.')):
+            continue
+        try:
+            if sys.version_info < (3, 10) and isinstance(finder, zipimport.zipimporter):
+                # zipimporter.load_module() is deprecated in 3.10 and removed in 3.12
+                # The exec_module branch below is the replacement for >= 3.10
+                # See: https://docs.python.org/3/library/zipimport.html#zipimport.zipimporter.exec_module
+                module = finder.load_module(module_name)
+            else:
+                spec = finder.find_spec(module_name)
+                module = importlib.util.module_from_spec(spec)
+                sys.modules[module_name] = module
+                spec.loader.exec_module(module)
+        except Exception:
+            write_string(f'Error while importing module {module_name!r}\n{traceback.format_exc(limit=-1)}')
+            continue
+        classes.update(load_module(module, module_name, suffix))
+
+    # Compat: old plugin system using __init__.py
+    # Note: plugins imported this way do not show up in directories()
+    # nor are considered part of the yt_dlp_plugins namespace package
+    with contextlib.suppress(FileNotFoundError):
+        spec = importlib.util.spec_from_file_location(
+            name, Path(get_executable_path(), COMPAT_PACKAGE_NAME, name, '__init__.py'))
+        plugins = importlib.util.module_from_spec(spec)
+        sys.modules[spec.name] = plugins
+        spec.loader.exec_module(plugins)
+        classes.update(load_module(plugins, spec.name, suffix))
+
+    return classes
+
+
+sys.meta_path.insert(0, PluginFinder(f'{PACKAGE_NAME}.extractor', f'{PACKAGE_NAME}.postprocessor'))
+
+__all__ = ['directories', 'load_plugins', 'PACKAGE_NAME', 'COMPAT_PACKAGE_NAME']

yt_dlp/postprocessor/__init__.py

@@ -33,14 +33,15 @@ from .movefilesafterdownload import MoveFilesAfterDownloadPP
 from .sponskrub import SponSkrubPP
 from .sponsorblock import SponsorBlockPP
 from .xattrpp import XAttrMetadataPP
-from ..utils import load_plugins
+from ..plugins import load_plugins
 
-_PLUGIN_CLASSES = load_plugins('postprocessor', 'PP', globals())
+_PLUGIN_CLASSES = load_plugins('postprocessor', 'PP')
 
 
 def get_postprocessor(key):
     return globals()[key + 'PP']
 
 
+globals().update(_PLUGIN_CLASSES)
 __all__ = [name for name in globals().keys() if name.endswith('PP')]
 __all__.extend(('PostProcessor', 'FFmpegPostProcessor'))

yt_dlp/utils.py

@@ -18,7 +18,6 @@ import html.entities
 import html.parser
 import http.client
 import http.cookiejar
-import importlib.util
 import inspect
 import io
 import itertools
@@ -5372,22 +5371,37 @@ def get_executable_path():
     return os.path.dirname(os.path.abspath(_get_variant_and_executable_path()[1]))
 
 
-def load_plugins(name, suffix, namespace):
-    classes = {}
-    with contextlib.suppress(FileNotFoundError):
-        plugins_spec = importlib.util.spec_from_file_location(
-            name, os.path.join(get_executable_path(), 'ytdlp_plugins', name, '__init__.py'))
-        plugins = importlib.util.module_from_spec(plugins_spec)
-        sys.modules[plugins_spec.name] = plugins
-        plugins_spec.loader.exec_module(plugins)
-        for name in dir(plugins):
-            if name in namespace:
-                continue
-            if not name.endswith(suffix):
-                continue
-            klass = getattr(plugins, name)
-            classes[name] = namespace[name] = klass
-    return classes
+def get_user_config_dirs(package_name):
+    locations = set()
+
+    # .config (e.g. ~/.config/package_name)
+    xdg_config_home = os.getenv('XDG_CONFIG_HOME') or compat_expanduser('~/.config')
+    config_dir = os.path.join(xdg_config_home, package_name)
+    if os.path.isdir(config_dir):
+        locations.add(config_dir)
+
+    # appdata (%APPDATA%/package_name)
+    appdata_dir = os.getenv('appdata')
+    if appdata_dir:
+        config_dir = os.path.join(appdata_dir, package_name)
+        if os.path.isdir(config_dir):
+            locations.add(config_dir)
+
+    # home (~/.package_name)
+    user_config_directory = os.path.join(compat_expanduser('~'), '.%s' % package_name)
+    if os.path.isdir(user_config_directory):
+        locations.add(user_config_directory)
+
+    return locations
+
+
+def get_system_config_dirs(package_name):
+    locations = set()
+    # /etc/package_name
+    system_config_directory = os.path.join('/etc', package_name)
+    if os.path.isdir(system_config_directory):
+        locations.add(system_config_directory)
+    return locations
 
 
 def traverse_obj(
@@ -6367,3 +6381,10 @@ class FormatSorter:
 # Deprecated
 has_certifi = bool(certifi)
 has_websockets = bool(websockets)
+
+
+def load_plugins(name, suffix, namespace):
+    from .plugins import load_plugins
+    ret = load_plugins(name, suffix)
+    namespace.update(ret)
+    return ret