Improve plugin architecture (#5553)

to make plugins easier to develop and use:
* Plugins are now loaded as namespace packages.
* Plugins can be loaded in any distribution of yt-dlp (binary, pip, source, etc.).
* Plugin packages can be installed and managed via pip, or dropped into any of the documented locations.
* Users do not need to edit any code files to install plugins.
* Backwards-compatible with previous plugin architecture.

As a side-effect, yt-dlp will now search in a few more locations for config files.

Closes https://github.com/yt-dlp/yt-dlp/issues/1389

Authored by: flashdagger, coletdjnz, pukkandan, Grub4K
Co-authored-by: Marcel <flashdagger@googlemail.com>
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
Co-authored-by: Simon Sawicki <accounts@grub4k.xyz>

yt_dlp/utils.py

@@ -18,7 +18,6 @@ import html.entities
 import html.parser
 import http.client
 import http.cookiejar
-import importlib.util
 import inspect
 import io
 import itertools
@@ -5372,22 +5371,37 @@ def get_executable_path():
     return os.path.dirname(os.path.abspath(_get_variant_and_executable_path()[1]))
 
 
-def load_plugins(name, suffix, namespace):
-    classes = {}
-    with contextlib.suppress(FileNotFoundError):
-        plugins_spec = importlib.util.spec_from_file_location(
-            name, os.path.join(get_executable_path(), 'ytdlp_plugins', name, '__init__.py'))
-        plugins = importlib.util.module_from_spec(plugins_spec)
-        sys.modules[plugins_spec.name] = plugins
-        plugins_spec.loader.exec_module(plugins)
-        for name in dir(plugins):
-            if name in namespace:
-                continue
-            if not name.endswith(suffix):
-                continue
-            klass = getattr(plugins, name)
-            classes[name] = namespace[name] = klass
-    return classes
+def get_user_config_dirs(package_name):
+    locations = set()
+
+    # .config (e.g. ~/.config/package_name)
+    xdg_config_home = os.getenv('XDG_CONFIG_HOME') or compat_expanduser('~/.config')
+    config_dir = os.path.join(xdg_config_home, package_name)
+    if os.path.isdir(config_dir):
+        locations.add(config_dir)
+
+    # appdata (%APPDATA%/package_name)
+    appdata_dir = os.getenv('appdata')
+    if appdata_dir:
+        config_dir = os.path.join(appdata_dir, package_name)
+        if os.path.isdir(config_dir):
+            locations.add(config_dir)
+
+    # home (~/.package_name)
+    user_config_directory = os.path.join(compat_expanduser('~'), '.%s' % package_name)
+    if os.path.isdir(user_config_directory):
+        locations.add(user_config_directory)
+
+    return locations
+
+
+def get_system_config_dirs(package_name):
+    locations = set()
+    # /etc/package_name
+    system_config_directory = os.path.join('/etc', package_name)
+    if os.path.isdir(system_config_directory):
+        locations.add(system_config_directory)
+    return locations
 
 
 def traverse_obj(
@@ -6367,3 +6381,10 @@ class FormatSorter:
 # Deprecated
 has_certifi = bool(certifi)
 has_websockets = bool(websockets)
+
+
+def load_plugins(name, suffix, namespace):
+    from .plugins import load_plugins
+    ret = load_plugins(name, suffix)
+    namespace.update(ret)
+    return ret