[devscripts] Improve install_deps script (#14766)

Authored by: bashonly

.github/workflows/build.yml

@@ -341,14 +341,14 @@ jobs:
           brew uninstall --ignore-dependencies python3
           python3 -m venv ~/yt-dlp-build-venv
           source ~/yt-dlp-build-venv/bin/activate
-          python3 devscripts/install_deps.py -o --include build
+          python3 devscripts/install_deps.py --only-optional-groups --include-group build
-          python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt
+          python3 devscripts/install_deps.py --print --include-group pyinstaller > requirements.txt
           # We need to ignore wheels otherwise we break universal2 builds
           python3 -m pip install -U --no-binary :all: -r requirements.txt
           # We need to fuse our own universal2 wheels for curl_cffi
           python3 -m pip install -U 'delocate==0.11.0'
           mkdir curl_cffi_whls curl_cffi_universal2
-          python3 devscripts/install_deps.py --print -o --include curl-cffi > requirements.txt
+          python3 devscripts/install_deps.py --print --only-optional-groups --include-group curl-cffi > requirements.txt
           for platform in "macosx_11_0_arm64" "macosx_11_0_x86_64"; do
             python3 -m pip download \
               --only-binary=:all: \
@@ -482,11 +482,11 @@ jobs:
           mkdir /pyi-wheels
           python -m pip download -d /pyi-wheels --no-deps --require-hashes "pyinstaller@${Env:PYI_URL}#sha256=${Env:PYI_HASH}"
           python -m pip install --force-reinstall -U "/pyi-wheels/${Env:PYI_WHEEL}"
-          python devscripts/install_deps.py -o --include build
+          python devscripts/install_deps.py --only-optional-groups --include-group build
           if ("${Env:ARCH}" -eq "x86") {
             python devscripts/install_deps.py
           } else {
-            python devscripts/install_deps.py --include curl-cffi
+            python devscripts/install_deps.py --include-group curl-cffi
           }
 
       - name: Prepare

.github/workflows/core.yml

@@ -59,7 +59,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install test requirements
-      run: python3 ./devscripts/install_deps.py --include test --include curl-cffi
+      run: python ./devscripts/install_deps.py --include-group test --include-group curl-cffi
     - name: Run tests
       timeout-minutes: 15
       continue-on-error: False

.github/workflows/download.yml

@@ -15,10 +15,10 @@ jobs:
       with:
         python-version: '3.10'
     - name: Install test requirements
-      run: python3 ./devscripts/install_deps.py --include dev
+      run: python ./devscripts/install_deps.py --include-group dev
     - name: Run tests
       continue-on-error: true
-      run: python3 ./devscripts/run_tests.py download
+      run: python ./devscripts/run_tests.py download
 
   full:
     name: Full Download Tests
@@ -42,7 +42,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install test requirements
-      run: python3 ./devscripts/install_deps.py --include dev
+      run: python ./devscripts/install_deps.py --include-group dev
     - name: Run tests
       continue-on-error: true
-      run: python3 ./devscripts/run_tests.py download
+      run: python ./devscripts/run_tests.py download

.github/workflows/quick-test.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         python-version: '3.10'
     - name: Install test requirements
-      run: python3 ./devscripts/install_deps.py -o --include test
+      run: python ./devscripts/install_deps.py --only-optional-groups --include-group test
     - name: Run tests
       timeout-minutes: 15
       run: |
@@ -31,9 +31,9 @@ jobs:
       with:
         python-version: '3.10'
     - name: Install dev dependencies
-      run: python3 ./devscripts/install_deps.py -o --include static-analysis
+      run: python ./devscripts/install_deps.py --only-optional-groups --include-group static-analysis
     - name: Make lazy extractors
-      run: python3 ./devscripts/make_lazy_extractors.py
+      run: python ./devscripts/make_lazy_extractors.py
     - name: Run ruff
       run: ruff check --output-format github .
     - name: Run autopep8

.github/workflows/release.yml

@@ -180,7 +180,7 @@ jobs:
       - name: Install Requirements
         run: |
           sudo apt -y install pandoc man
-          python devscripts/install_deps.py -o --include build
+          python devscripts/install_deps.py --only-optional-groups --include-group build
 
       - name: Prepare
         env:

.github/workflows/signature-tests.yml

@@ -33,7 +33,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install test requirements
-      run: python3 ./devscripts/install_deps.py --only-optional --include test
+      run: python ./devscripts/install_deps.py --only-optional-groups --include-group test
     - name: Run tests
       timeout-minutes: 15
       run: |

.github/workflows/test-workflows.yml

@@ -34,7 +34,7 @@ jobs:
         env:
           ACTIONLINT_TARBALL: ${{ format('actionlint_{0}_linux_amd64.tar.gz', env.ACTIONLINT_VERSION) }}
         run: |
-          python -m devscripts.install_deps -o --include test
+          python -m devscripts.install_deps --only-optional-groups --include-group test
           sudo apt -y install shellcheck
           python -m pip install -U pyflakes
           curl -LO "${ACTIONLINT_REPO}/releases/download/v${ACTIONLINT_VERSION}/${ACTIONLINT_TARBALL}"

bundle/docker/linux/build.sh

@@ -15,12 +15,12 @@ function venvpy {
 }
 
 INCLUDES=(
-    --include pyinstaller
+    --include-group pyinstaller
-    --include secretstorage
+    --include-group secretstorage
 )
 
 if [[ -z "${EXCLUDE_CURL_CFFI:-}" ]]; then
-    INCLUDES+=(--include curl-cffi)
+    INCLUDES+=(--include-group curl-cffi)
 fi
 
 runpy -m venv /yt-dlp-build-venv
@@ -28,7 +28,7 @@ runpy -m venv /yt-dlp-build-venv
 source /yt-dlp-build-venv/bin/activate
 # Inside the venv we use venvpy instead of runpy
 venvpy -m ensurepip --upgrade --default-pip
-venvpy -m devscripts.install_deps -o --include build
+venvpy -m devscripts.install_deps --only-optional-groups --include-group build
 venvpy -m devscripts.install_deps "${INCLUDES[@]}"
 venvpy -m devscripts.make_lazy_extractors
 venvpy devscripts/update-version.py -c "${CHANNEL}" -r "${ORIGIN}" "${VERSION}"

devscripts/install_deps.py

@@ -22,14 +22,19 @@ def parse_args():
         'input', nargs='?', metavar='TOMLFILE', default=Path(__file__).parent.parent / 'pyproject.toml',
         help='input file (default: %(default)s)')
     parser.add_argument(
-        '-e', '--exclude', metavar='DEPENDENCY', action='append',
+        '-e', '--exclude-dependency', metavar='DEPENDENCY', action='append',
-        help='exclude a dependency')
+        help='exclude a dependency (can be used multiple times)')
     parser.add_argument(
-        '-i', '--include', metavar='GROUP', action='append',
+        '-i', '--include-group', metavar='GROUP', action='append',
-        help='include an optional dependency group')
+        help='include an optional dependency group (can be used multiple times)')
     parser.add_argument(
-        '-o', '--only-optional', action='store_true',
+        '-c', '--cherry-pick', metavar='DEPENDENCY', action='append',
-        help='only install optional dependencies')
+        help=(
+            'only include a specific dependency from the resulting dependency list '
+            '(can be used multiple times)'))
+    parser.add_argument(
+        '-o', '--only-optional-groups', action='store_true',
+        help='omit default dependencies unless the "default" group is specified with --include-group')
     parser.add_argument(
         '-p', '--print', action='store_true',
         help='only print requirements to stdout')
@@ -39,30 +44,41 @@ def parse_args():
     return parser.parse_args()
 
 
+def uniq(arg) -> dict[str, None]:
+    return dict.fromkeys(map(str.lower, arg or ()))
+
+
 def main():
     args = parse_args()
     project_table = parse_toml(read_file(args.input))['project']
     recursive_pattern = re.compile(rf'{project_table["name"]}\[(?P<group_name>[\w-]+)\]')
     optional_groups = project_table['optional-dependencies']
-    excludes = args.exclude or []
+
+    excludes = uniq(args.exclude_dependency)
+    only_includes = uniq(args.cherry_pick)
+    include_groups = uniq(args.include_group)
 
     def yield_deps(group):
         for dep in group:
             if mobj := recursive_pattern.fullmatch(dep):
-                yield from optional_groups.get(mobj.group('group_name'), [])
+                yield from optional_groups.get(mobj.group('group_name'), ())
             else:
                 yield dep
 
-    targets = []
+    targets = {}
-    if not args.only_optional:  # `-o` should exclude 'dependencies' and the 'default' group
+    if not args.only_optional_groups:
-        targets.extend(project_table['dependencies'])
+        # legacy: 'dependencies' is empty now
-        if 'default' not in excludes:  # `--exclude default` should exclude entire 'default' group
+        targets.update(dict.fromkeys(project_table['dependencies']))
-            targets.extend(yield_deps(optional_groups['default']))
+        targets.update(dict.fromkeys(yield_deps(optional_groups['default'])))
 
-    for include in filter(None, map(optional_groups.get, args.include or [])):
+    for include in filter(None, map(optional_groups.get, include_groups)):
-        targets.extend(yield_deps(include))
+        targets.update(dict.fromkeys(yield_deps(include)))
 
-    targets = [t for t in targets if re.match(r'[\w-]+', t).group(0).lower() not in excludes]
+    def target_filter(target):
+        name = re.match(r'[\w-]+', target).group(0).lower()
+        return name not in excludes and (not only_includes or name in only_includes)
+
+    targets = list(filter(target_filter, targets))
 
     if args.print:
         for target in targets: