From 73922e66e437fb4bb618bdc119a96375081bf508 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 25 Oct 2025 17:47:00 -0500 Subject: [PATCH] [devscripts] Improve `install_deps` script (#14766) Authored by: bashonly --- .github/workflows/build.yml | 10 +++--- .github/workflows/core.yml | 2 +- .github/workflows/download.yml | 8 ++--- .github/workflows/quick-test.yml | 6 ++-- .github/workflows/release.yml | 2 +- .github/workflows/signature-tests.yml | 2 +- .github/workflows/test-workflows.yml | 2 +- bundle/docker/linux/build.sh | 8 ++--- devscripts/install_deps.py | 48 ++++++++++++++++++--------- 9 files changed, 52 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 29c18723c2..fc89dbfe05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -341,14 +341,14 @@ jobs: brew uninstall --ignore-dependencies python3 python3 -m venv ~/yt-dlp-build-venv source ~/yt-dlp-build-venv/bin/activate - python3 devscripts/install_deps.py -o --include build - python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt + python3 devscripts/install_deps.py --only-optional-groups --include-group build + python3 devscripts/install_deps.py --print --include-group pyinstaller > requirements.txt # We need to ignore wheels otherwise we break universal2 builds python3 -m pip install -U --no-binary :all: -r requirements.txt # We need to fuse our own universal2 wheels for curl_cffi python3 -m pip install -U 'delocate==0.11.0' mkdir curl_cffi_whls curl_cffi_universal2 - python3 devscripts/install_deps.py --print -o --include curl-cffi > requirements.txt + python3 devscripts/install_deps.py --print --only-optional-groups --include-group curl-cffi > requirements.txt for platform in "macosx_11_0_arm64" "macosx_11_0_x86_64"; do python3 -m pip download \ --only-binary=:all: \ @@ -482,11 +482,11 @@ jobs: mkdir /pyi-wheels python -m pip download -d /pyi-wheels --no-deps --require-hashes "pyinstaller@${Env:PYI_URL}#sha256=${Env:PYI_HASH}" python -m pip install --force-reinstall -U "/pyi-wheels/${Env:PYI_WHEEL}" - python devscripts/install_deps.py -o --include build + python devscripts/install_deps.py --only-optional-groups --include-group build if ("${Env:ARCH}" -eq "x86") { python devscripts/install_deps.py } else { - python devscripts/install_deps.py --include curl-cffi + python devscripts/install_deps.py --include-group curl-cffi } - name: Prepare diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index d196f59d83..e813b8f629 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -59,7 +59,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install test requirements - run: python3 ./devscripts/install_deps.py --include test --include curl-cffi + run: python ./devscripts/install_deps.py --include-group test --include-group curl-cffi - name: Run tests timeout-minutes: 15 continue-on-error: False diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml index 8163bd1a23..d075270d7b 100644 --- a/.github/workflows/download.yml +++ b/.github/workflows/download.yml @@ -15,10 +15,10 @@ jobs: with: python-version: '3.10' - name: Install test requirements - run: python3 ./devscripts/install_deps.py --include dev + run: python ./devscripts/install_deps.py --include-group dev - name: Run tests continue-on-error: true - run: python3 ./devscripts/run_tests.py download + run: python ./devscripts/run_tests.py download full: name: Full Download Tests @@ -42,7 +42,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install test requirements - run: python3 ./devscripts/install_deps.py --include dev + run: python ./devscripts/install_deps.py --include-group dev - name: Run tests continue-on-error: true - run: python3 ./devscripts/run_tests.py download + run: python ./devscripts/run_tests.py download diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml index c26628b421..a6e84b1d80 100644 --- a/.github/workflows/quick-test.yml +++ b/.github/workflows/quick-test.yml @@ -15,7 +15,7 @@ jobs: with: python-version: '3.10' - name: Install test requirements - run: python3 ./devscripts/install_deps.py -o --include test + run: python ./devscripts/install_deps.py --only-optional-groups --include-group test - name: Run tests timeout-minutes: 15 run: | @@ -31,9 +31,9 @@ jobs: with: python-version: '3.10' - name: Install dev dependencies - run: python3 ./devscripts/install_deps.py -o --include static-analysis + run: python ./devscripts/install_deps.py --only-optional-groups --include-group static-analysis - name: Make lazy extractors - run: python3 ./devscripts/make_lazy_extractors.py + run: python ./devscripts/make_lazy_extractors.py - name: Run ruff run: ruff check --output-format github . - name: Run autopep8 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b60a0650a5..b8f1ed78ea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -180,7 +180,7 @@ jobs: - name: Install Requirements run: | sudo apt -y install pandoc man - python devscripts/install_deps.py -o --include build + python devscripts/install_deps.py --only-optional-groups --include-group build - name: Prepare env: diff --git a/.github/workflows/signature-tests.yml b/.github/workflows/signature-tests.yml index 77f5e6a4c8..1b310db6aa 100644 --- a/.github/workflows/signature-tests.yml +++ b/.github/workflows/signature-tests.yml @@ -33,7 +33,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install test requirements - run: python3 ./devscripts/install_deps.py --only-optional --include test + run: python ./devscripts/install_deps.py --only-optional-groups --include-group test - name: Run tests timeout-minutes: 15 run: | diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index 6c993e6b34..37bf044d69 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -34,7 +34,7 @@ jobs: env: ACTIONLINT_TARBALL: ${{ format('actionlint_{0}_linux_amd64.tar.gz', env.ACTIONLINT_VERSION) }} run: | - python -m devscripts.install_deps -o --include test + python -m devscripts.install_deps --only-optional-groups --include-group test sudo apt -y install shellcheck python -m pip install -U pyflakes curl -LO "${ACTIONLINT_REPO}/releases/download/v${ACTIONLINT_VERSION}/${ACTIONLINT_TARBALL}" diff --git a/bundle/docker/linux/build.sh b/bundle/docker/linux/build.sh index 71adaad058..b30d40980e 100755 --- a/bundle/docker/linux/build.sh +++ b/bundle/docker/linux/build.sh @@ -15,12 +15,12 @@ function venvpy { } INCLUDES=( - --include pyinstaller - --include secretstorage + --include-group pyinstaller + --include-group secretstorage ) if [[ -z "${EXCLUDE_CURL_CFFI:-}" ]]; then - INCLUDES+=(--include curl-cffi) + INCLUDES+=(--include-group curl-cffi) fi runpy -m venv /yt-dlp-build-venv @@ -28,7 +28,7 @@ runpy -m venv /yt-dlp-build-venv source /yt-dlp-build-venv/bin/activate # Inside the venv we use venvpy instead of runpy venvpy -m ensurepip --upgrade --default-pip -venvpy -m devscripts.install_deps -o --include build +venvpy -m devscripts.install_deps --only-optional-groups --include-group build venvpy -m devscripts.install_deps "${INCLUDES[@]}" venvpy -m devscripts.make_lazy_extractors venvpy devscripts/update-version.py -c "${CHANNEL}" -r "${ORIGIN}" "${VERSION}" diff --git a/devscripts/install_deps.py b/devscripts/install_deps.py index d292505458..07c646a4c0 100755 --- a/devscripts/install_deps.py +++ b/devscripts/install_deps.py @@ -22,14 +22,19 @@ def parse_args(): 'input', nargs='?', metavar='TOMLFILE', default=Path(__file__).parent.parent / 'pyproject.toml', help='input file (default: %(default)s)') parser.add_argument( - '-e', '--exclude', metavar='DEPENDENCY', action='append', - help='exclude a dependency') + '-e', '--exclude-dependency', metavar='DEPENDENCY', action='append', + help='exclude a dependency (can be used multiple times)') parser.add_argument( - '-i', '--include', metavar='GROUP', action='append', - help='include an optional dependency group') + '-i', '--include-group', metavar='GROUP', action='append', + help='include an optional dependency group (can be used multiple times)') parser.add_argument( - '-o', '--only-optional', action='store_true', - help='only install optional dependencies') + '-c', '--cherry-pick', metavar='DEPENDENCY', action='append', + help=( + 'only include a specific dependency from the resulting dependency list ' + '(can be used multiple times)')) + parser.add_argument( + '-o', '--only-optional-groups', action='store_true', + help='omit default dependencies unless the "default" group is specified with --include-group') parser.add_argument( '-p', '--print', action='store_true', help='only print requirements to stdout') @@ -39,30 +44,41 @@ def parse_args(): return parser.parse_args() +def uniq(arg) -> dict[str, None]: + return dict.fromkeys(map(str.lower, arg or ())) + + def main(): args = parse_args() project_table = parse_toml(read_file(args.input))['project'] recursive_pattern = re.compile(rf'{project_table["name"]}\[(?P[\w-]+)\]') optional_groups = project_table['optional-dependencies'] - excludes = args.exclude or [] + + excludes = uniq(args.exclude_dependency) + only_includes = uniq(args.cherry_pick) + include_groups = uniq(args.include_group) def yield_deps(group): for dep in group: if mobj := recursive_pattern.fullmatch(dep): - yield from optional_groups.get(mobj.group('group_name'), []) + yield from optional_groups.get(mobj.group('group_name'), ()) else: yield dep - targets = [] - if not args.only_optional: # `-o` should exclude 'dependencies' and the 'default' group - targets.extend(project_table['dependencies']) - if 'default' not in excludes: # `--exclude default` should exclude entire 'default' group - targets.extend(yield_deps(optional_groups['default'])) + targets = {} + if not args.only_optional_groups: + # legacy: 'dependencies' is empty now + targets.update(dict.fromkeys(project_table['dependencies'])) + targets.update(dict.fromkeys(yield_deps(optional_groups['default']))) - for include in filter(None, map(optional_groups.get, args.include or [])): - targets.extend(yield_deps(include)) + for include in filter(None, map(optional_groups.get, include_groups)): + targets.update(dict.fromkeys(yield_deps(include))) - targets = [t for t in targets if re.match(r'[\w-]+', t).group(0).lower() not in excludes] + def target_filter(target): + name = re.match(r'[\w-]+', target).group(0).lower() + return name not in excludes and (not only_includes or name in only_includes) + + targets = list(filter(target_filter, targets)) if args.print: for target in targets: