From ade8c2b36ff300edef87d48fd1ba835ac35c5b63 Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Mon, 10 Nov 2025 01:45:58 +0100 Subject: [PATCH] [test] Skip flaky tests if source unchanged (#14970) Authored by: bashonly, Grub4K Co-authored-by: bashonly --- .github/workflows/core.yml | 23 ++++++++++++++++++++++- devscripts/run_tests.py | 24 ++++++++++++++++++++++-- test/conftest.py | 30 ++++++++++++++++++++++++++++++ test/test_http_proxy.py | 2 ++ test/test_networking.py | 3 +++ test/test_socks.py | 2 ++ test/test_websockets.py | 7 +++++++ 7 files changed, 88 insertions(+), 3 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index ae3dc95e1b..3cb17f2b7d 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -56,6 +56,8 @@ jobs: python-version: pypy-3.11 steps: - uses: actions/checkout@v5 + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -65,6 +67,25 @@ jobs: - name: Run tests timeout-minutes: 15 continue-on-error: False + env: + source: ${{ (github.event_name == 'push' && github.event.before) || 'origin/master' }} + target: ${{ (github.event_name == 'push' && github.event.after) || 'HEAD' }} + shell: bash run: | + flags=() + # Check if a networking file is involved + patterns="\ + ^yt_dlp/networking/ + ^yt_dlp/utils/networking\.py$ + ^test/test_http_proxy\.py$ + ^test/test_networking\.py$ + ^test/test_networking_utils\.py$ + ^test/test_socks\.py$ + ^test/test_websockets\.py$ + ^pyproject\.toml$ + " + if git diff --name-only "${source}" "${target}" | grep -Ef <(printf '%s' "${patterns}"); then + flags+=(--flaky) + fi python3 -m yt_dlp -v || true # Print debug head - python3 ./devscripts/run_tests.py --pytest-args '--reruns 2 --reruns-delay 3.0' core + python3 -m devscripts.run_tests "${flags[@]}" --pytest-args '--reruns 2 --reruns-delay 3.0' core diff --git a/devscripts/run_tests.py b/devscripts/run_tests.py index ebb3500b6c..3274abc39f 100755 --- a/devscripts/run_tests.py +++ b/devscripts/run_tests.py @@ -17,6 +17,18 @@ def parse_args(): parser = argparse.ArgumentParser(description='Run selected yt-dlp tests') parser.add_argument( 'test', help='an extractor test, test path, or one of "core" or "download"', nargs='*') + parser.add_argument( + '--flaky', + action='store_true', + default=None, + help='Allow running flaky tests. (default: run, unless in CI)', + ) + parser.add_argument( + '--no-flaky', + action='store_false', + dest='flaky', + help=argparse.SUPPRESS, + ) parser.add_argument( '-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION') parser.add_argument( @@ -24,10 +36,11 @@ def parse_args(): return parser.parse_args() -def run_tests(*tests, pattern=None, ci=False): +def run_tests(*tests, pattern=None, ci=False, flaky: bool | None = None): # XXX: hatch uses `tests` if no arguments are passed run_core = 'core' in tests or 'tests' in tests or (not pattern and not tests) run_download = 'download' in tests + run_flaky = flaky or (flaky is None and not ci) pytest_args = args.pytest_args or os.getenv('HATCH_TEST_ARGS', '') arguments = ['pytest', '-Werror', '--tb=short', *shlex.split(pytest_args)] @@ -44,6 +57,8 @@ def run_tests(*tests, pattern=None, ci=False): test if '/' in test else f'test/test_download.py::TestDownload::test_{fix_test_name(test)}' for test in tests) + if not run_flaky: + arguments.append('--disallow-flaky') print(f'Running {arguments}', flush=True) try: @@ -72,6 +87,11 @@ if __name__ == '__main__': args = parse_args() os.chdir(Path(__file__).parent.parent) - sys.exit(run_tests(*args.test, pattern=args.k, ci=bool(os.getenv('CI')))) + sys.exit(run_tests( + *args.test, + pattern=args.k, + ci=bool(os.getenv('CI')), + flaky=args.flaky, + )) except KeyboardInterrupt: pass diff --git a/test/conftest.py b/test/conftest.py index a8b92f811e..9d31986196 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -52,6 +52,33 @@ def skip_handlers_if(request, handler): pytest.skip(marker.args[1] if len(marker.args) > 1 else '') +@pytest.fixture(autouse=True) +def handler_flaky(request, handler): + """Mark a certain handler as being flaky. + + This will skip the test if pytest does not get run using `--allow-flaky` + + usage: + pytest.mark.handler_flaky('my_handler', os.name != 'nt', reason='reason') + """ + for marker in request.node.iter_markers(handler_flaky.__name__): + if ( + marker.args[0] == handler.RH_KEY + and (not marker.args[1:] or any(marker.args[1:])) + and request.config.getoption('disallow_flaky') + ): + reason = marker.kwargs.get('reason') + pytest.skip(f'flaky: {reason}' if reason else 'flaky') + + +def pytest_addoption(parser, pluginmanager): + parser.addoption( + '--disallow-flaky', + action='store_true', + help='disallow flaky tests from running.', + ) + + def pytest_configure(config): config.addinivalue_line( 'markers', 'skip_handler(handler): skip test for the given handler', @@ -62,3 +89,6 @@ def pytest_configure(config): config.addinivalue_line( 'markers', 'skip_handlers_if(handler): skip test for handlers when the condition is true', ) + config.addinivalue_line( + 'markers', 'handler_flaky(handler): mark handler as flaky if condition is true', + ) diff --git a/test/test_http_proxy.py b/test/test_http_proxy.py index e903ff8beb..22ce3ca5d7 100644 --- a/test/test_http_proxy.py +++ b/test/test_http_proxy.py @@ -247,6 +247,7 @@ def ctx(request): @pytest.mark.parametrize( 'handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults') @pytest.mark.parametrize('ctx', ['http'], indirect=True) # pure http proxy can only support http class TestHTTPProxy: def test_http_no_auth(self, handler, ctx): @@ -315,6 +316,7 @@ class TestHTTPProxy: ('Requests', 'https'), ('CurlCFFI', 'https'), ], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults') class TestHTTPConnectProxy: def test_http_connect_no_auth(self, handler, ctx): with ctx.http_server(HTTPConnectProxyHandler) as server_address: diff --git a/test/test_networking.py b/test/test_networking.py index e972f597b5..631e7458e6 100644 --- a/test/test_networking.py +++ b/test/test_networking.py @@ -312,6 +312,7 @@ class TestRequestHandlerBase: @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults') class TestHTTPRequestHandler(TestRequestHandlerBase): def test_verify_cert(self, handler): @@ -756,6 +757,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase): @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults') class TestClientCertificate: @classmethod def setup_class(cls): @@ -1060,6 +1062,7 @@ class TestRequestsRequestHandler(TestRequestHandlerBase): @pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults') class TestCurlCFFIRequestHandler(TestRequestHandlerBase): @pytest.mark.parametrize('params,extensions', [ diff --git a/test/test_socks.py b/test/test_socks.py index f601fc8a5e..4ec4733bd1 100644 --- a/test/test_socks.py +++ b/test/test_socks.py @@ -295,6 +295,7 @@ def ctx(request): ('Websockets', 'ws'), ('CurlCFFI', 'http'), ], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults') class TestSocks4Proxy: def test_socks4_no_auth(self, handler, ctx): with handler() as rh: @@ -370,6 +371,7 @@ class TestSocks4Proxy: ('Websockets', 'ws'), ('CurlCFFI', 'http'), ], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults') class TestSocks5Proxy: def test_socks5_no_auth(self, handler, ctx): diff --git a/test/test_websockets.py b/test/test_websockets.py index 167f67daa7..637b6ee2b2 100644 --- a/test/test_websockets.py +++ b/test/test_websockets.py @@ -38,6 +38,13 @@ from yt_dlp.utils.networking import HTTPHeaderDict TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +pytestmark = pytest.mark.handler_flaky( + 'Websockets', + os.name != 'nt' and sys.implementation.name == 'pypy', + reason='segfaults', +) + + def websocket_handler(websocket): for message in websocket: if isinstance(message, bytes):