[test] Skip flaky tests if source unchanged (#14970)

Authored by: bashonly, Grub4K
Co-authored-by: bashonly <bashonly@protonmail.com>

.github/workflows/core.yml

@@ -56,6 +56,8 @@ jobs:
           python-version: pypy-3.11
     steps:
     - uses: actions/checkout@v5
+      with:
+        fetch-depth: 0
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:
@@ -65,6 +67,25 @@ jobs:
     - name: Run tests
       timeout-minutes: 15
       continue-on-error: False
+      env:
+        source: ${{ (github.event_name == 'push' && github.event.before) || 'origin/master' }}
+        target: ${{ (github.event_name == 'push' && github.event.after) || 'HEAD' }}
+      shell: bash
       run: |
+        flags=()
+        # Check if a networking file is involved
+        patterns="\
+        ^yt_dlp/networking/
+        ^yt_dlp/utils/networking\.py$
+        ^test/test_http_proxy\.py$
+        ^test/test_networking\.py$
+        ^test/test_networking_utils\.py$
+        ^test/test_socks\.py$
+        ^test/test_websockets\.py$
+        ^pyproject\.toml$
+        "
+        if git diff --name-only "${source}" "${target}" | grep -Ef <(printf '%s' "${patterns}"); then
+          flags+=(--flaky)
+        fi
         python3 -m yt_dlp -v || true  # Print debug head
-        python3 ./devscripts/run_tests.py --pytest-args '--reruns 2 --reruns-delay 3.0' core
+        python3 -m devscripts.run_tests "${flags[@]}" --pytest-args '--reruns 2 --reruns-delay 3.0' core

devscripts/run_tests.py

@@ -17,6 +17,18 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Run selected yt-dlp tests')
     parser.add_argument(
         'test', help='an extractor test, test path, or one of "core" or "download"', nargs='*')
+    parser.add_argument(
+        '--flaky',
+        action='store_true',
+        default=None,
+        help='Allow running flaky tests. (default: run, unless in CI)',
+    )
+    parser.add_argument(
+        '--no-flaky',
+        action='store_false',
+        dest='flaky',
+        help=argparse.SUPPRESS,
+    )
     parser.add_argument(
         '-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
     parser.add_argument(
@@ -24,10 +36,11 @@ def parse_args():
     return parser.parse_args()
 
 
-def run_tests(*tests, pattern=None, ci=False):
+def run_tests(*tests, pattern=None, ci=False, flaky: bool | None = None):
     # XXX: hatch uses `tests` if no arguments are passed
     run_core = 'core' in tests or 'tests' in tests or (not pattern and not tests)
     run_download = 'download' in tests
+    run_flaky = flaky or (flaky is None and not ci)
 
     pytest_args = args.pytest_args or os.getenv('HATCH_TEST_ARGS', '')
     arguments = ['pytest', '-Werror', '--tb=short', *shlex.split(pytest_args)]
@@ -44,6 +57,8 @@ def run_tests(*tests, pattern=None, ci=False):
             test if '/' in test
             else f'test/test_download.py::TestDownload::test_{fix_test_name(test)}'
             for test in tests)
+    if not run_flaky:
+        arguments.append('--disallow-flaky')
 
     print(f'Running {arguments}', flush=True)
     try:
@@ -72,6 +87,11 @@ if __name__ == '__main__':
         args = parse_args()
 
         os.chdir(Path(__file__).parent.parent)
-        sys.exit(run_tests(*args.test, pattern=args.k, ci=bool(os.getenv('CI'))))
+        sys.exit(run_tests(
+            *args.test,
+            pattern=args.k,
+            ci=bool(os.getenv('CI')),
+            flaky=args.flaky,
+        ))
     except KeyboardInterrupt:
         pass

test/conftest.py

@@ -52,6 +52,33 @@ def skip_handlers_if(request, handler):
             pytest.skip(marker.args[1] if len(marker.args) > 1 else '')
 
 
+@pytest.fixture(autouse=True)
+def handler_flaky(request, handler):
+    """Mark a certain handler as being flaky.
+
+    This will skip the test if pytest does not get run using `--allow-flaky`
+
+    usage:
+        pytest.mark.handler_flaky('my_handler', os.name != 'nt', reason='reason')
+    """
+    for marker in request.node.iter_markers(handler_flaky.__name__):
+        if (
+            marker.args[0] == handler.RH_KEY
+            and (not marker.args[1:] or any(marker.args[1:]))
+            and request.config.getoption('disallow_flaky')
+        ):
+            reason = marker.kwargs.get('reason')
+            pytest.skip(f'flaky: {reason}' if reason else 'flaky')
+
+
+def pytest_addoption(parser, pluginmanager):
+    parser.addoption(
+        '--disallow-flaky',
+        action='store_true',
+        help='disallow flaky tests from running.',
+    )
+
+
 def pytest_configure(config):
     config.addinivalue_line(
         'markers', 'skip_handler(handler): skip test for the given handler',
@@ -62,3 +89,6 @@ def pytest_configure(config):
     config.addinivalue_line(
         'markers', 'skip_handlers_if(handler): skip test for handlers when the condition is true',
     )
+    config.addinivalue_line(
+        'markers', 'handler_flaky(handler): mark handler as flaky if condition is true',
+    )

test/test_http_proxy.py

@@ -247,6 +247,7 @@ def ctx(request):
 
 @pytest.mark.parametrize(
     'handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 @pytest.mark.parametrize('ctx', ['http'], indirect=True)  # pure http proxy can only support http
 class TestHTTPProxy:
     def test_http_no_auth(self, handler, ctx):
@@ -315,6 +316,7 @@ class TestHTTPProxy:
         ('Requests', 'https'),
         ('CurlCFFI', 'https'),
     ], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestHTTPConnectProxy:
     def test_http_connect_no_auth(self, handler, ctx):
         with ctx.http_server(HTTPConnectProxyHandler) as server_address:

test/test_networking.py

@@ -312,6 +312,7 @@ class TestRequestHandlerBase:
 
 
 @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults')
 class TestHTTPRequestHandler(TestRequestHandlerBase):
 
     def test_verify_cert(self, handler):
@@ -756,6 +757,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
 
 
 @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestClientCertificate:
     @classmethod
     def setup_class(cls):
@@ -1060,6 +1062,7 @@ class TestRequestsRequestHandler(TestRequestHandlerBase):
 
 
 @pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults')
 class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
 
     @pytest.mark.parametrize('params,extensions', [

test/test_socks.py

@@ -295,6 +295,7 @@ def ctx(request):
         ('Websockets', 'ws'),
         ('CurlCFFI', 'http'),
     ], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestSocks4Proxy:
     def test_socks4_no_auth(self, handler, ctx):
         with handler() as rh:
@@ -370,6 +371,7 @@ class TestSocks4Proxy:
         ('Websockets', 'ws'),
         ('CurlCFFI', 'http'),
     ], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestSocks5Proxy:
 
     def test_socks5_no_auth(self, handler, ctx):

test/test_websockets.py

@@ -38,6 +38,13 @@ from yt_dlp.utils.networking import HTTPHeaderDict
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
+pytestmark = pytest.mark.handler_flaky(
+    'Websockets',
+    os.name != 'nt' and sys.implementation.name == 'pypy',
+    reason='segfaults',
+)
+
+
 def websocket_handler(websocket):
     for message in websocket:
         if isinstance(message, bytes):