[test] Skip flaky tests if source unchanged (#14970)

Authored by: bashonly, Grub4K
Co-authored-by: bashonly <bashonly@protonmail.com>

test/conftest.py

@@ -52,6 +52,33 @@ def skip_handlers_if(request, handler):
             pytest.skip(marker.args[1] if len(marker.args) > 1 else '')
 
 
+@pytest.fixture(autouse=True)
+def handler_flaky(request, handler):
+    """Mark a certain handler as being flaky.
+
+    This will skip the test if pytest does not get run using `--allow-flaky`
+
+    usage:
+        pytest.mark.handler_flaky('my_handler', os.name != 'nt', reason='reason')
+    """
+    for marker in request.node.iter_markers(handler_flaky.__name__):
+        if (
+            marker.args[0] == handler.RH_KEY
+            and (not marker.args[1:] or any(marker.args[1:]))
+            and request.config.getoption('disallow_flaky')
+        ):
+            reason = marker.kwargs.get('reason')
+            pytest.skip(f'flaky: {reason}' if reason else 'flaky')
+
+
+def pytest_addoption(parser, pluginmanager):
+    parser.addoption(
+        '--disallow-flaky',
+        action='store_true',
+        help='disallow flaky tests from running.',
+    )
+
+
 def pytest_configure(config):
     config.addinivalue_line(
         'markers', 'skip_handler(handler): skip test for the given handler',
@@ -62,3 +89,6 @@ def pytest_configure(config):
     config.addinivalue_line(
         'markers', 'skip_handlers_if(handler): skip test for handlers when the condition is true',
     )
+    config.addinivalue_line(
+        'markers', 'handler_flaky(handler): mark handler as flaky if condition is true',
+    )

test/test_http_proxy.py

@@ -247,6 +247,7 @@ def ctx(request):
 
 @pytest.mark.parametrize(
     'handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 @pytest.mark.parametrize('ctx', ['http'], indirect=True)  # pure http proxy can only support http
 class TestHTTPProxy:
     def test_http_no_auth(self, handler, ctx):
@@ -315,6 +316,7 @@ class TestHTTPProxy:
         ('Requests', 'https'),
         ('CurlCFFI', 'https'),
     ], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestHTTPConnectProxy:
     def test_http_connect_no_auth(self, handler, ctx):
         with ctx.http_server(HTTPConnectProxyHandler) as server_address:

test/test_networking.py

@@ -312,6 +312,7 @@ class TestRequestHandlerBase:
 
 
 @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults')
 class TestHTTPRequestHandler(TestRequestHandlerBase):
 
     def test_verify_cert(self, handler):
@@ -756,6 +757,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
 
 
 @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestClientCertificate:
     @classmethod
     def setup_class(cls):
@@ -1060,6 +1062,7 @@ class TestRequestsRequestHandler(TestRequestHandlerBase):
 
 
 @pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults')
 class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
 
     @pytest.mark.parametrize('params,extensions', [

test/test_socks.py

@@ -295,6 +295,7 @@ def ctx(request):
         ('Websockets', 'ws'),
         ('CurlCFFI', 'http'),
     ], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestSocks4Proxy:
     def test_socks4_no_auth(self, handler, ctx):
         with handler() as rh:
@@ -370,6 +371,7 @@ class TestSocks4Proxy:
         ('Websockets', 'ws'),
         ('CurlCFFI', 'http'),
     ], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestSocks5Proxy:
 
     def test_socks5_no_auth(self, handler, ctx):

test/test_websockets.py

@@ -38,6 +38,13 @@ from yt_dlp.utils.networking import HTTPHeaderDict
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
+pytestmark = pytest.mark.handler_flaky(
+    'Websockets',
+    os.name != 'nt' and sys.implementation.name == 'pypy',
+    reason='segfaults',
+)
+
+
 def websocket_handler(websocket):
     for message in websocket:
         if isinstance(message, bytes):