[test] Skip flaky tests if source unchanged (#14970)

Authored by: bashonly, Grub4K
Co-authored-by: bashonly <bashonly@protonmail.com>

devscripts/run_tests.py

@@ -17,6 +17,18 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Run selected yt-dlp tests')
     parser.add_argument(
         'test', help='an extractor test, test path, or one of "core" or "download"', nargs='*')
+    parser.add_argument(
+        '--flaky',
+        action='store_true',
+        default=None,
+        help='Allow running flaky tests. (default: run, unless in CI)',
+    )
+    parser.add_argument(
+        '--no-flaky',
+        action='store_false',
+        dest='flaky',
+        help=argparse.SUPPRESS,
+    )
     parser.add_argument(
         '-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
     parser.add_argument(
@@ -24,10 +36,11 @@ def parse_args():
     return parser.parse_args()
 
 
-def run_tests(*tests, pattern=None, ci=False):
+def run_tests(*tests, pattern=None, ci=False, flaky: bool | None = None):
     # XXX: hatch uses `tests` if no arguments are passed
     run_core = 'core' in tests or 'tests' in tests or (not pattern and not tests)
     run_download = 'download' in tests
+    run_flaky = flaky or (flaky is None and not ci)
 
     pytest_args = args.pytest_args or os.getenv('HATCH_TEST_ARGS', '')
     arguments = ['pytest', '-Werror', '--tb=short', *shlex.split(pytest_args)]
@@ -44,6 +57,8 @@ def run_tests(*tests, pattern=None, ci=False):
             test if '/' in test
             else f'test/test_download.py::TestDownload::test_{fix_test_name(test)}'
             for test in tests)
+    if not run_flaky:
+        arguments.append('--disallow-flaky')
 
     print(f'Running {arguments}', flush=True)
     try:
@@ -72,6 +87,11 @@ if __name__ == '__main__':
         args = parse_args()
 
         os.chdir(Path(__file__).parent.parent)
-        sys.exit(run_tests(*args.test, pattern=args.k, ci=bool(os.getenv('CI'))))
+        sys.exit(run_tests(
+            *args.test,
+            pattern=args.k,
+            ci=bool(os.getenv('CI')),
+            flaky=args.flaky,
+        ))
     except KeyboardInterrupt:
         pass