From 6b20edd3c84e09154a4f9c8e75e6df101e06a2e4 Mon Sep 17 00:00:00 2001 From: ugo-studio Date: Fri, 21 Mar 2025 11:20:54 +0100 Subject: [PATCH 1/3] Add --url-prefix option to prepend a URL prefix to all request URLs --- README.md | 7 +++++++ yt_dlp/YoutubeDL.py | 3 ++- yt_dlp/__init__.py | 1 + yt_dlp/networking/_requests.py | 4 ++++ yt_dlp/networking/common.py | 7 ++++++- yt_dlp/options.py | 7 +++++++ 6 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e27b0a34..811c3619f 100644 --- a/README.md +++ b/README.md @@ -393,6 +393,13 @@ ## Network Options: e.g. socks5://user:pass@127.0.0.1:1080/. Pass in an empty string (--proxy "") for direct connection + --url-prefix URL Prepend the specified URL prefix to every request URL. + For example, if you set the URL prefix to + `http://127.0.0.1:1080/`, a request originally aimed at + `https://url.com` will be modified to + `http://127.0.0.1:1080/https://url.com`. + pass in an empty string (i.e., `--url-prefix ""`) to use + the original request URLs directly --socket-timeout SECONDS Time to wait before giving up, in seconds --source-address IP Client-side IP address to bind to --impersonate CLIENT[:OS] Client to impersonate for requests. E.g. diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 8790b326b..001cd689d 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -360,6 +360,7 @@ class YoutubeDL: enable_file_urls: Enable file:// URLs. This is disabled by default for security reasons. http_headers: A dictionary of custom headers to be used for all requests proxy: URL of the proxy server to use + url_prefix: Prepend the specified URL prefix to every request URL geo_verification_proxy: URL of the proxy to use for IP address verification on geo-restricted sites. socket_timeout: Time to wait for unresponsive hosts, in seconds @@ -4231,7 +4232,7 @@ def build_request_director(self, handlers, preferences=None): clean_headers(headers) clean_proxies(proxies, headers) - director = RequestDirector(logger=logger, verbose=self.params.get('debug_printtraffic')) + director = RequestDirector(url_prefix=self.params.get('url_prefix'), logger=logger, verbose=self.params.get('debug_printtraffic')) for handler in handlers: director.add_handler(handler( logger=logger, diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 7d8f10047..1a23f2b5f 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -920,6 +920,7 @@ def parse_options(argv=None): 'enable_file_urls': opts.enable_file_urls, 'http_headers': opts.headers, 'proxy': opts.proxy, + 'url_prefix': opts.url_prefix, 'socket_timeout': opts.socket_timeout, 'bidi_workaround': opts.bidi_workaround, 'debug_printtraffic': opts.debug_printtraffic, diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py index 5b6b264a6..a6a11162b 100644 --- a/yt_dlp/networking/_requests.py +++ b/yt_dlp/networking/_requests.py @@ -227,6 +227,10 @@ def rebuild_auth(self, prepared_request, response): del response._real_status_code return super().rebuild_auth(prepared_request, response) + def send(self, request, **kwargs): + print(f'Requesting URL: {request.url}') # Print each URL before sending the request + return super().send(request, **kwargs) + class Urllib3LoggingFilter(logging.Filter): diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py index ddceaa9a9..efebb84e3 100644 --- a/yt_dlp/networking/common.py +++ b/yt_dlp/networking/common.py @@ -61,9 +61,10 @@ class RequestDirector: @param verbose: Print debug request information to stdout. """ - def __init__(self, logger, verbose=False): + def __init__(self, url_prefix, logger, verbose=False): self.handlers: dict[str, RequestHandler] = {} self.preferences: set[Preference] = set() + self.url_prefix = url_prefix self.logger = logger # TODO(Grub4k): default logger self.verbose = verbose @@ -100,6 +101,10 @@ def send(self, request: Request) -> Response: assert isinstance(request, Request) + # Add url_prefix to request url + if self.url_prefix: + request.url = f'{self.url_prefix}{request.url}' + unexpected_errors = [] unsupported_errors = [] for handler in self._get_handlers(request): diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 91c2635a7..884f488a9 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -526,6 +526,13 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs): help=( 'Use the specified HTTP/HTTPS/SOCKS proxy. To enable SOCKS proxy, specify a proper scheme, ' 'e.g. socks5://user:pass@127.0.0.1:1080/. Pass in an empty string (--proxy "") for direct connection')) + network.add_option( + '--url-prefix', dest='url_prefix', + default=None, metavar='URL', + help=( + 'Prepend the specified URL prefix to every request URL. For example, if you set the URL prefix to `http://127.0.0.1:1080/`, ' + 'a request originally aimed at `https://request.url` will be modified to `http://127.0.0.1:1080/https://request.url`. ' + 'pass in an empty string (i.e., `--url-prefix ""`) to use the original request URLs directly')) network.add_option( '--socket-timeout', dest='socket_timeout', type=float, default=None, metavar='SECONDS', From 968ad0f5608692df2f9891e1c348cb58222b749c Mon Sep 17 00:00:00 2001 From: ugo-studio Date: Fri, 21 Mar 2025 12:25:36 +0100 Subject: [PATCH 2/3] fix: make 'url_prefix' optional --- yt_dlp/networking/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py index efebb84e3..2cba651c1 100644 --- a/yt_dlp/networking/common.py +++ b/yt_dlp/networking/common.py @@ -61,11 +61,11 @@ class RequestDirector: @param verbose: Print debug request information to stdout. """ - def __init__(self, url_prefix, logger, verbose=False): + def __init__(self, logger, url_prefix=None, verbose=False): self.handlers: dict[str, RequestHandler] = {} self.preferences: set[Preference] = set() - self.url_prefix = url_prefix self.logger = logger # TODO(Grub4k): default logger + self.url_prefix = url_prefix self.verbose = verbose def close(self): From f3b18714fbb17430a761ffc6dc6bc10e3b31ad14 Mon Sep 17 00:00:00 2001 From: ugo-studio <107785270+ugo-studio@users.noreply.github.com> Date: Fri, 21 Mar 2025 14:47:42 +0100 Subject: [PATCH 3/3] cleanup --- yt_dlp/networking/_requests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py index a6a11162b..5b6b264a6 100644 --- a/yt_dlp/networking/_requests.py +++ b/yt_dlp/networking/_requests.py @@ -227,10 +227,6 @@ def rebuild_auth(self, prepared_request, response): del response._real_status_code return super().rebuild_auth(prepared_request, response) - def send(self, request, **kwargs): - print(f'Requesting URL: {request.url}') # Print each URL before sending the request - return super().send(request, **kwargs) - class Urllib3LoggingFilter(logging.Filter):