mirrors/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2025-11-29 21:05:15 +00:00
Code Issues Projects Releases Wiki Activity

[compat, networking] Deprecate old functions (#2861)

Browse Source 
Authored by: coletdjnz, pukkandan
This commit is contained in:
coletdjnz
2023-07-09 13:23:02 +05:30
committed by pukkandan
parent 227bf1a33b
commit 3d2623a898
176 changed files with 707 additions and 729 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
yt_dlp/downloader/external.py
View File

@@ -10,6 +10,7 @@ import uuid
from .fragment import FragmentFD
from ..compat import functools
from ..networking import Request
from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor
from ..utils import (
Popen,
@@ -25,7 +26,6 @@ from ..utils import (
encodeFilename,
find_available_port,
remove_end,
sanitized_Request,
traverse_obj,
)
@@ -357,13 +357,12 @@ class Aria2cFD(ExternalFD):
'method': method,
'params': [f'token:{rpc_secret}', *params],
}).encode('utf-8')
request = sanitized_Request(
request = Request(
f'http://localhost:{rpc_port}/jsonrpc',
data=d, headers={
'Content-Type': 'application/json',
'Content-Length': f'{len(d)}',
'Ytdl-request-proxy': '__noproxy__',
})
}, proxies={'all': None})
with self.ydl.urlopen(request) as r:
resp = json.load(r)
assert resp.get('id') == sanitycheck, 'Something went wrong with RPC server'

8
yt_dlp/downloader/f4m.py
View File

@@ -3,11 +3,11 @@ import io
import itertools
import struct
import time
import urllib.error
import urllib.parse
from .fragment import FragmentFD
from ..compat import compat_etree_fromstring
from ..networking.exceptions import HTTPError
from ..utils import fix_xml_ampersands, xpath_text
@@ -312,7 +312,7 @@ class F4mFD(FragmentFD):
self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
man_url = urlh.geturl()
man_url = urlh.url
# Some manifests may be malformed, e.g. prosiebensat1 generated manifests
# (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244
# and https://github.com/ytdl-org/youtube-dl/issues/7823)
@@ -407,8 +407,8 @@ class F4mFD(FragmentFD):
if box_type == b'mdat':
self._append_fragment(ctx, box_data)
break
except urllib.error.HTTPError as err:
if live and (err.code == 404 or err.code == 410):
except HTTPError as err:
if live and (err.status == 404 or err.status == 410):
# We didn't keep up with the live window. Continue
# with the next available fragment.
msg = 'Fragment %d unavailable' % frag_i

19
yt_dlp/downloader/fragment.py
View File

@@ -1,24 +1,19 @@
import concurrent.futures
import contextlib
import http.client
import json
import math
import os
import struct
import time
import urllib.error
from .common import FileDownloader
from .http import HttpFD
from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
from ..compat import compat_os_name
from ..utils import (
DownloadError,
RetryManager,
encodeFilename,
sanitized_Request,
traverse_obj,
)
from ..networking import Request
from ..networking.exceptions import HTTPError, IncompleteRead
from ..utils import DownloadError, RetryManager, encodeFilename, traverse_obj
from ..utils.networking import HTTPHeaderDict
class HttpQuietDownloader(HttpFD):
@@ -75,7 +70,7 @@ class FragmentFD(FileDownloader):
def _prepare_url(self, info_dict, url):
headers = info_dict.get('http_headers')
return sanitized_Request(url, None, headers) if headers else url
return Request(url, None, headers) if headers else url
def _prepare_and_start_frag_download(self, ctx, info_dict):
self._prepare_frag_download(ctx)
@@ -457,7 +452,7 @@ class FragmentFD(FileDownloader):
frag_index = ctx['fragment_index'] = fragment['frag_index']
ctx['last_error'] = None
headers = info_dict.get('http_headers', {}).copy()
headers = HTTPHeaderDict(info_dict.get('http_headers'))
byte_range = fragment.get('byte_range')
if byte_range:
headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
@@ -477,7 +472,7 @@ class FragmentFD(FileDownloader):
if not self._download_fragment(
ctx, fragment['url'], info_dict, headers, info_dict.get('request_data')):
return
except (urllib.error.HTTPError, http.client.IncompleteRead) as err:
except (HTTPError, IncompleteRead) as err:
retry.error = err
continue
except DownloadError: # has own retry settings

2
yt_dlp/downloader/hls.py
View File

@@ -75,7 +75,7 @@ class HlsFD(FragmentFD):
self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
man_url = urlh.geturl()
man_url = urlh.url
s = urlh.read().decode('utf-8', 'ignore')
can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None

41
yt_dlp/downloader/http.py
View File

@@ -1,10 +1,14 @@
import os
import random
import time
import urllib.error
from .common import FileDownloader
from ..networking.exceptions import CertificateVerifyError, TransportError
from ..networking import Request
from ..networking.exceptions import (
CertificateVerifyError,
HTTPError,
TransportError,
)
from ..utils import (
ContentTooShortError,
RetryManager,
@@ -14,10 +18,10 @@ from ..utils import (
encodeFilename,
int_or_none,
parse_http_range,
sanitized_Request,
try_call,
write_xattr,
)
from ..utils.networking import HTTPHeaderDict
class HttpFD(FileDownloader):
@@ -36,10 +40,7 @@ class HttpFD(FileDownloader):
ctx.stream = None
# Disable compression
headers = {'Accept-Encoding': 'identity'}
add_headers = info_dict.get('http_headers')
if add_headers:
headers.update(add_headers)
headers = HTTPHeaderDict({'Accept-Encoding': 'identity'}, info_dict.get('http_headers'))
is_test = self.params.get('test', False)
chunk_size = self._TEST_FILE_SIZE if is_test else (
@@ -110,10 +111,10 @@ class HttpFD(FileDownloader):
if try_call(lambda: range_end >= ctx.content_len):
range_end = ctx.content_len - 1
request = sanitized_Request(url, request_data, headers)
request = Request(url, request_data, headers)
has_range = range_start is not None
if has_range:
request.add_header('Range', f'bytes={int(range_start)}-{int_or_none(range_end) or ""}')
request.headers['Range'] = f'bytes={int(range_start)}-{int_or_none(range_end) or ""}'
# Establish connection
try:
ctx.data = self.ydl.urlopen(request)
@@ -144,17 +145,17 @@ class HttpFD(FileDownloader):
self.report_unable_to_resume()
ctx.resume_len = 0
ctx.open_mode = 'wb'
ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
except urllib.error.HTTPError as err:
if err.code == 416:
ctx.data_len = ctx.content_len = int_or_none(ctx.data.headers.get('Content-length', None))
except HTTPError as err:
if err.status == 416:
# Unable to resume (requested range not satisfiable)
try:
# Open the connection again without the range header
ctx.data = self.ydl.urlopen(
sanitized_Request(url, request_data, headers))
content_length = ctx.data.info()['Content-Length']
except urllib.error.HTTPError as err:
if err.code < 500 or err.code >= 600:
Request(url, request_data, headers))
content_length = ctx.data.headers['Content-Length']
except HTTPError as err:
if err.status < 500 or err.status >= 600:
raise
else:
# Examine the reported length
@@ -182,7 +183,7 @@ class HttpFD(FileDownloader):
ctx.resume_len = 0
ctx.open_mode = 'wb'
return
elif err.code < 500 or err.code >= 600:
elif err.status < 500 or err.status >= 600:
# Unexpected HTTP error
raise
raise RetryDownload(err)
@@ -198,9 +199,9 @@ class HttpFD(FileDownloader):
ctx.stream = None
def download():
data_len = ctx.data.info().get('Content-length')
data_len = ctx.data.headers.get('Content-length')
if ctx.data.info().get('Content-encoding'):
if ctx.data.headers.get('Content-encoding'):
# Content-encoding is present, Content-length is not reliable anymore as we are
# doing auto decompression. (See: https://github.com/yt-dlp/yt-dlp/pull/6176)
data_len = None
@@ -345,7 +346,7 @@ class HttpFD(FileDownloader):
# Update file modification time
if self.params.get('updatetime', True):
info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))
info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.headers.get('last-modified', None))
self._hook_progress({
'downloaded_bytes': byte_counter,

4
yt_dlp/downloader/ism.py
View File

@@ -2,9 +2,9 @@ import binascii
import io
import struct
import time
import urllib.error
from .fragment import FragmentFD
from ..networking.exceptions import HTTPError
from ..utils import RetryManager
u8 = struct.Struct('>B')
@@ -271,7 +271,7 @@ class IsmFD(FragmentFD):
write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
extra_state['ism_track_written'] = True
self._append_fragment(ctx, frag_content)
except urllib.error.HTTPError as err:
except HTTPError as err:
retry.error = err
continue

11
yt_dlp/downloader/niconico.py
View File

@@ -5,13 +5,8 @@ import time
from . import get_suitable_downloader
from .common import FileDownloader
from .external import FFmpegFD
from ..utils import (
DownloadError,
WebSocketsWrapper,
sanitized_Request,
str_or_none,
try_get,
)
from ..networking import Request
from ..utils import DownloadError, WebSocketsWrapper, str_or_none, try_get
class NiconicoDmcFD(FileDownloader):
@@ -33,7 +28,7 @@ class NiconicoDmcFD(FileDownloader):
heartbeat_data = heartbeat_info_dict['data'].encode()
heartbeat_interval = heartbeat_info_dict.get('interval', 30)
request = sanitized_Request(heartbeat_url, heartbeat_data)
request = Request(heartbeat_url, heartbeat_data)
def heartbeat():
try:

10
yt_dlp/downloader/youtube_live_chat.py
View File

@@ -1,8 +1,8 @@
import json
import time
import urllib.error
from .fragment import FragmentFD
from ..networking.exceptions import HTTPError
from ..utils import (
RegexNotFoundError,
RetryManager,
@@ -10,6 +10,7 @@ from ..utils import (
int_or_none,
try_get,
)
from ..utils.networking import HTTPHeaderDict
class YoutubeLiveChatFD(FragmentFD):
@@ -37,10 +38,7 @@ class YoutubeLiveChatFD(FragmentFD):
start_time = int(time.time() * 1000)
def dl_fragment(url, data=None, headers=None):
http_headers = info_dict.get('http_headers', {})
if headers:
http_headers = http_headers.copy()
http_headers.update(headers)
http_headers = HTTPHeaderDict(info_dict.get('http_headers'), headers)
return self._download_fragment(ctx, url, info_dict, http_headers, data)
def parse_actions_replay(live_chat_continuation):
@@ -129,7 +127,7 @@ class YoutubeLiveChatFD(FragmentFD):
or frag_index == 1 and try_refresh_replay_beginning
or parse_actions_replay)
return (True, *func(live_chat_continuation))
except urllib.error.HTTPError as err:
except HTTPError as err:
retry.error = err
continue
return False, None, None, None