From a7d9a5eb79ceeecb851389f3f2c88597871ca3f2 Mon Sep 17 00:00:00 2001 From: Jan Baier <7996094+baierjan@users.noreply.github.com> Date: Sat, 17 May 2025 01:04:24 +0200 Subject: [PATCH 01/67] [ie/iprima] Fix login support (#12937) Closes #12387 Authored by: baierjan --- yt_dlp/extractor/iprima.py | 54 ++++++++------------------------------ 1 file changed, 11 insertions(+), 43 deletions(-) diff --git a/yt_dlp/extractor/iprima.py b/yt_dlp/extractor/iprima.py index 9b91a454b..bf2344f72 100644 --- a/yt_dlp/extractor/iprima.py +++ b/yt_dlp/extractor/iprima.py @@ -1,3 +1,4 @@ +import json import re import time @@ -6,9 +7,7 @@ ExtractorError, determine_ext, js_to_json, - parse_qs, traverse_obj, - urlencode_postdata, ) @@ -16,7 +15,6 @@ class IPrimaIE(InfoExtractor): _VALID_URL = r'https?://(?!cnn)(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P[^/?#&]+)' _GEO_BYPASS = False _NETRC_MACHINE = 'iprima' - _AUTH_ROOT = 'https://auth.iprima.cz' access_token = None _TESTS = [{ @@ -86,48 +84,18 @@ def _perform_login(self, username, password): if self.access_token: return - login_page = self._download_webpage( - f'{self._AUTH_ROOT}/oauth2/login', None, note='Downloading login page', - errnote='Downloading login page failed') - - login_form = self._hidden_inputs(login_page) - - login_form.update({ - '_email': username, - '_password': password}) - - profile_select_html, login_handle = self._download_webpage_handle( - f'{self._AUTH_ROOT}/oauth2/login', None, data=urlencode_postdata(login_form), - note='Logging in') - - # a profile may need to be selected first, even when there is only a single one - if '/profile-select' in login_handle.url: - profile_id = self._search_regex( - r'data-identifier\s*=\s*["\']?(\w+)', profile_select_html, 'profile id') - - login_handle = self._request_webpage( - f'{self._AUTH_ROOT}/user/profile-select-perform/{profile_id}', None, - query={'continueUrl': '/user/login?redirect_uri=/user/'}, note='Selecting profile') - - code = traverse_obj(login_handle.url, ({parse_qs}, 'code', 0)) - if not code: - raise ExtractorError('Login failed', expected=True) - - token_request_data = { - 'scope': 'openid+email+profile+phone+address+offline_access', - 'client_id': 'prima_sso', - 'grant_type': 'authorization_code', - 'code': code, - 'redirect_uri': f'{self._AUTH_ROOT}/sso/auth-check'} - token_data = self._download_json( - f'{self._AUTH_ROOT}/oauth2/token', None, - note='Downloading token', errnote='Downloading token failed', - data=urlencode_postdata(token_request_data)) + 'https://ucet.iprima.cz/api/session/create', None, + note='Logging in', errnote='Failed to log in', + data=json.dumps({ + 'email': username, + 'password': password, + 'deviceName': 'Windows Chrome', + }).encode(), headers={'content-type': 'application/json'}) - self.access_token = token_data.get('access_token') - if self.access_token is None: - raise ExtractorError('Getting token failed', expected=True) + self.access_token = token_data['accessToken']['value'] + if not self.access_token: + raise ExtractorError('Failed to fetch access token') def _real_initialize(self): if not self.access_token: From 41c0a1fb89628696f8bb88e2b9f3a68f355b8c26 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 16 May 2025 18:16:03 -0500 Subject: [PATCH 02/67] [ie/1tv] Fix extractor (#13168) Closes #13167 Authored by: bashonly --- yt_dlp/extractor/firsttv.py | 177 ++++++++++++++++-------------------- 1 file changed, 78 insertions(+), 99 deletions(-) diff --git a/yt_dlp/extractor/firsttv.py b/yt_dlp/extractor/firsttv.py index ac7697bb3..878732c49 100644 --- a/yt_dlp/extractor/firsttv.py +++ b/yt_dlp/extractor/firsttv.py @@ -2,11 +2,15 @@ from .common import InfoExtractor from ..utils import ( + determine_ext, int_or_none, - qualities, + join_nonempty, + mimetype2ext, + parse_qs, unified_strdate, url_or_none, ) +from ..utils.traversal import traverse_obj class FirstTVIE(InfoExtractor): @@ -15,40 +19,51 @@ class FirstTVIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?(?:sport)?1tv\.ru/(?:[^/?#]+/)+(?P[^/?#]+)' _TESTS = [{ - # single format - 'url': 'http://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015', - 'md5': 'a1b6b60d530ebcf8daacf4565762bbaf', + # single format; has item.id + 'url': 'https://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015', + 'md5': '8011ae8e88ff4150107ab9c5a8f5b659', 'info_dict': { 'id': '40049', 'ext': 'mp4', 'title': 'Гость Людмила Сенчина. Наедине со всеми. Выпуск от 12.02.2015', - 'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$', + 'thumbnail': r're:https?://.+/.+\.jpg', 'upload_date': '20150212', 'duration': 2694, }, + 'params': {'skip_download': 'm3u8'}, }, { - # multiple formats - 'url': 'http://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016', + # multiple formats; has item.id + 'url': 'https://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016', 'info_dict': { 'id': '364746', 'ext': 'mp4', 'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016', - 'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$', + 'thumbnail': r're:https?://.+/.+\.jpg', 'upload_date': '20160407', 'duration': 179, 'formats': 'mincount:3', }, - 'params': { - 'skip_download': True, - }, + 'params': {'skip_download': 'm3u8'}, }, { - 'url': 'http://www.1tv.ru/news/issue/2016-12-01/14:00', + 'url': 'https://www.1tv.ru/news/issue/2016-12-01/14:00', 'info_dict': { 'id': '14:00', - 'title': 'Выпуск новостей в 14:00 1 декабря 2016 года. Новости. Первый канал', - 'description': 'md5:2e921b948f8c1ff93901da78ebdb1dfd', + 'title': 'Выпуск программы «Время» в 20:00 1 декабря 2016 года. Новости. Первый канал', + 'thumbnail': 'https://static.1tv.ru/uploads/photo/image/8/big/338448_big_8fc7eb236f.jpg', }, 'playlist_count': 13, + }, { + # has timestamp; has item.uid but not item.id + 'url': 'https://www.1tv.ru/shows/segodnya-vecherom/vypuski/avtory-odnogo-hita-segodnya-vecherom-vypusk-ot-03-05-2025', + 'info_dict': { + 'id': '270411', + 'ext': 'mp4', + 'title': 'Авторы одного хита. Сегодня вечером. Выпуск от 03.05.2025', + 'thumbnail': r're:https?://.+/.+\.jpg', + 'timestamp': 1746286020, + 'upload_date': '20250503', + }, + 'params': {'skip_download': 'm3u8'}, }, { 'url': 'http://www.1tv.ru/shows/tochvtoch-supersezon/vystupleniya/evgeniy-dyatlov-vladimir-vysockiy-koni-priveredlivye-toch-v-toch-supersezon-fragment-vypuska-ot-06-11-2016', 'only_matching': True, @@ -57,96 +72,60 @@ class FirstTVIE(InfoExtractor): 'only_matching': True, }] + def _entries(self, items): + for item in items: + video_id = str(item.get('id') or item['uid']) + + formats, subtitles = [], {} + for f in traverse_obj(item, ('sources', lambda _, v: url_or_none(v['src']))): + src = f['src'] + ext = mimetype2ext(f.get('type'), default=determine_ext(src)) + if ext == 'm3u8': + fmts, subs = self._extract_m3u8_formats_and_subtitles( + src, video_id, 'mp4', m3u8_id='hls', fatal=False) + elif ext == 'mpd': + fmts, subs = self._extract_mpd_formats_and_subtitles( + src, video_id, mpd_id='dash', fatal=False) + else: + tbr = self._search_regex(fr'_(\d{{3,}})\.{ext}', src, 'tbr', default=None) + formats.append({ + 'url': src, + 'ext': ext, + 'format_id': join_nonempty('http', ext, tbr), + 'tbr': int_or_none(tbr), + # quality metadata of http formats may be incorrect + 'quality': -10, + }) + continue + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) + + yield { + **traverse_obj(item, { + 'title': ('title', {str}), + 'thumbnail': ('poster', {url_or_none}), + 'timestamp': ('dvr_begin_at', {int_or_none}), + 'upload_date': ('date_air', {unified_strdate}), + 'duration': ('duration', {int_or_none}), + }), + 'id': video_id, + 'formats': formats, + 'subtitles': subtitles, + } + def _real_extract(self, url): display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) - playlist_url = urllib.parse.urljoin(url, self._search_regex( + playlist_url = urllib.parse.urljoin(url, self._html_search_regex( r'data-playlist-url=(["\'])(?P(?:(?!\1).)+)\1', webpage, 'playlist url', group='url')) - parsed_url = urllib.parse.urlparse(playlist_url) - qs = urllib.parse.parse_qs(parsed_url.query) - item_ids = qs.get('videos_ids[]') or qs.get('news_ids[]') + item_ids = traverse_obj(parse_qs(playlist_url), 'video_id', 'videos_ids[]', 'news_ids[]') + items = traverse_obj( + self._download_json(playlist_url, display_id), + lambda _, v: v['uid'] and (str(v['uid']) in item_ids if item_ids else True)) - items = self._download_json(playlist_url, display_id) - - if item_ids: - items = [ - item for item in items - if item.get('uid') and str(item['uid']) in item_ids] - else: - items = [items[0]] - - entries = [] - QUALITIES = ('ld', 'sd', 'hd') - - for item in items: - title = item['title'] - quality = qualities(QUALITIES) - formats = [] - path = None - for f in item.get('mbr', []): - src = url_or_none(f.get('src')) - if not src: - continue - tbr = int_or_none(self._search_regex( - r'_(\d{3,})\.mp4', src, 'tbr', default=None)) - if not path: - path = self._search_regex( - r'//[^/]+/(.+?)_\d+\.mp4', src, - 'm3u8 path', default=None) - formats.append({ - 'url': src, - 'format_id': f.get('name'), - 'tbr': tbr, - 'source_preference': quality(f.get('name')), - # quality metadata of http formats may be incorrect - 'preference': -10, - }) - # m3u8 URL format is reverse engineered from [1] (search for - # master.m3u8). dashEdges (that is currently balancer-vod.1tv.ru) - # is taken from [2]. - # 1. http://static.1tv.ru/player/eump1tv-current/eump-1tv.all.min.js?rnd=9097422834:formatted - # 2. http://static.1tv.ru/player/eump1tv-config/config-main.js?rnd=9097422834 - if not path and len(formats) == 1: - path = self._search_regex( - r'//[^/]+/(.+?$)', formats[0]['url'], - 'm3u8 path', default=None) - if path: - if len(formats) == 1: - m3u8_path = ',' - else: - tbrs = [str(t) for t in sorted(f['tbr'] for f in formats)] - m3u8_path = '_,{},{}'.format(','.join(tbrs), '.mp4') - formats.extend(self._extract_m3u8_formats( - f'http://balancer-vod.1tv.ru/{path}{m3u8_path}.urlset/master.m3u8', - display_id, 'mp4', - entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)) - - thumbnail = item.get('poster') or self._og_search_thumbnail(webpage) - duration = int_or_none(item.get('duration') or self._html_search_meta( - 'video:duration', webpage, 'video duration', fatal=False)) - upload_date = unified_strdate(self._html_search_meta( - 'ya:ovs:upload_date', webpage, 'upload date', default=None)) - - entries.append({ - 'id': str(item.get('id') or item['uid']), - 'thumbnail': thumbnail, - 'title': title, - 'upload_date': upload_date, - 'duration': int_or_none(duration), - 'formats': formats, - }) - - title = self._html_search_regex( - (r'
\s*

([^<]*)', - r"'title'\s*:\s*'([^']+)'"), - webpage, 'title', default=None) or self._og_search_title( - webpage, default=None) - description = self._html_search_regex( - r'
\s*
 
\s*

([^<]*)

', - webpage, 'description', default=None) or self._html_search_meta( - 'description', webpage, 'description', default=None) - - return self.playlist_result(entries, display_id, title, description) + return self.playlist_result( + self._entries(items), display_id, self._og_search_title(webpage, default=None), + thumbnail=self._og_search_thumbnail(webpage, default=None)) From f475e8b529d18efdad603ffda02a56e707fe0e2c Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 16 May 2025 18:16:58 -0500 Subject: [PATCH 03/67] [ie/once] Remove extractor (#13164) Authored by: bashonly --- yt_dlp/extractor/espn.py | 7 ++---- yt_dlp/extractor/gamespot.py | 4 ++-- yt_dlp/extractor/once.py | 40 --------------------------------- yt_dlp/extractor/theplatform.py | 20 +++++++---------- yt_dlp/extractor/voxmedia.py | 20 +++-------------- 5 files changed, 15 insertions(+), 76 deletions(-) delete mode 100644 yt_dlp/extractor/once.py diff --git a/yt_dlp/extractor/espn.py b/yt_dlp/extractor/espn.py index 552f9af12..37e74bc08 100644 --- a/yt_dlp/extractor/espn.py +++ b/yt_dlp/extractor/espn.py @@ -5,7 +5,6 @@ from .adobepass import AdobePassIE from .common import InfoExtractor -from .once import OnceIE from ..utils import ( determine_ext, dict_get, @@ -16,7 +15,7 @@ ) -class ESPNIE(OnceIE): +class ESPNIE(InfoExtractor): _VALID_URL = r'''(?x) https?:// (?: @@ -131,9 +130,7 @@ def extract_source(source_url, source_id=None): return format_urls.add(source_url) ext = determine_ext(source_url) - if OnceIE.suitable(source_url): - formats.extend(self._extract_once_formats(source_url)) - elif ext == 'smil': + if ext == 'smil': formats.extend(self._extract_smil_formats( source_url, video_id, fatal=False)) elif ext == 'f4m': diff --git a/yt_dlp/extractor/gamespot.py b/yt_dlp/extractor/gamespot.py index cd3f9655d..2799a27ba 100644 --- a/yt_dlp/extractor/gamespot.py +++ b/yt_dlp/extractor/gamespot.py @@ -1,9 +1,9 @@ import urllib.parse -from .once import OnceIE +from .common import InfoExtractor -class GameSpotIE(OnceIE): +class GameSpotIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article|review)s/(?:[^/]+/\d+-|embed/)(?P\d+)' _TESTS = [{ 'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/', diff --git a/yt_dlp/extractor/once.py b/yt_dlp/extractor/once.py deleted file mode 100644 index 989f10abb..000000000 --- a/yt_dlp/extractor/once.py +++ /dev/null @@ -1,40 +0,0 @@ -import re - -from .common import InfoExtractor - - -class OnceIE(InfoExtractor): # XXX: Conventionally, base classes should end with BaseIE/InfoExtractor - _VALID_URL = r'https?://.+?\.unicornmedia\.com/now/(?:ads/vmap/)?[^/]+/[^/]+/(?P[^/]+)/(?P[^/]+)/(?:[^/]+/)?(?P[^/]+)/content\.(?:once|m3u8|mp4)' - ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8' - PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4' - - def _extract_once_formats(self, url, http_formats_preference=None): - domain_id, application_id, media_item_id = re.match( - OnceIE._VALID_URL, url).groups() - formats = self._extract_m3u8_formats( - self.ADAPTIVE_URL_TEMPLATE % ( - domain_id, application_id, media_item_id), - media_item_id, 'mp4', m3u8_id='hls', fatal=False) - progressive_formats = [] - for adaptive_format in formats: - # Prevent advertisement from embedding into m3u8 playlist (see - # https://github.com/ytdl-org/youtube-dl/issues/8893#issuecomment-199912684) - adaptive_format['url'] = re.sub( - r'\badsegmentlength=\d+', r'adsegmentlength=0', adaptive_format['url']) - rendition_id = self._search_regex( - r'/now/media/playlist/[^/]+/[^/]+/([^/]+)', - adaptive_format['url'], 'redition id', default=None) - if rendition_id: - progressive_format = adaptive_format.copy() - progressive_format.update({ - 'url': self.PROGRESSIVE_URL_TEMPLATE % ( - domain_id, application_id, rendition_id, media_item_id), - 'format_id': adaptive_format['format_id'].replace( - 'hls', 'http'), - 'protocol': 'http', - 'preference': http_formats_preference, - }) - progressive_formats.append(progressive_format) - self._check_formats(progressive_formats, media_item_id) - formats.extend(progressive_formats) - return formats diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py index b73bea18f..ebe2ac296 100644 --- a/yt_dlp/extractor/theplatform.py +++ b/yt_dlp/extractor/theplatform.py @@ -4,7 +4,6 @@ import time from .adobepass import AdobePassIE -from .once import OnceIE from ..networking import HEADRequest, Request from ..utils import ( ExtractorError, @@ -26,7 +25,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns}) -class ThePlatformBaseIE(OnceIE): +class ThePlatformBaseIE(AdobePassIE): _TP_TLD = 'com' def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'): @@ -54,16 +53,13 @@ def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL d formats = [] for _format in smil_formats: - if OnceIE.suitable(_format['url']): - formats.extend(self._extract_once_formats(_format['url'])) - else: - media_url = _format['url'] - if determine_ext(media_url) == 'm3u8': - hdnea2 = self._get_cookies(media_url).get('hdnea2') - if hdnea2: - _format['url'] = update_url_query(media_url, {'hdnea3': hdnea2.value}) + media_url = _format['url'] + if determine_ext(media_url) == 'm3u8': + hdnea2 = self._get_cookies(media_url).get('hdnea2') + if hdnea2: + _format['url'] = update_url_query(media_url, {'hdnea3': hdnea2.value}) - formats.append(_format) + formats.append(_format) return formats, subtitles @@ -129,7 +125,7 @@ def _extract_theplatform_metadata(self, path, video_id): return self._parse_theplatform_metadata(info) -class ThePlatformIE(ThePlatformBaseIE, AdobePassIE): +class ThePlatformIE(ThePlatformBaseIE): _VALID_URL = r'''(?x) (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P[^/]+)/ (?:(?:(?:[^/]+/)+select/)?(?Pmedia/(?:guid/\d+/)?)?|(?P(?:[^/\?]+/(?:swf|config)|onsite)/select/))? diff --git a/yt_dlp/extractor/voxmedia.py b/yt_dlp/extractor/voxmedia.py index e9b0047a4..995ef2185 100644 --- a/yt_dlp/extractor/voxmedia.py +++ b/yt_dlp/extractor/voxmedia.py @@ -1,7 +1,6 @@ import urllib.parse from .common import InfoExtractor -from .once import OnceIE from ..utils import ( ExtractorError, int_or_none, @@ -10,7 +9,7 @@ ) -class VoxMediaVolumeIE(OnceIE): +class VoxMediaVolumeIE(InfoExtractor): _VALID_URL = r'https?://volume\.vox-cdn\.com/embed/(?P[0-9a-f]{9})' def _real_extract(self, url): @@ -57,7 +56,8 @@ def _real_extract(self, url): if not provider_video_id: continue if provider_video_type == 'brightcove': - info['formats'] = self._extract_once_formats(provider_video_id) + # TODO: Find embed example or confirm that Vox has stopped using Brightcove + raise ExtractorError('Vox Brightcove embeds are currently unsupported') else: info.update({ '_type': 'url_transparent', @@ -155,20 +155,6 @@ class VoxMediaIE(InfoExtractor): }, }], 'skip': 'Page no longer contain videos', - }, { - # volume embed, Brightcove Once - 'url': 'https://www.recode.net/2014/6/17/11628066/post-post-pc-ceo-the-full-code-conference-video-of-microsofts-satya', - 'md5': '2dbc77b8b0bff1894c2fce16eded637d', - 'info_dict': { - 'id': '1231c973d', - 'ext': 'mp4', - 'title': 'Post-Post-PC CEO: The Full Code Conference Video of Microsoft\'s Satya Nadella', - 'description': 'The longtime veteran was chosen earlier this year as the software giant\'s third leader in its history.', - 'timestamp': 1402938000, - 'upload_date': '20140616', - 'duration': 4114, - }, - 'add_ie': ['VoxMediaVolume'], }] def _real_extract(self, url): From 20f288bdc2173c7cc58d709d25ca193c1f6001e7 Mon Sep 17 00:00:00 2001 From: Geoffrey Frogeye Date: Sat, 17 May 2025 01:24:30 +0200 Subject: [PATCH 04/67] [ie/nebula] Support `--mark-watched` (#13120) Authored by: GeoffreyFrogeye --- yt_dlp/extractor/nebula.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py index 42ef25f17..6ced19d6a 100644 --- a/yt_dlp/extractor/nebula.py +++ b/yt_dlp/extractor/nebula.py @@ -3,6 +3,7 @@ from .art19 import Art19IE from .common import InfoExtractor +from ..networking import PATCHRequest from ..networking.exceptions import HTTPError from ..utils import ( ExtractorError, @@ -74,7 +75,7 @@ def _extract_formats(self, content_id, slug): 'app_version': '23.10.0', 'platform': 'ios', }) - return {'formats': fmts, 'subtitles': subs} + break except ExtractorError as e: if isinstance(e.cause, HTTPError) and e.cause.status == 401: self.raise_login_required() @@ -84,6 +85,9 @@ def _extract_formats(self, content_id, slug): continue raise + self.mark_watched(content_id, slug) + return {'formats': fmts, 'subtitles': subs} + def _extract_video_metadata(self, episode): channel_url = traverse_obj( episode, (('channel_slug', 'class_slug'), {urljoin('https://nebula.tv/')}), get_all=False) @@ -111,6 +115,13 @@ def _extract_video_metadata(self, episode): 'uploader_url': channel_url, } + def _mark_watched(self, content_id, slug): + self._call_api( + PATCHRequest(f'https://content.api.nebula.app/{content_id.split(":")[0]}s/{content_id}/progress/'), + slug, 'Marking watched', 'Unable to mark watched', fatal=False, + data=json.dumps({'completed': True}).encode(), + headers={'content-type': 'application/json'}) + class NebulaIE(NebulaBaseIE): IE_NAME = 'nebula:video' @@ -322,6 +333,7 @@ def _real_extract(self, url): if not episode_url and metadata.get('premium'): self.raise_login_required() + self.mark_watched(metadata['id'], slug) if Art19IE.suitable(episode_url): return self.url_result(episode_url, Art19IE) return traverse_obj(metadata, { From abf58dcd6a09e14eec4ea82ae12f79a0337cb383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Povilas=20Balzaravi=C4=8Dius?= Date: Sat, 17 May 2025 23:37:00 +0300 Subject: [PATCH 05/67] [ie/LRTRadio] Fix extractor (#13200) Authored by: Pawka --- yt_dlp/extractor/lrt.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/yt_dlp/extractor/lrt.py b/yt_dlp/extractor/lrt.py index e50194f88..caff9125e 100644 --- a/yt_dlp/extractor/lrt.py +++ b/yt_dlp/extractor/lrt.py @@ -2,7 +2,6 @@ from ..utils import ( clean_html, merge_dicts, - str_or_none, traverse_obj, unified_timestamp, url_or_none, @@ -138,13 +137,15 @@ def _real_extract(self, url): 'https://www.lrt.lt/radioteka/api/media', video_id, query={'url': f'/mediateka/irasas/{video_id}/{path}'}) - return traverse_obj(media, { - 'id': ('id', {int}, {str_or_none}), - 'title': ('title', {str}), - 'tags': ('tags', ..., 'name', {str}), - 'categories': ('playlist_item', 'category', {str}, filter, all, filter), - 'description': ('content', {clean_html}, {str}), - 'timestamp': ('date', {lambda x: x.replace('.', '/')}, {unified_timestamp}), - 'thumbnail': ('playlist_item', 'image', {urljoin('https://www.lrt.lt')}), - 'formats': ('playlist_item', 'file', {lambda x: self._extract_m3u8_formats(x, video_id)}), - }) + return { + 'id': video_id, + 'formats': self._extract_m3u8_formats(media['playlist_item']['file'], video_id), + **traverse_obj(media, { + 'title': ('title', {str}), + 'tags': ('tags', ..., 'name', {str}), + 'categories': ('playlist_item', 'category', {str}, filter, all, filter), + 'description': ('content', {clean_html}, {str}), + 'timestamp': ('date', {lambda x: x.replace('.', '/')}, {unified_timestamp}), + 'thumbnail': ('playlist_item', 'image', {urljoin('https://www.lrt.lt')}), + }), + } From 2685654a37141cca63eda3a92da0e2706e23ccfd Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Sun, 18 May 2025 13:45:26 +1200 Subject: [PATCH 06/67] [ie/youtube] Add a PO Token Provider Framework (#12840) https://github.com/yt-dlp/yt-dlp/tree/master/yt_dlp/extractor/youtube/pot/README.md Authored by: coletdjnz --- README.md | 7 +- test/test_YoutubeDL.py | 21 + test/test_networking_utils.py | 3 +- test/test_pot/conftest.py | 71 + test/test_pot/test_pot_builtin_memorycache.py | 117 ++ test/test_pot/test_pot_builtin_utils.py | 46 + test/test_pot/test_pot_builtin_webpospec.py | 92 + test/test_pot/test_pot_director.py | 1529 +++++++++++++++++ test/test_pot/test_pot_framework.py | 629 +++++++ yt_dlp/YoutubeDL.py | 9 + yt_dlp/extractor/youtube/_video.py | 77 +- yt_dlp/extractor/youtube/pot/README.md | 309 ++++ yt_dlp/extractor/youtube/pot/__init__.py | 3 + .../youtube/pot/_builtin/__init__.py | 0 .../youtube/pot/_builtin/memory_cache.py | 78 + .../youtube/pot/_builtin/webpo_cachespec.py | 48 + yt_dlp/extractor/youtube/pot/_director.py | 468 +++++ yt_dlp/extractor/youtube/pot/_provider.py | 156 ++ yt_dlp/extractor/youtube/pot/_registry.py | 8 + yt_dlp/extractor/youtube/pot/cache.py | 97 ++ yt_dlp/extractor/youtube/pot/provider.py | 280 +++ yt_dlp/extractor/youtube/pot/utils.py | 73 + yt_dlp/networking/_curlcffi.py | 3 +- yt_dlp/networking/_helper.py | 14 - yt_dlp/networking/_requests.py | 3 +- yt_dlp/networking/_urllib.py | 3 +- yt_dlp/networking/_websockets.py | 2 +- yt_dlp/utils/networking.py | 16 +- 28 files changed, 4134 insertions(+), 28 deletions(-) create mode 100644 test/test_pot/conftest.py create mode 100644 test/test_pot/test_pot_builtin_memorycache.py create mode 100644 test/test_pot/test_pot_builtin_utils.py create mode 100644 test/test_pot/test_pot_builtin_webpospec.py create mode 100644 test/test_pot/test_pot_director.py create mode 100644 test/test_pot/test_pot_framework.py create mode 100644 yt_dlp/extractor/youtube/pot/README.md create mode 100644 yt_dlp/extractor/youtube/pot/__init__.py create mode 100644 yt_dlp/extractor/youtube/pot/_builtin/__init__.py create mode 100644 yt_dlp/extractor/youtube/pot/_builtin/memory_cache.py create mode 100644 yt_dlp/extractor/youtube/pot/_builtin/webpo_cachespec.py create mode 100644 yt_dlp/extractor/youtube/pot/_director.py create mode 100644 yt_dlp/extractor/youtube/pot/_provider.py create mode 100644 yt_dlp/extractor/youtube/pot/_registry.py create mode 100644 yt_dlp/extractor/youtube/pot/cache.py create mode 100644 yt_dlp/extractor/youtube/pot/provider.py create mode 100644 yt_dlp/extractor/youtube/pot/utils.py diff --git a/README.md b/README.md index a87b52832..9f542844e 100644 --- a/README.md +++ b/README.md @@ -1795,6 +1795,7 @@ #### youtube * `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `android`, `android_vr`, `tv` and `tv_embedded`. By default, `tv,ios,web` is used, or `tv,web` is used when authenticating with cookies. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only works if the video is embeddable. The `tv_embedded` and `web_creator` clients are added for age-restricted videos if account age-verification is required. Some clients, such as `web` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-ios` * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player), `initial_data` (skip initial data/next ep request). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause issues such as missing formats or metadata. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) and [#12826](https://github.com/yt-dlp/yt-dlp/issues/12826) for more details * `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp. +* `player_js_variant`: The player javascript variant to use for signature and nsig deciphering. The known variants are: `main`, `tce`, `tv`, `tv_es6`, `phone`, `tablet`. Only `main` is recommended as a possible workaround; the others are for debugging purposes. The default is to use what is prescribed by the site, and can be selected with `actual` * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side) * `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all` * E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total @@ -1805,7 +1806,11 @@ #### youtube * `data_sync_id`: Overrides the account Data Sync ID used in Innertube API requests. This may be needed if you are using an account with `youtube:player_skip=webpage,configs` or `youtubetab:skip=webpage` * `visitor_data`: Overrides the Visitor Data used in Innertube API requests. This should be used with `player_skip=webpage,configs` and without cookies. Note: this may have adverse effects if used improperly. If a session from a browser is wanted, you should pass cookies instead (which contain the Visitor ID) * `po_token`: Proof of Origin (PO) Token(s) to use. Comma seperated list of PO Tokens in the format `CLIENT.CONTEXT+PO_TOKEN`, e.g. `youtube:po_token=web.gvs+XXX,web.player=XXX,web_safari.gvs+YYY`. Context can be either `gvs` (Google Video Server URLs) or `player` (Innertube player request) -* `player_js_variant`: The player javascript variant to use for signature and nsig deciphering. The known variants are: `main`, `tce`, `tv`, `tv_es6`, `phone`, `tablet`. Only `main` is recommended as a possible workaround; the others are for debugging purposes. The default is to use what is prescribed by the site, and can be selected with `actual` +* `pot_trace`: Enable debug logging for PO Token fetching. Either `true` or `false` (default) +* `fetch_pot`: Policy to use for fetching a PO Token from providers. One of `always` (always try fetch a PO Token regardless if the client requires one for the given context), `never` (never fetch a PO Token), or `auto` (default; only fetch a PO Token if the client requires one for the given context) + +#### youtubepot-webpo +* `bind_to_visitor_id`: Whether to use the Visitor ID instead of Visitor Data for caching WebPO tokens. Either `true` (default) or `false` #### youtubetab (YouTube playlists, channels, feeds, etc.) * `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 708a04f92..91312e4e5 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -1435,6 +1435,27 @@ def test_load_plugins_compat(self): FakeYDL().close() assert all_plugins_loaded.value + def test_close_hooks(self): + # Should call all registered close hooks on close + close_hook_called = False + close_hook_two_called = False + + def close_hook(): + nonlocal close_hook_called + close_hook_called = True + + def close_hook_two(): + nonlocal close_hook_two_called + close_hook_two_called = True + + ydl = FakeYDL() + ydl.add_close_hook(close_hook) + ydl.add_close_hook(close_hook_two) + + ydl.close() + self.assertTrue(close_hook_called, 'Close hook was not called') + self.assertTrue(close_hook_two_called, 'Close hook two was not called') + if __name__ == '__main__': unittest.main() diff --git a/test/test_networking_utils.py b/test/test_networking_utils.py index 204fe87bd..a2feacba7 100644 --- a/test/test_networking_utils.py +++ b/test/test_networking_utils.py @@ -20,7 +20,6 @@ add_accept_encoding_header, get_redirect_method, make_socks_proxy_opts, - select_proxy, ssl_load_certs, ) from yt_dlp.networking.exceptions import ( @@ -28,7 +27,7 @@ IncompleteRead, ) from yt_dlp.socks import ProxyType -from yt_dlp.utils.networking import HTTPHeaderDict +from yt_dlp.utils.networking import HTTPHeaderDict, select_proxy TEST_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/test/test_pot/conftest.py b/test/test_pot/conftest.py new file mode 100644 index 000000000..ff0667e92 --- /dev/null +++ b/test/test_pot/conftest.py @@ -0,0 +1,71 @@ +import collections + +import pytest + +from yt_dlp import YoutubeDL +from yt_dlp.cookies import YoutubeDLCookieJar +from yt_dlp.extractor.common import InfoExtractor +from yt_dlp.extractor.youtube.pot._provider import IEContentProviderLogger +from yt_dlp.extractor.youtube.pot.provider import PoTokenRequest, PoTokenContext +from yt_dlp.utils.networking import HTTPHeaderDict + + +class MockLogger(IEContentProviderLogger): + + log_level = IEContentProviderLogger.LogLevel.TRACE + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.messages = collections.defaultdict(list) + + def trace(self, message: str): + self.messages['trace'].append(message) + + def debug(self, message: str): + self.messages['debug'].append(message) + + def info(self, message: str): + self.messages['info'].append(message) + + def warning(self, message: str, *, once=False): + self.messages['warning'].append(message) + + def error(self, message: str): + self.messages['error'].append(message) + + +@pytest.fixture +def ie() -> InfoExtractor: + ydl = YoutubeDL() + return ydl.get_info_extractor('Youtube') + + +@pytest.fixture +def logger() -> MockLogger: + return MockLogger() + + +@pytest.fixture() +def pot_request() -> PoTokenRequest: + return PoTokenRequest( + context=PoTokenContext.GVS, + innertube_context={'client': {'clientName': 'WEB'}}, + innertube_host='youtube.com', + session_index=None, + player_url=None, + is_authenticated=False, + video_webpage=None, + + visitor_data='example-visitor-data', + data_sync_id='example-data-sync-id', + video_id='example-video-id', + + request_cookiejar=YoutubeDLCookieJar(), + request_proxy=None, + request_headers=HTTPHeaderDict(), + request_timeout=None, + request_source_address=None, + request_verify_tls=True, + + bypass_cache=False, + ) diff --git a/test/test_pot/test_pot_builtin_memorycache.py b/test/test_pot/test_pot_builtin_memorycache.py new file mode 100644 index 000000000..ea19fbe29 --- /dev/null +++ b/test/test_pot/test_pot_builtin_memorycache.py @@ -0,0 +1,117 @@ +import threading +import time +from collections import OrderedDict +import pytest +from yt_dlp.extractor.youtube.pot._provider import IEContentProvider, BuiltinIEContentProvider +from yt_dlp.utils import bug_reports_message +from yt_dlp.extractor.youtube.pot._builtin.memory_cache import MemoryLRUPCP, memorylru_preference, initialize_global_cache +from yt_dlp.version import __version__ +from yt_dlp.extractor.youtube.pot._registry import _pot_cache_providers, _pot_memory_cache + + +class TestMemoryLRUPCS: + + def test_base_type(self): + assert issubclass(MemoryLRUPCP, IEContentProvider) + assert issubclass(MemoryLRUPCP, BuiltinIEContentProvider) + + @pytest.fixture + def pcp(self, ie, logger) -> MemoryLRUPCP: + return MemoryLRUPCP(ie, logger, {}, initialize_cache=lambda max_size: (OrderedDict(), threading.Lock(), max_size)) + + def test_is_registered(self): + assert _pot_cache_providers.value.get('MemoryLRU') == MemoryLRUPCP + + def test_initialization(self, pcp): + assert pcp.PROVIDER_NAME == 'memory' + assert pcp.PROVIDER_VERSION == __version__ + assert pcp.BUG_REPORT_MESSAGE == bug_reports_message(before='') + assert pcp.is_available() + + def test_store_and_get(self, pcp): + pcp.store('key1', 'value1', int(time.time()) + 60) + assert pcp.get('key1') == 'value1' + assert len(pcp.cache) == 1 + + def test_store_ignore_expired(self, pcp): + pcp.store('key1', 'value1', int(time.time()) - 1) + assert len(pcp.cache) == 0 + assert pcp.get('key1') is None + assert len(pcp.cache) == 0 + + def test_store_override_existing_key(self, ie, logger): + MAX_SIZE = 2 + pcp = MemoryLRUPCP(ie, logger, {}, initialize_cache=lambda max_size: (OrderedDict(), threading.Lock(), MAX_SIZE)) + pcp.store('key1', 'value1', int(time.time()) + 60) + pcp.store('key2', 'value2', int(time.time()) + 60) + assert len(pcp.cache) == 2 + pcp.store('key1', 'value2', int(time.time()) + 60) + # Ensure that the override key gets added to the end of the cache instead of in the same position + pcp.store('key3', 'value3', int(time.time()) + 60) + assert pcp.get('key1') == 'value2' + + def test_store_ignore_expired_existing_key(self, pcp): + pcp.store('key1', 'value2', int(time.time()) + 60) + pcp.store('key1', 'value1', int(time.time()) - 1) + assert len(pcp.cache) == 1 + assert pcp.get('key1') == 'value2' + assert len(pcp.cache) == 1 + + def test_get_key_expired(self, pcp): + pcp.store('key1', 'value1', int(time.time()) + 60) + assert pcp.get('key1') == 'value1' + assert len(pcp.cache) == 1 + pcp.cache['key1'] = ('value1', int(time.time()) - 1) + assert pcp.get('key1') is None + assert len(pcp.cache) == 0 + + def test_lru_eviction(self, ie, logger): + MAX_SIZE = 2 + provider = MemoryLRUPCP(ie, logger, {}, initialize_cache=lambda max_size: (OrderedDict(), threading.Lock(), MAX_SIZE)) + provider.store('key1', 'value1', int(time.time()) + 5) + provider.store('key2', 'value2', int(time.time()) + 5) + assert len(provider.cache) == 2 + + assert provider.get('key1') == 'value1' + + provider.store('key3', 'value3', int(time.time()) + 5) + assert len(provider.cache) == 2 + + assert provider.get('key2') is None + + provider.store('key4', 'value4', int(time.time()) + 5) + assert len(provider.cache) == 2 + + assert provider.get('key1') is None + assert provider.get('key3') == 'value3' + assert provider.get('key4') == 'value4' + + def test_delete(self, pcp): + pcp.store('key1', 'value1', int(time.time()) + 5) + assert len(pcp.cache) == 1 + assert pcp.get('key1') == 'value1' + pcp.delete('key1') + assert len(pcp.cache) == 0 + assert pcp.get('key1') is None + + def test_use_global_cache_default(self, ie, logger): + pcp = MemoryLRUPCP(ie, logger, {}) + assert pcp.max_size == _pot_memory_cache.value['max_size'] == 25 + assert pcp.cache is _pot_memory_cache.value['cache'] + assert pcp.lock is _pot_memory_cache.value['lock'] + + pcp2 = MemoryLRUPCP(ie, logger, {}) + assert pcp.max_size == pcp2.max_size == _pot_memory_cache.value['max_size'] == 25 + assert pcp.cache is pcp2.cache is _pot_memory_cache.value['cache'] + assert pcp.lock is pcp2.lock is _pot_memory_cache.value['lock'] + + def test_fail_max_size_change_global(self, ie, logger): + pcp = MemoryLRUPCP(ie, logger, {}) + assert pcp.max_size == _pot_memory_cache.value['max_size'] == 25 + with pytest.raises(ValueError, match='Cannot change max_size of initialized global memory cache'): + initialize_global_cache(50) + + assert pcp.max_size == _pot_memory_cache.value['max_size'] == 25 + + def test_memory_lru_preference(self, pcp, ie, pot_request): + assert memorylru_preference(pcp, pot_request) == 10000 diff --git a/test/test_pot/test_pot_builtin_utils.py b/test/test_pot/test_pot_builtin_utils.py new file mode 100644 index 000000000..1682e42a1 --- /dev/null +++ b/test/test_pot/test_pot_builtin_utils.py @@ -0,0 +1,46 @@ +import pytest +from yt_dlp.extractor.youtube.pot.provider import ( + PoTokenContext, + +) + +from yt_dlp.extractor.youtube.pot.utils import get_webpo_content_binding, ContentBindingType + + +class TestGetWebPoContentBinding: + + @pytest.mark.parametrize('client_name, context, is_authenticated, expected', [ + *[(client, context, is_authenticated, expected) for client in [ + 'WEB', 'MWEB', 'TVHTML5', 'WEB_EMBEDDED_PLAYER', 'WEB_CREATOR', 'TVHTML5_SIMPLY_EMBEDDED_PLAYER'] + for context, is_authenticated, expected in [ + (PoTokenContext.GVS, False, ('example-visitor-data', ContentBindingType.VISITOR_DATA)), + (PoTokenContext.PLAYER, False, ('example-video-id', ContentBindingType.VIDEO_ID)), + (PoTokenContext.GVS, True, ('example-data-sync-id', ContentBindingType.DATASYNC_ID)), + ]], + ('WEB_REMIX', PoTokenContext.GVS, False, ('example-visitor-data', ContentBindingType.VISITOR_DATA)), + ('WEB_REMIX', PoTokenContext.PLAYER, False, ('example-visitor-data', ContentBindingType.VISITOR_DATA)), + ('ANDROID', PoTokenContext.GVS, False, (None, None)), + ('IOS', PoTokenContext.GVS, False, (None, None)), + ]) + def test_get_webpo_content_binding(self, pot_request, client_name, context, is_authenticated, expected): + pot_request.innertube_context['client']['clientName'] = client_name + pot_request.context = context + pot_request.is_authenticated = is_authenticated + assert get_webpo_content_binding(pot_request) == expected + + def test_extract_visitor_id(self, pot_request): + pot_request.visitor_data = 'CgsxMjNhYmNYWVpfLSiA4s%2DqBg%3D%3D' + assert get_webpo_content_binding(pot_request, bind_to_visitor_id=True) == ('123abcXYZ_-', ContentBindingType.VISITOR_ID) + + def test_invalid_visitor_id(self, pot_request): + # visitor id not alphanumeric (i.e. protobuf extraction failed) + pot_request.visitor_data = 'CggxMjM0NTY3OCiA4s-qBg%3D%3D' + assert get_webpo_content_binding(pot_request, bind_to_visitor_id=True) == (pot_request.visitor_data, ContentBindingType.VISITOR_DATA) + + def test_no_visitor_id(self, pot_request): + pot_request.visitor_data = 'KIDiz6oG' + assert get_webpo_content_binding(pot_request, bind_to_visitor_id=True) == (pot_request.visitor_data, ContentBindingType.VISITOR_DATA) + + def test_invalid_base64(self, pot_request): + pot_request.visitor_data = 'invalid-base64' + assert get_webpo_content_binding(pot_request, bind_to_visitor_id=True) == (pot_request.visitor_data, ContentBindingType.VISITOR_DATA) diff --git a/test/test_pot/test_pot_builtin_webpospec.py b/test/test_pot/test_pot_builtin_webpospec.py new file mode 100644 index 000000000..c5fb6f382 --- /dev/null +++ b/test/test_pot/test_pot_builtin_webpospec.py @@ -0,0 +1,92 @@ +import pytest + +from yt_dlp.extractor.youtube.pot._provider import IEContentProvider, BuiltinIEContentProvider +from yt_dlp.extractor.youtube.pot.cache import CacheProviderWritePolicy +from yt_dlp.utils import bug_reports_message +from yt_dlp.extractor.youtube.pot.provider import ( + PoTokenRequest, + PoTokenContext, + +) +from yt_dlp.version import __version__ + +from yt_dlp.extractor.youtube.pot._builtin.webpo_cachespec import WebPoPCSP +from yt_dlp.extractor.youtube.pot._registry import _pot_pcs_providers + + +@pytest.fixture() +def pot_request(pot_request) -> PoTokenRequest: + pot_request.visitor_data = 'CgsxMjNhYmNYWVpfLSiA4s%2DqBg%3D%3D' # visitor_id=123abcXYZ_- + return pot_request + + +class TestWebPoPCSP: + def test_base_type(self): + assert issubclass(WebPoPCSP, IEContentProvider) + assert issubclass(WebPoPCSP, BuiltinIEContentProvider) + + def test_init(self, ie, logger): + pcs = WebPoPCSP(ie=ie, logger=logger, settings={}) + assert pcs.PROVIDER_NAME == 'webpo' + assert pcs.PROVIDER_VERSION == __version__ + assert pcs.BUG_REPORT_MESSAGE == bug_reports_message(before='') + assert pcs.is_available() + + def test_is_registered(self): + assert _pot_pcs_providers.value.get('WebPo') == WebPoPCSP + + @pytest.mark.parametrize('client_name, context, is_authenticated', [ + ('ANDROID', PoTokenContext.GVS, False), + ('IOS', PoTokenContext.GVS, False), + ('IOS', PoTokenContext.PLAYER, False), + ]) + def test_not_supports(self, ie, logger, pot_request, client_name, context, is_authenticated): + pcs = WebPoPCSP(ie=ie, logger=logger, settings={}) + pot_request.innertube_context['client']['clientName'] = client_name + pot_request.context = context + pot_request.is_authenticated = is_authenticated + assert pcs.generate_cache_spec(pot_request) is None + + @pytest.mark.parametrize('client_name, context, is_authenticated, remote_host, source_address, request_proxy, expected', [ + *[(client, context, is_authenticated, remote_host, source_address, request_proxy, expected) for client in [ + 'WEB', 'MWEB', 'TVHTML5', 'WEB_EMBEDDED_PLAYER', 'WEB_CREATOR', 'TVHTML5_SIMPLY_EMBEDDED_PLAYER'] + for context, is_authenticated, remote_host, source_address, request_proxy, expected in [ + (PoTokenContext.GVS, False, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': '123abcXYZ_-', 'cbt': 'visitor_id'}), + (PoTokenContext.PLAYER, False, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': '123abcXYZ_-', 'cbt': 'video_id'}), + (PoTokenContext.GVS, True, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': 'example-data-sync-id', 'cbt': 'datasync_id'}), + ]], + ('WEB_REMIX', PoTokenContext.PLAYER, False, 'example-remote-host', 'example-source-address', 'example-request-proxy', {'t': 'webpo', 'ip': 'example-remote-host', 'sa': 'example-source-address', 'px': 'example-request-proxy', 'cb': '123abcXYZ_-', 'cbt': 'visitor_id'}), + ('WEB', PoTokenContext.GVS, False, None, None, None, {'t': 'webpo', 'cb': '123abcXYZ_-', 'cbt': 'visitor_id', 'ip': None, 'sa': None, 'px': None}), + ('TVHTML5', PoTokenContext.PLAYER, False, None, None, 'http://example.com', {'t': 'webpo', 'cb': '123abcXYZ_-', 'cbt': 'video_id', 'ip': None, 'sa': None, 'px': 'http://example.com'}), + + ]) + def test_generate_key_bindings(self, ie, logger, pot_request, client_name, context, is_authenticated, remote_host, source_address, request_proxy, expected): + pcs = WebPoPCSP(ie=ie, logger=logger, settings={}) + pot_request.innertube_context['client']['clientName'] = client_name + pot_request.context = context + pot_request.is_authenticated = is_authenticated + pot_request.innertube_context['client']['remoteHost'] = remote_host + pot_request.request_source_address = source_address + pot_request.request_proxy = request_proxy + pot_request.video_id = '123abcXYZ_-' # same as visitor id to test type + + assert pcs.generate_cache_spec(pot_request).key_bindings == expected + + def test_no_bind_visitor_id(self, ie, logger, pot_request): + # Should not bind to visitor id if setting is set to False + pcs = WebPoPCSP(ie=ie, logger=logger, settings={'bind_to_visitor_id': ['false']}) + pot_request.innertube_context['client']['clientName'] = 'WEB' + pot_request.context = PoTokenContext.GVS + pot_request.is_authenticated = False + assert pcs.generate_cache_spec(pot_request).key_bindings == {'t': 'webpo', 'ip': None, 'sa': None, 'px': None, 'cb': 'CgsxMjNhYmNYWVpfLSiA4s%2DqBg%3D%3D', 'cbt': 'visitor_data'} + + def test_default_ttl(self, ie, logger, pot_request): + pcs = WebPoPCSP(ie=ie, logger=logger, settings={}) + assert pcs.generate_cache_spec(pot_request).default_ttl == 6 * 60 * 60 # should default to 6 hours + + def test_write_policy(self, ie, logger, pot_request): + pcs = WebPoPCSP(ie=ie, logger=logger, settings={}) + pot_request.context = PoTokenContext.GVS + assert pcs.generate_cache_spec(pot_request).write_policy == CacheProviderWritePolicy.WRITE_ALL + pot_request.context = PoTokenContext.PLAYER + assert pcs.generate_cache_spec(pot_request).write_policy == CacheProviderWritePolicy.WRITE_FIRST diff --git a/test/test_pot/test_pot_director.py b/test/test_pot/test_pot_director.py new file mode 100644 index 000000000..bbfdd0e98 --- /dev/null +++ b/test/test_pot/test_pot_director.py @@ -0,0 +1,1529 @@ +from __future__ import annotations +import abc +import base64 +import dataclasses +import hashlib +import json +import time +import pytest + +from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider, IEContentProvider + +from yt_dlp.extractor.youtube.pot.provider import ( + PoTokenRequest, + PoTokenContext, + PoTokenProviderError, + PoTokenProviderRejectedRequest, +) +from yt_dlp.extractor.youtube.pot._director import ( + PoTokenCache, + validate_cache_spec, + clean_pot, + validate_response, + PoTokenRequestDirector, + provider_display_list, +) + +from yt_dlp.extractor.youtube.pot.cache import ( + PoTokenCacheSpec, + PoTokenCacheSpecProvider, + PoTokenCacheProvider, + CacheProviderWritePolicy, + PoTokenCacheProviderError, +) + + +from yt_dlp.extractor.youtube.pot.provider import ( + PoTokenResponse, + PoTokenProvider, +) + + +class BaseMockPoTokenProvider(PoTokenProvider, abc.ABC): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.available_called_times = 0 + self.request_called_times = 0 + self.close_called = False + + def is_available(self) -> bool: + self.available_called_times += 1 + return True + + def request_pot(self, *args, **kwargs): + self.request_called_times += 1 + return super().request_pot(*args, **kwargs) + + def close(self): + self.close_called = True + super().close() + + +class ExamplePTP(BaseMockPoTokenProvider): + PROVIDER_NAME = 'example' + PROVIDER_VERSION = '0.0.1' + BUG_REPORT_LOCATION = 'https://example.com/issues' + + _SUPPORTED_CLIENTS = ('WEB',) + _SUPPORTED_CONTEXTS = (PoTokenContext.GVS, ) + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + if request.data_sync_id == 'example': + return PoTokenResponse(request.video_id) + return PoTokenResponse(EXAMPLE_PO_TOKEN) + + +def success_ptp(response: PoTokenResponse | None = None, key: str | None = None): + class SuccessPTP(BaseMockPoTokenProvider): + PROVIDER_NAME = 'success' + PROVIDER_VERSION = '0.0.1' + BUG_REPORT_LOCATION = 'https://success.example.com/issues' + + _SUPPORTED_CLIENTS = ('WEB',) + _SUPPORTED_CONTEXTS = (PoTokenContext.GVS,) + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + return response or PoTokenResponse(EXAMPLE_PO_TOKEN) + + if key: + SuccessPTP.PROVIDER_KEY = key + return SuccessPTP + + +@pytest.fixture +def pot_provider(ie, logger): + return success_ptp()(ie=ie, logger=logger, settings={}) + + +class UnavailablePTP(BaseMockPoTokenProvider): + PROVIDER_NAME = 'unavailable' + BUG_REPORT_LOCATION = 'https://unavailable.example.com/issues' + _SUPPORTED_CLIENTS = None + _SUPPORTED_CONTEXTS = None + + def is_available(self) -> bool: + super().is_available() + return False + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + raise PoTokenProviderError('something went wrong') + + +class UnsupportedPTP(BaseMockPoTokenProvider): + PROVIDER_NAME = 'unsupported' + BUG_REPORT_LOCATION = 'https://unsupported.example.com/issues' + _SUPPORTED_CLIENTS = None + _SUPPORTED_CONTEXTS = None + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + raise PoTokenProviderRejectedRequest('unsupported request') + + +class ErrorPTP(BaseMockPoTokenProvider): + PROVIDER_NAME = 'error' + BUG_REPORT_LOCATION = 'https://error.example.com/issues' + _SUPPORTED_CLIENTS = None + _SUPPORTED_CONTEXTS = None + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + expected = request.video_id == 'expected' + raise PoTokenProviderError('an error occurred', expected=expected) + + +class UnexpectedErrorPTP(BaseMockPoTokenProvider): + PROVIDER_NAME = 'unexpected_error' + BUG_REPORT_LOCATION = 'https://unexpected.example.com/issues' + _SUPPORTED_CLIENTS = None + _SUPPORTED_CONTEXTS = None + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + raise ValueError('an unexpected error occurred') + + +class InvalidPTP(BaseMockPoTokenProvider): + PROVIDER_NAME = 'invalid' + BUG_REPORT_LOCATION = 'https://invalid.example.com/issues' + _SUPPORTED_CLIENTS = None + _SUPPORTED_CONTEXTS = None + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + if request.video_id == 'invalid_type': + return 'invalid-response' + else: + return PoTokenResponse('example-token?', expires_at='123') + + +class BaseMockCacheSpecProvider(PoTokenCacheSpecProvider, abc.ABC): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.generate_called_times = 0 + self.is_available_called_times = 0 + self.close_called = False + + def is_available(self) -> bool: + self.is_available_called_times += 1 + return super().is_available() + + def generate_cache_spec(self, request: PoTokenRequest): + self.generate_called_times += 1 + + def close(self): + self.close_called = True + super().close() + + +class ExampleCacheSpecProviderPCSP(BaseMockCacheSpecProvider): + + PROVIDER_NAME = 'example' + PROVIDER_VERSION = '0.0.1' + BUG_REPORT_LOCATION = 'https://example.com/issues' + + def generate_cache_spec(self, request: PoTokenRequest): + super().generate_cache_spec(request) + return PoTokenCacheSpec( + key_bindings={'v': request.video_id, 'e': None}, + default_ttl=60, + ) + + +class UnavailableCacheSpecProviderPCSP(BaseMockCacheSpecProvider): + + PROVIDER_NAME = 'unavailable' + PROVIDER_VERSION = '0.0.1' + + def is_available(self) -> bool: + super().is_available() + return False + + def generate_cache_spec(self, request: PoTokenRequest): + super().generate_cache_spec(request) + return None + + +class UnsupportedCacheSpecProviderPCSP(BaseMockCacheSpecProvider): + + PROVIDER_NAME = 'unsupported' + PROVIDER_VERSION = '0.0.1' + + def generate_cache_spec(self, request: PoTokenRequest): + super().generate_cache_spec(request) + return None + + +class InvalidSpecCacheSpecProviderPCSP(BaseMockCacheSpecProvider): + + PROVIDER_NAME = 'invalid' + PROVIDER_VERSION = '0.0.1' + + def generate_cache_spec(self, request: PoTokenRequest): + super().generate_cache_spec(request) + return 'invalid-spec' + + +class ErrorSpecCacheSpecProviderPCSP(BaseMockCacheSpecProvider): + + PROVIDER_NAME = 'invalid' + PROVIDER_VERSION = '0.0.1' + + def generate_cache_spec(self, request: PoTokenRequest): + super().generate_cache_spec(request) + raise ValueError('something went wrong') + + +class BaseMockCacheProvider(PoTokenCacheProvider, abc.ABC): + BUG_REPORT_MESSAGE = 'example bug report message' + + def __init__(self, *args, available=True, **kwargs): + super().__init__(*args, **kwargs) + self.store_calls = 0 + self.delete_calls = 0 + self.get_calls = 0 + self.available_called_times = 0 + self.available = available + + def is_available(self) -> bool: + self.available_called_times += 1 + return self.available + + def store(self, *args, **kwargs): + self.store_calls += 1 + + def delete(self, *args, **kwargs): + self.delete_calls += 1 + + def get(self, *args, **kwargs): + self.get_calls += 1 + + def close(self): + self.close_called = True + super().close() + + +class ErrorPCP(BaseMockCacheProvider): + PROVIDER_NAME = 'error' + + def store(self, *args, **kwargs): + super().store(*args, **kwargs) + raise PoTokenCacheProviderError('something went wrong') + + def get(self, *args, **kwargs): + super().get(*args, **kwargs) + raise PoTokenCacheProviderError('something went wrong') + + +class UnexpectedErrorPCP(BaseMockCacheProvider): + PROVIDER_NAME = 'unexpected_error' + + def store(self, *args, **kwargs): + super().store(*args, **kwargs) + raise ValueError('something went wrong') + + def get(self, *args, **kwargs): + super().get(*args, **kwargs) + raise ValueError('something went wrong') + + +class MockMemoryPCP(BaseMockCacheProvider): + PROVIDER_NAME = 'memory' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cache = {} + + def store(self, key, value, expires_at): + super().store(key, value, expires_at) + self.cache[key] = (value, expires_at) + + def delete(self, key): + super().delete(key) + self.cache.pop(key, None) + + def get(self, key): + super().get(key) + return self.cache.get(key, [None])[0] + + +def create_memory_pcp(ie, logger, provider_key='memory', provider_name='memory', available=True): + cache = MockMemoryPCP(ie, logger, {}, available=available) + cache.PROVIDER_KEY = provider_key + cache.PROVIDER_NAME = provider_name + return cache + + +@pytest.fixture +def memorypcp(ie, logger) -> MockMemoryPCP: + return create_memory_pcp(ie, logger) + + +@pytest.fixture +def pot_cache(ie, logger): + class MockPoTokenCache(PoTokenCache): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.get_calls = 0 + self.store_calls = 0 + self.close_called = False + + def get(self, *args, **kwargs): + self.get_calls += 1 + return super().get(*args, **kwargs) + + def store(self, *args, **kwargs): + self.store_calls += 1 + return super().store(*args, **kwargs) + + def close(self): + self.close_called = True + super().close() + + return MockPoTokenCache( + cache_providers=[MockMemoryPCP(ie, logger, {})], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie, logger, settings={})], + logger=logger, + ) + + +EXAMPLE_PO_TOKEN = base64.urlsafe_b64encode(b'example-token').decode() + + +class TestPoTokenCache: + + def test_cache_success(self, memorypcp, pot_request, ie, logger): + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + + assert cache.get(pot_request) is None + cache.store(pot_request, response) + + cached_response = cache.get(pot_request) + assert cached_response is not None + assert cached_response.po_token == EXAMPLE_PO_TOKEN + assert cached_response.expires_at is not None + + assert cache.get(dataclasses.replace(pot_request, video_id='another-video-id')) is None + + def test_unsupported_cache_spec_no_fallback(self, memorypcp, pot_request, ie, logger): + unsupported_provider = UnsupportedCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[unsupported_provider], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + assert cache.get(pot_request) is None + assert unsupported_provider.generate_called_times == 1 + cache.store(pot_request, response) + assert len(memorypcp.cache) == 0 + assert unsupported_provider.generate_called_times == 2 + assert cache.get(pot_request) is None + assert unsupported_provider.generate_called_times == 3 + assert len(logger.messages.get('error', [])) == 0 + + def test_unsupported_cache_spec_fallback(self, memorypcp, pot_request, ie, logger): + unsupported_provider = UnsupportedCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + example_provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[unsupported_provider, example_provider], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + + assert cache.get(pot_request) is None + assert unsupported_provider.generate_called_times == 1 + assert example_provider.generate_called_times == 1 + + cache.store(pot_request, response) + assert unsupported_provider.generate_called_times == 2 + assert example_provider.generate_called_times == 2 + + cached_response = cache.get(pot_request) + assert unsupported_provider.generate_called_times == 3 + assert example_provider.generate_called_times == 3 + assert cached_response is not None + assert cached_response.po_token == EXAMPLE_PO_TOKEN + assert cached_response.expires_at is not None + + assert len(logger.messages.get('error', [])) == 0 + + def test_invalid_cache_spec_no_fallback(self, memorypcp, pot_request, ie, logger): + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[InvalidSpecCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + + assert cache.get(pot_request) is None + cache.store(pot_request, response) + + assert cache.get(pot_request) is None + + assert 'PoTokenCacheSpecProvider "InvalidSpecCacheSpecProvider" generate_cache_spec() returned invalid spec invalid-spec; please report this issue to the provider developer at (developer has not provided a bug report location) .' in logger.messages['error'] + + def test_invalid_cache_spec_fallback(self, memorypcp, pot_request, ie, logger): + + invalid_provider = InvalidSpecCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + example_provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[invalid_provider, example_provider], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + + assert cache.get(pot_request) is None + assert invalid_provider.generate_called_times == example_provider.generate_called_times == 1 + + cache.store(pot_request, response) + assert invalid_provider.generate_called_times == example_provider.generate_called_times == 2 + + cached_response = cache.get(pot_request) + assert invalid_provider.generate_called_times == example_provider.generate_called_times == 3 + assert cached_response is not None + assert cached_response.po_token == EXAMPLE_PO_TOKEN + assert cached_response.expires_at is not None + + assert 'PoTokenCacheSpecProvider "InvalidSpecCacheSpecProvider" generate_cache_spec() returned invalid spec invalid-spec; please report this issue to the provider developer at (developer has not provided a bug report location) .' in logger.messages['error'] + + def test_unavailable_cache_spec_no_fallback(self, memorypcp, pot_request, ie, logger): + unavailable_provider = UnavailableCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[unavailable_provider], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + + assert cache.get(pot_request) is None + cache.store(pot_request, response) + assert cache.get(pot_request) is None + assert unavailable_provider.generate_called_times == 0 + + def test_unavailable_cache_spec_fallback(self, memorypcp, pot_request, ie, logger): + unavailable_provider = UnavailableCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + example_provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[unavailable_provider, example_provider], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + + assert cache.get(pot_request) is None + assert unavailable_provider.generate_called_times == 0 + assert unavailable_provider.is_available_called_times == 1 + assert example_provider.generate_called_times == 1 + + cache.store(pot_request, response) + assert unavailable_provider.generate_called_times == 0 + assert unavailable_provider.is_available_called_times == 2 + assert example_provider.generate_called_times == 2 + + cached_response = cache.get(pot_request) + assert unavailable_provider.generate_called_times == 0 + assert unavailable_provider.is_available_called_times == 3 + assert example_provider.generate_called_times == 3 + assert example_provider.is_available_called_times == 3 + assert cached_response is not None + assert cached_response.po_token == EXAMPLE_PO_TOKEN + assert cached_response.expires_at is not None + + def test_unexpected_error_cache_spec(self, memorypcp, pot_request, ie, logger): + error_provider = ErrorSpecCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[error_provider], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + + assert cache.get(pot_request) is None + cache.store(pot_request, response) + assert cache.get(pot_request) is None + assert error_provider.generate_called_times == 3 + assert error_provider.is_available_called_times == 3 + + assert 'Error occurred with "invalid" PO Token cache spec provider: ValueError(\'something went wrong\'); please report this issue to the provider developer at (developer has not provided a bug report location) .' in logger.messages['error'] + + def test_unexpected_error_cache_spec_fallback(self, memorypcp, pot_request, ie, logger): + error_provider = ErrorSpecCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + example_provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[error_provider, example_provider], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + + assert cache.get(pot_request) is None + assert error_provider.generate_called_times == 1 + assert error_provider.is_available_called_times == 1 + assert example_provider.generate_called_times == 1 + + cache.store(pot_request, response) + assert error_provider.generate_called_times == 2 + assert error_provider.is_available_called_times == 2 + assert example_provider.generate_called_times == 2 + + cached_response = cache.get(pot_request) + assert error_provider.generate_called_times == 3 + assert error_provider.is_available_called_times == 3 + assert example_provider.generate_called_times == 3 + assert example_provider.is_available_called_times == 3 + assert cached_response is not None + assert cached_response.po_token == EXAMPLE_PO_TOKEN + assert cached_response.expires_at is not None + + assert 'Error occurred with "invalid" PO Token cache spec provider: ValueError(\'something went wrong\'); please report this issue to the provider developer at (developer has not provided a bug report location) .' in logger.messages['error'] + + def test_key_bindings_spec_provider(self, memorypcp, pot_request, ie, logger): + + class ExampleProviderPCSP(PoTokenCacheSpecProvider): + PROVIDER_NAME = 'example' + + def generate_cache_spec(self, request: PoTokenRequest): + return PoTokenCacheSpec( + key_bindings={'v': request.video_id}, + default_ttl=60, + ) + + class ExampleProviderTwoPCSP(ExampleProviderPCSP): + pass + + example_provider = ExampleProviderPCSP(ie=ie, logger=logger, settings={}) + example_provider_two = ExampleProviderTwoPCSP(ie=ie, logger=logger, settings={}) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[example_provider], + logger=logger, + ) + + assert cache.get(pot_request) is None + cache.store(pot_request, response) + assert len(memorypcp.cache) == 1 + assert hashlib.sha256( + f"{{'_dlp_cache': 'v1', '_p': 'ExampleProvider', 'v': '{pot_request.video_id}'}}".encode()).hexdigest() in memorypcp.cache + + # The second spec provider returns the exact same key bindings as the first one, + # however the PoTokenCache should use the provider key to differentiate between them + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[example_provider_two], + logger=logger, + ) + + assert cache.get(pot_request) is None + cache.store(pot_request, response) + assert len(memorypcp.cache) == 2 + assert hashlib.sha256( + f"{{'_dlp_cache': 'v1', '_p': 'ExampleProviderTwo', 'v': '{pot_request.video_id}'}}".encode()).hexdigest() in memorypcp.cache + + def test_cache_provider_preferences(self, pot_request, ie, logger): + pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one') + pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two') + + cache = PoTokenCache( + cache_providers=[pcp_one, pcp_two], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN), write_policy=CacheProviderWritePolicy.WRITE_FIRST) + assert len(pcp_one.cache) == 1 + assert len(pcp_two.cache) == 0 + + assert cache.get(pot_request) + assert pcp_one.get_calls == 1 + assert pcp_two.get_calls == 0 + + standard_preference_called = False + pcp_one_preference_claled = False + + def standard_preference(provider, request, *_, **__): + nonlocal standard_preference_called + standard_preference_called = True + assert isinstance(provider, PoTokenCacheProvider) + assert isinstance(request, PoTokenRequest) + return 1 + + def pcp_one_preference(provider, request, *_, **__): + nonlocal pcp_one_preference_claled + pcp_one_preference_claled = True + assert isinstance(provider, PoTokenCacheProvider) + assert isinstance(request, PoTokenRequest) + if provider.PROVIDER_KEY == pcp_one.PROVIDER_KEY: + return -100 + return 0 + + # test that it can hanldle multiple preferences + cache.cache_provider_preferences.append(standard_preference) + cache.cache_provider_preferences.append(pcp_one_preference) + + cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN), write_policy=CacheProviderWritePolicy.WRITE_FIRST) + assert cache.get(pot_request) + assert len(pcp_one.cache) == len(pcp_two.cache) == 1 + assert pcp_two.get_calls == pcp_one.get_calls == 1 + assert pcp_one.store_calls == pcp_two.store_calls == 1 + assert standard_preference_called + assert pcp_one_preference_claled + + def test_secondary_cache_provider_hit(self, pot_request, ie, logger): + pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one') + pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two') + + cache = PoTokenCache( + cache_providers=[pcp_two], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + # Given the lower priority provider has the cache hit, store the response in the higher priority provider + cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN)) + assert cache.get(pot_request) + + cache.cache_providers[pcp_one.PROVIDER_KEY] = pcp_one + + def pcp_one_pref(provider, *_, **__): + if provider.PROVIDER_KEY == pcp_one.PROVIDER_KEY: + return 1 + return -1 + + cache.cache_provider_preferences.append(pcp_one_pref) + + assert cache.get(pot_request) + assert pcp_one.get_calls == 1 + assert pcp_two.get_calls == 2 + # Should write back to pcp_one (now the highest priority cache provider) + assert pcp_one.store_calls == pcp_two.store_calls == 1 + assert 'Writing PO Token response to highest priority cache provider' in logger.messages['trace'] + + def test_cache_provider_no_hits(self, pot_request, ie, logger): + pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one') + pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two') + + cache = PoTokenCache( + cache_providers=[pcp_one, pcp_two], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + assert cache.get(pot_request) is None + assert pcp_one.get_calls == pcp_two.get_calls == 1 + + def test_get_invalid_po_token_response(self, pot_request, ie, logger): + # Test various scenarios where the po token response stored in the cache provider is invalid + pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one') + pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two') + + cache = PoTokenCache( + cache_providers=[pcp_one, pcp_two], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + valid_response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, valid_response) + assert len(pcp_one.cache) == len(pcp_two.cache) == 1 + # Overwrite the valid response with an invalid one in the cache + pcp_one.store(next(iter(pcp_one.cache.keys())), json.dumps(dataclasses.asdict(PoTokenResponse(None))), int(time.time() + 1000)) + assert cache.get(pot_request).po_token == valid_response.po_token + assert pcp_one.get_calls == pcp_two.get_calls == 1 + assert pcp_one.delete_calls == 1 # Invalid response should be deleted from cache + assert pcp_one.store_calls == 3 # Since response was fetched from second cache provider, it should be stored in the first one + assert len(pcp_one.cache) == 1 + assert 'Invalid PO Token response retrieved from cache provider "memory": {"po_token": null, "expires_at": null}; example bug report message' in logger.messages['error'] + + # Overwrite the valid response with an invalid json in the cache + pcp_one.store(next(iter(pcp_one.cache.keys())), 'invalid-json', int(time.time() + 1000)) + assert cache.get(pot_request).po_token == valid_response.po_token + assert pcp_one.get_calls == pcp_two.get_calls == 2 + assert pcp_one.delete_calls == 2 + assert pcp_one.store_calls == 5 # 3 + 1 store we made in the test + 1 store from lower priority cache provider + assert len(pcp_one.cache) == 1 + + assert 'Invalid PO Token response retrieved from cache provider "memory": invalid-json; example bug report message' in logger.messages['error'] + + # Valid json, but missing required fields + pcp_one.store(next(iter(pcp_one.cache.keys())), '{"unknown_param": 0}', int(time.time() + 1000)) + assert cache.get(pot_request).po_token == valid_response.po_token + assert pcp_one.get_calls == pcp_two.get_calls == 3 + assert pcp_one.delete_calls == 3 + assert pcp_one.store_calls == 7 # 5 + 1 store from test + 1 store from lower priority cache provider + assert len(pcp_one.cache) == 1 + + assert 'Invalid PO Token response retrieved from cache provider "memory": {"unknown_param": 0}; example bug report message' in logger.messages['error'] + + def test_store_invalid_po_token_response(self, pot_request, ie, logger): + # Should not store an invalid po token response + pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one') + + cache = PoTokenCache( + cache_providers=[pcp_one], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + cache.store(pot_request, PoTokenResponse(po_token=EXAMPLE_PO_TOKEN, expires_at=80)) + assert cache.get(pot_request) is None + assert pcp_one.store_calls == 0 + assert 'Invalid PO Token response provided to PoTokenCache.store()' in logger.messages['error'][0] + + def test_store_write_policy(self, pot_request, ie, logger): + pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one') + pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two') + + cache = PoTokenCache( + cache_providers=[pcp_one, pcp_two], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN), write_policy=CacheProviderWritePolicy.WRITE_FIRST) + assert pcp_one.store_calls == 1 + assert pcp_two.store_calls == 0 + + cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN), write_policy=CacheProviderWritePolicy.WRITE_ALL) + assert pcp_one.store_calls == 2 + assert pcp_two.store_calls == 1 + + def test_store_write_first_policy_cache_spec(self, pot_request, ie, logger): + pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one') + pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two') + + class WriteFirstPCSP(BaseMockCacheSpecProvider): + def generate_cache_spec(self, request: PoTokenRequest): + super().generate_cache_spec(request) + return PoTokenCacheSpec( + key_bindings={'v': request.video_id, 'e': None}, + default_ttl=60, + write_policy=CacheProviderWritePolicy.WRITE_FIRST, + ) + + cache = PoTokenCache( + cache_providers=[pcp_one, pcp_two], + cache_spec_providers=[WriteFirstPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN)) + assert pcp_one.store_calls == 1 + assert pcp_two.store_calls == 0 + + def test_store_write_all_policy_cache_spec(self, pot_request, ie, logger): + pcp_one = create_memory_pcp(ie, logger, provider_key='memory_pcp_one') + pcp_two = create_memory_pcp(ie, logger, provider_key='memory_pcp_two') + + class WriteAllPCSP(BaseMockCacheSpecProvider): + def generate_cache_spec(self, request: PoTokenRequest): + super().generate_cache_spec(request) + return PoTokenCacheSpec( + key_bindings={'v': request.video_id, 'e': None}, + default_ttl=60, + write_policy=CacheProviderWritePolicy.WRITE_ALL, + ) + + cache = PoTokenCache( + cache_providers=[pcp_one, pcp_two], + cache_spec_providers=[WriteAllPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN)) + assert pcp_one.store_calls == 1 + assert pcp_two.store_calls == 1 + + def test_expires_at_pot_response(self, pot_request, memorypcp, ie, logger): + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=10000000000) + cache.store(pot_request, response) + assert next(iter(memorypcp.cache.values()))[1] == 10000000000 + + def test_expires_at_default_spec(self, pot_request, memorypcp, ie, logger): + + class TtlPCSP(BaseMockCacheSpecProvider): + def generate_cache_spec(self, request: PoTokenRequest): + super().generate_cache_spec(request) + return PoTokenCacheSpec( + key_bindings={'v': request.video_id, 'e': None}, + default_ttl=10000000000, + ) + + cache = PoTokenCache( + cache_providers=[memorypcp], + cache_spec_providers=[TtlPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, response) + assert next(iter(memorypcp.cache.values()))[1] >= 10000000000 + + def test_cache_provider_error_no_fallback(self, pot_request, ie, logger): + error_pcp = ErrorPCP(ie, logger, {}) + cache = PoTokenCache( + cache_providers=[error_pcp], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, response) + assert cache.get(pot_request) is None + assert error_pcp.get_calls == 1 + assert error_pcp.store_calls == 1 + + assert logger.messages['warning'].count("Error from \"error\" PO Token cache provider: PoTokenCacheProviderError('something went wrong'); example bug report message") == 2 + + def test_cache_provider_error_fallback(self, pot_request, ie, logger): + error_pcp = ErrorPCP(ie, logger, {}) + memory_pcp = create_memory_pcp(ie, logger, provider_key='memory') + + cache = PoTokenCache( + cache_providers=[error_pcp, memory_pcp], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, response) + + # 1. Store fails for error_pcp, stored in memory_pcp + # 2. Get fails for error_pcp, fetched from memory_pcp + # 3. Since fetched from lower priority, it should be stored in the highest priority cache provider + # 4. Store fails in error_pcp. Since write policy is WRITE_FIRST, it should not try to store in memory_pcp regardless of if the store in error_pcp fails + + assert cache.get(pot_request) + assert error_pcp.get_calls == 1 + assert error_pcp.store_calls == 2 # since highest priority, when fetched from lower priority, it should be stored in the highest priority cache provider + assert memory_pcp.get_calls == 1 + assert memory_pcp.store_calls == 1 + + assert logger.messages['warning'].count("Error from \"error\" PO Token cache provider: PoTokenCacheProviderError('something went wrong'); example bug report message") == 3 + + def test_cache_provider_unexpected_error_no_fallback(self, pot_request, ie, logger): + error_pcp = UnexpectedErrorPCP(ie, logger, {}) + cache = PoTokenCache( + cache_providers=[error_pcp], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, response) + assert cache.get(pot_request) is None + assert error_pcp.get_calls == 1 + assert error_pcp.store_calls == 1 + + assert logger.messages['error'].count("Error occurred with \"unexpected_error\" PO Token cache provider: ValueError('something went wrong'); example bug report message") == 2 + + def test_cache_provider_unexpected_error_fallback(self, pot_request, ie, logger): + error_pcp = UnexpectedErrorPCP(ie, logger, {}) + memory_pcp = create_memory_pcp(ie, logger, provider_key='memory') + + cache = PoTokenCache( + cache_providers=[error_pcp, memory_pcp], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, response) + + # 1. Store fails for error_pcp, stored in memory_pcp + # 2. Get fails for error_pcp, fetched from memory_pcp + # 3. Since fetched from lower priority, it should be stored in the highest priority cache provider + # 4. Store fails in error_pcp. Since write policy is WRITE_FIRST, it should not try to store in memory_pcp regardless of if the store in error_pcp fails + + assert cache.get(pot_request) + assert error_pcp.get_calls == 1 + assert error_pcp.store_calls == 2 # since highest priority, when fetched from lower priority, it should be stored in the highest priority cache provider + assert memory_pcp.get_calls == 1 + assert memory_pcp.store_calls == 1 + + assert logger.messages['error'].count("Error occurred with \"unexpected_error\" PO Token cache provider: ValueError('something went wrong'); example bug report message") == 3 + + def test_cache_provider_unavailable_no_fallback(self, pot_request, ie, logger): + provider = create_memory_pcp(ie, logger, available=False) + + cache = PoTokenCache( + cache_providers=[provider], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, response) + assert cache.get(pot_request) is None + assert provider.get_calls == 0 + assert provider.store_calls == 0 + assert provider.available_called_times + + def test_cache_provider_unavailable_fallback(self, pot_request, ie, logger): + provider_unavailable = create_memory_pcp(ie, logger, provider_key='unavailable', provider_name='unavailable', available=False) + provider_available = create_memory_pcp(ie, logger, provider_key='available', provider_name='available') + + cache = PoTokenCache( + cache_providers=[provider_unavailable, provider_available], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, response) + assert cache.get(pot_request) is not None + assert provider_unavailable.get_calls == 0 + assert provider_unavailable.store_calls == 0 + assert provider_available.get_calls == 1 + assert provider_available.store_calls == 1 + assert provider_unavailable.available_called_times + assert provider_available.available_called_times + + # should not even try to use the provider for the request + assert 'Attempting to fetch a PO Token response from "unavailable" provider' not in logger.messages['trace'] + assert 'Attempting to fetch a PO Token response from "available" provider' not in logger.messages['trace'] + + def test_available_not_called(self, ie, pot_request, logger): + # Test that the available method is not called when provider higher in the list is available + provider_unavailable = create_memory_pcp( + ie, logger, provider_key='unavailable', provider_name='unavailable', available=False) + provider_available = create_memory_pcp(ie, logger, provider_key='available', provider_name='available') + + logger.log_level = logger.LogLevel.INFO + + cache = PoTokenCache( + cache_providers=[provider_available, provider_unavailable], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, response, write_policy=CacheProviderWritePolicy.WRITE_FIRST) + assert cache.get(pot_request) is not None + assert provider_unavailable.get_calls == 0 + assert provider_unavailable.store_calls == 0 + assert provider_available.get_calls == 1 + assert provider_available.store_calls == 1 + assert provider_unavailable.available_called_times == 0 + assert provider_available.available_called_times + assert 'PO Token Cache Providers: available-0.0.0 (external), unavailable-0.0.0 (external, unavailable)' not in logger.messages.get('trace', []) + + def test_available_called_trace(self, ie, pot_request, logger): + # But if logging level is trace should call available (as part of debug logging) + provider_unavailable = create_memory_pcp( + ie, logger, provider_key='unavailable', provider_name='unavailable', available=False) + provider_available = create_memory_pcp(ie, logger, provider_key='available', provider_name='available') + + logger.log_level = logger.LogLevel.TRACE + + cache = PoTokenCache( + cache_providers=[provider_available, provider_unavailable], + cache_spec_providers=[ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={})], + logger=logger, + ) + + response = PoTokenResponse(EXAMPLE_PO_TOKEN) + cache.store(pot_request, response, write_policy=CacheProviderWritePolicy.WRITE_FIRST) + assert cache.get(pot_request) is not None + assert provider_unavailable.get_calls == 0 + assert provider_unavailable.store_calls == 0 + assert provider_available.get_calls == 1 + assert provider_available.store_calls == 1 + assert provider_unavailable.available_called_times + assert provider_available.available_called_times + assert 'PO Token Cache Providers: available-0.0.0 (external), unavailable-0.0.0 (external, unavailable)' in logger.messages.get('trace', []) + + def test_close(self, ie, pot_request, logger): + # Should call close on the cache providers and cache specs + memory_pcp = create_memory_pcp(ie, logger, provider_key='memory') + memory2_pcp = create_memory_pcp(ie, logger, provider_key='memory2') + + spec1 = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + spec2 = UnavailableCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + + cache = PoTokenCache( + cache_providers=[memory2_pcp, memory_pcp], + cache_spec_providers=[spec1, spec2], + logger=logger, + ) + + cache.close() + assert memory_pcp.close_called + assert memory2_pcp.close_called + assert spec1.close_called + assert spec2.close_called + + +class TestPoTokenRequestDirector: + + def test_request_pot_success(self, ie, pot_request, pot_cache, pot_provider, logger): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + director.register_provider(pot_provider) + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + + def test_request_and_cache(self, ie, pot_request, pot_cache, pot_provider, logger): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + director.register_provider(pot_provider) + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert pot_provider.request_called_times == 1 + assert pot_cache.get_calls == 1 + assert pot_cache.store_calls == 1 + + # Second request, should be cached + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert pot_cache.get_calls == 2 + assert pot_cache.store_calls == 1 + assert pot_provider.request_called_times == 1 + + def test_bypass_cache(self, ie, pot_request, pot_cache, logger, pot_provider): + pot_request.bypass_cache = True + + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + director.register_provider(pot_provider) + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert pot_provider.request_called_times == 1 + assert pot_cache.get_calls == 0 + assert pot_cache.store_calls == 1 + + # Second request, should not get from cache + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert pot_provider.request_called_times == 2 + assert pot_cache.get_calls == 0 + assert pot_cache.store_calls == 2 + + # POT is still cached, should get from cache + pot_request.bypass_cache = False + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert pot_provider.request_called_times == 2 + assert pot_cache.get_calls == 1 + assert pot_cache.store_calls == 2 + + def test_clean_pot_generate(self, ie, pot_request, pot_cache, logger): + # Token should be cleaned before returning + base_token = base64.urlsafe_b64encode(b'token').decode() + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = success_ptp(PoTokenResponse(base_token + '?extra=params'))(ie, logger, settings={}) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response == base_token + assert provider.request_called_times == 1 + + # Confirm the cleaned version was stored in the cache + cached_token = pot_cache.get(pot_request) + assert cached_token.po_token == base_token + + def test_clean_pot_cache(self, ie, pot_request, pot_cache, logger, pot_provider): + # Token retrieved from cache should be cleaned before returning + base_token = base64.urlsafe_b64encode(b'token').decode() + pot_cache.store(pot_request, PoTokenResponse(base_token + '?extra=params')) + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + director.register_provider(pot_provider) + + response = director.get_po_token(pot_request) + assert response == base_token + assert pot_cache.get_calls == 1 + assert pot_provider.request_called_times == 0 + + def test_cache_expires_at_none(self, ie, pot_request, pot_cache, logger, pot_provider): + # Should cache if expires_at=None in the response + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = success_ptp(PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=None))(ie, logger, settings={}) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert pot_cache.store_calls == 1 + assert pot_cache.get(pot_request).po_token == EXAMPLE_PO_TOKEN + + def test_cache_expires_at_positive(self, ie, pot_request, pot_cache, logger, pot_provider): + # Should cache if expires_at is a positive number in the response + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = success_ptp(PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=99999999999))(ie, logger, settings={}) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert pot_cache.store_calls == 1 + assert pot_cache.get(pot_request).po_token == EXAMPLE_PO_TOKEN + + @pytest.mark.parametrize('expires_at', [0, -1]) + def test_not_cache_expires_at(self, ie, pot_request, pot_cache, logger, pot_provider, expires_at): + # Should not cache if expires_at <= 0 in the response + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = success_ptp(PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=expires_at))(ie, logger, settings={}) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert pot_cache.store_calls == 0 + assert pot_cache.get(pot_request) is None + + def test_no_providers(self, ie, pot_request, pot_cache, logger): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + response = director.get_po_token(pot_request) + assert response is None + + def test_try_cache_no_providers(self, ie, pot_request, pot_cache, logger): + # Should still try the cache even if no providers are configured + pot_cache.store(pot_request, PoTokenResponse(EXAMPLE_PO_TOKEN)) + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + + def test_close(self, ie, pot_request, pot_cache, pot_provider, logger): + # Should call close on the pot cache and any providers + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + + provider2 = UnavailablePTP(ie, logger, {}) + director.register_provider(pot_provider) + director.register_provider(provider2) + + director.close() + assert pot_provider.close_called + assert provider2.close_called + assert pot_cache.close_called + + def test_pot_provider_preferences(self, pot_request, pot_cache, ie, logger): + pot_request.bypass_cache = True + provider_two_pot = base64.urlsafe_b64encode(b'token2').decode() + + example_provider = success_ptp(response=PoTokenResponse(EXAMPLE_PO_TOKEN), key='exampleone')(ie, logger, settings={}) + example_provider_two = success_ptp(response=PoTokenResponse(provider_two_pot), key='exampletwo')(ie, logger, settings={}) + + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + director.register_provider(example_provider) + director.register_provider(example_provider_two) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert example_provider.request_called_times == 1 + assert example_provider_two.request_called_times == 0 + + standard_preference_called = False + example_preference_called = False + + # Test that the provider preferences are respected + def standard_preference(provider, request, *_, **__): + nonlocal standard_preference_called + standard_preference_called = True + assert isinstance(provider, PoTokenProvider) + assert isinstance(request, PoTokenRequest) + return 1 + + def example_preference(provider, request, *_, **__): + nonlocal example_preference_called + example_preference_called = True + assert isinstance(provider, PoTokenProvider) + assert isinstance(request, PoTokenRequest) + if provider.PROVIDER_KEY == example_provider.PROVIDER_KEY: + return -100 + return 0 + + # test that it can handle multiple preferences + director.register_preference(example_preference) + director.register_preference(standard_preference) + + response = director.get_po_token(pot_request) + assert response == provider_two_pot + assert example_provider.request_called_times == 1 + assert example_provider_two.request_called_times == 1 + assert standard_preference_called + assert example_preference_called + + def test_unsupported_request_no_fallback(self, ie, logger, pot_cache, pot_request): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = UnsupportedPTP(ie, logger, {}) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response is None + assert provider.request_called_times == 1 + + def test_unsupported_request_fallback(self, ie, logger, pot_cache, pot_request, pot_provider): + # Should fallback to the next provider if the first one does not support the request + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = UnsupportedPTP(ie, logger, {}) + director.register_provider(provider) + director.register_provider(pot_provider) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert provider.request_called_times == 1 + assert pot_provider.request_called_times == 1 + assert 'PO Token Provider "unsupported" rejected this request, trying next available provider. Reason: unsupported request' in logger.messages['trace'] + + def test_unavailable_request_no_fallback(self, ie, logger, pot_cache, pot_request): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = UnavailablePTP(ie, logger, {}) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response is None + assert provider.request_called_times == 0 + assert provider.available_called_times + + def test_unavailable_request_fallback(self, ie, logger, pot_cache, pot_request, pot_provider): + # Should fallback to the next provider if the first one is unavailable + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = UnavailablePTP(ie, logger, {}) + director.register_provider(provider) + director.register_provider(pot_provider) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert provider.request_called_times == 0 + assert provider.available_called_times + assert pot_provider.request_called_times == 1 + assert pot_provider.available_called_times + # should not even try use the provider for the request + assert 'Attempting to fetch a PO Token from "unavailable" provider' not in logger.messages['trace'] + assert 'Attempting to fetch a PO Token from "success" provider' in logger.messages['trace'] + + def test_available_not_called(self, ie, logger, pot_cache, pot_request, pot_provider): + # Test that the available method is not called when provider higher in the list is available + logger.log_level = logger.LogLevel.INFO + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = UnavailablePTP(ie, logger, {}) + director.register_provider(pot_provider) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert provider.request_called_times == 0 + assert provider.available_called_times == 0 + assert pot_provider.request_called_times == 1 + assert pot_provider.available_called_times == 2 + assert 'PO Token Providers: success-0.0.1 (external), unavailable-0.0.0 (external, unavailable)' not in logger.messages.get('trace', []) + + def test_available_called_trace(self, ie, logger, pot_cache, pot_request, pot_provider): + # But if logging level is trace should call available (as part of debug logging) + logger.log_level = logger.LogLevel.TRACE + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = UnavailablePTP(ie, logger, {}) + director.register_provider(pot_provider) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert provider.request_called_times == 0 + assert provider.available_called_times == 1 + assert pot_provider.request_called_times == 1 + assert pot_provider.available_called_times == 3 + assert 'PO Token Providers: success-0.0.1 (external), unavailable-0.0.0 (external, unavailable)' in logger.messages['trace'] + + def test_provider_error_no_fallback_unexpected(self, ie, logger, pot_cache, pot_request): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = ErrorPTP(ie, logger, {}) + director.register_provider(provider) + pot_request.video_id = 'unexpected' + response = director.get_po_token(pot_request) + assert response is None + assert provider.request_called_times == 1 + assert "Error fetching PO Token from \"error\" provider: PoTokenProviderError('an error occurred'); please report this issue to the provider developer at https://error.example.com/issues ." in logger.messages['warning'] + + def test_provider_error_no_fallback_expected(self, ie, logger, pot_cache, pot_request): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = ErrorPTP(ie, logger, {}) + director.register_provider(provider) + pot_request.video_id = 'expected' + response = director.get_po_token(pot_request) + assert response is None + assert provider.request_called_times == 1 + assert "Error fetching PO Token from \"error\" provider: PoTokenProviderError('an error occurred')" in logger.messages['warning'] + + def test_provider_error_fallback(self, ie, logger, pot_cache, pot_request, pot_provider): + # Should fallback to the next provider if the first one raises an error + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = ErrorPTP(ie, logger, {}) + director.register_provider(provider) + director.register_provider(pot_provider) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert provider.request_called_times == 1 + assert pot_provider.request_called_times == 1 + assert "Error fetching PO Token from \"error\" provider: PoTokenProviderError('an error occurred'); please report this issue to the provider developer at https://error.example.com/issues ." in logger.messages['warning'] + + def test_provider_unexpected_error_no_fallback(self, ie, logger, pot_cache, pot_request): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = UnexpectedErrorPTP(ie, logger, {}) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response is None + assert provider.request_called_times == 1 + assert "Unexpected error when fetching PO Token from \"unexpected_error\" provider: ValueError('an unexpected error occurred'); please report this issue to the provider developer at https://unexpected.example.com/issues ." in logger.messages['error'] + + def test_provider_unexpected_error_fallback(self, ie, logger, pot_cache, pot_request, pot_provider): + # Should fallback to the next provider if the first one raises an unexpected error + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = UnexpectedErrorPTP(ie, logger, {}) + director.register_provider(provider) + director.register_provider(pot_provider) + + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert provider.request_called_times == 1 + assert pot_provider.request_called_times == 1 + assert "Unexpected error when fetching PO Token from \"unexpected_error\" provider: ValueError('an unexpected error occurred'); please report this issue to the provider developer at https://unexpected.example.com/issues ." in logger.messages['error'] + + def test_invalid_po_token_response_type(self, ie, logger, pot_cache, pot_request, pot_provider): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = InvalidPTP(ie, logger, {}) + director.register_provider(provider) + + pot_request.video_id = 'invalid_type' + + response = director.get_po_token(pot_request) + assert response is None + assert provider.request_called_times == 1 + assert 'Invalid PO Token response received from "invalid" provider: invalid-response; please report this issue to the provider developer at https://invalid.example.com/issues .' in logger.messages['error'] + + # Should fallback to next available provider + director.register_provider(pot_provider) + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert provider.request_called_times == 2 + assert pot_provider.request_called_times == 1 + + def test_invalid_po_token_response(self, ie, logger, pot_cache, pot_request, pot_provider): + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + provider = InvalidPTP(ie, logger, {}) + director.register_provider(provider) + + response = director.get_po_token(pot_request) + assert response is None + assert provider.request_called_times == 1 + assert "Invalid PO Token response received from \"invalid\" provider: PoTokenResponse(po_token='example-token?', expires_at='123'); please report this issue to the provider developer at https://invalid.example.com/issues ." in logger.messages['error'] + + # Should fallback to next available provider + director.register_provider(pot_provider) + response = director.get_po_token(pot_request) + assert response == EXAMPLE_PO_TOKEN + assert provider.request_called_times == 2 + assert pot_provider.request_called_times == 1 + + def test_copy_request_provider(self, ie, logger, pot_cache, pot_request): + + class BadProviderPTP(BaseMockPoTokenProvider): + _SUPPORTED_CONTEXTS = None + _SUPPORTED_CLIENTS = None + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + # Providers should not modify the request object, but we should guard against it + request.video_id = 'bad' + raise PoTokenProviderRejectedRequest('bad request') + + class GoodProviderPTP(BaseMockPoTokenProvider): + _SUPPORTED_CONTEXTS = None + _SUPPORTED_CLIENTS = None + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + return PoTokenResponse(base64.urlsafe_b64encode(request.video_id.encode()).decode()) + + director = PoTokenRequestDirector(logger=logger, cache=pot_cache) + + bad_provider = BadProviderPTP(ie, logger, {}) + good_provider = GoodProviderPTP(ie, logger, {}) + + director.register_provider(bad_provider) + director.register_provider(good_provider) + + pot_request.video_id = 'good' + response = director.get_po_token(pot_request) + assert response == base64.urlsafe_b64encode(b'good').decode() + assert bad_provider.request_called_times == 1 + assert good_provider.request_called_times == 1 + assert pot_request.video_id == 'good' + + +@pytest.mark.parametrize('spec, expected', [ + (None, False), + (PoTokenCacheSpec(key_bindings={'v': 'video-id'}, default_ttl=60, write_policy=None), False), # type: ignore + (PoTokenCacheSpec(key_bindings={'v': 'video-id'}, default_ttl='invalid'), False), # type: ignore + (PoTokenCacheSpec(key_bindings='invalid', default_ttl=60), False), # type: ignore + (PoTokenCacheSpec(key_bindings={2: 'video-id'}, default_ttl=60), False), # type: ignore + (PoTokenCacheSpec(key_bindings={'v': 2}, default_ttl=60), False), # type: ignore + (PoTokenCacheSpec(key_bindings={'v': None}, default_ttl=60), False), # type: ignore + + (PoTokenCacheSpec(key_bindings={'v': 'video_id', 'e': None}, default_ttl=60), True), + (PoTokenCacheSpec(key_bindings={'v': 'video_id'}, default_ttl=60, write_policy=CacheProviderWritePolicy.WRITE_FIRST), True), +]) +def test_validate_cache_spec(spec, expected): + assert validate_cache_spec(spec) == expected + + +@pytest.mark.parametrize('po_token', [ + 'invalid-token?', + '123', +]) +def test_clean_pot_fail(po_token): + with pytest.raises(ValueError, match='Invalid PO Token'): + clean_pot(po_token) + + +@pytest.mark.parametrize('po_token,expected', [ + ('TwAA/+8=', 'TwAA_-8='), + ('TwAA%5F%2D9VA6Q92v%5FvEQ4==?extra-param=2', 'TwAA_-9VA6Q92v_vEQ4='), +]) +def test_clean_pot(po_token, expected): + assert clean_pot(po_token) == expected + + +@pytest.mark.parametrize( + 'response, expected', + [ + (None, False), + (PoTokenResponse(None), False), + (PoTokenResponse(1), False), + (PoTokenResponse('invalid-token?'), False), + (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at='abc'), False), # type: ignore + (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=100), False), + (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=time.time() + 10000.0), False), # type: ignore + (PoTokenResponse(EXAMPLE_PO_TOKEN), True), + (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=-1), True), + (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=0), True), + (PoTokenResponse(EXAMPLE_PO_TOKEN, expires_at=int(time.time()) + 10000), True), + ], +) +def test_validate_pot_response(response, expected): + assert validate_response(response) == expected + + +def test_built_in_provider(ie, logger): + class BuiltinProviderDefaultT(BuiltinIEContentProvider, suffix='T'): + def is_available(self): + return True + + class BuiltinProviderCustomNameT(BuiltinIEContentProvider, suffix='T'): + PROVIDER_NAME = 'CustomName' + + def is_available(self): + return True + + class ExternalProviderDefaultT(IEContentProvider, suffix='T'): + def is_available(self): + return True + + class ExternalProviderCustomT(IEContentProvider, suffix='T'): + PROVIDER_NAME = 'custom' + PROVIDER_VERSION = '5.4b2' + + def is_available(self): + return True + + class ExternalProviderUnavailableT(IEContentProvider, suffix='T'): + def is_available(self) -> bool: + return False + + class BuiltinProviderUnavailableT(IEContentProvider, suffix='T'): + def is_available(self) -> bool: + return False + + built_in_default = BuiltinProviderDefaultT(ie=ie, logger=logger, settings={}) + built_in_custom_name = BuiltinProviderCustomNameT(ie=ie, logger=logger, settings={}) + built_in_unavailable = BuiltinProviderUnavailableT(ie=ie, logger=logger, settings={}) + external_default = ExternalProviderDefaultT(ie=ie, logger=logger, settings={}) + external_custom = ExternalProviderCustomT(ie=ie, logger=logger, settings={}) + external_unavailable = ExternalProviderUnavailableT(ie=ie, logger=logger, settings={}) + + assert provider_display_list([]) == 'none' + assert provider_display_list([built_in_default]) == 'BuiltinProviderDefault' + assert provider_display_list([external_unavailable]) == 'ExternalProviderUnavailable-0.0.0 (external, unavailable)' + assert provider_display_list([ + built_in_default, + built_in_custom_name, + external_default, + external_custom, + external_unavailable, + built_in_unavailable], + ) == 'BuiltinProviderDefault, CustomName, ExternalProviderDefault-0.0.0 (external), custom-5.4b2 (external), ExternalProviderUnavailable-0.0.0 (external, unavailable), BuiltinProviderUnavailable-0.0.0 (external, unavailable)' diff --git a/test/test_pot/test_pot_framework.py b/test/test_pot/test_pot_framework.py new file mode 100644 index 000000000..bc94653f4 --- /dev/null +++ b/test/test_pot/test_pot_framework.py @@ -0,0 +1,629 @@ +import pytest + +from yt_dlp.extractor.youtube.pot._provider import IEContentProvider +from yt_dlp.cookies import YoutubeDLCookieJar +from yt_dlp.utils.networking import HTTPHeaderDict +from yt_dlp.extractor.youtube.pot.provider import ( + PoTokenRequest, + PoTokenContext, + ExternalRequestFeature, + +) + +from yt_dlp.extractor.youtube.pot.cache import ( + PoTokenCacheProvider, + PoTokenCacheSpec, + PoTokenCacheSpecProvider, + CacheProviderWritePolicy, +) + +import yt_dlp.extractor.youtube.pot.cache as cache + +from yt_dlp.networking import Request +from yt_dlp.extractor.youtube.pot.provider import ( + PoTokenResponse, + PoTokenProvider, + PoTokenProviderRejectedRequest, + provider_bug_report_message, + register_provider, + register_preference, +) + +from yt_dlp.extractor.youtube.pot._registry import _pot_providers, _ptp_preferences, _pot_pcs_providers, _pot_cache_providers, _pot_cache_provider_preferences + + +class ExamplePTP(PoTokenProvider): + PROVIDER_NAME = 'example' + PROVIDER_VERSION = '0.0.1' + BUG_REPORT_LOCATION = 'https://example.com/issues' + + _SUPPORTED_CLIENTS = ('WEB',) + _SUPPORTED_CONTEXTS = (PoTokenContext.GVS, ) + + _SUPPORTED_EXTERNAL_REQUEST_FEATURES = ( + ExternalRequestFeature.PROXY_SCHEME_HTTP, + ExternalRequestFeature.PROXY_SCHEME_SOCKS5H, + ) + + def is_available(self) -> bool: + return True + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + return PoTokenResponse('example-token', expires_at=123) + + +class ExampleCacheProviderPCP(PoTokenCacheProvider): + + PROVIDER_NAME = 'example' + PROVIDER_VERSION = '0.0.1' + BUG_REPORT_LOCATION = 'https://example.com/issues' + + def is_available(self) -> bool: + return True + + def get(self, key: str): + return 'example-cache' + + def store(self, key: str, value: str, expires_at: int): + pass + + def delete(self, key: str): + pass + + +class ExampleCacheSpecProviderPCSP(PoTokenCacheSpecProvider): + + PROVIDER_NAME = 'example' + PROVIDER_VERSION = '0.0.1' + BUG_REPORT_LOCATION = 'https://example.com/issues' + + def generate_cache_spec(self, request: PoTokenRequest): + return PoTokenCacheSpec( + key_bindings={'field': 'example-key'}, + default_ttl=60, + write_policy=CacheProviderWritePolicy.WRITE_FIRST, + ) + + +class TestPoTokenProvider: + + def test_base_type(self): + assert issubclass(PoTokenProvider, IEContentProvider) + + def test_create_provider_missing_fetch_method(self, ie, logger): + class MissingMethodsPTP(PoTokenProvider): + def is_available(self) -> bool: + return True + + with pytest.raises(TypeError): + MissingMethodsPTP(ie=ie, logger=logger, settings={}) + + def test_create_provider_missing_available_method(self, ie, logger): + class MissingMethodsPTP(PoTokenProvider): + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + raise PoTokenProviderRejectedRequest('Not implemented') + + with pytest.raises(TypeError): + MissingMethodsPTP(ie=ie, logger=logger, settings={}) + + def test_barebones_provider(self, ie, logger): + class BarebonesProviderPTP(PoTokenProvider): + def is_available(self) -> bool: + return True + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + raise PoTokenProviderRejectedRequest('Not implemented') + + provider = BarebonesProviderPTP(ie=ie, logger=logger, settings={}) + assert provider.PROVIDER_NAME == 'BarebonesProvider' + assert provider.PROVIDER_KEY == 'BarebonesProvider' + assert provider.PROVIDER_VERSION == '0.0.0' + assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at (developer has not provided a bug report location) .' + + def test_example_provider_success(self, ie, logger, pot_request): + provider = ExamplePTP(ie=ie, logger=logger, settings={}) + assert provider.PROVIDER_NAME == 'example' + assert provider.PROVIDER_KEY == 'Example' + assert provider.PROVIDER_VERSION == '0.0.1' + assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at https://example.com/issues .' + assert provider.is_available() + + response = provider.request_pot(pot_request) + + assert response.po_token == 'example-token' + assert response.expires_at == 123 + + def test_provider_unsupported_context(self, ie, logger, pot_request): + provider = ExamplePTP(ie=ie, logger=logger, settings={}) + pot_request.context = PoTokenContext.PLAYER + + with pytest.raises(PoTokenProviderRejectedRequest): + provider.request_pot(pot_request) + + def test_provider_unsupported_client(self, ie, logger, pot_request): + provider = ExamplePTP(ie=ie, logger=logger, settings={}) + pot_request.innertube_context['client']['clientName'] = 'ANDROID' + + with pytest.raises(PoTokenProviderRejectedRequest): + provider.request_pot(pot_request) + + def test_provider_unsupported_proxy_scheme(self, ie, logger, pot_request): + provider = ExamplePTP(ie=ie, logger=logger, settings={}) + pot_request.request_proxy = 'socks4://example.com' + + with pytest.raises( + PoTokenProviderRejectedRequest, + match='External requests by "example" provider do not support proxy scheme "socks4". Supported proxy ' + 'schemes: http, socks5h', + ): + provider.request_pot(pot_request) + + pot_request.request_proxy = 'http://example.com' + + assert provider.request_pot(pot_request) + + def test_provider_ignore_external_request_features(self, ie, logger, pot_request): + class InternalPTP(ExamplePTP): + _SUPPORTED_EXTERNAL_REQUEST_FEATURES = None + + provider = InternalPTP(ie=ie, logger=logger, settings={}) + + pot_request.request_proxy = 'socks5://example.com' + assert provider.request_pot(pot_request) + pot_request.request_source_address = '0.0.0.0' + assert provider.request_pot(pot_request) + + def test_provider_unsupported_external_request_source_address(self, ie, logger, pot_request): + class InternalPTP(ExamplePTP): + _SUPPORTED_EXTERNAL_REQUEST_FEATURES = tuple() + + provider = InternalPTP(ie=ie, logger=logger, settings={}) + + pot_request.request_source_address = None + assert provider.request_pot(pot_request) + + pot_request.request_source_address = '0.0.0.0' + with pytest.raises( + PoTokenProviderRejectedRequest, + match='External requests by "example" provider do not support setting source address', + ): + provider.request_pot(pot_request) + + def test_provider_supported_external_request_source_address(self, ie, logger, pot_request): + class InternalPTP(ExamplePTP): + _SUPPORTED_EXTERNAL_REQUEST_FEATURES = ( + ExternalRequestFeature.SOURCE_ADDRESS, + ) + + provider = InternalPTP(ie=ie, logger=logger, settings={}) + + pot_request.request_source_address = None + assert provider.request_pot(pot_request) + + pot_request.request_source_address = '0.0.0.0' + assert provider.request_pot(pot_request) + + def test_provider_unsupported_external_request_tls_verification(self, ie, logger, pot_request): + class InternalPTP(ExamplePTP): + _SUPPORTED_EXTERNAL_REQUEST_FEATURES = tuple() + + provider = InternalPTP(ie=ie, logger=logger, settings={}) + + pot_request.request_verify_tls = True + assert provider.request_pot(pot_request) + + pot_request.request_verify_tls = False + with pytest.raises( + PoTokenProviderRejectedRequest, + match='External requests by "example" provider do not support ignoring TLS certificate failures', + ): + provider.request_pot(pot_request) + + def test_provider_supported_external_request_tls_verification(self, ie, logger, pot_request): + class InternalPTP(ExamplePTP): + _SUPPORTED_EXTERNAL_REQUEST_FEATURES = ( + ExternalRequestFeature.DISABLE_TLS_VERIFICATION, + ) + + provider = InternalPTP(ie=ie, logger=logger, settings={}) + + pot_request.request_verify_tls = True + assert provider.request_pot(pot_request) + + pot_request.request_verify_tls = False + assert provider.request_pot(pot_request) + + def test_provider_request_webpage(self, ie, logger, pot_request): + provider = ExamplePTP(ie=ie, logger=logger, settings={}) + + cookiejar = YoutubeDLCookieJar() + pot_request.request_headers = HTTPHeaderDict({'User-Agent': 'example-user-agent'}) + pot_request.request_proxy = 'socks5://example-proxy.com' + pot_request.request_cookiejar = cookiejar + + def mock_urlopen(request): + return request + + ie._downloader.urlopen = mock_urlopen + + sent_request = provider._request_webpage(Request( + 'https://example.com', + ), pot_request=pot_request) + + assert sent_request.url == 'https://example.com' + assert sent_request.headers['User-Agent'] == 'example-user-agent' + assert sent_request.proxies == {'all': 'socks5://example-proxy.com'} + assert sent_request.extensions['cookiejar'] is cookiejar + assert 'Requesting webpage' in logger.messages['info'] + + def test_provider_request_webpage_override(self, ie, logger, pot_request): + provider = ExamplePTP(ie=ie, logger=logger, settings={}) + + cookiejar_request = YoutubeDLCookieJar() + pot_request.request_headers = HTTPHeaderDict({'User-Agent': 'example-user-agent'}) + pot_request.request_proxy = 'socks5://example-proxy.com' + pot_request.request_cookiejar = cookiejar_request + + def mock_urlopen(request): + return request + + ie._downloader.urlopen = mock_urlopen + + sent_request = provider._request_webpage(Request( + 'https://example.com', + headers={'User-Agent': 'override-user-agent-override'}, + proxies={'http': 'http://example-proxy-override.com'}, + extensions={'cookiejar': YoutubeDLCookieJar()}, + ), pot_request=pot_request, note='Custom requesting webpage') + + assert sent_request.url == 'https://example.com' + assert sent_request.headers['User-Agent'] == 'override-user-agent-override' + assert sent_request.proxies == {'http': 'http://example-proxy-override.com'} + assert sent_request.extensions['cookiejar'] is not cookiejar_request + assert 'Custom requesting webpage' in logger.messages['info'] + + def test_provider_request_webpage_no_log(self, ie, logger, pot_request): + provider = ExamplePTP(ie=ie, logger=logger, settings={}) + + def mock_urlopen(request): + return request + + ie._downloader.urlopen = mock_urlopen + + sent_request = provider._request_webpage(Request( + 'https://example.com', + ), note=False) + + assert sent_request.url == 'https://example.com' + assert 'info' not in logger.messages + + def test_provider_request_webpage_no_pot_request(self, ie, logger): + provider = ExamplePTP(ie=ie, logger=logger, settings={}) + + def mock_urlopen(request): + return request + + ie._downloader.urlopen = mock_urlopen + + sent_request = provider._request_webpage(Request( + 'https://example.com', + ), pot_request=None) + + assert sent_request.url == 'https://example.com' + + def test_get_config_arg(self, ie, logger): + provider = ExamplePTP(ie=ie, logger=logger, settings={'abc': ['123D'], 'xyz': ['456a', '789B']}) + + assert provider._configuration_arg('abc') == ['123d'] + assert provider._configuration_arg('abc', default=['default']) == ['123d'] + assert provider._configuration_arg('ABC', default=['default']) == ['default'] + assert provider._configuration_arg('abc', casesense=True) == ['123D'] + assert provider._configuration_arg('xyz', casesense=False) == ['456a', '789b'] + + def test_require_class_end_with_suffix(self, ie, logger): + class InvalidSuffix(PoTokenProvider): + PROVIDER_NAME = 'invalid-suffix' + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + raise PoTokenProviderRejectedRequest('Not implemented') + + def is_available(self) -> bool: + return True + + provider = InvalidSuffix(ie=ie, logger=logger, settings={}) + + with pytest.raises(AssertionError): + provider.PROVIDER_KEY # noqa: B018 + + +class TestPoTokenCacheProvider: + + def test_base_type(self): + assert issubclass(PoTokenCacheProvider, IEContentProvider) + + def test_create_provider_missing_get_method(self, ie, logger): + class MissingMethodsPCP(PoTokenCacheProvider): + def store(self, key: str, value: str, expires_at: int): + pass + + def delete(self, key: str): + pass + + def is_available(self) -> bool: + return True + + with pytest.raises(TypeError): + MissingMethodsPCP(ie=ie, logger=logger, settings={}) + + def test_create_provider_missing_store_method(self, ie, logger): + class MissingMethodsPCP(PoTokenCacheProvider): + def get(self, key: str): + pass + + def delete(self, key: str): + pass + + def is_available(self) -> bool: + return True + + with pytest.raises(TypeError): + MissingMethodsPCP(ie=ie, logger=logger, settings={}) + + def test_create_provider_missing_delete_method(self, ie, logger): + class MissingMethodsPCP(PoTokenCacheProvider): + def get(self, key: str): + pass + + def store(self, key: str, value: str, expires_at: int): + pass + + def is_available(self) -> bool: + return True + + with pytest.raises(TypeError): + MissingMethodsPCP(ie=ie, logger=logger, settings={}) + + def test_create_provider_missing_is_available_method(self, ie, logger): + class MissingMethodsPCP(PoTokenCacheProvider): + def get(self, key: str): + pass + + def store(self, key: str, value: str, expires_at: int): + pass + + def delete(self, key: str): + pass + + with pytest.raises(TypeError): + MissingMethodsPCP(ie=ie, logger=logger, settings={}) + + def test_barebones_provider(self, ie, logger): + class BarebonesProviderPCP(PoTokenCacheProvider): + + def is_available(self) -> bool: + return True + + def get(self, key: str): + return 'example-cache' + + def store(self, key: str, value: str, expires_at: int): + pass + + def delete(self, key: str): + pass + + provider = BarebonesProviderPCP(ie=ie, logger=logger, settings={}) + assert provider.PROVIDER_NAME == 'BarebonesProvider' + assert provider.PROVIDER_KEY == 'BarebonesProvider' + assert provider.PROVIDER_VERSION == '0.0.0' + assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at (developer has not provided a bug report location) .' + + def test_create_provider_example(self, ie, logger): + provider = ExampleCacheProviderPCP(ie=ie, logger=logger, settings={}) + assert provider.PROVIDER_NAME == 'example' + assert provider.PROVIDER_KEY == 'ExampleCacheProvider' + assert provider.PROVIDER_VERSION == '0.0.1' + assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at https://example.com/issues .' + assert provider.is_available() + + def test_get_config_arg(self, ie, logger): + provider = ExampleCacheProviderPCP(ie=ie, logger=logger, settings={'abc': ['123D'], 'xyz': ['456a', '789B']}) + assert provider._configuration_arg('abc') == ['123d'] + assert provider._configuration_arg('abc', default=['default']) == ['123d'] + assert provider._configuration_arg('ABC', default=['default']) == ['default'] + assert provider._configuration_arg('abc', casesense=True) == ['123D'] + assert provider._configuration_arg('xyz', casesense=False) == ['456a', '789b'] + + def test_require_class_end_with_suffix(self, ie, logger): + class InvalidSuffix(PoTokenCacheProvider): + def get(self, key: str): + return 'example-cache' + + def store(self, key: str, value: str, expires_at: int): + pass + + def delete(self, key: str): + pass + + def is_available(self) -> bool: + return True + + provider = InvalidSuffix(ie=ie, logger=logger, settings={}) + + with pytest.raises(AssertionError): + provider.PROVIDER_KEY # noqa: B018 + + +class TestPoTokenCacheSpecProvider: + + def test_base_type(self): + assert issubclass(PoTokenCacheSpecProvider, IEContentProvider) + + def test_create_provider_missing_supports_method(self, ie, logger): + class MissingMethodsPCS(PoTokenCacheSpecProvider): + pass + + with pytest.raises(TypeError): + MissingMethodsPCS(ie=ie, logger=logger, settings={}) + + def test_create_provider_barebones(self, ie, pot_request, logger): + class BarebonesProviderPCSP(PoTokenCacheSpecProvider): + def generate_cache_spec(self, request: PoTokenRequest): + return PoTokenCacheSpec( + default_ttl=100, + key_bindings={}, + ) + + provider = BarebonesProviderPCSP(ie=ie, logger=logger, settings={}) + assert provider.PROVIDER_NAME == 'BarebonesProvider' + assert provider.PROVIDER_KEY == 'BarebonesProvider' + assert provider.PROVIDER_VERSION == '0.0.0' + assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at (developer has not provided a bug report location) .' + assert provider.is_available() + assert provider.generate_cache_spec(request=pot_request).default_ttl == 100 + assert provider.generate_cache_spec(request=pot_request).key_bindings == {} + assert provider.generate_cache_spec(request=pot_request).write_policy == CacheProviderWritePolicy.WRITE_ALL + + def test_create_provider_example(self, ie, pot_request, logger): + provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={}) + assert provider.PROVIDER_NAME == 'example' + assert provider.PROVIDER_KEY == 'ExampleCacheSpecProvider' + assert provider.PROVIDER_VERSION == '0.0.1' + assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at https://example.com/issues .' + assert provider.is_available() + assert provider.generate_cache_spec(pot_request) + assert provider.generate_cache_spec(pot_request).key_bindings == {'field': 'example-key'} + assert provider.generate_cache_spec(pot_request).default_ttl == 60 + assert provider.generate_cache_spec(pot_request).write_policy == CacheProviderWritePolicy.WRITE_FIRST + + def test_get_config_arg(self, ie, logger): + provider = ExampleCacheSpecProviderPCSP(ie=ie, logger=logger, settings={'abc': ['123D'], 'xyz': ['456a', '789B']}) + + assert provider._configuration_arg('abc') == ['123d'] + assert provider._configuration_arg('abc', default=['default']) == ['123d'] + assert provider._configuration_arg('ABC', default=['default']) == ['default'] + assert provider._configuration_arg('abc', casesense=True) == ['123D'] + assert provider._configuration_arg('xyz', casesense=False) == ['456a', '789b'] + + def test_require_class_end_with_suffix(self, ie, logger): + class InvalidSuffix(PoTokenCacheSpecProvider): + def generate_cache_spec(self, request: PoTokenRequest): + return None + + provider = InvalidSuffix(ie=ie, logger=logger, settings={}) + + with pytest.raises(AssertionError): + provider.PROVIDER_KEY # noqa: B018 + + +class TestPoTokenRequest: + def test_copy_request(self, pot_request): + copied_request = pot_request.copy() + + assert copied_request is not pot_request + assert copied_request.context == pot_request.context + assert copied_request.innertube_context == pot_request.innertube_context + assert copied_request.innertube_context is not pot_request.innertube_context + copied_request.innertube_context['client']['clientName'] = 'ANDROID' + assert pot_request.innertube_context['client']['clientName'] != 'ANDROID' + assert copied_request.innertube_host == pot_request.innertube_host + assert copied_request.session_index == pot_request.session_index + assert copied_request.player_url == pot_request.player_url + assert copied_request.is_authenticated == pot_request.is_authenticated + assert copied_request.visitor_data == pot_request.visitor_data + assert copied_request.data_sync_id == pot_request.data_sync_id + assert copied_request.video_id == pot_request.video_id + assert copied_request.request_cookiejar is pot_request.request_cookiejar + assert copied_request.request_proxy == pot_request.request_proxy + assert copied_request.request_headers == pot_request.request_headers + assert copied_request.request_headers is not pot_request.request_headers + assert copied_request.request_timeout == pot_request.request_timeout + assert copied_request.request_source_address == pot_request.request_source_address + assert copied_request.request_verify_tls == pot_request.request_verify_tls + assert copied_request.bypass_cache == pot_request.bypass_cache + + +def test_provider_bug_report_message(ie, logger): + provider = ExamplePTP(ie=ie, logger=logger, settings={}) + assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at https://example.com/issues .' + + message = provider_bug_report_message(provider) + assert message == '; please report this issue to the provider developer at https://example.com/issues .' + + message_before = provider_bug_report_message(provider, before='custom message!') + assert message_before == 'custom message! Please report this issue to the provider developer at https://example.com/issues .' + + +def test_register_provider(ie): + + @register_provider + class UnavailableProviderPTP(PoTokenProvider): + def is_available(self) -> bool: + return False + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + raise PoTokenProviderRejectedRequest('Not implemented') + + assert _pot_providers.value.get('UnavailableProvider') == UnavailableProviderPTP + _pot_providers.value.pop('UnavailableProvider') + + +def test_register_pot_preference(ie): + before = len(_ptp_preferences.value) + + @register_preference(ExamplePTP) + def unavailable_preference(provider: PoTokenProvider, request: PoTokenRequest): + return 1 + + assert len(_ptp_preferences.value) == before + 1 + + +def test_register_cache_provider(ie): + + @cache.register_provider + class UnavailableCacheProviderPCP(PoTokenCacheProvider): + def is_available(self) -> bool: + return False + + def get(self, key: str): + return 'example-cache' + + def store(self, key: str, value: str, expires_at: int): + pass + + def delete(self, key: str): + pass + + assert _pot_cache_providers.value.get('UnavailableCacheProvider') == UnavailableCacheProviderPCP + _pot_cache_providers.value.pop('UnavailableCacheProvider') + + +def test_register_cache_provider_spec(ie): + + @cache.register_spec + class UnavailableCacheProviderPCSP(PoTokenCacheSpecProvider): + def is_available(self) -> bool: + return False + + def generate_cache_spec(self, request: PoTokenRequest): + return None + + assert _pot_pcs_providers.value.get('UnavailableCacheProvider') == UnavailableCacheProviderPCSP + _pot_pcs_providers.value.pop('UnavailableCacheProvider') + + +def test_register_cache_provider_preference(ie): + before = len(_pot_cache_provider_preferences.value) + + @cache.register_preference(ExampleCacheProviderPCP) + def unavailable_preference(provider: PoTokenCacheProvider, request: PoTokenRequest): + return 1 + + assert len(_pot_cache_provider_preferences.value) == before + 1 + + +def test_logger_log_level(logger): + assert logger.LogLevel('INFO') == logger.LogLevel.INFO + assert logger.LogLevel('debuG') == logger.LogLevel.DEBUG + assert logger.LogLevel(10) == logger.LogLevel.DEBUG + assert logger.LogLevel('UNKNOWN') == logger.LogLevel.INFO diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 63e6e11b2..ea6264a0d 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -640,6 +640,7 @@ def __init__(self, params=None, auto_init=True): self._printed_messages = set() self._first_webpage_request = True self._post_hooks = [] + self._close_hooks = [] self._progress_hooks = [] self._postprocessor_hooks = [] self._download_retcode = 0 @@ -908,6 +909,11 @@ def add_post_hook(self, ph): """Add the post hook""" self._post_hooks.append(ph) + def add_close_hook(self, ch): + """Add a close hook, called when YoutubeDL.close() is called""" + assert callable(ch), 'Close hook must be callable' + self._close_hooks.append(ch) + def add_progress_hook(self, ph): """Add the download progress hook""" self._progress_hooks.append(ph) @@ -1016,6 +1022,9 @@ def close(self): self._request_director.close() del self._request_director + for close_hook in self._close_hooks: + close_hook() + def trouble(self, message=None, tb=None, is_error=True): """Determine action to take when a download problem appears. diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index 548e3aa93..28fff1969 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -23,6 +23,8 @@ _split_innertube_client, short_client_name, ) +from .pot._director import initialize_pot_director +from .pot.provider import PoTokenContext, PoTokenRequest from ..openload import PhantomJSwrapper from ...jsinterp import JSInterpreter from ...networking.exceptions import HTTPError @@ -66,6 +68,7 @@ urljoin, variadic, ) +from ...utils.networking import clean_headers, clean_proxies, select_proxy STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client' STREAMING_DATA_INITIAL_PO_TOKEN = '__yt_dlp_po_token' @@ -1809,6 +1812,11 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._code_cache = {} self._player_cache = {} + self._pot_director = None + + def _real_initialize(self): + super()._real_initialize() + self._pot_director = initialize_pot_director(self) def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live): lock = threading.Lock() @@ -2855,7 +2863,7 @@ def _get_config_po_token(self, client: str, context: _PoTokenContext): continue def fetch_po_token(self, client='web', context=_PoTokenContext.GVS, ytcfg=None, visitor_data=None, - data_sync_id=None, session_index=None, player_url=None, video_id=None, **kwargs): + data_sync_id=None, session_index=None, player_url=None, video_id=None, webpage=None, **kwargs): """ Fetch a PO Token for a given client and context. This function will validate required parameters for a given context and client. @@ -2869,10 +2877,14 @@ def fetch_po_token(self, client='web', context=_PoTokenContext.GVS, ytcfg=None, @param session_index: session index. @param player_url: player URL. @param video_id: video ID. + @param webpage: video webpage. @param kwargs: Additional arguments to pass down. May be more added in the future. @return: The fetched PO Token. None if it could not be fetched. """ + # TODO(future): This validation should be moved into pot framework. + # Some sort of middleware or validation provider perhaps? + # GVS WebPO Token is bound to visitor_data / Visitor ID when logged out. # Must have visitor_data for it to function. if player_url and context == _PoTokenContext.GVS and not visitor_data and not self.is_authenticated: @@ -2894,6 +2906,7 @@ def fetch_po_token(self, client='web', context=_PoTokenContext.GVS, ytcfg=None, f'Got a GVS PO Token for {client} client, but missing Data Sync ID for account. Formats may not work.' f'You may need to pass a Data Sync ID with --extractor-args "youtube:data_sync_id=XXX"') + self.write_debug(f'{video_id}: Retrieved a {context.value} PO Token for {client} client from config') return config_po_token # Require GVS WebPO Token if logged in for external fetching @@ -2903,7 +2916,7 @@ def fetch_po_token(self, client='web', context=_PoTokenContext.GVS, ytcfg=None, f'You may need to pass a Data Sync ID with --extractor-args "youtube:data_sync_id=XXX"') return - return self._fetch_po_token( + po_token = self._fetch_po_token( client=client, context=context.value, ytcfg=ytcfg, @@ -2912,11 +2925,66 @@ def fetch_po_token(self, client='web', context=_PoTokenContext.GVS, ytcfg=None, session_index=session_index, player_url=player_url, video_id=video_id, + video_webpage=webpage, **kwargs, ) + if po_token: + self.write_debug(f'{video_id}: Retrieved a {context.value} PO Token for {client} client') + return po_token + def _fetch_po_token(self, client, **kwargs): - """(Unstable) External PO Token fetch stub""" + context = kwargs.get('context') + + # Avoid fetching PO Tokens when not required + fetch_pot_policy = self._configuration_arg('fetch_pot', [''], ie_key=YoutubeIE)[0] + if fetch_pot_policy not in ('never', 'auto', 'always'): + fetch_pot_policy = 'auto' + if ( + fetch_pot_policy == 'never' + or ( + fetch_pot_policy == 'auto' + and _PoTokenContext(context) not in self._get_default_ytcfg(client)['PO_TOKEN_REQUIRED_CONTEXTS'] + ) + ): + return None + + headers = self.get_param('http_headers').copy() + proxies = self._downloader.proxies.copy() + clean_headers(headers) + clean_proxies(proxies, headers) + + innertube_host = self._select_api_hostname(None, default_client=client) + + pot_request = PoTokenRequest( + context=PoTokenContext(context), + innertube_context=traverse_obj(kwargs, ('ytcfg', 'INNERTUBE_CONTEXT')), + innertube_host=innertube_host, + internal_client_name=client, + session_index=kwargs.get('session_index'), + player_url=kwargs.get('player_url'), + video_webpage=kwargs.get('video_webpage'), + is_authenticated=self.is_authenticated, + visitor_data=kwargs.get('visitor_data'), + data_sync_id=kwargs.get('data_sync_id'), + video_id=kwargs.get('video_id'), + request_cookiejar=self._downloader.cookiejar, + + # All requests that would need to be proxied should be in the + # context of www.youtube.com or the innertube host + request_proxy=( + select_proxy('https://www.youtube.com', proxies) + or select_proxy(f'https://{innertube_host}', proxies) + ), + request_headers=headers, + request_timeout=self.get_param('socket_timeout'), + request_verify_tls=not self.get_param('nocheckcertificate'), + request_source_address=self.get_param('source_address'), + + bypass_cache=False, + ) + + return self._pot_director.get_po_token(pot_request) @staticmethod def _is_agegated(player_response): @@ -3074,8 +3142,9 @@ def append_client(*client_names): 'video_id': video_id, 'data_sync_id': data_sync_id if self.is_authenticated else None, 'player_url': player_url if require_js_player else None, + 'webpage': webpage, 'session_index': self._extract_session_index(master_ytcfg, player_ytcfg), - 'ytcfg': player_ytcfg, + 'ytcfg': player_ytcfg or self._get_default_ytcfg(client), } player_po_token = self.fetch_po_token( diff --git a/yt_dlp/extractor/youtube/pot/README.md b/yt_dlp/extractor/youtube/pot/README.md new file mode 100644 index 000000000..f39e29071 --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/README.md @@ -0,0 +1,309 @@ +# YoutubeIE PO Token Provider Framework + +As part of the YouTube extractor, we have a framework for providing PO Tokens programmatically. This can be used by plugins. + +Refer to the [PO Token Guide](https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide) for more information on PO Tokens. + +> [!TIP] +> If publishing a PO Token Provider plugin to GitHub, add the [yt-dlp-pot-provider](https://github.com/topics/yt-dlp-pot-provider) topic to your repository to help users find it. + + +## Public APIs + +- `yt_dlp.extractor.youtube.pot.cache` +- `yt_dlp.extractor.youtube.pot.provider` +- `yt_dlp.extractor.youtube.pot.utils` + +Everything else is internal-only and no guarantees are made about the API stability. + +> [!WARNING] +> We will try our best to maintain stability with the public APIs. +> However, due to the nature of extractors and YouTube, we may need to remove or change APIs in the future. +> If you are using these APIs outside yt-dlp plugins, please account for this by importing them safely. + +## PO Token Provider + +`yt_dlp.extractor.youtube.pot.provider` + +```python +from yt_dlp.extractor.youtube.pot.provider import ( + PoTokenRequest, + PoTokenContext, + PoTokenProvider, + PoTokenResponse, + PoTokenProviderError, + PoTokenProviderRejectedRequest, + register_provider, + register_preference, + ExternalRequestFeature, +) +from yt_dlp.networking.common import Request +from yt_dlp.extractor.youtube.pot.utils import get_webpo_content_binding +from yt_dlp.utils import traverse_obj +from yt_dlp.networking.exceptions import RequestError +import json + + +@register_provider +class MyPoTokenProviderPTP(PoTokenProvider): # Provider class name must end with "PTP" + PROVIDER_VERSION = '0.2.1' + # Define a unique display name for the provider + PROVIDER_NAME = 'my-provider' + BUG_REPORT_LOCATION = 'https://issues.example.com/report' + + # -- Validation shortcuts. Set these to None to disable. -- + + # Innertube Client Name. + # For example, "WEB", "ANDROID", "TVHTML5". + # For a list of WebPO client names, + # see yt_dlp.extractor.youtube.pot.utils.WEBPO_CLIENTS. + # Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS + # for a list of client names currently supported by the YouTube extractor. + _SUPPORTED_CLIENTS = ('WEB', 'TVHTML5') + + _SUPPORTED_CONTEXTS = ( + PoTokenContext.GVS, + ) + + # If your provider makes external requests to websites (i.e. to youtube.com) + # using another library or service (i.e., not _request_webpage), + # set the request features that are supported here. + # If only using _request_webpage to make external requests, set this to None. + _SUPPORTED_EXTERNAL_REQUEST_FEATURES = ( + ExternalRequestFeature.PROXY_SCHEME_HTTP, + ExternalRequestFeature.SOURCE_ADDRESS, + ExternalRequestFeature.DISABLE_TLS_VERIFICATION + ) + + def is_available(self) -> bool: + """ + Check if the provider is available (e.g. all required dependencies are available) + This is used to determine if the provider should be used and to provide debug information. + + IMPORTANT: This method SHOULD NOT make any network requests or perform any expensive operations. + + Since this is called multiple times, we recommend caching the result. + """ + return True + + def close(self): + # Optional close hook, called when YoutubeDL is closed. + pass + + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + # ℹ️ If you need to validate the request before making the request to the external source. + # Raise yt_dlp.extractor.youtube.pot.provider.PoTokenProviderRejectedRequest if the request is not supported. + if request.is_authenticated: + raise PoTokenProviderRejectedRequest( + 'This provider does not support authenticated requests' + ) + + # ℹ️ Settings are pulled from extractor args passed to yt-dlp with the key `youtubepot-`. + # For this example, the extractor arg would be: + # `--extractor-args "youtubepot-mypotokenprovider:url=https://custom.example.com/get_pot"` + external_provider_url = self._configuration_arg( + 'url', default=['https://provider.example.com/get_pot'])[0] + + # See below for logging guidelines + self.logger.trace(f'Using external provider URL: {external_provider_url}') + + # You should use the internal HTTP client to make requests where possible, + # as it will handle cookies and other networking settings passed to yt-dlp. + try: + # See docstring in _request_webpage method for request tips + response = self._request_webpage( + Request(external_provider_url, data=json.dumps({ + 'content_binding': get_webpo_content_binding(request), + 'proxy': request.request_proxy, + 'headers': request.request_headers, + 'source_address': request.request_source_address, + 'verify_tls': request.request_verify_tls, + # Important: If your provider has its own caching, please respect `bypass_cache`. + # This may be used in the future to request a fresh PO Token if required. + 'do_not_cache': request.bypass_cache, + }).encode(), proxies={'all': None}), + pot_request=request, + note=( + f'Requesting {request.context.value} PO Token ' + f'for {request.internal_client_name} client from external provider'), + ) + + except RequestError as e: + # ℹ️ If there is an error, raise PoTokenProviderError. + # You can specify whether it is expected or not. If it is unexpected, + # the log will include a link to the bug report location (BUG_REPORT_LOCATION). + raise PoTokenProviderError( + 'Networking error while fetching to get PO Token from external provider', + expected=True + ) from e + + # Note: PO Token is expected to be base64url encoded + po_token = traverse_obj(response, 'po_token') + if not po_token: + raise PoTokenProviderError( + 'Bad PO Token Response from external provider', + expected=False + ) + + return PoTokenResponse( + po_token=po_token, + # Optional, add a custom expiration timestamp for the token. Use for caching. + # By default, yt-dlp will use the default ttl from a registered cache spec (see below) + # Set to 0 or -1 to not cache this response. + expires_at=None, + ) + + +# If there are multiple PO Token Providers that can handle the same PoTokenRequest, +# you can define a preference function to increase/decrease the priority of providers. + +@register_preference(MyPoTokenProviderPTP) +def my_provider_preference(provider: PoTokenProvider, request: PoTokenRequest) -> int: + return 50 +``` + +## Logging Guidelines + +- Use the `self.logger` object to log messages. +- When making HTTP requests or any other expensive operation, use `self.logger.info` to log a message to standard non-verbose output. + - This lets users know what is happening when a time-expensive operation is taking place. + - It is recommended to include the PO Token context and internal client name in the message if possible. + - For example, `self.logger.info(f'Requesting {request.context.value} PO Token for {request.internal_client_name} client from external provider')`. +- Use `self.logger.debug` to log a message to the verbose output (`--verbose`). + - For debugging information visible to users posting verbose logs. + - Try to not log too much, prefer using trace logging for detailed debug messages. +- Use `self.logger.trace` to log a message to the PO Token debug output (`--extractor-args "youtube:pot_trace=true"`). + - Log as much as you like here as needed for debugging your provider. +- Avoid logging PO Tokens or any sensitive information to debug or info output. + +## Debugging + +- Use `-v --extractor-args "youtube:pot_trace=true"` to enable PO Token debug output. + +## Caching + +> [!WARNING] +> The following describes more advance features that most users/developers will not need to use. + +> [!IMPORTANT] +> yt-dlp currently has a built-in LRU Memory Cache Provider and a cache spec provider for WebPO Tokens. +> You should only need to implement cache providers if you want an external cache, or a cache spec if you are handling non-WebPO Tokens. + +### Cache Providers + +`yt_dlp.extractor.youtube.pot.cache` + +```python +from yt_dlp.extractor.youtube.pot.cache import ( + PoTokenCacheProvider, + register_preference, + register_provider +) + +from yt_dlp.extractor.youtube.pot.provider import PoTokenRequest + + +@register_provider +class MyCacheProviderPCP(PoTokenCacheProvider): # Provider class name must end with "PCP" + PROVIDER_VERSION = '0.1.0' + # Define a unique display name for the provider + PROVIDER_NAME = 'my-cache-provider' + BUG_REPORT_LOCATION = 'https://issues.example.com/report' + + def is_available(self) -> bool: + """ + Check if the provider is available (e.g. all required dependencies are available) + This is used to determine if the provider should be used and to provide debug information. + + IMPORTANT: This method SHOULD NOT make any network requests or perform any expensive operations. + + Since this is called multiple times, we recommend caching the result. + """ + return True + + def get(self, key: str): + # ℹ️ Similar to PO Token Providers, Cache Providers and Cache Spec Providers + # are passed down extractor args matching key youtubepot-. + some_setting = self._configuration_arg('some_setting', default=['default_value'])[0] + return self.my_cache.get(key) + + def store(self, key: str, value: str, expires_at: int): + # ⚠ expires_at MUST be respected. + # Cache entries should not be returned if they have expired. + self.my_cache.store(key, value, expires_at) + + def delete(self, key: str): + self.my_cache.delete(key) + + def close(self): + # Optional close hook, called when the YoutubeDL instance is closed. + pass + +# If there are multiple PO Token Cache Providers available, you can +# define a preference function to increase/decrease the priority of providers. + +# IMPORTANT: Providers should be in preference of cache lookup time. +# For example, a memory cache should have a higher preference than a disk cache. + +# VERY IMPORTANT: yt-dlp has a built-in memory cache with a priority of 10000. +# Your cache provider should be lower than this. + + +@register_preference(MyCacheProviderPCP) +def my_cache_preference(provider: PoTokenCacheProvider, request: PoTokenRequest) -> int: + return 50 +``` + +### Cache Specs + +`yt_dlp.extractor.youtube.pot.cache` + +These are used to provide information on how to cache a particular PO Token Request. +You might have a different cache spec for different kinds of PO Tokens. + +```python +from yt_dlp.extractor.youtube.pot.cache import ( + PoTokenCacheSpec, + PoTokenCacheSpecProvider, + CacheProviderWritePolicy, + register_spec, +) +from yt_dlp.utils import traverse_obj +from yt_dlp.extractor.youtube.pot.provider import PoTokenRequest + + +@register_spec +class MyCacheSpecProviderPCSP(PoTokenCacheSpecProvider): # Provider class name must end with "PCSP" + PROVIDER_VERSION = '0.1.0' + # Define a unique display name for the provider + PROVIDER_NAME = 'mycachespec' + BUG_REPORT_LOCATION = 'https://issues.example.com/report' + + def generate_cache_spec(self, request: PoTokenRequest): + + client_name = traverse_obj(request.innertube_context, ('client', 'clientName')) + if client_name != 'ANDROID': + # ℹ️ If the request is not supported by the cache spec, return None + return None + + # Generate a cache spec for the request + return PoTokenCacheSpec( + # Key bindings to uniquely identify the request. These are used to generate a cache key. + key_bindings={ + 'client_name': client_name, + 'content_binding': 'unique_content_binding', + 'ip': traverse_obj(request.innertube_context, ('client', 'remoteHost')), + 'source_address': request.request_source_address, + 'proxy': request.request_proxy, + }, + # Default Cache TTL in seconds + default_ttl=21600, + + # Optional: Specify a write policy. + # WRITE_FIRST will write to the highest priority provider only, + # whereas WRITE_ALL will write to all providers. + # WRITE_FIRST may be useful if the PO Token is short-lived + # and there is no use writing to all providers. + write_policy=CacheProviderWritePolicy.WRITE_ALL, + ) +``` \ No newline at end of file diff --git a/yt_dlp/extractor/youtube/pot/__init__.py b/yt_dlp/extractor/youtube/pot/__init__.py new file mode 100644 index 000000000..febcee010 --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/__init__.py @@ -0,0 +1,3 @@ +# Trigger import of built-in providers +from ._builtin.memory_cache import MemoryLRUPCP as _MemoryLRUPCP # noqa: F401 +from ._builtin.webpo_cachespec import WebPoPCSP as _WebPoPCSP # noqa: F401 diff --git a/yt_dlp/extractor/youtube/pot/_builtin/__init__.py b/yt_dlp/extractor/youtube/pot/_builtin/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/yt_dlp/extractor/youtube/pot/_builtin/memory_cache.py b/yt_dlp/extractor/youtube/pot/_builtin/memory_cache.py new file mode 100644 index 000000000..9c913e8c9 --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/_builtin/memory_cache.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import datetime as dt +import typing +from threading import Lock + +from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider +from yt_dlp.extractor.youtube.pot._registry import _pot_memory_cache +from yt_dlp.extractor.youtube.pot.cache import ( + PoTokenCacheProvider, + register_preference, + register_provider, +) + + +def initialize_global_cache(max_size: int): + if _pot_memory_cache.value.get('cache') is None: + _pot_memory_cache.value['cache'] = {} + _pot_memory_cache.value['lock'] = Lock() + _pot_memory_cache.value['max_size'] = max_size + + if _pot_memory_cache.value['max_size'] != max_size: + raise ValueError('Cannot change max_size of initialized global memory cache') + + return ( + _pot_memory_cache.value['cache'], + _pot_memory_cache.value['lock'], + _pot_memory_cache.value['max_size'], + ) + + +@register_provider +class MemoryLRUPCP(PoTokenCacheProvider, BuiltinIEContentProvider): + PROVIDER_NAME = 'memory' + DEFAULT_CACHE_SIZE = 25 + + def __init__( + self, + *args, + initialize_cache: typing.Callable[[int], tuple[dict[str, tuple[str, int]], Lock, int]] = initialize_global_cache, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.cache, self.lock, self.max_size = initialize_cache(self.DEFAULT_CACHE_SIZE) + + def is_available(self) -> bool: + return True + + def get(self, key: str) -> str | None: + with self.lock: + if key not in self.cache: + return None + value, expires_at = self.cache.pop(key) + if expires_at < int(dt.datetime.now(dt.timezone.utc).timestamp()): + return None + self.cache[key] = (value, expires_at) + return value + + def store(self, key: str, value: str, expires_at: int): + with self.lock: + if expires_at < int(dt.datetime.now(dt.timezone.utc).timestamp()): + return + if key in self.cache: + self.cache.pop(key) + self.cache[key] = (value, expires_at) + if len(self.cache) > self.max_size: + oldest_key = next(iter(self.cache)) + self.cache.pop(oldest_key) + + def delete(self, key: str): + with self.lock: + self.cache.pop(key, None) + + +@register_preference(MemoryLRUPCP) +def memorylru_preference(*_, **__): + # Memory LRU Cache SHOULD be the highest priority + return 10000 diff --git a/yt_dlp/extractor/youtube/pot/_builtin/webpo_cachespec.py b/yt_dlp/extractor/youtube/pot/_builtin/webpo_cachespec.py new file mode 100644 index 000000000..426b815c7 --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/_builtin/webpo_cachespec.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider +from yt_dlp.extractor.youtube.pot.cache import ( + CacheProviderWritePolicy, + PoTokenCacheSpec, + PoTokenCacheSpecProvider, + register_spec, +) +from yt_dlp.extractor.youtube.pot.provider import ( + PoTokenRequest, +) +from yt_dlp.extractor.youtube.pot.utils import ContentBindingType, get_webpo_content_binding +from yt_dlp.utils import traverse_obj + + +@register_spec +class WebPoPCSP(PoTokenCacheSpecProvider, BuiltinIEContentProvider): + PROVIDER_NAME = 'webpo' + + def generate_cache_spec(self, request: PoTokenRequest) -> PoTokenCacheSpec | None: + bind_to_visitor_id = self._configuration_arg( + 'bind_to_visitor_id', default=['true'])[0] == 'true' + + content_binding, content_binding_type = get_webpo_content_binding( + request, bind_to_visitor_id=bind_to_visitor_id) + + if not content_binding or not content_binding_type: + return None + + write_policy = CacheProviderWritePolicy.WRITE_ALL + if content_binding_type == ContentBindingType.VIDEO_ID: + write_policy = CacheProviderWritePolicy.WRITE_FIRST + + return PoTokenCacheSpec( + key_bindings={ + 't': 'webpo', + 'cb': content_binding, + 'cbt': content_binding_type.value, + 'ip': traverse_obj(request.innertube_context, ('client', 'remoteHost')), + 'sa': request.request_source_address, + 'px': request.request_proxy, + }, + # Integrity token response usually states it has a ttl of 12 hours (43200 seconds). + # We will default to 6 hours to be safe. + default_ttl=21600, + write_policy=write_policy, + ) diff --git a/yt_dlp/extractor/youtube/pot/_director.py b/yt_dlp/extractor/youtube/pot/_director.py new file mode 100644 index 000000000..aaf1d5290 --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/_director.py @@ -0,0 +1,468 @@ +from __future__ import annotations + +import base64 +import binascii +import dataclasses +import datetime as dt +import hashlib +import json +import typing +import urllib.parse +from collections.abc import Iterable + +from yt_dlp.extractor.youtube.pot._provider import ( + BuiltinIEContentProvider, + IEContentProvider, + IEContentProviderLogger, +) +from yt_dlp.extractor.youtube.pot._registry import ( + _pot_cache_provider_preferences, + _pot_cache_providers, + _pot_pcs_providers, + _pot_providers, + _ptp_preferences, +) +from yt_dlp.extractor.youtube.pot.cache import ( + CacheProviderWritePolicy, + PoTokenCacheProvider, + PoTokenCacheProviderError, + PoTokenCacheSpec, + PoTokenCacheSpecProvider, +) +from yt_dlp.extractor.youtube.pot.provider import ( + PoTokenProvider, + PoTokenProviderError, + PoTokenProviderRejectedRequest, + PoTokenRequest, + PoTokenResponse, + provider_bug_report_message, +) +from yt_dlp.utils import bug_reports_message, format_field, join_nonempty + +if typing.TYPE_CHECKING: + from yt_dlp.extractor.youtube.pot.cache import CacheProviderPreference + from yt_dlp.extractor.youtube.pot.provider import Preference + + +class YoutubeIEContentProviderLogger(IEContentProviderLogger): + def __init__(self, ie, prefix, log_level: IEContentProviderLogger.LogLevel | None = None): + self.__ie = ie + self.prefix = prefix + self.log_level = log_level if log_level is not None else self.LogLevel.INFO + + def _format_msg(self, message: str): + prefixstr = format_field(self.prefix, None, '[%s] ') + return f'{prefixstr}{message}' + + def trace(self, message: str): + if self.log_level <= self.LogLevel.TRACE: + self.__ie.write_debug(self._format_msg('TRACE: ' + message)) + + def debug(self, message: str): + if self.log_level <= self.LogLevel.DEBUG: + self.__ie.write_debug(self._format_msg(message)) + + def info(self, message: str): + if self.log_level <= self.LogLevel.INFO: + self.__ie.to_screen(self._format_msg(message)) + + def warning(self, message: str, *, once=False): + if self.log_level <= self.LogLevel.WARNING: + self.__ie.report_warning(self._format_msg(message), only_once=once) + + def error(self, message: str): + if self.log_level <= self.LogLevel.ERROR: + self.__ie._downloader.report_error(self._format_msg(message), is_error=False) + + +class PoTokenCache: + + def __init__( + self, + logger: IEContentProviderLogger, + cache_providers: list[PoTokenCacheProvider], + cache_spec_providers: list[PoTokenCacheSpecProvider], + cache_provider_preferences: list[CacheProviderPreference] | None = None, + ): + self.cache_providers: dict[str, PoTokenCacheProvider] = { + provider.PROVIDER_KEY: provider for provider in (cache_providers or [])} + self.cache_provider_preferences: list[CacheProviderPreference] = cache_provider_preferences or [] + self.cache_spec_providers: dict[str, PoTokenCacheSpecProvider] = { + provider.PROVIDER_KEY: provider for provider in (cache_spec_providers or [])} + self.logger = logger + + def _get_cache_providers(self, request: PoTokenRequest) -> Iterable[PoTokenCacheProvider]: + """Sorts available cache providers by preference, given a request""" + preferences = { + provider: sum(pref(provider, request) for pref in self.cache_provider_preferences) + for provider in self.cache_providers.values() + } + if self.logger.log_level <= self.logger.LogLevel.TRACE: + # calling is_available() for every PO Token provider upfront may have some overhead + self.logger.trace(f'PO Token Cache Providers: {provider_display_list(self.cache_providers.values())}') + self.logger.trace('Cache Provider preferences for this request: {}'.format(', '.join( + f'{provider.PROVIDER_KEY}={pref}' for provider, pref in preferences.items()))) + + return ( + provider for provider in sorted( + self.cache_providers.values(), key=preferences.get, reverse=True) if provider.is_available()) + + def _get_cache_spec(self, request: PoTokenRequest) -> PoTokenCacheSpec | None: + for provider in self.cache_spec_providers.values(): + if not provider.is_available(): + continue + try: + spec = provider.generate_cache_spec(request) + if not spec: + continue + if not validate_cache_spec(spec): + self.logger.error( + f'PoTokenCacheSpecProvider "{provider.PROVIDER_KEY}" generate_cache_spec() ' + f'returned invalid spec {spec}{provider_bug_report_message(provider)}') + continue + spec = dataclasses.replace(spec, _provider=provider) + self.logger.trace( + f'Retrieved cache spec {spec} from cache spec provider "{provider.PROVIDER_NAME}"') + return spec + except Exception as e: + self.logger.error( + f'Error occurred with "{provider.PROVIDER_NAME}" PO Token cache spec provider: ' + f'{e!r}{provider_bug_report_message(provider)}') + continue + return None + + def _generate_key_bindings(self, spec: PoTokenCacheSpec) -> dict[str, str]: + bindings_cleaned = { + **{k: v for k, v in spec.key_bindings.items() if v is not None}, + # Allow us to invalidate caches if such need arises + '_dlp_cache': 'v1', + } + if spec._provider: + bindings_cleaned['_p'] = spec._provider.PROVIDER_KEY + self.logger.trace(f'Generated cache key bindings: {bindings_cleaned}') + return bindings_cleaned + + def _generate_key(self, bindings: dict) -> str: + binding_string = ''.join(repr(dict(sorted(bindings.items())))) + return hashlib.sha256(binding_string.encode()).hexdigest() + + def get(self, request: PoTokenRequest) -> PoTokenResponse | None: + spec = self._get_cache_spec(request) + if not spec: + self.logger.trace('No cache spec available for this request, unable to fetch from cache') + return None + + cache_key = self._generate_key(self._generate_key_bindings(spec)) + self.logger.trace(f'Attempting to access PO Token cache using key: {cache_key}') + + for idx, provider in enumerate(self._get_cache_providers(request)): + try: + self.logger.trace( + f'Attempting to fetch PO Token response from "{provider.PROVIDER_NAME}" cache provider') + cache_response = provider.get(cache_key) + if not cache_response: + continue + try: + po_token_response = PoTokenResponse(**json.loads(cache_response)) + except (TypeError, ValueError, json.JSONDecodeError): + po_token_response = None + if not validate_response(po_token_response): + self.logger.error( + f'Invalid PO Token response retrieved from cache provider "{provider.PROVIDER_NAME}": ' + f'{cache_response}{provider_bug_report_message(provider)}') + provider.delete(cache_key) + continue + self.logger.trace( + f'PO Token response retrieved from cache using "{provider.PROVIDER_NAME}" provider: ' + f'{po_token_response}') + if idx > 0: + # Write back to the highest priority cache provider, + # so we stop trying to fetch from lower priority providers + self.logger.trace('Writing PO Token response to highest priority cache provider') + self.store(request, po_token_response, write_policy=CacheProviderWritePolicy.WRITE_FIRST) + + return po_token_response + except PoTokenCacheProviderError as e: + self.logger.warning( + f'Error from "{provider.PROVIDER_NAME}" PO Token cache provider: ' + f'{e!r}{provider_bug_report_message(provider) if not e.expected else ""}') + continue + except Exception as e: + self.logger.error( + f'Error occurred with "{provider.PROVIDER_NAME}" PO Token cache provider: ' + f'{e!r}{provider_bug_report_message(provider)}', + ) + continue + return None + + def store( + self, + request: PoTokenRequest, + response: PoTokenResponse, + write_policy: CacheProviderWritePolicy | None = None, + ): + spec = self._get_cache_spec(request) + if not spec: + self.logger.trace('No cache spec available for this request. Not caching.') + return + + if not validate_response(response): + self.logger.error( + f'Invalid PO Token response provided to PoTokenCache.store(): ' + f'{response}{bug_reports_message()}') + return + + cache_key = self._generate_key(self._generate_key_bindings(spec)) + self.logger.trace(f'Attempting to access PO Token cache using key: {cache_key}') + + default_expires_at = int(dt.datetime.now(dt.timezone.utc).timestamp()) + spec.default_ttl + cache_response = dataclasses.replace(response, expires_at=response.expires_at or default_expires_at) + + write_policy = write_policy or spec.write_policy + self.logger.trace(f'Using write policy: {write_policy}') + + for idx, provider in enumerate(self._get_cache_providers(request)): + try: + self.logger.trace( + f'Caching PO Token response in "{provider.PROVIDER_NAME}" cache provider ' + f'(key={cache_key}, expires_at={cache_response.expires_at})') + provider.store( + key=cache_key, + value=json.dumps(dataclasses.asdict(cache_response)), + expires_at=cache_response.expires_at) + except PoTokenCacheProviderError as e: + self.logger.warning( + f'Error from "{provider.PROVIDER_NAME}" PO Token cache provider: ' + f'{e!r}{provider_bug_report_message(provider) if not e.expected else ""}') + except Exception as e: + self.logger.error( + f'Error occurred with "{provider.PROVIDER_NAME}" PO Token cache provider: ' + f'{e!r}{provider_bug_report_message(provider)}') + + # WRITE_FIRST should not write to lower priority providers in the case the highest priority provider fails + if idx == 0 and write_policy == CacheProviderWritePolicy.WRITE_FIRST: + return + + def close(self): + for provider in self.cache_providers.values(): + provider.close() + for spec_provider in self.cache_spec_providers.values(): + spec_provider.close() + + +class PoTokenRequestDirector: + + def __init__(self, logger: IEContentProviderLogger, cache: PoTokenCache): + self.providers: dict[str, PoTokenProvider] = {} + self.preferences: list[Preference] = [] + self.cache = cache + self.logger = logger + + def register_provider(self, provider: PoTokenProvider): + self.providers[provider.PROVIDER_KEY] = provider + + def register_preference(self, preference: Preference): + self.preferences.append(preference) + + def _get_providers(self, request: PoTokenRequest) -> Iterable[PoTokenProvider]: + """Sorts available providers by preference, given a request""" + preferences = { + provider: sum(pref(provider, request) for pref in self.preferences) + for provider in self.providers.values() + } + if self.logger.log_level <= self.logger.LogLevel.TRACE: + # calling is_available() for every PO Token provider upfront may have some overhead + self.logger.trace(f'PO Token Providers: {provider_display_list(self.providers.values())}') + self.logger.trace('Provider preferences for this request: {}'.format(', '.join( + f'{provider.PROVIDER_NAME}={pref}' for provider, pref in preferences.items()))) + + return ( + provider for provider in sorted( + self.providers.values(), key=preferences.get, reverse=True) + if provider.is_available() + ) + + def _get_po_token(self, request) -> PoTokenResponse | None: + for provider in self._get_providers(request): + try: + self.logger.trace( + f'Attempting to fetch a PO Token from "{provider.PROVIDER_NAME}" provider') + response = provider.request_pot(request.copy()) + except PoTokenProviderRejectedRequest as e: + self.logger.trace( + f'PO Token Provider "{provider.PROVIDER_NAME}" rejected this request, ' + f'trying next available provider. Reason: {e}') + continue + except PoTokenProviderError as e: + self.logger.warning( + f'Error fetching PO Token from "{provider.PROVIDER_NAME}" provider: ' + f'{e!r}{provider_bug_report_message(provider) if not e.expected else ""}') + continue + except Exception as e: + self.logger.error( + f'Unexpected error when fetching PO Token from "{provider.PROVIDER_NAME}" provider: ' + f'{e!r}{provider_bug_report_message(provider)}') + continue + + self.logger.trace(f'PO Token response from "{provider.PROVIDER_NAME}" provider: {response}') + + if not validate_response(response): + self.logger.error( + f'Invalid PO Token response received from "{provider.PROVIDER_NAME}" provider: ' + f'{response}{provider_bug_report_message(provider)}') + continue + + return response + + self.logger.trace('No PO Token providers were able to provide a valid PO Token') + return None + + def get_po_token(self, request: PoTokenRequest) -> str | None: + if not request.bypass_cache: + if pot_response := self.cache.get(request): + return clean_pot(pot_response.po_token) + + if not self.providers: + self.logger.trace('No PO Token providers registered') + return None + + pot_response = self._get_po_token(request) + if not pot_response: + return None + + pot_response.po_token = clean_pot(pot_response.po_token) + + if pot_response.expires_at is None or pot_response.expires_at > 0: + self.cache.store(request, pot_response) + else: + self.logger.trace( + f'PO Token response will not be cached (expires_at={pot_response.expires_at})') + + return pot_response.po_token + + def close(self): + for provider in self.providers.values(): + provider.close() + self.cache.close() + + +EXTRACTOR_ARG_PREFIX = 'youtubepot' + + +def initialize_pot_director(ie): + assert ie._downloader is not None, 'Downloader not set' + + enable_trace = ie._configuration_arg( + 'pot_trace', ['false'], ie_key='youtube', casesense=False)[0] == 'true' + + if enable_trace: + log_level = IEContentProviderLogger.LogLevel.TRACE + elif ie.get_param('verbose', False): + log_level = IEContentProviderLogger.LogLevel.DEBUG + else: + log_level = IEContentProviderLogger.LogLevel.INFO + + def get_provider_logger_and_settings(provider, logger_key): + logger_prefix = f'{logger_key}:{provider.PROVIDER_NAME}' + extractor_key = f'{EXTRACTOR_ARG_PREFIX}-{provider.PROVIDER_KEY.lower()}' + return ( + YoutubeIEContentProviderLogger(ie, logger_prefix, log_level=log_level), + ie.get_param('extractor_args', {}).get(extractor_key, {})) + + cache_providers = [] + for cache_provider in _pot_cache_providers.value.values(): + logger, settings = get_provider_logger_and_settings(cache_provider, 'pot:cache') + cache_providers.append(cache_provider(ie, logger, settings)) + cache_spec_providers = [] + for cache_spec_provider in _pot_pcs_providers.value.values(): + logger, settings = get_provider_logger_and_settings(cache_spec_provider, 'pot:cache:spec') + cache_spec_providers.append(cache_spec_provider(ie, logger, settings)) + + cache = PoTokenCache( + logger=YoutubeIEContentProviderLogger(ie, 'pot:cache', log_level=log_level), + cache_providers=cache_providers, + cache_spec_providers=cache_spec_providers, + cache_provider_preferences=list(_pot_cache_provider_preferences.value), + ) + + director = PoTokenRequestDirector( + logger=YoutubeIEContentProviderLogger(ie, 'pot', log_level=log_level), + cache=cache, + ) + + ie._downloader.add_close_hook(director.close) + + for provider in _pot_providers.value.values(): + logger, settings = get_provider_logger_and_settings(provider, 'pot') + director.register_provider(provider(ie, logger, settings)) + + for preference in _ptp_preferences.value: + director.register_preference(preference) + + if director.logger.log_level <= director.logger.LogLevel.DEBUG: + # calling is_available() for every PO Token provider upfront may have some overhead + director.logger.debug(f'PO Token Providers: {provider_display_list(director.providers.values())}') + director.logger.debug(f'PO Token Cache Providers: {provider_display_list(cache.cache_providers.values())}') + director.logger.debug(f'PO Token Cache Spec Providers: {provider_display_list(cache.cache_spec_providers.values())}') + director.logger.trace(f'Registered {len(director.preferences)} provider preferences') + director.logger.trace(f'Registered {len(cache.cache_provider_preferences)} cache provider preferences') + + return director + + +def provider_display_list(providers: Iterable[IEContentProvider]): + def provider_display_name(provider): + display_str = join_nonempty( + provider.PROVIDER_NAME, + provider.PROVIDER_VERSION if not isinstance(provider, BuiltinIEContentProvider) else None) + statuses = [] + if not isinstance(provider, BuiltinIEContentProvider): + statuses.append('external') + if not provider.is_available(): + statuses.append('unavailable') + if statuses: + display_str += f' ({", ".join(statuses)})' + return display_str + + return ', '.join(provider_display_name(provider) for provider in providers) or 'none' + + +def clean_pot(po_token: str): + # Clean and validate the PO Token. This will strip invalid characters off + # (e.g. additional url params the user may accidentally include) + try: + return base64.urlsafe_b64encode( + base64.urlsafe_b64decode(urllib.parse.unquote(po_token))).decode() + except (binascii.Error, ValueError): + raise ValueError('Invalid PO Token') + + +def validate_response(response: PoTokenResponse | None): + if ( + not isinstance(response, PoTokenResponse) + or not isinstance(response.po_token, str) + or not response.po_token + ): # noqa: SIM103 + return False + + try: + clean_pot(response.po_token) + except ValueError: + return False + + if not isinstance(response.expires_at, int): + return response.expires_at is None + + return response.expires_at <= 0 or response.expires_at > int(dt.datetime.now(dt.timezone.utc).timestamp()) + + +def validate_cache_spec(spec: PoTokenCacheSpec): + return ( + isinstance(spec, PoTokenCacheSpec) + and isinstance(spec.write_policy, CacheProviderWritePolicy) + and isinstance(spec.default_ttl, int) + and isinstance(spec.key_bindings, dict) + and all(isinstance(k, str) for k in spec.key_bindings) + and all(v is None or isinstance(v, str) for v in spec.key_bindings.values()) + and bool([v for v in spec.key_bindings.values() if v is not None]) + ) diff --git a/yt_dlp/extractor/youtube/pot/_provider.py b/yt_dlp/extractor/youtube/pot/_provider.py new file mode 100644 index 000000000..af7034d22 --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/_provider.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import abc +import enum +import functools + +from yt_dlp.extractor.common import InfoExtractor +from yt_dlp.utils import NO_DEFAULT, bug_reports_message, classproperty, traverse_obj +from yt_dlp.version import __version__ + +# xxx: these could be generalized outside YoutubeIE eventually + + +class IEContentProviderLogger(abc.ABC): + + class LogLevel(enum.IntEnum): + TRACE = 0 + DEBUG = 10 + INFO = 20 + WARNING = 30 + ERROR = 40 + + @classmethod + def _missing_(cls, value): + if isinstance(value, str): + value = value.upper() + if value in dir(cls): + return cls[value] + + return cls.INFO + + log_level = LogLevel.INFO + + @abc.abstractmethod + def trace(self, message: str): + pass + + @abc.abstractmethod + def debug(self, message: str): + pass + + @abc.abstractmethod + def info(self, message: str): + pass + + @abc.abstractmethod + def warning(self, message: str, *, once=False): + pass + + @abc.abstractmethod + def error(self, message: str): + pass + + +class IEContentProviderError(Exception): + def __init__(self, msg=None, expected=False): + super().__init__(msg) + self.expected = expected + + +class IEContentProvider(abc.ABC): + PROVIDER_VERSION: str = '0.0.0' + BUG_REPORT_LOCATION: str = '(developer has not provided a bug report location)' + + def __init__( + self, + ie: InfoExtractor, + logger: IEContentProviderLogger, + settings: dict[str, list[str]], *_, **__, + ): + self.ie = ie + self.settings = settings or {} + self.logger = logger + super().__init__() + + @classmethod + def __init_subclass__(cls, *, suffix=None, **kwargs): + if suffix: + cls._PROVIDER_KEY_SUFFIX = suffix + return super().__init_subclass__(**kwargs) + + @classproperty + def PROVIDER_NAME(cls) -> str: + return cls.__name__[:-len(cls._PROVIDER_KEY_SUFFIX)] + + @classproperty + def BUG_REPORT_MESSAGE(cls): + return f'please report this issue to the provider developer at {cls.BUG_REPORT_LOCATION} .' + + @classproperty + def PROVIDER_KEY(cls) -> str: + assert hasattr(cls, '_PROVIDER_KEY_SUFFIX'), 'Content Provider implementation must define a suffix for the provider key' + assert cls.__name__.endswith(cls._PROVIDER_KEY_SUFFIX), f'PoTokenProvider class names must end with "{cls._PROVIDER_KEY_SUFFIX}"' + return cls.__name__[:-len(cls._PROVIDER_KEY_SUFFIX)] + + @abc.abstractmethod + def is_available(self) -> bool: + """ + Check if the provider is available (e.g. all required dependencies are available) + This is used to determine if the provider should be used and to provide debug information. + + IMPORTANT: This method should not make any network requests or perform any expensive operations. + It is called multiple times. + """ + raise NotImplementedError + + def close(self): # noqa: B027 + pass + + def _configuration_arg(self, key, default=NO_DEFAULT, *, casesense=False): + """ + @returns A list of values for the setting given by "key" + or "default" if no such key is present + @param default The default value to return when the key is not present (default: []) + @param casesense When false, the values are converted to lower case + """ + val = traverse_obj(self.settings, key) + if val is None: + return [] if default is NO_DEFAULT else default + return list(val) if casesense else [x.lower() for x in val] + + +class BuiltinIEContentProvider(IEContentProvider, abc.ABC): + PROVIDER_VERSION = __version__ + BUG_REPORT_MESSAGE = bug_reports_message(before='') + + +def register_provider_generic( + provider, + base_class, + registry, +): + """Generic function to register a provider class""" + assert issubclass(provider, base_class), f'{provider} must be a subclass of {base_class.__name__}' + assert provider.PROVIDER_KEY not in registry, f'{base_class.__name__} {provider.PROVIDER_KEY} already registered' + registry[provider.PROVIDER_KEY] = provider + return provider + + +def register_preference_generic( + base_class, + registry, + *providers, +): + """Generic function to register a preference for a provider""" + assert all(issubclass(provider, base_class) for provider in providers) + + def outer(preference): + @functools.wraps(preference) + def inner(provider, *args, **kwargs): + if not providers or isinstance(provider, providers): + return preference(provider, *args, **kwargs) + return 0 + registry.add(inner) + return preference + return outer diff --git a/yt_dlp/extractor/youtube/pot/_registry.py b/yt_dlp/extractor/youtube/pot/_registry.py new file mode 100644 index 000000000..c72a622c1 --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/_registry.py @@ -0,0 +1,8 @@ +from yt_dlp.globals import Indirect + +_pot_providers = Indirect({}) +_ptp_preferences = Indirect(set()) +_pot_pcs_providers = Indirect({}) +_pot_cache_providers = Indirect({}) +_pot_cache_provider_preferences = Indirect(set()) +_pot_memory_cache = Indirect({}) diff --git a/yt_dlp/extractor/youtube/pot/cache.py b/yt_dlp/extractor/youtube/pot/cache.py new file mode 100644 index 000000000..6d69316ad --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/cache.py @@ -0,0 +1,97 @@ +"""PUBLIC API""" + +from __future__ import annotations + +import abc +import dataclasses +import enum +import typing + +from yt_dlp.extractor.youtube.pot._provider import ( + IEContentProvider, + IEContentProviderError, + register_preference_generic, + register_provider_generic, +) +from yt_dlp.extractor.youtube.pot._registry import ( + _pot_cache_provider_preferences, + _pot_cache_providers, + _pot_pcs_providers, +) +from yt_dlp.extractor.youtube.pot.provider import PoTokenRequest + + +class PoTokenCacheProviderError(IEContentProviderError): + """An error occurred while fetching a PO Token""" + + +class PoTokenCacheProvider(IEContentProvider, abc.ABC, suffix='PCP'): + @abc.abstractmethod + def get(self, key: str) -> str | None: + pass + + @abc.abstractmethod + def store(self, key: str, value: str, expires_at: int): + pass + + @abc.abstractmethod + def delete(self, key: str): + pass + + +class CacheProviderWritePolicy(enum.Enum): + WRITE_ALL = enum.auto() # Write to all cache providers + WRITE_FIRST = enum.auto() # Write to only the first cache provider + + +@dataclasses.dataclass +class PoTokenCacheSpec: + key_bindings: dict[str, str | None] + default_ttl: int + write_policy: CacheProviderWritePolicy = CacheProviderWritePolicy.WRITE_ALL + + # Internal + _provider: PoTokenCacheSpecProvider | None = None + + +class PoTokenCacheSpecProvider(IEContentProvider, abc.ABC, suffix='PCSP'): + + def is_available(self) -> bool: + return True + + @abc.abstractmethod + def generate_cache_spec(self, request: PoTokenRequest) -> PoTokenCacheSpec | None: + """Generate a cache spec for the given request""" + pass + + +def register_provider(provider: type[PoTokenCacheProvider]): + """Register a PoTokenCacheProvider class""" + return register_provider_generic( + provider=provider, + base_class=PoTokenCacheProvider, + registry=_pot_cache_providers.value, + ) + + +def register_spec(provider: type[PoTokenCacheSpecProvider]): + """Register a PoTokenCacheSpecProvider class""" + return register_provider_generic( + provider=provider, + base_class=PoTokenCacheSpecProvider, + registry=_pot_pcs_providers.value, + ) + + +def register_preference( + *providers: type[PoTokenCacheProvider]) -> typing.Callable[[CacheProviderPreference], CacheProviderPreference]: + """Register a preference for a PoTokenCacheProvider""" + return register_preference_generic( + PoTokenCacheProvider, + _pot_cache_provider_preferences.value, + *providers, + ) + + +if typing.TYPE_CHECKING: + CacheProviderPreference = typing.Callable[[PoTokenCacheProvider, PoTokenRequest], int] diff --git a/yt_dlp/extractor/youtube/pot/provider.py b/yt_dlp/extractor/youtube/pot/provider.py new file mode 100644 index 000000000..53af92d30 --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/provider.py @@ -0,0 +1,280 @@ +"""PUBLIC API""" + +from __future__ import annotations + +import abc +import copy +import dataclasses +import enum +import functools +import typing +import urllib.parse + +from yt_dlp.cookies import YoutubeDLCookieJar +from yt_dlp.extractor.youtube.pot._provider import ( + IEContentProvider, + IEContentProviderError, + register_preference_generic, + register_provider_generic, +) +from yt_dlp.extractor.youtube.pot._registry import _pot_providers, _ptp_preferences +from yt_dlp.networking import Request, Response +from yt_dlp.utils import traverse_obj +from yt_dlp.utils.networking import HTTPHeaderDict + +__all__ = [ + 'ExternalRequestFeature', + 'PoTokenContext', + 'PoTokenProvider', + 'PoTokenProviderError', + 'PoTokenProviderRejectedRequest', + 'PoTokenRequest', + 'PoTokenResponse', + 'provider_bug_report_message', + 'register_preference', + 'register_provider', +] + + +class PoTokenContext(enum.Enum): + GVS = 'gvs' + PLAYER = 'player' + + +@dataclasses.dataclass +class PoTokenRequest: + # YouTube parameters + context: PoTokenContext + innertube_context: InnertubeContext + innertube_host: str | None = None + session_index: str | None = None + player_url: str | None = None + is_authenticated: bool = False + video_webpage: str | None = None + internal_client_name: str | None = None + + # Content binding parameters + visitor_data: str | None = None + data_sync_id: str | None = None + video_id: str | None = None + + # Networking parameters + request_cookiejar: YoutubeDLCookieJar = dataclasses.field(default_factory=YoutubeDLCookieJar) + request_proxy: str | None = None + request_headers: HTTPHeaderDict = dataclasses.field(default_factory=HTTPHeaderDict) + request_timeout: float | None = None + request_source_address: str | None = None + request_verify_tls: bool = True + + # Generate a new token, do not used a cached token + # The token should still be cached for future requests + bypass_cache: bool = False + + def copy(self): + return dataclasses.replace( + self, + request_headers=HTTPHeaderDict(self.request_headers), + innertube_context=copy.deepcopy(self.innertube_context), + ) + + +@dataclasses.dataclass +class PoTokenResponse: + po_token: str + expires_at: int | None = None + + +class PoTokenProviderRejectedRequest(IEContentProviderError): + """Reject the PoTokenRequest (cannot handle the request)""" + + +class PoTokenProviderError(IEContentProviderError): + """An error occurred while fetching a PO Token""" + + +class ExternalRequestFeature(enum.Enum): + PROXY_SCHEME_HTTP = enum.auto() + PROXY_SCHEME_HTTPS = enum.auto() + PROXY_SCHEME_SOCKS4 = enum.auto() + PROXY_SCHEME_SOCKS4A = enum.auto() + PROXY_SCHEME_SOCKS5 = enum.auto() + PROXY_SCHEME_SOCKS5H = enum.auto() + SOURCE_ADDRESS = enum.auto() + DISABLE_TLS_VERIFICATION = enum.auto() + + +class PoTokenProvider(IEContentProvider, abc.ABC, suffix='PTP'): + + # Set to None to disable the check + _SUPPORTED_CONTEXTS: tuple[PoTokenContext] | None = () + + # Innertube Client Name. + # For example, "WEB", "ANDROID", "TVHTML5". + # For a list of WebPO client names, see yt_dlp.extractor.youtube.pot.utils.WEBPO_CLIENTS. + # Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS + # for a list of client names currently supported by the YouTube extractor. + _SUPPORTED_CLIENTS: tuple[str] | None = () + + # If making external requests to websites (i.e. to youtube.com) + # using another library or service (i.e., not _request_webpage), + # add the request features that are supported. + # If only using _request_webpage to make external requests, set this to None. + _SUPPORTED_EXTERNAL_REQUEST_FEATURES: tuple[ExternalRequestFeature] | None = () + + def __validate_request(self, request: PoTokenRequest): + if not self.is_available(): + raise PoTokenProviderRejectedRequest(f'{self.PROVIDER_NAME} is not available') + + # Validate request using built-in settings + if ( + self._SUPPORTED_CONTEXTS is not None + and request.context not in self._SUPPORTED_CONTEXTS + ): + raise PoTokenProviderRejectedRequest( + f'PO Token Context "{request.context}" is not supported by {self.PROVIDER_NAME}') + + if self._SUPPORTED_CLIENTS is not None: + client_name = traverse_obj( + request.innertube_context, ('client', 'clientName')) + if client_name not in self._SUPPORTED_CLIENTS: + raise PoTokenProviderRejectedRequest( + f'Client "{client_name}" is not supported by {self.PROVIDER_NAME}. ' + f'Supported clients: {", ".join(self._SUPPORTED_CLIENTS) or "none"}') + + self.__validate_external_request_features(request) + + @functools.cached_property + def _supported_proxy_schemes(self): + return { + scheme: feature + for scheme, feature in { + 'http': ExternalRequestFeature.PROXY_SCHEME_HTTP, + 'https': ExternalRequestFeature.PROXY_SCHEME_HTTPS, + 'socks4': ExternalRequestFeature.PROXY_SCHEME_SOCKS4, + 'socks4a': ExternalRequestFeature.PROXY_SCHEME_SOCKS4A, + 'socks5': ExternalRequestFeature.PROXY_SCHEME_SOCKS5, + 'socks5h': ExternalRequestFeature.PROXY_SCHEME_SOCKS5H, + }.items() + if feature in (self._SUPPORTED_EXTERNAL_REQUEST_FEATURES or []) + } + + def __validate_external_request_features(self, request: PoTokenRequest): + if self._SUPPORTED_EXTERNAL_REQUEST_FEATURES is None: + return + + if request.request_proxy: + scheme = urllib.parse.urlparse(request.request_proxy).scheme + if scheme.lower() not in self._supported_proxy_schemes: + raise PoTokenProviderRejectedRequest( + f'External requests by "{self.PROVIDER_NAME}" provider do not ' + f'support proxy scheme "{scheme}". Supported proxy schemes: ' + f'{", ".join(self._supported_proxy_schemes) or "none"}') + + if ( + request.request_source_address + and ExternalRequestFeature.SOURCE_ADDRESS not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES + ): + raise PoTokenProviderRejectedRequest( + f'External requests by "{self.PROVIDER_NAME}" provider ' + f'do not support setting source address') + + if ( + not request.request_verify_tls + and ExternalRequestFeature.DISABLE_TLS_VERIFICATION not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES + ): + raise PoTokenProviderRejectedRequest( + f'External requests by "{self.PROVIDER_NAME}" provider ' + f'do not support ignoring TLS certificate failures') + + def request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + self.__validate_request(request) + return self._real_request_pot(request) + + @abc.abstractmethod + def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: + """To be implemented by subclasses""" + pass + + # Helper functions + + def _request_webpage(self, request: Request, pot_request: PoTokenRequest | None = None, note=None, **kwargs) -> Response: + """Make a request using the internal HTTP Client. + Use this instead of calling requests, urllib3 or other HTTP client libraries directly! + + YouTube cookies will be automatically applied if this request is made to YouTube. + + @param request: The request to make + @param pot_request: The PoTokenRequest to use. Request parameters will be merged from it. + @param note: Custom log message to display when making the request. Set to `False` to disable logging. + + Tips: + - Disable proxy (e.g. if calling local service): Request(..., proxies={'all': None}) + - Set request timeout: Request(..., extensions={'timeout': 5.0}) + """ + req = request.copy() + + # Merge some ctx request settings into the request + # Most of these will already be used by the configured ydl instance, + # however, the YouTube extractor may override some. + if pot_request is not None: + req.headers = HTTPHeaderDict(pot_request.request_headers, req.headers) + req.proxies = req.proxies or ({'all': pot_request.request_proxy} if pot_request.request_proxy else {}) + + if pot_request.request_cookiejar is not None: + req.extensions['cookiejar'] = req.extensions.get('cookiejar', pot_request.request_cookiejar) + + if note is not False: + self.logger.info(str(note) if note else 'Requesting webpage') + return self.ie._downloader.urlopen(req) + + +def register_provider(provider: type[PoTokenProvider]): + """Register a PoTokenProvider class""" + return register_provider_generic( + provider=provider, + base_class=PoTokenProvider, + registry=_pot_providers.value, + ) + + +def provider_bug_report_message(provider: IEContentProvider, before=';'): + msg = provider.BUG_REPORT_MESSAGE + + before = before.rstrip() + if not before or before.endswith(('.', '!', '?')): + msg = msg[0].title() + msg[1:] + + return f'{before} {msg}' if before else msg + + +def register_preference(*providers: type[PoTokenProvider]) -> typing.Callable[[Preference], Preference]: + """Register a preference for a PoTokenProvider""" + return register_preference_generic( + PoTokenProvider, + _ptp_preferences.value, + *providers, + ) + + +if typing.TYPE_CHECKING: + Preference = typing.Callable[[PoTokenProvider, PoTokenRequest], int] + __all__.append('Preference') + + # Barebones innertube context. There may be more fields. + class ClientInfo(typing.TypedDict, total=False): + hl: str | None + gl: str | None + remoteHost: str | None + deviceMake: str | None + deviceModel: str | None + visitorData: str | None + userAgent: str | None + clientName: str + clientVersion: str + osName: str | None + osVersion: str | None + + class InnertubeContext(typing.TypedDict, total=False): + client: ClientInfo + request: dict + user: dict diff --git a/yt_dlp/extractor/youtube/pot/utils.py b/yt_dlp/extractor/youtube/pot/utils.py new file mode 100644 index 000000000..1c0db243b --- /dev/null +++ b/yt_dlp/extractor/youtube/pot/utils.py @@ -0,0 +1,73 @@ +"""PUBLIC API""" + +from __future__ import annotations + +import base64 +import contextlib +import enum +import re +import urllib.parse + +from yt_dlp.extractor.youtube.pot.provider import PoTokenContext, PoTokenRequest +from yt_dlp.utils import traverse_obj + +__all__ = ['WEBPO_CLIENTS', 'ContentBindingType', 'get_webpo_content_binding'] + +WEBPO_CLIENTS = ( + 'WEB', + 'MWEB', + 'TVHTML5', + 'WEB_EMBEDDED_PLAYER', + 'WEB_CREATOR', + 'WEB_REMIX', + 'TVHTML5_SIMPLY_EMBEDDED_PLAYER', +) + + +class ContentBindingType(enum.Enum): + VISITOR_DATA = 'visitor_data' + DATASYNC_ID = 'datasync_id' + VIDEO_ID = 'video_id' + VISITOR_ID = 'visitor_id' + + +def get_webpo_content_binding( + request: PoTokenRequest, + webpo_clients=WEBPO_CLIENTS, + bind_to_visitor_id=False, +) -> tuple[str | None, ContentBindingType | None]: + + client_name = traverse_obj(request.innertube_context, ('client', 'clientName')) + if not client_name or client_name not in webpo_clients: + return None, None + + if request.context == PoTokenContext.GVS or client_name in ('WEB_REMIX', ): + if request.is_authenticated: + return request.data_sync_id, ContentBindingType.DATASYNC_ID + else: + if bind_to_visitor_id: + visitor_id = _extract_visitor_id(request.visitor_data) + if visitor_id: + return visitor_id, ContentBindingType.VISITOR_ID + return request.visitor_data, ContentBindingType.VISITOR_DATA + + elif request.context == PoTokenContext.PLAYER or client_name != 'WEB_REMIX': + return request.video_id, ContentBindingType.VIDEO_ID + + return None, None + + +def _extract_visitor_id(visitor_data): + if not visitor_data: + return None + + # Attempt to extract the visitor ID from the visitor_data protobuf + # xxx: ideally should use a protobuf parser + with contextlib.suppress(Exception): + visitor_id = base64.urlsafe_b64decode( + urllib.parse.unquote_plus(visitor_data))[2:13].decode() + # check that visitor id is all letters and numbers + if re.fullmatch(r'[A-Za-z0-9_-]{11}', visitor_id): + return visitor_id + + return None diff --git a/yt_dlp/networking/_curlcffi.py b/yt_dlp/networking/_curlcffi.py index c800f2c09..747879da8 100644 --- a/yt_dlp/networking/_curlcffi.py +++ b/yt_dlp/networking/_curlcffi.py @@ -6,7 +6,8 @@ import re import urllib.parse -from ._helper import InstanceStoreMixin, select_proxy +from ._helper import InstanceStoreMixin +from ..utils.networking import select_proxy from .common import ( Features, Request, diff --git a/yt_dlp/networking/_helper.py b/yt_dlp/networking/_helper.py index b86d3606d..ef9c8bafa 100644 --- a/yt_dlp/networking/_helper.py +++ b/yt_dlp/networking/_helper.py @@ -13,7 +13,6 @@ from .exceptions import RequestError from ..dependencies import certifi from ..socks import ProxyType, sockssocket -from ..utils import format_field, traverse_obj if typing.TYPE_CHECKING: from collections.abc import Iterable @@ -82,19 +81,6 @@ def unquote_if_non_empty(s): } -def select_proxy(url, proxies): - """Unified proxy selector for all backends""" - url_components = urllib.parse.urlparse(url) - if 'no' in proxies: - hostport = url_components.hostname + format_field(url_components.port, None, ':%s') - if urllib.request.proxy_bypass_environment(hostport, {'no': proxies['no']}): - return - elif urllib.request.proxy_bypass(hostport): # check system settings - return - - return traverse_obj(proxies, url_components.scheme or 'http', 'all') - - def get_redirect_method(method, status): """Unified redirect method handling""" diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py index 5b6b264a6..d02e976b5 100644 --- a/yt_dlp/networking/_requests.py +++ b/yt_dlp/networking/_requests.py @@ -10,7 +10,7 @@ from ..dependencies import brotli, requests, urllib3 from ..utils import bug_reports_message, int_or_none, variadic -from ..utils.networking import normalize_url +from ..utils.networking import normalize_url, select_proxy if requests is None: raise ImportError('requests module is not installed') @@ -41,7 +41,6 @@ create_socks_proxy_socket, get_redirect_method, make_socks_proxy_opts, - select_proxy, ) from .common import ( Features, diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py index a188b35f5..cb7a430bb 100644 --- a/yt_dlp/networking/_urllib.py +++ b/yt_dlp/networking/_urllib.py @@ -26,7 +26,6 @@ create_socks_proxy_socket, get_redirect_method, make_socks_proxy_opts, - select_proxy, ) from .common import Features, RequestHandler, Response, register_rh from .exceptions import ( @@ -41,7 +40,7 @@ from ..dependencies import brotli from ..socks import ProxyError as SocksProxyError from ..utils import update_url_query -from ..utils.networking import normalize_url +from ..utils.networking import normalize_url, select_proxy SUPPORTED_ENCODINGS = ['gzip', 'deflate'] CONTENT_DECODE_ERRORS = [zlib.error, OSError] diff --git a/yt_dlp/networking/_websockets.py b/yt_dlp/networking/_websockets.py index d29f8e45a..fd8730ac7 100644 --- a/yt_dlp/networking/_websockets.py +++ b/yt_dlp/networking/_websockets.py @@ -11,8 +11,8 @@ create_connection, create_socks_proxy_socket, make_socks_proxy_opts, - select_proxy, ) +from ..utils.networking import select_proxy from .common import Features, Response, register_rh from .exceptions import ( CertificateVerifyError, diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py index 542abace8..9fcab6456 100644 --- a/yt_dlp/utils/networking.py +++ b/yt_dlp/utils/networking.py @@ -10,7 +10,8 @@ if typing.TYPE_CHECKING: T = typing.TypeVar('T') -from ._utils import NO_DEFAULT, remove_start +from ._utils import NO_DEFAULT, remove_start, format_field +from .traversal import traverse_obj def random_user_agent(): @@ -278,3 +279,16 @@ def normalize_url(url): query=escape_rfc3986(url_parsed.query), fragment=escape_rfc3986(url_parsed.fragment), ).geturl() + + +def select_proxy(url, proxies): + """Unified proxy selector for all backends""" + url_components = urllib.parse.urlparse(url) + if 'no' in proxies: + hostport = url_components.hostname + format_field(url_components.port, None, ':%s') + if urllib.request.proxy_bypass_environment(hostport, {'no': proxies['no']}): + return + elif urllib.request.proxy_bypass(hostport): # check system settings + return + + return traverse_obj(proxies, url_components.scheme or 'http', 'all') From f569be4602c2a857087e495d5d7ed6060cd97abe Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Tue, 20 May 2025 13:25:27 -0500 Subject: [PATCH 07/67] [ie/niconico] Fix error handling (#13236) Closes #11430 Authored by: bashonly --- yt_dlp/extractor/niconico.py | 47 +++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py index fc050c383..0d0f7ceef 100644 --- a/yt_dlp/extractor/niconico.py +++ b/yt_dlp/extractor/niconico.py @@ -32,7 +32,7 @@ urlencode_postdata, urljoin, ) -from ..utils.traversal import find_element, traverse_obj +from ..utils.traversal import find_element, require, traverse_obj class NiconicoBaseIE(InfoExtractor): @@ -283,35 +283,54 @@ def _yield_dms_formats(self, api_data, video_id): lambda _, v: v['id'] == video_fmt['format_id'], 'qualityLevel', {int_or_none}, any)) or -1 yield video_fmt + def _extract_server_response(self, webpage, video_id, fatal=True): + try: + return traverse_obj( + self._parse_json(self._html_search_meta('server-response', webpage) or '', video_id), + ('data', 'response', {dict}, {require('server response')})) + except ExtractorError: + if not fatal: + return {} + raise + def _real_extract(self, url): video_id = self._match_id(url) try: webpage, handle = self._download_webpage_handle( - 'https://www.nicovideo.jp/watch/' + video_id, video_id) + f'https://www.nicovideo.jp/watch/{video_id}', video_id, + headers=self.geo_verification_headers()) if video_id.startswith('so'): video_id = self._match_id(handle.url) - api_data = traverse_obj( - self._parse_json(self._html_search_meta('server-response', webpage) or '', video_id), - ('data', 'response', {dict})) - if not api_data: - raise ExtractorError('Server response data not found') + api_data = self._extract_server_response(webpage, video_id) except ExtractorError as e: try: api_data = self._download_json( - f'https://www.nicovideo.jp/api/watch/v3/{video_id}?_frontendId=6&_frontendVersion=0&actionTrackId=AAAAAAAAAA_{round(time.time() * 1000)}', video_id, - note='Downloading API JSON', errnote='Unable to fetch data')['data'] + f'https://www.nicovideo.jp/api/watch/v3/{video_id}', video_id, + 'Downloading API JSON', 'Unable to fetch data', query={ + '_frontendId': '6', + '_frontendVersion': '0', + 'actionTrackId': f'AAAAAAAAAA_{round(time.time() * 1000)}', + }, headers=self.geo_verification_headers())['data'] except ExtractorError: if not isinstance(e.cause, HTTPError): + # Raise if original exception was from _parse_json or utils.traversal.require raise + # The webpage server response has more detailed error info than the API response webpage = e.cause.response.read().decode('utf-8', 'replace') - error_msg = self._html_search_regex( - r'(?s)(.+?)', - webpage, 'error reason', default=None) - if not error_msg: + reason_code = self._extract_server_response( + webpage, video_id, fatal=False).get('reasonCode') + if not reason_code: raise - raise ExtractorError(clean_html(error_msg), expected=True) + if reason_code in ('DOMESTIC_VIDEO', 'HIGH_RISK_COUNTRY_VIDEO'): + self.raise_geo_restricted(countries=self._GEO_COUNTRIES) + elif reason_code == 'HIDDEN_VIDEO': + raise ExtractorError( + 'The viewing period of this video has expired', expected=True) + elif reason_code == 'DELETED_VIDEO': + raise ExtractorError('This video has been deleted', expected=True) + raise ExtractorError(f'Niconico says: {reason_code}') availability = self._availability(**(traverse_obj(api_data, ('payment', 'video', { 'needs_premium': ('isPremium', {bool}), From 545c1a5b6f2fe88722b41aef0e7485bf3be3f3f9 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Tue, 20 May 2025 13:28:34 -0500 Subject: [PATCH 08/67] [ie/vimeo:event] Add extractor (#13216) Closes #1608 Authored by: bashonly --- yt_dlp/extractor/_extractors.py | 1 + yt_dlp/extractor/vimeo.py | 428 ++++++++++++++++++++++++++++++-- 2 files changed, 413 insertions(+), 16 deletions(-) diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index e7dcb9853..14a006893 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -2369,6 +2369,7 @@ VHXEmbedIE, VimeoAlbumIE, VimeoChannelIE, + VimeoEventIE, VimeoGroupsIE, VimeoIE, VimeoLikesIE, diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py index fb9af7acf..09497b699 100644 --- a/yt_dlp/extractor/vimeo.py +++ b/yt_dlp/extractor/vimeo.py @@ -3,6 +3,7 @@ import itertools import json import re +import time import urllib.parse from .common import InfoExtractor @@ -13,10 +14,12 @@ OnDemandPagedList, clean_html, determine_ext, + filter_dict, get_element_by_class, int_or_none, join_nonempty, js_to_json, + jwt_decode_hs256, merge_dicts, parse_filesize, parse_iso8601, @@ -39,6 +42,9 @@ class VimeoBaseInfoExtractor(InfoExtractor): _NETRC_MACHINE = 'vimeo' _LOGIN_REQUIRED = False _LOGIN_URL = 'https://vimeo.com/log_in' + _REFERER_HINT = ( + 'Cannot download embed-only video without embedding URL. Please call yt-dlp ' + 'with the URL of the page that embeds this video.') _IOS_CLIENT_AUTH = 'MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==' _IOS_CLIENT_HEADERS = { 'Accept': 'application/vnd.vimeo.*+json; version=3.4.10', @@ -47,6 +53,7 @@ class VimeoBaseInfoExtractor(InfoExtractor): } _IOS_OAUTH_CACHE_KEY = 'oauth-token-ios' _ios_oauth_token = None + _viewer_info = None @staticmethod def _smuggle_referrer(url, referrer_url): @@ -60,8 +67,21 @@ def _unsmuggle_headers(self, url): headers['Referer'] = data['referer'] return url, data, headers + def _jwt_is_expired(self, token): + return jwt_decode_hs256(token)['exp'] - time.time() < 120 + + def _fetch_viewer_info(self, display_id=None, fatal=True): + if self._viewer_info and not self._jwt_is_expired(self._viewer_info['jwt']): + return self._viewer_info + + self._viewer_info = self._download_json( + 'https://vimeo.com/_next/viewer', display_id, 'Downloading web token info', + 'Failed to download web token info', fatal=fatal, headers={'Accept': 'application/json'}) + + return self._viewer_info + def _perform_login(self, username, password): - viewer = self._download_json('https://vimeo.com/_next/viewer', None, 'Downloading login token') + viewer = self._fetch_viewer_info() data = { 'action': 'login', 'email': username, @@ -96,11 +116,10 @@ def _get_video_password(self): expected=True) return password - def _verify_video_password(self, video_id): + def _verify_video_password(self, video_id, path=None): video_password = self._get_video_password() - token = self._download_json( - 'https://vimeo.com/_next/viewer', video_id, 'Downloading viewer info')['xsrft'] - url = f'https://vimeo.com/{video_id}' + token = self._fetch_viewer_info(video_id)['xsrft'] + url = join_nonempty('https://vimeo.com', path, video_id, delim='/') try: self._request_webpage( f'{url}/password', video_id, @@ -117,6 +136,10 @@ def _verify_video_password(self, video_id): raise ExtractorError('Wrong password', expected=True) raise + def _extract_config_url(self, webpage, **kwargs): + return self._html_search_regex( + r'\bdata-config-url="([^"]+)"', webpage, 'config URL', **kwargs) + def _extract_vimeo_config(self, webpage, video_id, *args, **kwargs): vimeo_config = self._search_regex( r'vimeo\.config\s*=\s*(?:({.+?})|_extend\([^,]+,\s+({.+?})\));', @@ -164,6 +187,7 @@ def _parse_config(self, config, video_id): sep_pattern = r'/sep/video/' for files_type in ('hls', 'dash'): for cdn_name, cdn_data in (try_get(config_files, lambda x: x[files_type]['cdns']) or {}).items(): + # TODO: Also extract 'avc_url'? Investigate if there are 'hevc_url', 'av1_url'? manifest_url = cdn_data.get('url') if not manifest_url: continue @@ -244,7 +268,10 @@ def _parse_config(self, config, video_id): 'formats': formats, 'subtitles': subtitles, 'live_status': live_status, - 'release_timestamp': traverse_obj(live_event, ('ingest', 'scheduled_start_time', {parse_iso8601})), + 'release_timestamp': traverse_obj(live_event, ('ingest', ( + ('scheduled_start_time', {parse_iso8601}), + ('start_time', {int_or_none}), + ), any)), # Note: Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps # at the same time without actual units specified. '_format_sort_fields': ('quality', 'res', 'fps', 'hdr:12', 'source'), @@ -353,7 +380,7 @@ class VimeoIE(VimeoBaseInfoExtractor): (?: (?Puser)| (?!(?:channels|album|showcase)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/) - (?:.*?/)?? + (?:(?!event/).*?/)?? (?P (?: play_redirect_hls| @@ -933,8 +960,7 @@ def _try_album_password(self, url): r'vimeo\.com/(?:album|showcase)/([^/]+)', url, 'album id', default=None) if not album_id: return - viewer = self._download_json( - 'https://vimeo.com/_rv/viewer', album_id, fatal=False) + viewer = self._fetch_viewer_info(album_id, fatal=False) if not viewer: webpage = self._download_webpage(url, album_id) viewer = self._parse_json(self._search_regex( @@ -992,9 +1018,7 @@ def _real_extract(self, url): raise errmsg = error.cause.response.read() if b'Because of its privacy settings, this video cannot be played here' in errmsg: - raise ExtractorError( - 'Cannot download embed-only video without embedding URL. Please call yt-dlp ' - 'with the URL of the page that embeds this video.', expected=True) + raise ExtractorError(self._REFERER_HINT, expected=True) # 403 == vimeo.com TLS fingerprint or DC IP block; 429 == player.vimeo.com TLS FP block status = error.cause.status dcip_msg = 'If you are using a data center IP or VPN/proxy, your IP may be blocked' @@ -1039,8 +1063,7 @@ def _real_extract(self, url): channel_id = self._search_regex( r'vimeo\.com/channels/([^/]+)', url, 'channel id', default=None) if channel_id: - config_url = self._html_search_regex( - r'\bdata-config-url="([^"]+)"', webpage, 'config URL', default=None) + config_url = self._extract_config_url(webpage, default=None) video_description = clean_html(get_element_by_class('description', webpage)) info_dict.update({ 'channel_id': channel_id, @@ -1333,8 +1356,7 @@ def _fetch_page(self, album_id, authorization, hashed_pass, page): def _real_extract(self, url): album_id = self._match_id(url) - viewer = self._download_json( - 'https://vimeo.com/_rv/viewer', album_id, fatal=False) + viewer = self._fetch_viewer_info(album_id, fatal=False) if not viewer: webpage = self._download_webpage(url, album_id) viewer = self._parse_json(self._search_regex( @@ -1626,3 +1648,377 @@ def _real_extract(self, url): return self.url_result(vimeo_url, VimeoIE, video_id, url_transparent=True, description=description) + + +class VimeoEventIE(VimeoBaseInfoExtractor): + IE_NAME = 'vimeo:event' + _VALID_URL = r'''(?x) + https?://(?:www\.)?vimeo\.com/event/(?P\d+)(?:/ + (?: + (?:embed/)?(?P[\da-f]{10})| + videos/(?P\d+) + ) + )?''' + _EMBED_REGEX = [r']+\bsrc=["\'](?Phttps?://vimeo\.com/event/\d+/embed(?:[/?][^"\']*)?)["\'][^>]*>'] + _TESTS = [{ + # stream_privacy.view: 'anybody' + 'url': 'https://vimeo.com/event/5116195', + 'info_dict': { + 'id': '1082194134', + 'ext': 'mp4', + 'display_id': '5116195', + 'title': 'Skidmore College Commencement 2025', + 'description': 'md5:1902dd5165d21f98aa198297cc729d23', + 'uploader': 'Skidmore College', + 'uploader_id': 'user116066434', + 'uploader_url': 'https://vimeo.com/user116066434', + 'comment_count': int, + 'like_count': int, + 'duration': 9810, + 'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d', + 'timestamp': 1747502974, + 'upload_date': '20250517', + 'release_timestamp': 1747502998, + 'release_date': '20250517', + 'live_status': 'was_live', + }, + 'params': {'skip_download': 'm3u8'}, + 'expected_warnings': ['Failed to parse XML: not well-formed'], + }, { + # stream_privacy.view: 'embed_only' + 'url': 'https://vimeo.com/event/5034253/embed', + 'info_dict': { + 'id': '1071439154', + 'ext': 'mp4', + 'display_id': '5034253', + 'title': 'Advancing Humans with AI', + 'description': r're:AI is here to stay, but how do we ensure that people flourish in a world of pervasive AI use.{322}$', + 'uploader': 'MIT Media Lab', + 'uploader_id': 'mitmedialab', + 'uploader_url': 'https://vimeo.com/mitmedialab', + 'duration': 23235, + 'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d', + 'chapters': 'count:37', + 'release_timestamp': 1744290000, + 'release_date': '20250410', + 'live_status': 'was_live', + }, + 'params': { + 'skip_download': 'm3u8', + 'http_headers': {'Referer': 'https://www.media.mit.edu/events/aha-symposium/'}, + }, + 'expected_warnings': ['Failed to parse XML: not well-formed'], + }, { + # Last entry on 2nd page of the 37 video playlist, but use clip_to_play_id API param shortcut + 'url': 'https://vimeo.com/event/4753126/videos/1046153257', + 'info_dict': { + 'id': '1046153257', + 'ext': 'mp4', + 'display_id': '4753126', + 'title': 'January 12, 2025 The True Vine (Pastor John Mindrup)', + 'description': 'The True Vine (Pastor \tJohn Mindrup)', + 'uploader': 'Salem United Church of Christ', + 'uploader_id': 'user230181094', + 'uploader_url': 'https://vimeo.com/user230181094', + 'comment_count': int, + 'like_count': int, + 'duration': 4962, + 'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d', + 'timestamp': 1736702464, + 'upload_date': '20250112', + 'release_timestamp': 1736702543, + 'release_date': '20250112', + 'live_status': 'was_live', + }, + 'params': {'skip_download': 'm3u8'}, + 'expected_warnings': ['Failed to parse XML: not well-formed'], + }, { + # "24/7" livestream + 'url': 'https://vimeo.com/event/4768062', + 'info_dict': { + 'id': '1079901414', + 'ext': 'mp4', + 'display_id': '4768062', + 'title': r're:GRACELAND CAM \d{4}-\d{2}-\d{2} \d{2}:\d{2}$', + 'description': '24/7 camera at Graceland Mansion', + 'uploader': 'Elvis Presley\'s Graceland', + 'uploader_id': 'visitgraceland', + 'uploader_url': 'https://vimeo.com/visitgraceland', + 'release_timestamp': 1745975450, + 'release_date': '20250430', + 'live_status': 'is_live', + }, + 'params': {'skip_download': 'livestream'}, + }, { + # stream_privacy.view: 'unlisted' with unlisted_hash in URL path (stream_privacy.embed: 'whitelist') + 'url': 'https://vimeo.com/event/4259978/3db517c479', + 'info_dict': { + 'id': '939104114', + 'ext': 'mp4', + 'display_id': '4259978', + 'title': 'Enhancing Credibility in Your Community Science Project', + 'description': 'md5:eab953341168b9c146bc3cfe3f716070', + 'uploader': 'NOAA Research', + 'uploader_id': 'noaaresearch', + 'uploader_url': 'https://vimeo.com/noaaresearch', + 'comment_count': int, + 'like_count': int, + 'duration': 3961, + 'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d', + 'timestamp': 1716408008, + 'upload_date': '20240522', + 'release_timestamp': 1716408062, + 'release_date': '20240522', + 'live_status': 'was_live', + }, + 'params': {'skip_download': 'm3u8'}, + 'expected_warnings': ['Failed to parse XML: not well-formed'], + }, { + # "done" event with video_id in URL and unlisted_hash in VimeoIE URL + 'url': 'https://vimeo.com/event/595460/videos/498149131/', + 'info_dict': { + 'id': '498149131', + 'ext': 'mp4', + 'display_id': '595460', + 'title': '2021 Eighth Annual John Cardinal Foley Lecture on Social Communications', + 'description': 'Replay: https://vimeo.com/catholicphilly/review/498149131/544f26a12f', + 'uploader': 'Kearns Media Consulting LLC', + 'uploader_id': 'kearnsmediaconsulting', + 'uploader_url': 'https://vimeo.com/kearnsmediaconsulting', + 'comment_count': int, + 'like_count': int, + 'duration': 4466, + 'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d', + 'timestamp': 1612228466, + 'upload_date': '20210202', + 'release_timestamp': 1612228538, + 'release_date': '20210202', + 'live_status': 'was_live', + }, + 'params': {'skip_download': 'm3u8'}, + 'expected_warnings': ['Failed to parse XML: not well-formed'], + }, { + # stream_privacy.view: 'password'; stream_privacy.embed: 'public' + 'url': 'https://vimeo.com/event/4940578', + 'info_dict': { + 'id': '1059263570', + 'ext': 'mp4', + 'display_id': '4940578', + 'title': 'TMAC AKC AGILITY 2-22-2025', + 'uploader': 'Paws \'N Effect', + 'uploader_id': 'pawsneffect', + 'uploader_url': 'https://vimeo.com/pawsneffect', + 'comment_count': int, + 'like_count': int, + 'duration': 33115, + 'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d', + 'timestamp': 1740261836, + 'upload_date': '20250222', + 'release_timestamp': 1740261873, + 'release_date': '20250222', + 'live_status': 'was_live', + }, + 'params': { + 'videopassword': '22', + 'skip_download': 'm3u8', + }, + 'expected_warnings': ['Failed to parse XML: not well-formed'], + }, { + # API serves a playlist of 37 videos, but the site only streams the newest one (changes every Sunday) + 'url': 'https://vimeo.com/event/4753126', + 'only_matching': True, + }, { + # Scheduled for 2025.05.15 but never started; "unavailable"; stream_privacy.view: "anybody" + 'url': 'https://vimeo.com/event/5120811/embed', + 'only_matching': True, + }, { + 'url': 'https://vimeo.com/event/5112969/embed?muted=1', + 'only_matching': True, + }, { + 'url': 'https://vimeo.com/event/5097437/embed/interaction?muted=1', + 'only_matching': True, + }, { + 'url': 'https://vimeo.com/event/5113032/embed?autoplay=1&muted=1', + 'only_matching': True, + }, { + # Ended livestream with video_id + 'url': 'https://vimeo.com/event/595460/videos/507329569/', + 'only_matching': True, + }, { + # stream_privacy.view: 'unlisted' with unlisted_hash in URL path (stream_privacy.embed: 'public') + 'url': 'https://vimeo.com/event/4606123/embed/358d60ce2e', + 'only_matching': True, + }] + _WEBPAGE_TESTS = [{ + # Same result as https://vimeo.com/event/5034253/embed + 'url': 'https://www.media.mit.edu/events/aha-symposium/', + 'info_dict': { + 'id': '1071439154', + 'ext': 'mp4', + 'display_id': '5034253', + 'title': 'Advancing Humans with AI', + 'description': r're:AI is here to stay, but how do we ensure that people flourish in a world of pervasive AI use.{322}$', + 'uploader': 'MIT Media Lab', + 'uploader_id': 'mitmedialab', + 'uploader_url': 'https://vimeo.com/mitmedialab', + 'duration': 23235, + 'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d', + 'chapters': 'count:37', + 'release_timestamp': 1744290000, + 'release_date': '20250410', + 'live_status': 'was_live', + }, + 'params': {'skip_download': 'm3u8'}, + 'expected_warnings': ['Failed to parse XML: not well-formed'], + }] + + _EVENT_FIELDS = ( + 'title', 'uri', 'schedule', 'stream_description', 'stream_privacy.embed', 'stream_privacy.view', + 'clip_to_play.name', 'clip_to_play.uri', 'clip_to_play.config_url', 'clip_to_play.live.status', + 'clip_to_play.privacy.embed', 'clip_to_play.privacy.view', 'clip_to_play.password', + 'streamable_clip.name', 'streamable_clip.uri', 'streamable_clip.config_url', 'streamable_clip.live.status', + ) + _VIDEOS_FIELDS = ('items', 'uri', 'name', 'config_url', 'duration', 'live.status') + + def _call_events_api( + self, event_id, ep=None, unlisted_hash=None, note=None, + fields=(), referrer=None, query=None, headers=None, + ): + resource = join_nonempty('event', ep, note, 'API JSON', delim=' ') + + return self._download_json( + join_nonempty( + 'https://api.vimeo.com/live_events', + join_nonempty(event_id, unlisted_hash, delim=':'), ep, delim='/'), + event_id, f'Downloading {resource}', f'Failed to download {resource}', + query=filter_dict({ + 'fields': ','.join(fields) or [], + # Correct spelling with 4 R's is deliberate + 'referrer': referrer, + **(query or {}), + }), headers=filter_dict({ + 'Accept': 'application/json', + 'Authorization': f'jwt {self._fetch_viewer_info(event_id)["jwt"]}', + 'Referer': referrer, + **(headers or {}), + })) + + @staticmethod + def _extract_video_id_and_unlisted_hash(video): + if not traverse_obj(video, ('uri', {lambda x: x.startswith('/videos/')})): + return None, None + video_id, _, unlisted_hash = video['uri'][8:].partition(':') + return video_id, unlisted_hash or None + + def _vimeo_url_result(self, video_id, unlisted_hash=None, event_id=None): + # VimeoIE can extract more metadata and formats for was_live event videos + return self.url_result( + join_nonempty('https://vimeo.com', video_id, unlisted_hash, delim='/'), VimeoIE, + video_id, display_id=event_id, live_status='was_live', url_transparent=True) + + @classmethod + def _extract_embed_urls(cls, url, webpage): + for embed_url in super()._extract_embed_urls(url, webpage): + yield cls._smuggle_referrer(embed_url, url) + + def _real_extract(self, url): + url, _, headers = self._unsmuggle_headers(url) + # XXX: Keep key name in sync with _unsmuggle_headers + referrer = headers.get('Referer') + event_id, unlisted_hash, video_id = self._match_valid_url(url).group('id', 'unlisted_hash', 'video_id') + + for retry in (False, True): + try: + live_event_data = self._call_events_api( + event_id, unlisted_hash=unlisted_hash, fields=self._EVENT_FIELDS, + referrer=referrer, query={'clip_to_play_id': video_id or '0'}, + headers={'Accept': 'application/vnd.vimeo.*+json;version=3.4.9'}) + break + except ExtractorError as e: + if retry or not isinstance(e.cause, HTTPError) or e.cause.status not in (400, 403): + raise + response = traverse_obj(e.cause.response.read(), ({json.loads}, {dict})) or {} + error_code = response.get('error_code') + if error_code == 2204: + self._verify_video_password(event_id, path='event') + continue + if error_code == 3200: + raise ExtractorError(self._REFERER_HINT, expected=True) + if error_msg := response.get('error'): + raise ExtractorError(f'Vimeo says: {error_msg}', expected=True) + raise + + # stream_privacy.view can be: 'anybody', 'embed_only', 'nobody', 'password', 'unlisted' + view_policy = live_event_data['stream_privacy']['view'] + if view_policy == 'nobody': + raise ExtractorError('This event has not been made available to anyone', expected=True) + + clip_data = traverse_obj(live_event_data, ('clip_to_play', {dict})) or {} + # live.status can be: 'streaming' (is_live), 'done' (was_live), 'unavailable' (is_upcoming OR dead) + clip_status = traverse_obj(clip_data, ('live', 'status', {str})) + start_time = traverse_obj(live_event_data, ('schedule', 'start_time', {str})) + release_timestamp = parse_iso8601(start_time) + + if clip_status == 'unavailable' and release_timestamp and release_timestamp > time.time(): + self.raise_no_formats(f'This live event is scheduled for {start_time}', expected=True) + live_status = 'is_upcoming' + config_url = None + + elif view_policy == 'embed_only': + webpage = self._download_webpage( + join_nonempty('https://vimeo.com/event', event_id, 'embed', unlisted_hash, delim='/'), + event_id, 'Downloading embed iframe webpage', impersonate=True, headers=headers) + # The _parse_config result will overwrite live_status w/ 'is_live' if livestream is active + live_status = 'was_live' + config_url = self._extract_config_url(webpage) + + else: # view_policy in ('anybody', 'password', 'unlisted') + if video_id: + clip_id, clip_hash = self._extract_video_id_and_unlisted_hash(clip_data) + if video_id == clip_id and clip_status == 'done' and (clip_hash or view_policy != 'unlisted'): + return self._vimeo_url_result(clip_id, clip_hash, event_id) + + video_filter = lambda _, v: self._extract_video_id_and_unlisted_hash(v)[0] == video_id + else: + video_filter = lambda _, v: v['live']['status'] in ('streaming', 'done') + + for page in itertools.count(1): + videos_data = self._call_events_api( + event_id, 'videos', unlisted_hash=unlisted_hash, note=f'page {page}', + fields=self._VIDEOS_FIELDS, referrer=referrer, query={'page': page}, + headers={'Accept': 'application/vnd.vimeo.*;version=3.4.1'}) + + video = traverse_obj(videos_data, ('data', video_filter, any)) + if video or not traverse_obj(videos_data, ('paging', 'next', {str})): + break + + live_status = { + 'streaming': 'is_live', + 'done': 'was_live', + }.get(traverse_obj(video, ('live', 'status', {str}))) + + if not live_status: # requested video_id is unavailable or no videos are available + raise ExtractorError('This event video is unavailable', expected=True) + elif live_status == 'was_live': + return self._vimeo_url_result(*self._extract_video_id_and_unlisted_hash(video), event_id) + config_url = video['config_url'] + + if config_url: # view_policy == 'embed_only' or live_status == 'is_live' + info = filter_dict(self._parse_config( + self._download_json(config_url, event_id, 'Downloading config JSON'), event_id)) + else: # live_status == 'is_upcoming' + info = {'id': event_id} + + if info.get('live_status') == 'post_live': + self.report_warning('This live event recently ended and some formats may not yet be available') + + return { + **traverse_obj(live_event_data, { + 'title': ('title', {str}), + 'description': ('stream_description', {str}), + }), + 'display_id': event_id, + 'live_status': live_status, + 'release_timestamp': release_timestamp, + **info, + } From 31e090cb787f3504ec25485adff9a2a51d056734 Mon Sep 17 00:00:00 2001 From: Subrat Lima <74418100+subrat-lima@users.noreply.github.com> Date: Wed, 21 May 2025 03:07:21 +0530 Subject: [PATCH 09/67] [ie/picarto:vod] Support `/profile/` video URLs (#13227) Closes #13192 Authored by: subrat-lima --- yt_dlp/extractor/picarto.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/picarto.py b/yt_dlp/extractor/picarto.py index 72e89c31e..92431fa24 100644 --- a/yt_dlp/extractor/picarto.py +++ b/yt_dlp/extractor/picarto.py @@ -10,7 +10,8 @@ class PicartoIE(InfoExtractor): - _VALID_URL = r'https?://(?:www.)?picarto\.tv/(?P[a-zA-Z0-9]+)' + IE_NAME = 'picarto' + _VALID_URL = r'https?://(?:www.)?picarto\.tv/(?P[^/#?]+)/?(?:$|[?#])' _TEST = { 'url': 'https://picarto.tv/Setz', 'info_dict': { @@ -89,7 +90,8 @@ def _real_extract(self, url): class PicartoVodIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?picarto\.tv/(?:videopopout|\w+/videos)/(?P[^/?#&]+)' + IE_NAME = 'picarto:vod' + _VALID_URL = r'https?://(?:www\.)?picarto\.tv/(?:videopopout|\w+(?:/profile)?/videos)/(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://picarto.tv/videopopout/ArtofZod_2017.12.12.00.13.23.flv', 'md5': '3ab45ba4352c52ee841a28fb73f2d9ca', @@ -111,6 +113,18 @@ class PicartoVodIE(InfoExtractor): 'channel': 'ArtofZod', 'age_limit': 18, }, + }, { + 'url': 'https://picarto.tv/DrechuArt/profile/videos/400347', + 'md5': 'f9ea54868b1d9dec40eb554b484cc7bf', + 'info_dict': { + 'id': '400347', + 'ext': 'mp4', + 'title': 'Welcome to the Show', + 'thumbnail': r're:^https?://.*\.jpg', + 'channel': 'DrechuArt', + 'age_limit': 0, + }, + }, { 'url': 'https://picarto.tv/videopopout/Plague', 'only_matching': True, From c7e575e31608c19c5b26c10a4229db89db5fc9a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20=C3=96zel?= Date: Tue, 20 May 2025 23:39:27 +0200 Subject: [PATCH 10/67] [ie/youtube] Fix geo-restriction error handling (#13217) Authored by: yozel --- yt_dlp/extractor/youtube/_video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index 28fff1969..b4c6ba453 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -3755,7 +3755,7 @@ def feed_entry(name): reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason') subreason = clean_html(self._get_text(pemr, 'subreason') or '') if subreason: - if subreason == 'The uploader has not made this video available in your country.': + if subreason.startswith('The uploader has not made this video available in your country'): countries = get_first(microformats, 'availableCountries') if not countries: regions_allowed = search_meta('regionsAllowed') From 00b1bec55249cf2ad6271d36492c51b34b6459d1 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Tue, 20 May 2025 16:53:54 -0500 Subject: [PATCH 11/67] [ie/twitch] Support `--live-from-start` (#13202) Closes #10520 Authored by: bashonly --- yt_dlp/extractor/twitch.py | 72 ++++++++++++++++++++++++++++---------- yt_dlp/options.py | 2 +- 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py index 4f4c59627..e4f2aec46 100644 --- a/yt_dlp/extractor/twitch.py +++ b/yt_dlp/extractor/twitch.py @@ -187,7 +187,7 @@ def _get_thumbnails(self, thumbnail): 'url': thumbnail, }] if thumbnail else None - def _extract_twitch_m3u8_formats(self, path, video_id, token, signature): + def _extract_twitch_m3u8_formats(self, path, video_id, token, signature, live_from_start=False): formats = self._extract_m3u8_formats( f'{self._USHER_BASE}/{path}/{video_id}.m3u8', video_id, 'mp4', query={ 'allow_source': 'true', @@ -204,7 +204,10 @@ def _extract_twitch_m3u8_formats(self, path, video_id, token, signature): for fmt in formats: if fmt.get('vcodec') and fmt['vcodec'].startswith('av01'): # mpegts does not yet have proper support for av1 - fmt['downloader_options'] = {'ffmpeg_args_out': ['-f', 'mp4']} + fmt.setdefault('downloader_options', {}).update({'ffmpeg_args_out': ['-f', 'mp4']}) + if live_from_start: + fmt.setdefault('downloader_options', {}).update({'ffmpeg_args': ['-live_start_index', '0']}) + fmt['is_from_start'] = True return formats @@ -550,7 +553,8 @@ def _real_extract(self, url): access_token = self._download_access_token(vod_id, 'video', 'id') formats = self._extract_twitch_m3u8_formats( - 'vod', vod_id, access_token['value'], access_token['signature']) + 'vod', vod_id, access_token['value'], access_token['signature'], + live_from_start=self.get_param('live_from_start')) formats.extend(self._extract_storyboard(vod_id, video.get('storyboard'), info.get('duration'))) self._prefer_source(formats) @@ -633,6 +637,10 @@ class TwitchPlaylistBaseIE(TwitchBaseIE): _PAGE_LIMIT = 100 def _entries(self, channel_name, *args): + """ + Subclasses must define _make_variables() and _extract_entry(), + as well as set _OPERATION_NAME, _ENTRY_KIND, _EDGE_KIND, and _NODE_KIND + """ cursor = None variables_common = self._make_variables(channel_name, *args) entries_key = f'{self._ENTRY_KIND}s' @@ -672,7 +680,22 @@ def _entries(self, channel_name, *args): break -class TwitchVideosIE(TwitchPlaylistBaseIE): +class TwitchVideosBaseIE(TwitchPlaylistBaseIE): + _OPERATION_NAME = 'FilterableVideoTower_Videos' + _ENTRY_KIND = 'video' + _EDGE_KIND = 'VideoEdge' + _NODE_KIND = 'Video' + + @staticmethod + def _make_variables(channel_name, broadcast_type, sort): + return { + 'channelOwnerLogin': channel_name, + 'broadcastType': broadcast_type, + 'videoSort': sort.upper(), + } + + +class TwitchVideosIE(TwitchVideosBaseIE): _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P[^/]+)/(?:videos|profile)' _TESTS = [{ @@ -751,11 +774,6 @@ class TwitchVideosIE(TwitchPlaylistBaseIE): 'views': 'Popular', } - _OPERATION_NAME = 'FilterableVideoTower_Videos' - _ENTRY_KIND = 'video' - _EDGE_KIND = 'VideoEdge' - _NODE_KIND = 'Video' - @classmethod def suitable(cls, url): return (False @@ -764,14 +782,6 @@ def suitable(cls, url): TwitchVideosCollectionsIE)) else super().suitable(url)) - @staticmethod - def _make_variables(channel_name, broadcast_type, sort): - return { - 'channelOwnerLogin': channel_name, - 'broadcastType': broadcast_type, - 'videoSort': sort.upper(), - } - @staticmethod def _extract_entry(node): return _make_video_result(node) @@ -919,7 +929,7 @@ def _real_extract(self, url): playlist_title=f'{channel_name} - Collections') -class TwitchStreamIE(TwitchBaseIE): +class TwitchStreamIE(TwitchVideosBaseIE): IE_NAME = 'twitch:stream' _VALID_URL = r'''(?x) https?:// @@ -982,6 +992,7 @@ class TwitchStreamIE(TwitchBaseIE): 'skip_download': 'Livestream', }, }] + _PAGE_LIMIT = 1 @classmethod def suitable(cls, url): @@ -995,6 +1006,20 @@ def suitable(cls, url): TwitchClipsIE)) else super().suitable(url)) + @staticmethod + def _extract_entry(node): + if not isinstance(node, dict) or not node.get('id'): + return None + video_id = node['id'] + return { + '_type': 'url', + 'ie_key': TwitchVodIE.ie_key(), + 'id': 'v' + video_id, + 'url': f'https://www.twitch.tv/videos/{video_id}', + 'title': node.get('title'), + 'timestamp': unified_timestamp(node.get('publishedAt')) or 0, + } + def _real_extract(self, url): channel_name = self._match_id(url).lower() @@ -1029,6 +1054,16 @@ def _real_extract(self, url): if not stream: raise UserNotLive(video_id=channel_name) + timestamp = unified_timestamp(stream.get('createdAt')) + + if self.get_param('live_from_start'): + self.to_screen(f'{channel_name}: Extracting VOD to download live from start') + entry = next(self._entries(channel_name, None, 'time'), None) + if entry and entry.pop('timestamp') >= (timestamp or float('inf')): + return entry + self.report_warning( + 'Unable to extract the VOD associated with this livestream', video_id=channel_name) + access_token = self._download_access_token( channel_name, 'stream', 'channelName') @@ -1038,7 +1073,6 @@ def _real_extract(self, url): self._prefer_source(formats) view_count = stream.get('viewers') - timestamp = unified_timestamp(stream.get('createdAt')) sq_user = try_get(gql, lambda x: x[1]['data']['user'], dict) or {} uploader = sq_user.get('displayName') diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 76d401cea..19f16e725 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -470,7 +470,7 @@ def _preset_alias_callback(option, opt_str, value, parser): general.add_option( '--live-from-start', action='store_true', dest='live_from_start', - help='Download livestreams from the start. Currently only supported for YouTube (Experimental)') + help='Download livestreams from the start. Currently only supported for YouTube (experimental) and Twitch') general.add_option( '--no-live-from-start', action='store_false', dest='live_from_start', From 83fabf352489d52843f67e6e9cc752db86d27e6e Mon Sep 17 00:00:00 2001 From: garret1317 Date: Wed, 21 May 2025 19:29:35 +0100 Subject: [PATCH 12/67] [ie/xinpianchang] Fix extractor (#13245) Closes #12737 Authored by: garret1317 --- yt_dlp/extractor/xinpianchang.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/xinpianchang.py b/yt_dlp/extractor/xinpianchang.py index 23ed9270d..a4263579a 100644 --- a/yt_dlp/extractor/xinpianchang.py +++ b/yt_dlp/extractor/xinpianchang.py @@ -45,7 +45,7 @@ class XinpianchangIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id=video_id) + webpage = self._download_webpage(url, video_id=video_id, headers={'Referer': url}) video_data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['detail']['video'] data = self._download_json( From 167d7a9f0ffd1b4fe600193441bdb7358db2740b Mon Sep 17 00:00:00 2001 From: sepro Date: Thu, 22 May 2025 00:27:07 +0200 Subject: [PATCH 13/67] [jsinterp] Fix increment/decrement evaluation (#13238) Closes #13241 Authored by: seproDev, bashonly Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com> --- test/test_jsinterp.py | 8 +++++ test/test_youtube_signature.py | 4 +++ yt_dlp/jsinterp.py | 61 +++++++++++++++++----------------- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index b14069ccc..2e3cdc2a5 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -478,6 +478,14 @@ def test_extract_function_with_global_stack(self): func = jsi.extract_function('c', {'e': 10}, {'f': 100, 'g': 1000}) self.assertEqual(func([1]), 1111) + def test_increment_decrement(self): + self._test('function f() { var x = 1; return ++x; }', 2) + self._test('function f() { var x = 1; return x++; }', 1) + self._test('function f() { var x = 1; x--; return x }', 0) + self._test('function f() { var y; var x = 1; x++, --x, x--, x--, y="z", "abc", x++; return --x }', -1) + self._test('function f() { var a = "test--"; return a; }', 'test--') + self._test('function f() { var b = 1; var a = "b--"; return a; }', 'b--') + if __name__ == '__main__': unittest.main() diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 0f0885366..3f777aed7 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -316,6 +316,10 @@ 'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js', 'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE', ), + ( + 'https://www.youtube.com/s/player/59b252b9/player_ias.vflset/en_US/base.js', + 'D3XWVpYgwhLLKNK4AGX', 'aZrQ1qWJ5yv5h', + ), ] diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py index b59fb2c61..45aeffa22 100644 --- a/yt_dlp/jsinterp.py +++ b/yt_dlp/jsinterp.py @@ -590,39 +590,12 @@ def dict_item(key, val): return ret, True return ret, False - for m in re.finditer(rf'''(?x) - (?P\+\+|--)(?P{_NAME_RE})| - (?P{_NAME_RE})(?P\+\+|--)''', expr): - var = m.group('var1') or m.group('var2') - start, end = m.span() - sign = m.group('pre_sign') or m.group('post_sign') - ret = local_vars[var] - local_vars[var] += 1 if sign[0] == '+' else -1 - if m.group('pre_sign'): - ret = local_vars[var] - expr = expr[:start] + self._dump(ret, local_vars) + expr[end:] - - if not expr: - return None, should_return - m = re.match(fr'''(?x) - (?P (?P{_NAME_RE})(?:\[(?P{_NESTED_BRACKETS})\])?\s* (?P{"|".join(map(re.escape, set(_OPERATORS) - _COMP_OPERATORS))})? =(?!=)(?P.*)$ - )|(?P - (?!if|return|true|false|null|undefined|NaN)(?P{_NAME_RE})$ - )|(?P - (?P{_NAME_RE})(?: - (?P\?)?\.(?P[^(]+)| - \[(?P{_NESTED_BRACKETS})\] - )\s* - )|(?P - (?P{_NAME_RE})\[(?P.+)\]$ - )|(?P - (?P{_NAME_RE})\((?P.*)\)$ - )''', expr) - if m and m.group('assign'): + ''', expr) + if m: # We are assigning a value to a variable left_val = local_vars.get(m.group('out')) if not m.group('index'): @@ -640,7 +613,35 @@ def dict_item(key, val): m.group('op'), self._index(left_val, idx), m.group('expr'), expr, local_vars, allow_recursion) return left_val[idx], should_return - elif expr.isdigit(): + for m in re.finditer(rf'''(?x) + (?P\+\+|--)(?P{_NAME_RE})| + (?P{_NAME_RE})(?P\+\+|--)''', expr): + var = m.group('var1') or m.group('var2') + start, end = m.span() + sign = m.group('pre_sign') or m.group('post_sign') + ret = local_vars[var] + local_vars[var] += 1 if sign[0] == '+' else -1 + if m.group('pre_sign'): + ret = local_vars[var] + expr = expr[:start] + self._dump(ret, local_vars) + expr[end:] + + if not expr: + return None, should_return + + m = re.match(fr'''(?x) + (?P + (?!if|return|true|false|null|undefined|NaN)(?P{_NAME_RE})$ + )|(?P + (?P{_NAME_RE})(?: + (?P\?)?\.(?P[^(]+)| + \[(?P{_NESTED_BRACKETS})\] + )\s* + )|(?P + (?P{_NAME_RE})\[(?P.+)\]$ + )|(?P + (?P{_NAME_RE})\((?P.*)\)$ + )''', expr) + if expr.isdigit(): return int(expr), should_return elif expr == 'break': From 32ed5f107c6c641958d1cd2752e130de4db55a13 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Thu, 22 May 2025 04:13:42 -0500 Subject: [PATCH 14/67] [ie/youtube] Add PO token support for subtitles (#13234) Closes #13075 Authored by: bashonly, coletdjnz Co-authored-by: coletdjnz --- README.md | 2 +- test/test_pot/test_pot_builtin_utils.py | 1 + yt_dlp/extractor/youtube/_base.py | 2 + yt_dlp/extractor/youtube/_video.py | 172 +++++++++++++++++------ yt_dlp/extractor/youtube/pot/provider.py | 1 + yt_dlp/extractor/youtube/pot/utils.py | 2 +- 6 files changed, 138 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 9f542844e..aaa3beb71 100644 --- a/README.md +++ b/README.md @@ -1805,7 +1805,7 @@ #### youtube * `raise_incomplete_data`: `Incomplete Data Received` raises an error instead of reporting a warning * `data_sync_id`: Overrides the account Data Sync ID used in Innertube API requests. This may be needed if you are using an account with `youtube:player_skip=webpage,configs` or `youtubetab:skip=webpage` * `visitor_data`: Overrides the Visitor Data used in Innertube API requests. This should be used with `player_skip=webpage,configs` and without cookies. Note: this may have adverse effects if used improperly. If a session from a browser is wanted, you should pass cookies instead (which contain the Visitor ID) -* `po_token`: Proof of Origin (PO) Token(s) to use. Comma seperated list of PO Tokens in the format `CLIENT.CONTEXT+PO_TOKEN`, e.g. `youtube:po_token=web.gvs+XXX,web.player=XXX,web_safari.gvs+YYY`. Context can be either `gvs` (Google Video Server URLs) or `player` (Innertube player request) +* `po_token`: Proof of Origin (PO) Token(s) to use. Comma seperated list of PO Tokens in the format `CLIENT.CONTEXT+PO_TOKEN`, e.g. `youtube:po_token=web.gvs+XXX,web.player=XXX,web_safari.gvs+YYY`. Context can be any of `gvs` (Google Video Server URLs), `player` (Innertube player request) or `subs` (Subtitles) * `pot_trace`: Enable debug logging for PO Token fetching. Either `true` or `false` (default) * `fetch_pot`: Policy to use for fetching a PO Token from providers. One of `always` (always try fetch a PO Token regardless if the client requires one for the given context), `never` (never fetch a PO Token), or `auto` (default; only fetch a PO Token if the client requires one for the given context) diff --git a/test/test_pot/test_pot_builtin_utils.py b/test/test_pot/test_pot_builtin_utils.py index 1682e42a1..a95fc4e15 100644 --- a/test/test_pot/test_pot_builtin_utils.py +++ b/test/test_pot/test_pot_builtin_utils.py @@ -15,6 +15,7 @@ class TestGetWebPoContentBinding: for context, is_authenticated, expected in [ (PoTokenContext.GVS, False, ('example-visitor-data', ContentBindingType.VISITOR_DATA)), (PoTokenContext.PLAYER, False, ('example-video-id', ContentBindingType.VIDEO_ID)), + (PoTokenContext.SUBS, False, ('example-video-id', ContentBindingType.VIDEO_ID)), (PoTokenContext.GVS, True, ('example-data-sync-id', ContentBindingType.DATASYNC_ID)), ]], ('WEB_REMIX', PoTokenContext.GVS, False, ('example-visitor-data', ContentBindingType.VISITOR_DATA)), diff --git a/yt_dlp/extractor/youtube/_base.py b/yt_dlp/extractor/youtube/_base.py index 4194e1c21..9c5bb75fe 100644 --- a/yt_dlp/extractor/youtube/_base.py +++ b/yt_dlp/extractor/youtube/_base.py @@ -35,6 +35,7 @@ class _PoTokenContext(enum.Enum): PLAYER = 'player' GVS = 'gvs' + SUBS = 'subs' # any clients starting with _ cannot be explicitly requested by the user @@ -787,6 +788,7 @@ def _download_webpage_with_retries(self, *args, retry_fatal=False, retry_on_stat def _download_ytcfg(self, client, video_id): url = { + 'mweb': 'https://m.youtube.com', 'web': 'https://www.youtube.com', 'web_music': 'https://music.youtube.com', 'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1', diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index b4c6ba453..9f929664f 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -72,6 +72,9 @@ STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client' STREAMING_DATA_INITIAL_PO_TOKEN = '__yt_dlp_po_token' +STREAMING_DATA_FETCH_SUBS_PO_TOKEN = '__yt_dlp_fetch_subs_po_token' +STREAMING_DATA_INNERTUBE_CONTEXT = '__yt_dlp_innertube_context' + PO_TOKEN_GUIDE_URL = 'https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide' @@ -2863,7 +2866,8 @@ def _get_config_po_token(self, client: str, context: _PoTokenContext): continue def fetch_po_token(self, client='web', context=_PoTokenContext.GVS, ytcfg=None, visitor_data=None, - data_sync_id=None, session_index=None, player_url=None, video_id=None, webpage=None, **kwargs): + data_sync_id=None, session_index=None, player_url=None, video_id=None, webpage=None, + required=False, **kwargs): """ Fetch a PO Token for a given client and context. This function will validate required parameters for a given context and client. @@ -2878,6 +2882,7 @@ def fetch_po_token(self, client='web', context=_PoTokenContext.GVS, ytcfg=None, @param player_url: player URL. @param video_id: video ID. @param webpage: video webpage. + @param required: Whether the PO Token is required (i.e. try to fetch unless policy is "never"). @param kwargs: Additional arguments to pass down. May be more added in the future. @return: The fetched PO Token. None if it could not be fetched. """ @@ -2926,6 +2931,7 @@ def fetch_po_token(self, client='web', context=_PoTokenContext.GVS, ytcfg=None, player_url=player_url, video_id=video_id, video_webpage=webpage, + required=required, **kwargs, ) @@ -2945,6 +2951,7 @@ def _fetch_po_token(self, client, **kwargs): or ( fetch_pot_policy == 'auto' and _PoTokenContext(context) not in self._get_default_ytcfg(client)['PO_TOKEN_REQUIRED_CONTEXTS'] + and not kwargs.get('required', False) ) ): return None @@ -3133,6 +3140,8 @@ def append_client(*client_names): player_url = self._download_player_url(video_id) tried_iframe_fallback = True + pr = initial_pr if client == 'web' else None + visitor_data = visitor_data or self._extract_visitor_data(master_ytcfg, initial_pr, player_ytcfg) data_sync_id = data_sync_id or self._extract_data_sync_id(master_ytcfg, initial_pr, player_ytcfg) @@ -3147,12 +3156,19 @@ def append_client(*client_names): 'ytcfg': player_ytcfg or self._get_default_ytcfg(client), } - player_po_token = self.fetch_po_token( + # Don't need a player PO token for WEB if using player response from webpage + player_po_token = None if pr else self.fetch_po_token( context=_PoTokenContext.PLAYER, **fetch_po_token_args) gvs_po_token = self.fetch_po_token( context=_PoTokenContext.GVS, **fetch_po_token_args) + fetch_subs_po_token_func = functools.partial( + self.fetch_po_token, + context=_PoTokenContext.SUBS, + **fetch_po_token_args, + ) + required_pot_contexts = self._get_default_ytcfg(client)['PO_TOKEN_REQUIRED_CONTEXTS'] if ( @@ -3179,7 +3195,6 @@ def append_client(*client_names): only_once=True) deprioritize_pr = True - pr = initial_pr if client == 'web' else None try: pr = pr or self._extract_player_response( client, video_id, @@ -3197,10 +3212,13 @@ def append_client(*client_names): if pr_id := self._invalid_player_response(pr, video_id): skipped_clients[client] = pr_id elif pr: - # Save client name for introspection later - sd = traverse_obj(pr, ('streamingData', {dict})) or {} + # Save client details for introspection later + innertube_context = traverse_obj(player_ytcfg or self._get_default_ytcfg(client), 'INNERTUBE_CONTEXT') + sd = pr.setdefault('streamingData', {}) sd[STREAMING_DATA_CLIENT_NAME] = client sd[STREAMING_DATA_INITIAL_PO_TOKEN] = gvs_po_token + sd[STREAMING_DATA_INNERTUBE_CONTEXT] = innertube_context + sd[STREAMING_DATA_FETCH_SUBS_PO_TOKEN] = fetch_subs_po_token_func for f in traverse_obj(sd, (('formats', 'adaptiveFormats'), ..., {dict})): f[STREAMING_DATA_CLIENT_NAME] = client f[STREAMING_DATA_INITIAL_PO_TOKEN] = gvs_po_token @@ -3262,6 +3280,25 @@ def _report_pot_format_skipped(self, video_id, client_name, proto): else: self.report_warning(msg, only_once=True) + def _report_pot_subtitles_skipped(self, video_id, client_name, msg=None): + msg = msg or ( + f'{video_id}: Some {client_name} client subtitles require a PO Token which was not provided. ' + 'They will be discarded since they are not downloadable as-is. ' + f'You can manually pass a Subtitles PO Token for this client with ' + f'--extractor-args "youtube:po_token={client_name}.subs+XXX" . ' + f'For more information, refer to {PO_TOKEN_GUIDE_URL}') + + subs_wanted = any(( + self.get_param('writesubtitles'), + self.get_param('writeautomaticsub'), + self.get_param('listsubtitles'))) + + # Only raise a warning for non-default clients, to not confuse users. + if not subs_wanted or client_name in (*self._DEFAULT_CLIENTS, *self._DEFAULT_AUTHED_CLIENTS): + self.write_debug(msg, only_once=True) + else: + self.report_warning(msg, only_once=True) + def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration): CHUNK_SIZE = 10 << 20 PREFERRED_LANG_VALUE = 10 @@ -3553,6 +3590,9 @@ def process_manifest_format(f, proto, client_name, itag, po_token): hls_manifest_url = hls_manifest_url.rstrip('/') + f'/pot/{po_token}' fmts, subs = self._extract_m3u8_formats_and_subtitles( hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live') + for sub in traverse_obj(subs, (..., ..., {dict})): + # HLS subs (m3u8) do not need a PO token; save client name for debugging + sub[STREAMING_DATA_CLIENT_NAME] = client_name subtitles = self._merge_subtitles(subs, subtitles) for f in fmts: if process_manifest_format(f, 'hls', client_name, self._search_regex( @@ -3564,6 +3604,9 @@ def process_manifest_format(f, proto, client_name, itag, po_token): if po_token: dash_manifest_url = dash_manifest_url.rstrip('/') + f'/pot/{po_token}' formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False) + for sub in traverse_obj(subs, (..., ..., {dict})): + # TODO: Investigate if DASH subs ever need a PO token; save client name for debugging + sub[STREAMING_DATA_CLIENT_NAME] = client_name subtitles = self._merge_subtitles(subs, subtitles) # Prioritize HLS subs over DASH for f in formats: if process_manifest_format(f, 'dash', client_name, f['format_id'], po_token): @@ -3890,47 +3933,81 @@ def is_bad_format(fmt): 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec', 'channels', 'acodec', 'lang', 'proto'), } + def get_lang_code(track): + return (remove_start(track.get('vssId') or '', '.').replace('.', '-') + or track.get('languageCode')) + + def process_language(container, base_url, lang_code, sub_name, client_name, query): + lang_subs = container.setdefault(lang_code, []) + for fmt in self._SUBTITLE_FORMATS: + query = {**query, 'fmt': fmt} + lang_subs.append({ + 'ext': fmt, + 'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)), + 'name': sub_name, + STREAMING_DATA_CLIENT_NAME: client_name, + }) + subtitles = {} - pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict) - if pctr: - def get_lang_code(track): - return (remove_start(track.get('vssId') or '', '.').replace('.', '-') - or track.get('languageCode')) + skipped_subs_clients = set() + prs = traverse_obj(player_responses, ( + # Filter out initial_pr which does not have streamingData (smuggled client context) + lambda _, v: v['streamingData'] and v['captions']['playerCaptionsTracklistRenderer'])) - # Converted into dicts to remove duplicates - captions = { - get_lang_code(sub): sub - for sub in traverse_obj(pctr, (..., 'captionTracks', ...))} - translation_languages = { - lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1) - for lang in traverse_obj(pctr, (..., 'translationLanguages', ...))} + pctrs = traverse_obj(prs, (..., 'captions', 'playerCaptionsTracklistRenderer', {dict})) + translation_languages = { + lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1) + for lang in traverse_obj(pctrs, (..., 'translationLanguages', ..., {dict}))} + # NB: Constructing the full subtitle dictionary is slow + get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and ( + self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles')) - def process_language(container, base_url, lang_code, sub_name, query): - lang_subs = container.setdefault(lang_code, []) - for fmt in self._SUBTITLE_FORMATS: - query.update({ - 'fmt': fmt, - }) - lang_subs.append({ - 'ext': fmt, - 'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)), - 'name': sub_name, - }) + all_captions = traverse_obj(pctrs, (..., 'captionTracks', ..., {dict})) + need_subs_langs = {get_lang_code(sub) for sub in all_captions if sub.get('kind') != 'asr'} + need_caps_langs = { + remove_start(get_lang_code(sub), 'a-') + for sub in all_captions if sub.get('kind') == 'asr'} - # NB: Constructing the full subtitle dictionary is slow - get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and ( - self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles')) - for lang_code, caption_track in captions.items(): - base_url = caption_track.get('baseUrl') - orig_lang = parse_qs(base_url).get('lang', [None])[-1] - if not base_url: - continue + for pr in prs: + pctr = pr['captions']['playerCaptionsTracklistRenderer'] + client_name = pr['streamingData'][STREAMING_DATA_CLIENT_NAME] + innertube_client_name = pr['streamingData'][STREAMING_DATA_INNERTUBE_CONTEXT]['client']['clientName'] + required_contexts = self._get_default_ytcfg(client_name)['PO_TOKEN_REQUIRED_CONTEXTS'] + fetch_subs_po_token_func = pr['streamingData'][STREAMING_DATA_FETCH_SUBS_PO_TOKEN] + + pot_params = {} + already_fetched_pot = False + + for caption_track in traverse_obj(pctr, ('captionTracks', lambda _, v: v['baseUrl'])): + base_url = caption_track['baseUrl'] + qs = parse_qs(base_url) + lang_code = get_lang_code(caption_track) + requires_pot = ( + # We can detect the experiment for now + any(e in traverse_obj(qs, ('exp', ...)) for e in ('xpe', 'xpv')) + or _PoTokenContext.SUBS in required_contexts) + + if not already_fetched_pot: + already_fetched_pot = True + if subs_po_token := fetch_subs_po_token_func(required=requires_pot): + pot_params.update({ + 'pot': subs_po_token, + 'potc': '1', + 'c': innertube_client_name, + }) + + if not pot_params and requires_pot: + skipped_subs_clients.add(client_name) + self._report_pot_subtitles_skipped(video_id, client_name) + break + + orig_lang = qs.get('lang', [None])[-1] lang_name = self._get_text(caption_track, 'name', max_runs=1) if caption_track.get('kind') != 'asr': if not lang_code: continue process_language( - subtitles, base_url, lang_code, lang_name, {}) + subtitles, base_url, lang_code, lang_name, client_name, pot_params) if not caption_track.get('isTranslatable'): continue for trans_code, trans_name in translation_languages.items(): @@ -3950,10 +4027,25 @@ def process_language(container, base_url, lang_code, sub_name, query): # Add an "-orig" label to the original language so that it can be distinguished. # The subs are returned without "-orig" as well for compatibility process_language( - automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {}) + automatic_captions, base_url, f'{trans_code}-orig', + f'{trans_name} (Original)', client_name, pot_params) # Setting tlang=lang returns damaged subtitles. - process_language(automatic_captions, base_url, trans_code, trans_name, - {} if orig_lang == orig_trans_code else {'tlang': trans_code}) + process_language( + automatic_captions, base_url, trans_code, trans_name, client_name, + pot_params if orig_lang == orig_trans_code else {'tlang': trans_code, **pot_params}) + + # Avoid duplication if we've already got everything we need + need_subs_langs.difference_update(subtitles) + need_caps_langs.difference_update(automatic_captions) + if not (need_subs_langs or need_caps_langs): + break + + if skipped_subs_clients and (need_subs_langs or need_caps_langs): + self._report_pot_subtitles_skipped(video_id, True, msg=join_nonempty( + f'{video_id}: There are missing subtitles languages because a PO token was not provided.', + need_subs_langs and f'Subtitles for these languages are missing: {", ".join(need_subs_langs)}.', + need_caps_langs and f'Automatic captions for {len(need_caps_langs)} languages are missing.', + delim=' ')) info['automatic_captions'] = automatic_captions info['subtitles'] = subtitles diff --git a/yt_dlp/extractor/youtube/pot/provider.py b/yt_dlp/extractor/youtube/pot/provider.py index 53af92d30..13b3b1f9b 100644 --- a/yt_dlp/extractor/youtube/pot/provider.py +++ b/yt_dlp/extractor/youtube/pot/provider.py @@ -39,6 +39,7 @@ class PoTokenContext(enum.Enum): GVS = 'gvs' PLAYER = 'player' + SUBS = 'subs' @dataclasses.dataclass diff --git a/yt_dlp/extractor/youtube/pot/utils.py b/yt_dlp/extractor/youtube/pot/utils.py index 1c0db243b..7a5b7d4ab 100644 --- a/yt_dlp/extractor/youtube/pot/utils.py +++ b/yt_dlp/extractor/youtube/pot/utils.py @@ -51,7 +51,7 @@ def get_webpo_content_binding( return visitor_id, ContentBindingType.VISITOR_ID return request.visitor_data, ContentBindingType.VISITOR_DATA - elif request.context == PoTokenContext.PLAYER or client_name != 'WEB_REMIX': + elif request.context in (PoTokenContext.PLAYER, PoTokenContext.SUBS): return request.video_id, ContentBindingType.VIDEO_ID return None, None From e491fd4d090db3af52a82863fb0553dd5e17fb85 Mon Sep 17 00:00:00 2001 From: Matt Broadway Date: Thu, 22 May 2025 10:22:11 +0100 Subject: [PATCH 15/67] [cookies] Fix Linux desktop environment detection (#13197) Closes #12885 Authored by: mbway --- test/test_cookies.py | 8 +++++++ yt_dlp/cookies.py | 53 ++++++++++++++++++++++---------------------- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/test/test_cookies.py b/test/test_cookies.py index 4b9b9b5a9..f956ab187 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -58,6 +58,14 @@ def test_get_desktop_environment(self): ({'DESKTOP_SESSION': 'kde'}, _LinuxDesktopEnvironment.KDE3), ({'DESKTOP_SESSION': 'xfce'}, _LinuxDesktopEnvironment.XFCE), + ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'gnome'}, _LinuxDesktopEnvironment.GNOME), + ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'mate'}, _LinuxDesktopEnvironment.GNOME), + ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE4), + ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'kde'}, _LinuxDesktopEnvironment.KDE3), + ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'xfce'}, _LinuxDesktopEnvironment.XFCE), + + ({'XDG_CURRENT_DESKTOP': 'my_custom_de', 'DESKTOP_SESSION': 'my_custom_de', 'GNOME_DESKTOP_SESSION_ID': 1}, _LinuxDesktopEnvironment.GNOME), + ({'GNOME_DESKTOP_SESSION_ID': 1}, _LinuxDesktopEnvironment.GNOME), ({'KDE_FULL_SESSION': 1}, _LinuxDesktopEnvironment.KDE3), ({'KDE_FULL_SESSION': 1, 'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE4), diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py index fad323c90..5675445ac 100644 --- a/yt_dlp/cookies.py +++ b/yt_dlp/cookies.py @@ -764,11 +764,11 @@ def _get_linux_desktop_environment(env, logger): GetDesktopEnvironment """ xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None) - desktop_session = env.get('DESKTOP_SESSION', None) + desktop_session = env.get('DESKTOP_SESSION', '') if xdg_current_desktop is not None: for part in map(str.strip, xdg_current_desktop.split(':')): if part == 'Unity': - if desktop_session is not None and 'gnome-fallback' in desktop_session: + if 'gnome-fallback' in desktop_session: return _LinuxDesktopEnvironment.GNOME else: return _LinuxDesktopEnvironment.UNITY @@ -797,35 +797,34 @@ def _get_linux_desktop_environment(env, logger): return _LinuxDesktopEnvironment.UKUI elif part == 'LXQt': return _LinuxDesktopEnvironment.LXQT - logger.info(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"') + logger.debug(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"') - elif desktop_session is not None: - if desktop_session == 'deepin': - return _LinuxDesktopEnvironment.DEEPIN - elif desktop_session in ('mate', 'gnome'): - return _LinuxDesktopEnvironment.GNOME - elif desktop_session in ('kde4', 'kde-plasma'): + if desktop_session == 'deepin': + return _LinuxDesktopEnvironment.DEEPIN + elif desktop_session in ('mate', 'gnome'): + return _LinuxDesktopEnvironment.GNOME + elif desktop_session in ('kde4', 'kde-plasma'): + return _LinuxDesktopEnvironment.KDE4 + elif desktop_session == 'kde': + if 'KDE_SESSION_VERSION' in env: return _LinuxDesktopEnvironment.KDE4 - elif desktop_session == 'kde': - if 'KDE_SESSION_VERSION' in env: - return _LinuxDesktopEnvironment.KDE4 - else: - return _LinuxDesktopEnvironment.KDE3 - elif 'xfce' in desktop_session or desktop_session == 'xubuntu': - return _LinuxDesktopEnvironment.XFCE - elif desktop_session == 'ukui': - return _LinuxDesktopEnvironment.UKUI else: - logger.info(f'DESKTOP_SESSION is set to an unknown value: "{desktop_session}"') - + return _LinuxDesktopEnvironment.KDE3 + elif 'xfce' in desktop_session or desktop_session == 'xubuntu': + return _LinuxDesktopEnvironment.XFCE + elif desktop_session == 'ukui': + return _LinuxDesktopEnvironment.UKUI else: - if 'GNOME_DESKTOP_SESSION_ID' in env: - return _LinuxDesktopEnvironment.GNOME - elif 'KDE_FULL_SESSION' in env: - if 'KDE_SESSION_VERSION' in env: - return _LinuxDesktopEnvironment.KDE4 - else: - return _LinuxDesktopEnvironment.KDE3 + logger.debug(f'DESKTOP_SESSION is set to an unknown value: "{desktop_session}"') + + if 'GNOME_DESKTOP_SESSION_ID' in env: + return _LinuxDesktopEnvironment.GNOME + elif 'KDE_FULL_SESSION' in env: + if 'KDE_SESSION_VERSION' in env: + return _LinuxDesktopEnvironment.KDE4 + else: + return _LinuxDesktopEnvironment.KDE3 + return _LinuxDesktopEnvironment.OTHER From 7977b329ed97b216e37bd402f4935f28c00eac9e Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Thu, 22 May 2025 04:33:11 -0500 Subject: [PATCH 16/67] [cleanup] Misc (#13166) Authored by: bashonly --- README.md | 1 + yt_dlp/extractor/playsuisse.py | 3 +-- yt_dlp/extractor/soundcloud.py | 2 +- yt_dlp/extractor/twitter.py | 2 +- yt_dlp/options.py | 11 +++++++---- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index aaa3beb71..518c63eef 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ * [Post-processing Options](#post-processing-options) * [SponsorBlock Options](#sponsorblock-options) * [Extractor Options](#extractor-options) + * [Preset Aliases](#preset-aliases) * [CONFIGURATION](#configuration) * [Configuration file encoding](#configuration-file-encoding) * [Authentication with netrc](#authentication-with-netrc) diff --git a/yt_dlp/extractor/playsuisse.py b/yt_dlp/extractor/playsuisse.py index 9bf5765fa..46e3a5b8f 100644 --- a/yt_dlp/extractor/playsuisse.py +++ b/yt_dlp/extractor/playsuisse.py @@ -9,11 +9,10 @@ int_or_none, join_nonempty, parse_qs, - traverse_obj, update_url_query, urlencode_postdata, ) -from ..utils.traversal import unpack +from ..utils.traversal import traverse_obj, unpack class PlaySuisseIE(InfoExtractor): diff --git a/yt_dlp/extractor/soundcloud.py b/yt_dlp/extractor/soundcloud.py index c70940a60..3496a08ef 100644 --- a/yt_dlp/extractor/soundcloud.py +++ b/yt_dlp/extractor/soundcloud.py @@ -697,7 +697,7 @@ def _real_extract(self, url): try: return self._extract_info_dict(info, full_title, token) except ExtractorError as e: - if not isinstance(e.cause, HTTPError) or not e.cause.status == 429: + if not isinstance(e.cause, HTTPError) or e.cause.status != 429: raise self.report_warning( 'You have reached the API rate limit, which is ~600 requests per ' diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py index 5eee3e726..ad3e74588 100644 --- a/yt_dlp/extractor/twitter.py +++ b/yt_dlp/extractor/twitter.py @@ -1342,7 +1342,7 @@ def _extract_status(self, twid): 'tweet_mode': 'extended', }) except ExtractorError as e: - if not isinstance(e.cause, HTTPError) or not e.cause.status == 429: + if not isinstance(e.cause, HTTPError) or e.cause.status != 429: raise self.report_warning('Rate-limit exceeded; falling back to syndication endpoint') status = self._call_syndication_api(twid) diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 19f16e725..b4d3d4d66 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -230,6 +230,9 @@ def format_option_help(self, formatter=None): formatter.indent() heading = formatter.format_heading('Preset Aliases') formatter.indent() + description = formatter.format_description( + 'Predefined aliases for convenience and ease of use. Note that future versions of yt-dlp ' + 'may add or adjust presets, but the existing preset names will not be changed or removed') result = [] for name, args in _PRESET_ALIASES.items(): option = optparse.Option('-t', help=shlex.join(args)) @@ -238,7 +241,7 @@ def format_option_help(self, formatter=None): formatter.dedent() formatter.dedent() help_lines = '\n'.join(result) - return f'{formatted_help}\n{heading}{help_lines}' + return f'{formatted_help}\n{heading}{description}\n{help_lines}' def create_parser(): @@ -470,7 +473,7 @@ def _preset_alias_callback(option, opt_str, value, parser): general.add_option( '--live-from-start', action='store_true', dest='live_from_start', - help='Download livestreams from the start. Currently only supported for YouTube (experimental) and Twitch') + help='Download livestreams from the start. Currently experimental and only supported for YouTube and Twitch') general.add_option( '--no-live-from-start', action='store_false', dest='live_from_start', @@ -545,9 +548,9 @@ def _preset_alias_callback(option, opt_str, value, parser): help=( 'Create aliases for an option string. Unless an alias starts with a dash "-", it is prefixed with "--". ' 'Arguments are parsed according to the Python string formatting mini-language. ' - 'E.g. --alias get-audio,-X "-S=aext:{0},abr -x --audio-format {0}" creates options ' + 'E.g. --alias get-audio,-X "-S aext:{0},abr -x --audio-format {0}" creates options ' '"--get-audio" and "-X" that takes an argument (ARG0) and expands to ' - '"-S=aext:ARG0,abr -x --audio-format ARG0". All defined aliases are listed in the --help output. ' + '"-S aext:ARG0,abr -x --audio-format ARG0". All defined aliases are listed in the --help output. ' 'Alias options can trigger more aliases; so be careful to avoid defining recursive options. ' f'As a safety measure, each alias may be triggered a maximum of {_YoutubeDLOptionParser.ALIAS_TRIGGER_LIMIT} times. ' 'This option can be used multiple times')) From 415b4c9f955b1a0391204bd24a7132590e7b3bdb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 May 2025 09:49:11 +0000 Subject: [PATCH 17/67] Release 2025.05.22 Created by: bashonly :ci skip all --- CONTRIBUTORS | 5 +++++ Changelog.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ README.md | 20 ++++++++++++-------- supportedsites.md | 10 ++++++---- yt_dlp/version.py | 6 +++--- 5 files changed, 72 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 5710f9a9e..6aa52c595 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -770,3 +770,8 @@ NeonMan pj47x troex WouterGordts +baierjan +GeoffreyFrogeye +Pawka +v3DJG6GL +yozel diff --git a/Changelog.md b/Changelog.md index 513724bf4..80b72da05 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,52 @@ # Changelog # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master --> +### 2025.05.22 + +#### Core changes +- **cookies**: [Fix Linux desktop environment detection](https://github.com/yt-dlp/yt-dlp/commit/e491fd4d090db3af52a82863fb0553dd5e17fb85) ([#13197](https://github.com/yt-dlp/yt-dlp/issues/13197)) by [mbway](https://github.com/mbway) +- **jsinterp**: [Fix increment/decrement evaluation](https://github.com/yt-dlp/yt-dlp/commit/167d7a9f0ffd1b4fe600193441bdb7358db2740b) ([#13238](https://github.com/yt-dlp/yt-dlp/issues/13238)) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev) + +#### Extractor changes +- **1tv**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/41c0a1fb89628696f8bb88e2b9f3a68f355b8c26) ([#13168](https://github.com/yt-dlp/yt-dlp/issues/13168)) by [bashonly](https://github.com/bashonly) +- **amcnetworks**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/464c84fedf78eef822a431361155f108b5df96d7) ([#13147](https://github.com/yt-dlp/yt-dlp/issues/13147)) by [bashonly](https://github.com/bashonly) +- **bitchute**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/1d0f6539c47e5d5c68c3c47cdb7075339e2885ac) ([#13081](https://github.com/yt-dlp/yt-dlp/issues/13081)) by [bashonly](https://github.com/bashonly) +- **cartoonnetwork**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/7dbb47f84f0ee1266a3a01f58c9bc4c76d76794a) ([#13148](https://github.com/yt-dlp/yt-dlp/issues/13148)) by [bashonly](https://github.com/bashonly) +- **iprima**: [Fix login support](https://github.com/yt-dlp/yt-dlp/commit/a7d9a5eb79ceeecb851389f3f2c88597871ca3f2) ([#12937](https://github.com/yt-dlp/yt-dlp/issues/12937)) by [baierjan](https://github.com/baierjan) +- **jiosaavn** + - artist: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/586b557b124f954d3f625360ebe970989022ad97) ([#12803](https://github.com/yt-dlp/yt-dlp/issues/12803)) by [subrat-lima](https://github.com/subrat-lima) + - playlist, show: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/317f4b8006c2c0f0f64f095b1485163ad97c9053) ([#12803](https://github.com/yt-dlp/yt-dlp/issues/12803)) by [subrat-lima](https://github.com/subrat-lima) + - show: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/6839276496d8814cf16f58b637e45663467928e6) ([#12803](https://github.com/yt-dlp/yt-dlp/issues/12803)) by [subrat-lima](https://github.com/subrat-lima) +- **lrtradio**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/abf58dcd6a09e14eec4ea82ae12f79a0337cb383) ([#13200](https://github.com/yt-dlp/yt-dlp/issues/13200)) by [Pawka](https://github.com/Pawka) +- **nebula**: [Support `--mark-watched`](https://github.com/yt-dlp/yt-dlp/commit/20f288bdc2173c7cc58d709d25ca193c1f6001e7) ([#13120](https://github.com/yt-dlp/yt-dlp/issues/13120)) by [GeoffreyFrogeye](https://github.com/GeoffreyFrogeye) +- **niconico** + - [Fix error handling](https://github.com/yt-dlp/yt-dlp/commit/f569be4602c2a857087e495d5d7ed6060cd97abe) ([#13236](https://github.com/yt-dlp/yt-dlp/issues/13236)) by [bashonly](https://github.com/bashonly) + - live: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/7a7b85c9014d96421e18aa7ea5f4c1bee5ceece0) ([#13045](https://github.com/yt-dlp/yt-dlp/issues/13045)) by [doe1080](https://github.com/doe1080) +- **nytimesarticle**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/b26bc32579c00ef579d75a835807ccc87d20ee0a) ([#13104](https://github.com/yt-dlp/yt-dlp/issues/13104)) by [bashonly](https://github.com/bashonly) +- **once**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/f475e8b529d18efdad603ffda02a56e707fe0e2c) ([#13164](https://github.com/yt-dlp/yt-dlp/issues/13164)) by [bashonly](https://github.com/bashonly) +- **picarto**: vod: [Support `/profile/` video URLs](https://github.com/yt-dlp/yt-dlp/commit/31e090cb787f3504ec25485adff9a2a51d056734) ([#13227](https://github.com/yt-dlp/yt-dlp/issues/13227)) by [subrat-lima](https://github.com/subrat-lima) +- **playsuisse**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/d880e060803ae8ed5a047e578cca01e1f0e630ce) ([#12466](https://github.com/yt-dlp/yt-dlp/issues/12466)) by [v3DJG6GL](https://github.com/v3DJG6GL) +- **sprout**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/cbcfe6378dde33a650e3852ab17ad4503b8e008d) ([#13149](https://github.com/yt-dlp/yt-dlp/issues/13149)) by [bashonly](https://github.com/bashonly) +- **svtpage**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/ea8498ed534642dd7e925961b97b934987142fd3) ([#12957](https://github.com/yt-dlp/yt-dlp/issues/12957)) by [diman8](https://github.com/diman8) +- **twitch**: [Support `--live-from-start`](https://github.com/yt-dlp/yt-dlp/commit/00b1bec55249cf2ad6271d36492c51b34b6459d1) ([#13202](https://github.com/yt-dlp/yt-dlp/issues/13202)) by [bashonly](https://github.com/bashonly) +- **vimeo**: event: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/545c1a5b6f2fe88722b41aef0e7485bf3be3f3f9) ([#13216](https://github.com/yt-dlp/yt-dlp/issues/13216)) by [bashonly](https://github.com/bashonly) +- **wat.tv**: [Improve error handling](https://github.com/yt-dlp/yt-dlp/commit/f123cc83b3aea45053f5fa1d9141048b01fc2774) ([#13111](https://github.com/yt-dlp/yt-dlp/issues/13111)) by [bashonly](https://github.com/bashonly) +- **weverse**: [Fix live extraction](https://github.com/yt-dlp/yt-dlp/commit/5328eda8820cc5f21dcf917684d23fbdca41831d) ([#13084](https://github.com/yt-dlp/yt-dlp/issues/13084)) by [bashonly](https://github.com/bashonly) +- **xinpianchang**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/83fabf352489d52843f67e6e9cc752db86d27e6e) ([#13245](https://github.com/yt-dlp/yt-dlp/issues/13245)) by [garret1317](https://github.com/garret1317) +- **youtube** + - [Add PO token support for subtitles](https://github.com/yt-dlp/yt-dlp/commit/32ed5f107c6c641958d1cd2752e130de4db55a13) ([#13234](https://github.com/yt-dlp/yt-dlp/issues/13234)) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz) + - [Add `web_embedded` client for age-restricted videos](https://github.com/yt-dlp/yt-dlp/commit/0feec6dc131f488428bf881519e7c69766fbb9ae) ([#13089](https://github.com/yt-dlp/yt-dlp/issues/13089)) by [bashonly](https://github.com/bashonly) + - [Add a PO Token Provider Framework](https://github.com/yt-dlp/yt-dlp/commit/2685654a37141cca63eda3a92da0e2706e23ccfd) ([#12840](https://github.com/yt-dlp/yt-dlp/issues/12840)) by [coletdjnz](https://github.com/coletdjnz) + - [Extract `media_type` for all videos](https://github.com/yt-dlp/yt-dlp/commit/ded11ebc9afba6ba33923375103e9be2d7c804e7) ([#13136](https://github.com/yt-dlp/yt-dlp/issues/13136)) by [bashonly](https://github.com/bashonly) + - [Fix `--live-from-start` support for premieres](https://github.com/yt-dlp/yt-dlp/commit/8f303afb43395be360cafd7ad4ce2b6e2eedfb8a) ([#13079](https://github.com/yt-dlp/yt-dlp/issues/13079)) by [arabcoders](https://github.com/arabcoders) + - [Fix geo-restriction error handling](https://github.com/yt-dlp/yt-dlp/commit/c7e575e31608c19c5b26c10a4229db89db5fc9a8) ([#13217](https://github.com/yt-dlp/yt-dlp/issues/13217)) by [yozel](https://github.com/yozel) + +#### Misc. changes +- **build** + - [Bump PyInstaller to v6.13.0](https://github.com/yt-dlp/yt-dlp/commit/17cf9088d0d535e4a7feffbf02bd49cd9dae5ab9) ([#13082](https://github.com/yt-dlp/yt-dlp/issues/13082)) by [bashonly](https://github.com/bashonly) + - [Bump run-on-arch-action to v3](https://github.com/yt-dlp/yt-dlp/commit/9064d2482d1fe722bbb4a49731fe0711c410d1c8) ([#13088](https://github.com/yt-dlp/yt-dlp/issues/13088)) by [bashonly](https://github.com/bashonly) +- **cleanup**: Miscellaneous: [7977b32](https://github.com/yt-dlp/yt-dlp/commit/7977b329ed97b216e37bd402f4935f28c00eac9e) by [bashonly](https://github.com/bashonly) + ### 2025.04.30 #### Important changes diff --git a/README.md b/README.md index 518c63eef..6e2dc6243 100644 --- a/README.md +++ b/README.md @@ -349,8 +349,8 @@ ## General Options: --no-flat-playlist Fully extract the videos of a playlist (default) --live-from-start Download livestreams from the start. - Currently only supported for YouTube - (Experimental) + Currently experimental and only supported + for YouTube and Twitch --no-live-from-start Download livestreams from the current time (default) --wait-for-video MIN[-MAX] Wait for scheduled streams to become @@ -376,12 +376,12 @@ ## General Options: an alias starts with a dash "-", it is prefixed with "--". Arguments are parsed according to the Python string formatting - mini-language. E.g. --alias get-audio,-X - "-S=aext:{0},abr -x --audio-format {0}" - creates options "--get-audio" and "-X" that - takes an argument (ARG0) and expands to - "-S=aext:ARG0,abr -x --audio-format ARG0". - All defined aliases are listed in the --help + mini-language. E.g. --alias get-audio,-X "-S + aext:{0},abr -x --audio-format {0}" creates + options "--get-audio" and "-X" that takes an + argument (ARG0) and expands to "-S + aext:ARG0,abr -x --audio-format ARG0". All + defined aliases are listed in the --help output. Alias options can trigger more aliases; so be careful to avoid defining recursive options. As a safety measure, each @@ -1106,6 +1106,10 @@ ## Extractor Options: arguments for different extractors ## Preset Aliases: +Predefined aliases for convenience and ease of use. Note that future + versions of yt-dlp may add or adjust presets, but the existing preset + names will not be changed or removed + -t mp3 -f 'ba[acodec^=mp3]/ba/b' -x --audio-format mp3 diff --git a/supportedsites.md b/supportedsites.md index 03bd8a7c3..c2d7b4555 100644 --- a/supportedsites.md +++ b/supportedsites.md @@ -246,7 +246,6 @@ # Supported sites - **Canalplus**: mycanal.fr and piwiplus.fr - **Canalsurmas** - **CaracolTvPlay**: [*caracoltv-play*](## "netrc machine") - - **CartoonNetwork** - **cbc.ca** - **cbc.ca:player** - **cbc.ca:​player:playlist** @@ -649,7 +648,10 @@ # Supported sites - **jiocinema**: [*jiocinema*](## "netrc machine") - **jiocinema:series**: [*jiocinema*](## "netrc machine") - **jiosaavn:album** + - **jiosaavn:artist** - **jiosaavn:playlist** + - **jiosaavn:show** + - **jiosaavn:​show:playlist** - **jiosaavn:song** - **Joj** - **JoqrAg**: 超!A&G+ 文化放送 (f.k.a. AGQR) Nippon Cultural Broadcasting, Inc. (JOQR) @@ -1081,8 +1083,8 @@ # Supported sites - **Photobucket** - **PiaLive** - **Piapro**: [*piapro*](## "netrc machine") - - **Picarto** - - **PicartoVod** + - **picarto** + - **picarto:vod** - **Piksel** - **Pinkbike** - **Pinterest** @@ -1390,7 +1392,6 @@ # Supported sites - **Spreaker** - **SpreakerShow** - **SpringboardPlatform** - - **Sprout** - **SproutVideo** - **sr:mediathek**: Saarländischer Rundfunk (**Currently broken**) - **SRGSSR** @@ -1656,6 +1657,7 @@ # Supported sites - **vimeo**: [*vimeo*](## "netrc machine") - **vimeo:album**: [*vimeo*](## "netrc machine") - **vimeo:channel**: [*vimeo*](## "netrc machine") + - **vimeo:event**: [*vimeo*](## "netrc machine") - **vimeo:group**: [*vimeo*](## "netrc machine") - **vimeo:likes**: [*vimeo*](## "netrc machine") Vimeo user likes - **vimeo:ondemand**: [*vimeo*](## "netrc machine") diff --git a/yt_dlp/version.py b/yt_dlp/version.py index e8b2bf170..c375cc6ad 100644 --- a/yt_dlp/version.py +++ b/yt_dlp/version.py @@ -1,8 +1,8 @@ # Autogenerated by devscripts/update-version.py -__version__ = '2025.04.30' +__version__ = '2025.05.22' -RELEASE_GIT_HEAD = '505b400795af557bdcfd9d4fa7e9133b26ef431c' +RELEASE_GIT_HEAD = '7977b329ed97b216e37bd402f4935f28c00eac9e' VARIANT = None @@ -12,4 +12,4 @@ ORIGIN = 'yt-dlp/yt-dlp' -_pkg_version = '2025.04.30' +_pkg_version = '2025.05.22' From 53ea743a9c158f8ca2d75a09ca44ba68606042d8 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Thu, 22 May 2025 17:41:31 -0500 Subject: [PATCH 18/67] [ie/youtube] Fix automatic captions for some client combinations (#13268) Fix 32ed5f107c6c641958d1cd2752e130de4db55a13 Authored by: bashonly --- yt_dlp/extractor/youtube/_video.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index 9f929664f..840829be6 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -3950,19 +3950,23 @@ def process_language(container, base_url, lang_code, sub_name, client_name, quer subtitles = {} skipped_subs_clients = set() - prs = traverse_obj(player_responses, ( - # Filter out initial_pr which does not have streamingData (smuggled client context) - lambda _, v: v['streamingData'] and v['captions']['playerCaptionsTracklistRenderer'])) - pctrs = traverse_obj(prs, (..., 'captions', 'playerCaptionsTracklistRenderer', {dict})) + # Only web/mweb clients provide translationLanguages, so include initial_pr in the traversal translation_languages = { - lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1) - for lang in traverse_obj(pctrs, (..., 'translationLanguages', ..., {dict}))} + lang['languageCode']: self._get_text(lang['languageName'], max_runs=1) + for lang in traverse_obj(player_responses, ( + ..., 'captions', 'playerCaptionsTracklistRenderer', 'translationLanguages', + lambda _, v: v['languageCode'] and v['languageName'])) + } # NB: Constructing the full subtitle dictionary is slow get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and ( self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles')) - all_captions = traverse_obj(pctrs, (..., 'captionTracks', ..., {dict})) + # Filter out initial_pr which does not have streamingData (smuggled client context) + prs = traverse_obj(player_responses, ( + lambda _, v: v['streamingData'] and v['captions']['playerCaptionsTracklistRenderer'])) + all_captions = traverse_obj(prs, ( + ..., 'captions', 'playerCaptionsTracklistRenderer', 'captionTracks', ..., {dict})) need_subs_langs = {get_lang_code(sub) for sub in all_captions if sub.get('kind') != 'asr'} need_caps_langs = { remove_start(get_lang_code(sub), 'a-') From e0d6c0822930f6e63f574d46d946a58b73ecd10c Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Thu, 22 May 2025 17:42:42 -0500 Subject: [PATCH 19/67] [ie/patreon] Fix m3u8 formats extraction (#13266) Closes #13263 Authored by: bashonly --- yt_dlp/extractor/patreon.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/patreon.py b/yt_dlp/extractor/patreon.py index 7794cae6c..dddb09c91 100644 --- a/yt_dlp/extractor/patreon.py +++ b/yt_dlp/extractor/patreon.py @@ -341,7 +341,7 @@ def _real_extract(self, url): })) # all-lowercase 'referer' so we can smuggle it to Generic, SproutVideo, Vimeo - headers = {'referer': 'https://patreon.com/'} + headers = {'referer': url} # handle Vimeo embeds if traverse_obj(attributes, ('embed', 'provider')) == 'Vimeo': @@ -379,11 +379,13 @@ def _real_extract(self, url): 'url': post_file['url'], }) elif name == 'video' or determine_ext(post_file.get('url')) == 'm3u8': - formats, subtitles = self._extract_m3u8_formats_and_subtitles(post_file['url'], video_id) + formats, subtitles = self._extract_m3u8_formats_and_subtitles( + post_file['url'], video_id, headers=headers) entries.append({ 'id': video_id, 'formats': formats, 'subtitles': subtitles, + 'http_headers': headers, }) can_view_post = traverse_obj(attributes, 'current_user_can_view') From 1a8a03ea8d827107319a18076ee3505090667c5a Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 23 May 2025 07:53:36 -0500 Subject: [PATCH 20/67] [ie/patreon] Fix referer header used for embeds (#13276) Fix e0d6c0822930f6e63f574d46d946a58b73ecd10c Closes #13263 Authored by: bashonly --- yt_dlp/extractor/patreon.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/patreon.py b/yt_dlp/extractor/patreon.py index dddb09c91..2c1436cac 100644 --- a/yt_dlp/extractor/patreon.py +++ b/yt_dlp/extractor/patreon.py @@ -340,8 +340,9 @@ def _real_extract(self, url): 'channel_follower_count': ('attributes', 'patron_count', {int_or_none}), })) - # all-lowercase 'referer' so we can smuggle it to Generic, SproutVideo, Vimeo - headers = {'referer': url} + # Must be all-lowercase 'referer' so we can smuggle it to Generic, SproutVideo, and Vimeo. + # patreon.com URLs redirect to www.patreon.com; this matters when requesting mux.com m3u8s + headers = {'referer': 'https://www.patreon.com/'} # handle Vimeo embeds if traverse_obj(attributes, ('embed', 'provider')) == 'Vimeo': @@ -352,7 +353,7 @@ def _real_extract(self, url): v_url, video_id, 'Checking Vimeo embed URL', headers=headers, fatal=False, errnote=False, expected_status=429): # 429 is TLS fingerprint rejection entries.append(self.url_result( - VimeoIE._smuggle_referrer(v_url, 'https://patreon.com/'), + VimeoIE._smuggle_referrer(v_url, headers['referer']), VimeoIE, url_transparent=True)) embed_url = traverse_obj(attributes, ('embed', 'url', {url_or_none})) From 52f9729c9a92ad4656d746ff0b1acecb87b3e96d Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 23 May 2025 07:58:53 -0500 Subject: [PATCH 21/67] [ie/twitcasting] Fix password-protected livestream support (#13097) Closes #13096 Authored by: bashonly --- yt_dlp/extractor/twitcasting.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py index 0a7f95c21..ebc2963b0 100644 --- a/yt_dlp/extractor/twitcasting.py +++ b/yt_dlp/extractor/twitcasting.py @@ -1,4 +1,5 @@ import base64 +import hashlib import itertools import re @@ -16,6 +17,7 @@ str_to_int, try_get, unified_timestamp, + update_url_query, url_or_none, urlencode_postdata, urljoin, @@ -171,6 +173,10 @@ def find_dmu(x): 'player': 'pc_web', }) + password_params = { + 'word': hashlib.md5(video_password.encode()).hexdigest(), + } if video_password else None + formats = [] # low: 640x360, medium: 1280x720, high: 1920x1080 qq = qualities(['low', 'medium', 'high']) @@ -178,7 +184,7 @@ def find_dmu(x): 'tc-hls', 'streams', {dict.items}, lambda _, v: url_or_none(v[1]), )): formats.append({ - 'url': m3u8_url, + 'url': update_url_query(m3u8_url, password_params), 'format_id': f'hls-{quality}', 'ext': 'mp4', 'quality': qq(quality), @@ -192,7 +198,7 @@ def find_dmu(x): 'llfmp4', 'streams', {dict.items}, lambda _, v: url_or_none(v[1]), )): formats.append({ - 'url': ws_url, + 'url': update_url_query(ws_url, password_params), 'format_id': f'ws-{mode}', 'ext': 'mp4', 'quality': qq(mode), From f8051e3a61686c5db1de5f5746366ecfbc3ad20c Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Sat, 24 May 2025 02:29:55 +0900 Subject: [PATCH 22/67] [ie/toutiao] Add extractor (#13246) Closes #12125 Authored by: doe1080 --- yt_dlp/extractor/_extractors.py | 1 + yt_dlp/extractor/toutiao.py | 121 ++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 yt_dlp/extractor/toutiao.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 14a006893..c516c79ce 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -2147,6 +2147,7 @@ from .toggo import ToggoIE from .tonline import TOnlineIE from .toongoggles import ToonGogglesIE +from .toutiao import ToutiaoIE from .toutv import TouTvIE from .toypics import ( ToypicsIE, diff --git a/yt_dlp/extractor/toutiao.py b/yt_dlp/extractor/toutiao.py new file mode 100644 index 000000000..b2a5aa236 --- /dev/null +++ b/yt_dlp/extractor/toutiao.py @@ -0,0 +1,121 @@ +import json +import urllib.parse + +from .common import InfoExtractor +from ..utils import ( + float_or_none, + int_or_none, + str_or_none, + try_call, + url_or_none, +) +from ..utils.traversal import find_element, traverse_obj + + +class ToutiaoIE(InfoExtractor): + IE_NAME = 'toutiao' + IE_DESC = '今日头条' + + _VALID_URL = r'https?://www\.toutiao\.com/video/(?P\d+)/?(?:[?#]|$)' + _TESTS = [{ + 'url': 'https://www.toutiao.com/video/7505382061495176511/', + 'info_dict': { + 'id': '7505382061495176511', + 'ext': 'mp4', + 'title': '新疆多地现不明飞行物，目击者称和月亮一样亮，几秒内突然加速消失，气象部门回应', + 'comment_count': int, + 'duration': 9.753, + 'like_count': int, + 'release_date': '20250517', + 'release_timestamp': 1747483344, + 'thumbnail': r're:https?://p\d+-sign\.toutiaoimg\.com/.+$', + 'uploader': '极目新闻', + 'uploader_id': 'MS4wLjABAAAAeateBb9Su8I3MJOZozmvyzWktmba5LMlliRDz1KffnM', + 'view_count': int, + }, + }, { + 'url': 'https://www.toutiao.com/video/7479446610359878153/', + 'info_dict': { + 'id': '7479446610359878153', + 'ext': 'mp4', + 'title': '小伙竟然利用两块磁铁制作成磁力减震器，简直太有创意了！', + 'comment_count': int, + 'duration': 118.374, + 'like_count': int, + 'release_date': '20250308', + 'release_timestamp': 1741444368, + 'thumbnail': r're:https?://p\d+-sign\.toutiaoimg\.com/.+$', + 'uploader': '小莉创意发明', + 'uploader_id': 'MS4wLjABAAAA4f7d4mwtApALtHIiq-QM20dwXqe32NUz0DeWF7wbHKw', + 'view_count': int, + }, + }] + + def _real_initialize(self): + if self._get_cookies('https://www.toutiao.com').get('ttwid'): + return + + urlh = self._request_webpage( + 'https://ttwid.bytedance.com/ttwid/union/register/', None, + 'Fetching ttwid', 'Unable to fetch ttwid', headers={ + 'Content-Type': 'application/json', + }, data=json.dumps({ + 'aid': 24, + 'needFid': False, + 'region': 'cn', + 'service': 'www.toutiao.com', + 'union': True, + }).encode(), + ) + + if ttwid := try_call(lambda: self._get_cookies(urlh.url)['ttwid'].value): + self._set_cookie('.toutiao.com', 'ttwid', ttwid) + return + + self.raise_login_required() + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + video_data = traverse_obj(webpage, ( + {find_element(tag='script', id='RENDER_DATA')}, + {urllib.parse.unquote}, {json.loads}, 'data', 'initialVideo', + )) + + formats = [] + for video in traverse_obj(video_data, ( + 'videoPlayInfo', 'video_list', lambda _, v: v['main_url'], + )): + formats.append({ + 'url': video['main_url'], + **traverse_obj(video, ('video_meta', { + 'acodec': ('audio_profile', {str}), + 'asr': ('audio_sample_rate', {int_or_none}), + 'audio_channels': ('audio_channels', {float_or_none}, {int_or_none}), + 'ext': ('vtype', {str}), + 'filesize': ('size', {int_or_none}), + 'format_id': ('definition', {str}), + 'fps': ('fps', {int_or_none}), + 'height': ('vheight', {int_or_none}), + 'tbr': ('real_bitrate', {float_or_none(scale=1000)}), + 'vcodec': ('codec_type', {str}), + 'width': ('vwidth', {int_or_none}), + })), + }) + + return { + 'id': video_id, + 'formats': formats, + **traverse_obj(video_data, { + 'comment_count': ('commentCount', {int_or_none}), + 'duration': ('videoPlayInfo', 'video_duration', {float_or_none}), + 'like_count': ('repinCount', {int_or_none}), + 'release_timestamp': ('publishTime', {int_or_none}), + 'thumbnail': (('poster', 'coverUrl'), {url_or_none}, any), + 'title': ('title', {str}), + 'uploader': ('userInfo', 'name', {str}), + 'uploader_id': ('userInfo', 'userId', {str_or_none}), + 'view_count': ('playCount', {int_or_none}), + 'webpage_url': ('detailUrl', {url_or_none}), + }), + } From 538eb305673c26bff6a2b12f1c96375fe02ce41a Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 23 May 2025 12:42:24 -0500 Subject: [PATCH 23/67] [ie/podchaser] Fix extractor (#13271) Closes #13269 Authored by: bashonly --- yt_dlp/extractor/podchaser.py | 59 ++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/yt_dlp/extractor/podchaser.py b/yt_dlp/extractor/podchaser.py index 4570f0f17..6c125f9ba 100644 --- a/yt_dlp/extractor/podchaser.py +++ b/yt_dlp/extractor/podchaser.py @@ -5,11 +5,13 @@ from ..utils import ( OnDemandPagedList, float_or_none, + int_or_none, + orderedSet, str_or_none, - str_to_int, - traverse_obj, unified_timestamp, + url_or_none, ) +from ..utils.traversal import require, traverse_obj class PodchaserIE(InfoExtractor): @@ -21,24 +23,25 @@ class PodchaserIE(InfoExtractor): 'id': '104365585', 'title': 'Ep. 285 – freeze me off', 'description': 'cam ahn', - 'thumbnail': r're:^https?://.*\.jpg$', + 'thumbnail': r're:https?://.+/.+\.jpg', 'ext': 'mp3', - 'categories': ['Comedy'], + 'categories': ['Comedy', 'News', 'Politics', 'Arts'], 'tags': ['comedy', 'dark humor'], - 'series': 'Cum Town', + 'series': 'The Adam Friedland Show Podcast', 'duration': 3708, 'timestamp': 1636531259, 'upload_date': '20211110', 'average_rating': 4.0, + 'series_id': '36924', }, }, { 'url': 'https://www.podchaser.com/podcasts/the-bone-zone-28853', 'info_dict': { 'id': '28853', 'title': 'The Bone Zone', - 'description': 'Podcast by The Bone Zone', + 'description': r're:The official home of the Bone Zone podcast.+', }, - 'playlist_count': 275, + 'playlist_mincount': 275, }, { 'url': 'https://www.podchaser.com/podcasts/sean-carrolls-mindscape-scienc-699349/episodes', 'info_dict': { @@ -51,19 +54,33 @@ class PodchaserIE(InfoExtractor): @staticmethod def _parse_episode(episode, podcast): - return { - 'id': str(episode.get('id')), - 'title': episode.get('title'), - 'description': episode.get('description'), - 'url': episode.get('audio_url'), - 'thumbnail': episode.get('image_url'), - 'duration': str_to_int(episode.get('length')), - 'timestamp': unified_timestamp(episode.get('air_date')), - 'average_rating': float_or_none(episode.get('rating')), - 'categories': list(set(traverse_obj(podcast, (('summary', None), 'categories', ..., 'text')))), - 'tags': traverse_obj(podcast, ('tags', ..., 'text')), - 'series': podcast.get('title'), - } + info = traverse_obj(episode, { + 'id': ('id', {int}, {str_or_none}, {require('episode ID')}), + 'title': ('title', {str}), + 'description': ('description', {str}), + 'url': ('audio_url', {url_or_none}), + 'thumbnail': ('image_url', {url_or_none}), + 'duration': ('length', {int_or_none}), + 'timestamp': ('air_date', {unified_timestamp}), + 'average_rating': ('rating', {float_or_none}), + }) + info.update(traverse_obj(podcast, { + 'series': ('title', {str}), + 'series_id': ('id', {int}, {str_or_none}), + 'categories': (('summary', None), 'categories', ..., 'text', {str}, filter, all, {orderedSet}), + 'tags': ('tags', ..., 'text', {str}), + })) + info['vcodec'] = 'none' + + if info.get('series_id'): + podcast_slug = traverse_obj(podcast, ('slug', {str})) or 'podcast' + episode_slug = traverse_obj(episode, ('slug', {str})) or 'episode' + info['webpage_url'] = '/'.join(( + 'https://www.podchaser.com/podcasts', + '-'.join((podcast_slug[:30].rstrip('-'), info['series_id'])), + '-'.join((episode_slug[:30].rstrip('-'), info['id'])))) + + return info def _call_api(self, path, *args, **kwargs): return self._download_json(f'https://api.podchaser.com/{path}', *args, **kwargs) @@ -93,5 +110,5 @@ def _real_extract(self, url): OnDemandPagedList(functools.partial(self._fetch_page, podcast_id, podcast), self._PAGE_SIZE), str_or_none(podcast.get('id')), podcast.get('title'), podcast.get('description')) - episode = self._call_api(f'episodes/{episode_id}', episode_id) + episode = self._call_api(f'podcasts/{podcast_id}/episodes/{episode_id}/player_ids', episode_id) return self._parse_episode(episode, podcast) From 7794374de8afb20499b023107e2abfd4e6b93ee4 Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Sat, 24 May 2025 04:25:56 +0900 Subject: [PATCH 24/67] [ie/twitter:broadcast] Support events URLs (#13248) Closes #12989 Authored by: doe1080 --- yt_dlp/extractor/twitter.py | 56 +++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py index ad3e74588..65182b971 100644 --- a/yt_dlp/extractor/twitter.py +++ b/yt_dlp/extractor/twitter.py @@ -20,7 +20,6 @@ remove_end, str_or_none, strip_or_none, - traverse_obj, truncate_string, try_call, try_get, @@ -29,6 +28,7 @@ url_or_none, xpath_text, ) +from ..utils.traversal import require, traverse_obj class TwitterBaseIE(InfoExtractor): @@ -1596,8 +1596,8 @@ def _find_dimension(target): class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE): IE_NAME = 'twitter:broadcast' - _VALID_URL = TwitterBaseIE._BASE_REGEX + r'i/broadcasts/(?P[0-9a-zA-Z]{13})' + _VALID_URL = TwitterBaseIE._BASE_REGEX + r'i/(?Pbroadcasts|events)/(?P\w+)' _TESTS = [{ # untitled Periscope video 'url': 'https://twitter.com/i/broadcasts/1yNGaQLWpejGj', @@ -1605,6 +1605,7 @@ class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE): 'id': '1yNGaQLWpejGj', 'ext': 'mp4', 'title': 'Andrea May Sahouri - Periscope Broadcast', + 'display_id': '1yNGaQLWpejGj', 'uploader': 'Andrea May Sahouri', 'uploader_id': 'andreamsahouri', 'uploader_url': 'https://twitter.com/andreamsahouri', @@ -1612,6 +1613,8 @@ class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE): 'upload_date': '20200601', 'thumbnail': r're:^https?://[^?#]+\.jpg\?token=', 'view_count': int, + 'concurrent_view_count': int, + 'live_status': 'was_live', }, }, { 'url': 'https://twitter.com/i/broadcasts/1ZkKzeyrPbaxv', @@ -1619,6 +1622,7 @@ class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE): 'id': '1ZkKzeyrPbaxv', 'ext': 'mp4', 'title': 'Starship | SN10 | High-Altitude Flight Test', + 'display_id': '1ZkKzeyrPbaxv', 'uploader': 'SpaceX', 'uploader_id': 'SpaceX', 'uploader_url': 'https://twitter.com/SpaceX', @@ -1626,6 +1630,8 @@ class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE): 'upload_date': '20210303', 'thumbnail': r're:^https?://[^?#]+\.jpg\?token=', 'view_count': int, + 'concurrent_view_count': int, + 'live_status': 'was_live', }, }, { 'url': 'https://twitter.com/i/broadcasts/1OyKAVQrgzwGb', @@ -1633,6 +1639,7 @@ class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE): 'id': '1OyKAVQrgzwGb', 'ext': 'mp4', 'title': 'Starship Flight Test', + 'display_id': '1OyKAVQrgzwGb', 'uploader': 'SpaceX', 'uploader_id': 'SpaceX', 'uploader_url': 'https://twitter.com/SpaceX', @@ -1640,21 +1647,58 @@ class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE): 'upload_date': '20230420', 'thumbnail': r're:^https?://[^?#]+\.jpg\?token=', 'view_count': int, + 'concurrent_view_count': int, + 'live_status': 'was_live', + }, + }, { + 'url': 'https://x.com/i/events/1910629646300762112', + 'info_dict': { + 'id': '1LyxBWDRNqyKN', + 'ext': 'mp4', + 'title': '#ガンニバル ウォッチパーティー', + 'concurrent_view_count': int, + 'display_id': '1910629646300762112', + 'live_status': 'was_live', + 'release_date': '20250423', + 'release_timestamp': 1745409000, + 'tags': ['ガンニバル'], + 'thumbnail': r're:https?://[^?#]+\.jpg\?token=', + 'timestamp': 1745403328, + 'upload_date': '20250423', + 'uploader': 'ディズニープラス公式', + 'uploader_id': 'DisneyPlusJP', + 'uploader_url': 'https://twitter.com/DisneyPlusJP', + 'view_count': int, }, }] def _real_extract(self, url): - broadcast_id = self._match_id(url) + broadcast_type, display_id = self._match_valid_url(url).group('type', 'id') + + if broadcast_type == 'events': + timeline = self._call_api( + f'live_event/1/{display_id}/timeline.json', display_id) + broadcast_id = traverse_obj(timeline, ( + 'twitter_objects', 'broadcasts', ..., ('id', 'broadcast_id'), + {str}, any, {require('broadcast ID')})) + else: + broadcast_id = display_id + broadcast = self._call_api( 'broadcasts/show.json', broadcast_id, {'ids': broadcast_id})['broadcasts'][broadcast_id] if not broadcast: raise ExtractorError('Broadcast no longer exists', expected=True) info = self._parse_broadcast_data(broadcast, broadcast_id) - info['title'] = broadcast.get('status') or info.get('title') - info['uploader_id'] = broadcast.get('twitter_username') or info.get('uploader_id') - info['uploader_url'] = format_field(broadcast, 'twitter_username', 'https://twitter.com/%s', default=None) + info.update({ + 'display_id': display_id, + 'title': broadcast.get('status') or info.get('title'), + 'uploader_id': broadcast.get('twitter_username') or info.get('uploader_id'), + 'uploader_url': format_field( + broadcast, 'twitter_username', 'https://twitter.com/%s', default=None), + }) if info['live_status'] == 'is_upcoming': + self.raise_no_formats('This live broadcast has not yet started', expected=True) return info media_key = broadcast['media_key'] From 0ee1102268cf31b07f8a8318a47424c66b2f7378 Mon Sep 17 00:00:00 2001 From: bashonly Date: Mon, 26 May 2025 13:34:20 -0500 Subject: [PATCH 25/67] [ie/adobepass] Always add newer user-agent when required (#13131) Fix dcfeea4dd5e5686821350baa6c7767a011944867 Closes #516 Authored by: bashonly --- yt_dlp/extractor/adobepass.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py index f1b877927..f580dfda7 100644 --- a/yt_dlp/extractor/adobepass.py +++ b/yt_dlp/extractor/adobepass.py @@ -45,6 +45,7 @@ 'name': 'Comcast XFINITY', 'username_field': 'user', 'password_field': 'passwd', + 'needs_newer_ua': True, }, 'TWC': { 'name': 'Time Warner Cable | Spectrum', @@ -1355,7 +1356,6 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should end with BaseIE/InfoExtractor _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s' _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0' - _MODERN_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; rv:131.0) Gecko/20100101 Firefox/131.0' _MVPD_CACHE = 'ap-mvpd' _DOWNLOADING_LOGIN_PAGE = 'Downloading Provider Login Page' @@ -1367,6 +1367,14 @@ def _download_webpage_handle(self, *args, **kwargs): return super()._download_webpage_handle( *args, **kwargs) + @staticmethod + def _get_mso_headers(mso_info): + # yt-dlp's default user-agent is usually too old for some MSO's like Comcast_SSO + # See: https://github.com/yt-dlp/yt-dlp/issues/10848 + return { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:131.0) Gecko/20100101 Firefox/131.0', + } if mso_info.get('needs_newer_ua') else {} + @staticmethod def _get_mvpd_resource(provider_id, title, guid, rating): channel = etree.Element('channel') @@ -1383,6 +1391,12 @@ def _get_mvpd_resource(provider_id, title, guid, rating): return '' + etree.tostring(channel).decode() + '' def _extract_mvpd_auth(self, url, video_id, requestor_id, resource): + mso_id = self.get_param('ap_mso') + if mso_id: + mso_info = MSO_INFO[mso_id] + else: + mso_info = {} + def xml_text(xml_str, tag): return self._search_regex( f'<{tag}>(.+?)', xml_str, tag) @@ -1400,6 +1414,7 @@ def post_form(form_page_res, note, data={}): form_data.update(data) return self._download_webpage_handle( post_url, video_id, note, data=urlencode_postdata(form_data), headers={ + **self._get_mso_headers(mso_info), 'Content-Type': 'application/x-www-form-urlencoded', }) @@ -1439,12 +1454,10 @@ def extract_redirect_url(html, url=None, fatal=False): if authn_token and is_expired(authn_token, 'simpleTokenExpires'): authn_token = None if not authn_token: - mso_id = self.get_param('ap_mso') if mso_id: username, password = self._get_login_info('ap_username', 'ap_password', mso_id) if not username or not password: raise_mvpd_required() - mso_info = MSO_INFO[mso_id] provider_redirect_page_res = self._download_webpage_handle( self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id, @@ -1455,11 +1468,7 @@ def extract_redirect_url(html, url=None, fatal=False): 'no_iframe': 'false', 'domain_name': 'adobe.com', 'redirect_url': url, - }, headers={ - # yt-dlp's default user-agent is usually too old for Comcast_SSO - # See: https://github.com/yt-dlp/yt-dlp/issues/10848 - 'User-Agent': self._MODERN_USER_AGENT, - } if mso_id == 'Comcast_SSO' else None) + }, headers=self._get_mso_headers(mso_info)) elif not self._cookies_passed: raise_mvpd_required() @@ -1489,8 +1498,8 @@ def extract_redirect_url(html, url=None, fatal=False): oauth_redirect_url = extract_redirect_url( provider_redirect_page, fatal=True) provider_login_page_res = self._download_webpage_handle( - oauth_redirect_url, video_id, - self._DOWNLOADING_LOGIN_PAGE) + oauth_redirect_url, video_id, self._DOWNLOADING_LOGIN_PAGE, + headers=self._get_mso_headers(mso_info)) else: provider_login_page_res = post_form( provider_redirect_page_res, From 89c1b349ad81318d9d3bea76c01c891696e58d38 Mon Sep 17 00:00:00 2001 From: bashonly Date: Mon, 26 May 2025 13:48:10 -0500 Subject: [PATCH 26/67] [ie/adobepass] Validate login URL before sending credentials (#13131) Authored by: bashonly --- yt_dlp/extractor/adobepass.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py index f580dfda7..8a5e7d9b5 100644 --- a/yt_dlp/extractor/adobepass.py +++ b/yt_dlp/extractor/adobepass.py @@ -45,6 +45,7 @@ 'name': 'Comcast XFINITY', 'username_field': 'user', 'password_field': 'passwd', + 'login_hostname': 'login.xfinity.com', 'needs_newer_ua': True, }, 'TWC': { @@ -75,6 +76,7 @@ 'name': 'Verizon FiOS', 'username_field': 'IDToken1', 'password_field': 'IDToken2', + 'login_hostname': 'ssoauth.verizon.com', }, 'Cablevision': { 'name': 'Optimum/Cablevision', @@ -1339,6 +1341,7 @@ 'name': 'Sling TV', 'username_field': 'username', 'password_field': 'password', + 'login_hostname': 'identity.sling.com', }, 'Suddenlink': { 'name': 'Suddenlink', @@ -1405,11 +1408,22 @@ def is_expired(token, date_ele): token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(token, date_ele))) return token_expires and token_expires <= int(time.time()) - def post_form(form_page_res, note, data={}): + def post_form(form_page_res, note, data={}, validate_url=False): form_page, urlh = form_page_res post_url = self._html_search_regex(r']+action=(["\'])(?P.+?)\1', form_page, 'post url', group='url') if not re.match(r'https?://', post_url): post_url = urllib.parse.urljoin(urlh.url, post_url) + if validate_url: + # This request is submitting credentials so we should validate it when possible + url_parsed = urllib.parse.urlparse(post_url) + expected_hostname = mso_info.get('login_hostname') + if expected_hostname and expected_hostname != url_parsed.hostname: + raise ExtractorError( + f'Unexpected login URL hostname; expected "{expected_hostname}" but got ' + f'"{url_parsed.hostname}". Aborting before submitting credentials') + if url_parsed.scheme != 'https': + self.write_debug('Upgrading login URL scheme to https') + post_url = urllib.parse.urlunparse(url_parsed._replace(scheme='https')) form_data = self._hidden_inputs(form_page) form_data.update(data) return self._download_webpage_handle( @@ -1509,7 +1523,7 @@ def extract_redirect_url(html, url=None, fatal=False): provider_login_page_res, 'Logging in', { mso_info['username_field']: username, mso_info['password_field']: password, - }) + }, validate_url=True) mvpd_confirm_page, urlh = mvpd_confirm_page_res if '' in mvpd_confirm_page: post_form(mvpd_confirm_page_res, 'Confirming Login') @@ -1548,7 +1562,7 @@ def extract_redirect_url(html, url=None, fatal=False): provider_redirect_page_res, 'Logging in', { mso_info['username_field']: username, mso_info['password_field']: password, - }) + }, validate_url=True) saml_login_page, urlh = saml_login_page_res if 'Please try again.' in saml_login_page: raise ExtractorError( @@ -1569,7 +1583,7 @@ def extract_redirect_url(html, url=None, fatal=False): [saml_login_page, saml_redirect_url], 'Logging in', { mso_info['username_field']: username, mso_info['password_field']: password, - }) + }, validate_url=True) if 'Please try again.' in saml_login_page: raise ExtractorError( 'Failed to login, incorrect User ID or Password.') @@ -1640,7 +1654,7 @@ def extract_redirect_url(html, url=None, fatal=False): provider_login_page_res, 'Logging in', { mso_info['username_field']: username, mso_info['password_field']: password, - }) + }, validate_url=True) provider_refresh_redirect_url = extract_redirect_url( provider_association_redirect, url=urlh.url) @@ -1691,7 +1705,7 @@ def extract_redirect_url(html, url=None, fatal=False): provider_login_page_res, 'Logging in', { mso_info['username_field']: username, mso_info['password_field']: password, - }) + }, validate_url=True) provider_refresh_redirect_url = extract_redirect_url( provider_association_redirect, url=urlh.url) @@ -1726,7 +1740,8 @@ def extract_redirect_url(html, url=None, fatal=False): } if mso_id in ('Cablevision', 'AlticeOne'): form_data['_eventId_proceed'] = '' - mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', form_data) + mvpd_confirm_page_res = post_form( + provider_login_page_res, 'Logging in', form_data, validate_url=True) if mso_id != 'Rogers': post_form(mvpd_confirm_page_res, 'Confirming Login') From 711c5d5d098fee2992a1a624b1c4b30364b91426 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 26 May 2025 13:57:20 -0500 Subject: [PATCH 27/67] [ie/adobepass] Rework to require software statement (#13131) * Also removes broken cookie support Closes #11811 Authored by: maxbin123, bashonly Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com> --- yt_dlp/extractor/adobepass.py | 87 +++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 25 deletions(-) diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py index 8a5e7d9b5..080fe319e 100644 --- a/yt_dlp/extractor/adobepass.py +++ b/yt_dlp/extractor/adobepass.py @@ -3,6 +3,7 @@ import re import time import urllib.parse +import uuid import xml.etree.ElementTree as etree from .common import InfoExtractor @@ -1393,7 +1394,7 @@ def _get_mvpd_resource(provider_id, title, guid, rating): resource_rating.text = rating return '' + etree.tostring(channel).decode() + '' - def _extract_mvpd_auth(self, url, video_id, requestor_id, resource): + def _extract_mvpd_auth(self, url, video_id, requestor_id, resource, software_statement): mso_id = self.get_param('ap_mso') if mso_id: mso_info = MSO_INFO[mso_id] @@ -1461,34 +1462,72 @@ def extract_redirect_url(html, url=None, fatal=False): } guid = xml_text(resource, 'guid') if '<' in resource else resource - count = 0 - while count < 2: + for _ in range(2): requestor_info = self.cache.load(self._MVPD_CACHE, requestor_id) or {} authn_token = requestor_info.get('authn_token') if authn_token and is_expired(authn_token, 'simpleTokenExpires'): authn_token = None if not authn_token: - if mso_id: - username, password = self._get_login_info('ap_username', 'ap_password', mso_id) - if not username or not password: - raise_mvpd_required() - - provider_redirect_page_res = self._download_webpage_handle( - self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id, - 'Downloading Provider Redirect Page', query={ - 'noflash': 'true', - 'mso_id': mso_id, - 'requestor_id': requestor_id, - 'no_iframe': 'false', - 'domain_name': 'adobe.com', - 'redirect_url': url, - }, headers=self._get_mso_headers(mso_info)) - elif not self._cookies_passed: + if not mso_id: + raise_mvpd_required() + username, password = self._get_login_info('ap_username', 'ap_password', mso_id) + if not username or not password: raise_mvpd_required() - if not mso_id: - pass - elif mso_id == 'Comcast_SSO': + device_info, urlh = self._download_json_handle( + 'https://sp.auth.adobe.com/indiv/devices', + video_id, 'Registering device with Adobe', + data=json.dumps({'fingerprint': uuid.uuid4().hex}).encode(), + headers={'Content-Type': 'application/json; charset=UTF-8'}) + + device_id = device_info['deviceId'] + mvpd_headers['pass_sfp'] = urlh.get_header('pass_sfp') + mvpd_headers['Ap_21'] = device_id + + registration = self._download_json( + 'https://sp.auth.adobe.com/o/client/register', + video_id, 'Registering client with Adobe', + data=json.dumps({'software_statement': software_statement}).encode(), + headers={'Content-Type': 'application/json; charset=UTF-8'}) + + access_token = self._download_json( + 'https://sp.auth.adobe.com/o/client/token', video_id, + 'Obtaining access token', data=urlencode_postdata({ + 'grant_type': 'client_credentials', + 'client_id': registration['client_id'], + 'client_secret': registration['client_secret'], + }), + headers={ + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + })['access_token'] + mvpd_headers['Authorization'] = f'Bearer {access_token}' + + reg_code = self._download_json( + f'https://sp.auth.adobe.com/reggie/v1/{requestor_id}/regcode', + video_id, 'Obtaining registration code', + data=urlencode_postdata({ + 'requestor': requestor_id, + 'deviceId': device_id, + 'format': 'json', + }), + headers={ + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'Authorization': f'Bearer {access_token}', + })['code'] + + provider_redirect_page_res = self._download_webpage_handle( + self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id, + 'Downloading Provider Redirect Page', query={ + 'noflash': 'true', + 'mso_id': mso_id, + 'requestor_id': requestor_id, + 'no_iframe': 'false', + 'domain_name': 'adobe.com', + 'redirect_url': url, + 'reg_code': reg_code, + }, headers=self._get_mso_headers(mso_info)) + + if mso_id == 'Comcast_SSO': # Comcast page flow varies by video site and whether you # are on Comcast's network. provider_redirect_page, urlh = provider_redirect_page_res @@ -1751,6 +1790,7 @@ def extract_redirect_url(html, url=None, fatal=False): 'Retrieving Session', data=urlencode_postdata({ '_method': 'GET', 'requestor_id': requestor_id, + 'reg_code': reg_code, }), headers=mvpd_headers) except ExtractorError as e: if not mso_id and isinstance(e.cause, HTTPError) and e.cause.status == 401: @@ -1758,7 +1798,6 @@ def extract_redirect_url(html, url=None, fatal=False): raise if ' Date: Mon, 26 May 2025 14:03:39 -0500 Subject: [PATCH 28/67] [ie/adobepass] Add Fubo MSO (#13131) Closes #8287 Authored by: maxbin123 --- yt_dlp/extractor/adobepass.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py index 080fe319e..91c40b32e 100644 --- a/yt_dlp/extractor/adobepass.py +++ b/yt_dlp/extractor/adobepass.py @@ -11,6 +11,7 @@ from ..utils import ( NO_DEFAULT, ExtractorError, + parse_qs, unescapeHTML, unified_timestamp, urlencode_postdata, @@ -79,6 +80,11 @@ 'password_field': 'IDToken2', 'login_hostname': 'ssoauth.verizon.com', }, + 'Fubo': { + 'name': 'Fubo', + 'username_field': 'username', + 'password_field': 'password', + }, 'Cablevision': { 'name': 'Optimum/Cablevision', 'username_field': 'j_username', @@ -1761,6 +1767,27 @@ def extract_redirect_url(html, url=None, fatal=False): query=hidden_data) post_form(mvpd_confirm_page_res, 'Confirming Login') + elif mso_id == 'Fubo': + _, urlh = provider_redirect_page_res + + fubo_response = self._download_json( + 'https://api.fubo.tv/partners/tve/connect', video_id, + 'Authenticating with Fubo', 'Unable to authenticate with Fubo', + query=parse_qs(urlh.url), data=json.dumps({ + 'username': username, + 'password': password, + }).encode(), headers={ + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }) + + self._request_webpage( + 'https://sp.auth.adobe.com/adobe-services/oauth2', video_id, + 'Authenticating with Adobe', 'Failed to authenticate with Adobe', + query={ + 'code': fubo_response['code'], + 'state': fubo_response['state'], + }) else: # Some providers (e.g. DIRECTV NOW) have another meta refresh # based redirect that should be followed. From ed108b3ea481c6a4b5215a9302ba92d74baa2425 Mon Sep 17 00:00:00 2001 From: bashonly Date: Mon, 26 May 2025 14:06:39 -0500 Subject: [PATCH 29/67] [ie/theplatform] Improve metadata extraction (#13131) Authored by: bashonly --- yt_dlp/extractor/theplatform.py | 95 +++++++++++++++------------------ 1 file changed, 44 insertions(+), 51 deletions(-) diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py index ebe2ac296..0f8cdfed9 100644 --- a/yt_dlp/extractor/theplatform.py +++ b/yt_dlp/extractor/theplatform.py @@ -12,11 +12,13 @@ float_or_none, int_or_none, mimetype2ext, + parse_age_limit, parse_qs, traverse_obj, unsmuggle_url, update_url, update_url_query, + url_or_none, urlhandle_detect_ext, xpath_with_ns, ) @@ -63,62 +65,53 @@ def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL d return formats, subtitles - def _download_theplatform_metadata(self, path, video_id): - info_url = f'http://link.theplatform.{self._TP_TLD}/s/{path}?format=preview' - return self._download_json(info_url, video_id) + def _download_theplatform_metadata(self, path, video_id, fatal=True): + return self._download_json( + f'https://link.theplatform.{self._TP_TLD}/s/{path}', video_id, + fatal=fatal, query={'format': 'preview'}) or {} - def _parse_theplatform_metadata(self, info): - subtitles = {} - captions = info.get('captions') - if isinstance(captions, list): - for caption in captions: - lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type') - subtitles.setdefault(lang, []).append({ - 'ext': mimetype2ext(mime), - 'url': src, - }) + @staticmethod + def _parse_theplatform_metadata(tp_metadata): + def site_specific_filter(*fields): + return lambda k, v: v and k.endswith(tuple(f'${f}' for f in fields)) - duration = info.get('duration') - tp_chapters = info.get('chapters', []) - chapters = [] - if tp_chapters: - def _add_chapter(start_time, end_time): - start_time = float_or_none(start_time, 1000) - end_time = float_or_none(end_time, 1000) - if start_time is None or end_time is None: - return - chapters.append({ - 'start_time': start_time, - 'end_time': end_time, - }) + info = traverse_obj(tp_metadata, { + 'title': ('title', {str}), + 'episode': ('title', {str}), + 'description': ('description', {str}), + 'thumbnail': ('defaultThumbnailUrl', {url_or_none}), + 'duration': ('duration', {float_or_none(scale=1000)}), + 'timestamp': ('pubDate', {float_or_none(scale=1000)}), + 'uploader': ('billingCode', {str}), + 'creators': ('author', {str}, filter, all, filter), + 'categories': ( + 'categories', lambda _, v: v.get('label') in ['category', None], + 'name', {str}, filter, all, filter), + 'tags': ('keywords', {str}, filter, {lambda x: re.split(r'[;,]\s?', x)}, filter), + 'age_limit': ('ratings', ..., 'rating', {parse_age_limit}, any), + 'season_number': (site_specific_filter('seasonNumber'), {int_or_none}, any), + 'episode_number': (site_specific_filter('episodeNumber', 'airOrder'), {int_or_none}, any), + 'series': (site_specific_filter('show', 'seriesTitle', 'seriesShortTitle'), (None, ...), {str}, any), + 'location': (site_specific_filter('region'), {str}, any), + 'media_type': (site_specific_filter('programmingType', 'type'), {str}, any), + }) - for chapter in tp_chapters[:-1]: - _add_chapter(chapter.get('startTime'), chapter.get('endTime')) - _add_chapter(tp_chapters[-1].get('startTime'), tp_chapters[-1].get('endTime') or duration) + chapters = traverse_obj(tp_metadata, ('chapters', ..., { + 'start_time': ('startTime', {float_or_none(scale=1000)}), + 'end_time': ('endTime', {float_or_none(scale=1000)}), + })) + # Ignore pointless single chapters from short videos that span the entire video's duration + if len(chapters) > 1 or traverse_obj(chapters, (0, 'end_time')): + info['chapters'] = chapters - def extract_site_specific_field(field): - # A number of sites have custom-prefixed keys, e.g. 'cbc$seasonNumber' - return traverse_obj(info, lambda k, v: v and k.endswith(f'${field}'), get_all=False) + info['subtitles'] = {} + for caption in traverse_obj(tp_metadata, ('captions', lambda _, v: url_or_none(v['src']))): + info['subtitles'].setdefault(caption.get('lang') or 'en', []).append({ + 'url': caption['src'], + 'ext': mimetype2ext(caption.get('type')), + }) - return { - 'title': info['title'], - 'subtitles': subtitles, - 'description': info['description'], - 'thumbnail': info['defaultThumbnailUrl'], - 'duration': float_or_none(duration, 1000), - 'timestamp': int_or_none(info.get('pubDate'), 1000) or None, - 'uploader': info.get('billingCode'), - 'chapters': chapters, - 'creator': traverse_obj(info, ('author', {str})) or None, - 'categories': traverse_obj(info, ( - 'categories', lambda _, v: v.get('label') in ('category', None), 'name', {str})) or None, - 'tags': traverse_obj(info, ('keywords', {lambda x: re.split(r'[;,]\s?', x) if x else None})), - 'location': extract_site_specific_field('region'), - 'series': extract_site_specific_field('show') or extract_site_specific_field('seriesTitle'), - 'season_number': int_or_none(extract_site_specific_field('seasonNumber')), - 'episode_number': int_or_none(extract_site_specific_field('episodeNumber')), - 'media_type': extract_site_specific_field('programmingType') or extract_site_specific_field('type'), - } + return info def _extract_theplatform_metadata(self, path, video_id): info = self._download_theplatform_metadata(path, video_id) From 2d7949d5642bc37d1e71bf00c9a55260e5505d58 Mon Sep 17 00:00:00 2001 From: bashonly Date: Mon, 26 May 2025 14:14:16 -0500 Subject: [PATCH 30/67] [ie/nbc] Rework and adapt extractors to new AdobePass flow (#13131) Closes #1032, Closes #10874, Closes #11148, Closes #12432 Authored by: bashonly --- yt_dlp/extractor/_extractors.py | 4 +- yt_dlp/extractor/bravotv.py | 188 ------------- yt_dlp/extractor/nbc.py | 478 ++++++++++++++++++++++++-------- yt_dlp/extractor/syfy.py | 58 ---- 4 files changed, 365 insertions(+), 363 deletions(-) delete mode 100644 yt_dlp/extractor/bravotv.py delete mode 100644 yt_dlp/extractor/syfy.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index c516c79ce..b0c52e0fc 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -300,7 +300,6 @@ BrainPOPIlIE, BrainPOPJrIE, ) -from .bravotv import BravoTVIE from .breitbart import BreitBartIE from .brightcove import ( BrightcoveLegacyIE, @@ -1262,6 +1261,7 @@ ) from .nbc import ( NBCIE, + BravoTVIE, NBCNewsIE, NBCOlympicsIE, NBCOlympicsStreamIE, @@ -1269,6 +1269,7 @@ NBCSportsStreamIE, NBCSportsVPlayerIE, NBCStationsIE, + SyfyIE, ) from .ndr import ( NDRIE, @@ -2022,7 +2023,6 @@ SVTSeriesIE, ) from .swearnet import SwearnetEpisodeIE -from .syfy import SyfyIE from .syvdk import SYVDKIE from .sztvhu import SztvHuIE from .tagesschau import TagesschauIE diff --git a/yt_dlp/extractor/bravotv.py b/yt_dlp/extractor/bravotv.py deleted file mode 100644 index 0b2c44798..000000000 --- a/yt_dlp/extractor/bravotv.py +++ /dev/null @@ -1,188 +0,0 @@ -from .adobepass import AdobePassIE -from ..networking import HEADRequest -from ..utils import ( - extract_attributes, - float_or_none, - get_element_html_by_class, - int_or_none, - merge_dicts, - parse_age_limit, - remove_end, - str_or_none, - traverse_obj, - unescapeHTML, - unified_timestamp, - update_url_query, - url_or_none, -) - - -class BravoTVIE(AdobePassIE): - _VALID_URL = r'https?://(?:www\.)?(?Pbravotv|oxygen)\.com/(?:[^/]+/)+(?P[^/?#]+)' - _TESTS = [{ - 'url': 'https://www.bravotv.com/top-chef/season-16/episode-15/videos/the-top-chef-season-16-winner-is', - 'info_dict': { - 'id': '3923059', - 'ext': 'mp4', - 'title': 'The Top Chef Season 16 Winner Is...', - 'description': 'Find out who takes the title of Top Chef!', - 'upload_date': '20190314', - 'timestamp': 1552591860, - 'season_number': 16, - 'episode_number': 15, - 'series': 'Top Chef', - 'episode': 'The Top Chef Season 16 Winner Is...', - 'duration': 190.357, - 'season': 'Season 16', - 'thumbnail': r're:^https://.+\.jpg', - }, - 'params': {'skip_download': 'm3u8'}, - }, { - 'url': 'https://www.bravotv.com/top-chef/season-20/episode-1/london-calling', - 'info_dict': { - 'id': '9000234570', - 'ext': 'mp4', - 'title': 'London Calling', - 'description': 'md5:5af95a8cbac1856bd10e7562f86bb759', - 'upload_date': '20230310', - 'timestamp': 1678410000, - 'season_number': 20, - 'episode_number': 1, - 'series': 'Top Chef', - 'episode': 'London Calling', - 'duration': 3266.03, - 'season': 'Season 20', - 'chapters': 'count:7', - 'thumbnail': r're:^https://.+\.jpg', - 'age_limit': 14, - }, - 'params': {'skip_download': 'm3u8'}, - 'skip': 'This video requires AdobePass MSO credentials', - }, { - 'url': 'https://www.oxygen.com/in-ice-cold-blood/season-1/closing-night', - 'info_dict': { - 'id': '3692045', - 'ext': 'mp4', - 'title': 'Closing Night', - 'description': 'md5:3170065c5c2f19548d72a4cbc254af63', - 'upload_date': '20180401', - 'timestamp': 1522623600, - 'season_number': 1, - 'episode_number': 1, - 'series': 'In Ice Cold Blood', - 'episode': 'Closing Night', - 'duration': 2629.051, - 'season': 'Season 1', - 'chapters': 'count:6', - 'thumbnail': r're:^https://.+\.jpg', - 'age_limit': 14, - }, - 'params': {'skip_download': 'm3u8'}, - 'skip': 'This video requires AdobePass MSO credentials', - }, { - 'url': 'https://www.oxygen.com/in-ice-cold-blood/season-2/episode-16/videos/handling-the-horwitz-house-after-the-murder-season-2', - 'info_dict': { - 'id': '3974019', - 'ext': 'mp4', - 'title': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)', - 'description': 'md5:f9d638dd6946a1c1c0533a9c6100eae5', - 'upload_date': '20190617', - 'timestamp': 1560790800, - 'season_number': 2, - 'episode_number': 16, - 'series': 'In Ice Cold Blood', - 'episode': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)', - 'duration': 68.235, - 'season': 'Season 2', - 'thumbnail': r're:^https://.+\.jpg', - 'age_limit': 14, - }, - 'params': {'skip_download': 'm3u8'}, - }, { - 'url': 'https://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1', - 'only_matching': True, - }] - - def _real_extract(self, url): - site, display_id = self._match_valid_url(url).group('site', 'id') - webpage = self._download_webpage(url, display_id) - settings = self._search_json( - r']+data-drupal-selector="drupal-settings-json"[^>]*>', webpage, 'settings', display_id) - tve = extract_attributes(get_element_html_by_class('tve-video-deck-app', webpage) or '') - query = { - 'manifest': 'm3u', - 'formats': 'm3u,mpeg4', - } - - if tve: - account_pid = tve.get('data-mpx-media-account-pid') or 'HNK2IC' - account_id = tve['data-mpx-media-account-id'] - metadata = self._parse_json( - tve.get('data-normalized-video', ''), display_id, fatal=False, transform_source=unescapeHTML) - video_id = tve.get('data-guid') or metadata['guid'] - if tve.get('data-entitlement') == 'auth': - auth = traverse_obj(settings, ('tve_adobe_auth', {dict})) or {} - site = remove_end(site, 'tv') - release_pid = tve['data-release-pid'] - resource = self._get_mvpd_resource( - tve.get('data-adobe-pass-resource-id') or auth.get('adobePassResourceId') or site, - tve['data-title'], release_pid, tve.get('data-rating')) - query.update({ - 'switch': 'HLSServiceSecure', - 'auth': self._extract_mvpd_auth( - url, release_pid, auth.get('adobePassRequestorId') or site, resource), - }) - - else: - ls_playlist = traverse_obj(settings, ('ls_playlist', ..., {dict}), get_all=False) or {} - account_pid = ls_playlist.get('mpxMediaAccountPid') or 'PHSl-B' - account_id = ls_playlist['mpxMediaAccountId'] - video_id = ls_playlist['defaultGuid'] - metadata = traverse_obj( - ls_playlist, ('videos', lambda _, v: v['guid'] == video_id, {dict}), get_all=False) - - tp_url = f'https://link.theplatform.com/s/{account_pid}/media/guid/{account_id}/{video_id}' - tp_metadata = self._download_json( - update_url_query(tp_url, {'format': 'preview'}), video_id, fatal=False) - - chapters = traverse_obj(tp_metadata, ('chapters', ..., { - 'start_time': ('startTime', {float_or_none(scale=1000)}), - 'end_time': ('endTime', {float_or_none(scale=1000)}), - })) - # prune pointless single chapters that span the entire duration from short videos - if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')): - chapters = None - - m3u8_url = self._request_webpage(HEADRequest( - update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').url - if 'mpeg_cenc' in m3u8_url: - self.report_drm(video_id) - formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls') - - return { - 'id': video_id, - 'formats': formats, - 'subtitles': subtitles, - 'chapters': chapters, - **merge_dicts(traverse_obj(tp_metadata, { - 'title': 'title', - 'description': 'description', - 'duration': ('duration', {float_or_none(scale=1000)}), - 'timestamp': ('pubDate', {float_or_none(scale=1000)}), - 'season_number': (('pl1$seasonNumber', 'nbcu$seasonNumber'), {int_or_none}), - 'episode_number': (('pl1$episodeNumber', 'nbcu$episodeNumber'), {int_or_none}), - 'series': (('pl1$show', 'nbcu$show'), (None, ...), {str}), - 'episode': (('title', 'pl1$episodeNumber', 'nbcu$episodeNumber'), {str_or_none}), - 'age_limit': ('ratings', ..., 'rating', {parse_age_limit}), - }, get_all=False), traverse_obj(metadata, { - 'title': 'title', - 'description': 'description', - 'duration': ('durationInSeconds', {int_or_none}), - 'timestamp': ('airDate', {unified_timestamp}), - 'thumbnail': ('thumbnailUrl', {url_or_none}), - 'season_number': ('seasonNumber', {int_or_none}), - 'episode_number': ('episodeNumber', {int_or_none}), - 'episode': 'episodeTitle', - 'series': 'show', - })), - } diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py index d9aded09e..bd4862bde 100644 --- a/yt_dlp/extractor/nbc.py +++ b/yt_dlp/extractor/nbc.py @@ -6,7 +6,7 @@ from .adobepass import AdobePassIE from .common import InfoExtractor -from .theplatform import ThePlatformIE, default_ns +from .theplatform import ThePlatformBaseIE, ThePlatformIE, default_ns from ..networking import HEADRequest from ..utils import ( ExtractorError, @@ -14,26 +14,130 @@ UserNotLive, clean_html, determine_ext, + extract_attributes, float_or_none, + get_element_html_by_class, int_or_none, join_nonempty, + make_archive_id, mimetype2ext, parse_age_limit, parse_duration, + parse_iso8601, remove_end, - smuggle_url, - traverse_obj, try_get, unescapeHTML, unified_timestamp, update_url_query, url_basename, + url_or_none, ) +from ..utils.traversal import require, traverse_obj -class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE - _VALID_URL = r'https?(?P://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/]+/video/[^/]+/(?P(?:NBCE|n)?\d+))' +class NBCUniversalBaseIE(ThePlatformBaseIE): + _GEO_COUNTRIES = ['US'] + _GEO_BYPASS = False + _M3U8_RE = r'https?://[^/?#]+/prod/[\w-]+/(?P[^?#]+/)cmaf/mpeg_(?:cbcs|cenc)\w*/master_cmaf\w*\.m3u8' + def _download_nbcu_smil_and_extract_m3u8_url(self, tp_path, video_id, query): + smil = self._download_xml( + f'https://link.theplatform.com/s/{tp_path}', video_id, + 'Downloading SMIL manifest', 'Failed to download SMIL manifest', query={ + **query, + 'format': 'SMIL', # XXX: Do not confuse "format" with "formats" + 'manifest': 'm3u', + 'switch': 'HLSServiceSecure', # Or else we get broken mp4 http URLs instead of HLS + }, headers=self.geo_verification_headers()) + + ns = f'//{{{default_ns}}}' + if url := traverse_obj(smil, (f'{ns}video/@src', lambda _, v: determine_ext(v) == 'm3u8', any)): + return url + + exc = traverse_obj(smil, (f'{ns}param', lambda _, v: v.get('name') == 'exception', '@value', any)) + if exc == 'GeoLocationBlocked': + self.raise_geo_restricted(countries=self._GEO_COUNTRIES) + raise ExtractorError(traverse_obj(smil, (f'{ns}ref/@abstract', ..., any)), expected=exc == 'Expired') + + def _extract_nbcu_formats_and_subtitles(self, tp_path, video_id, query): + # formats='mpeg4' will return either a working m3u8 URL or an m3u8 template for non-DRM HLS + # formats='m3u+none,mpeg4' may return DRM HLS but w/the "folders" needed for non-DRM template + query['formats'] = 'm3u+none,mpeg4' + m3u8_url = self._download_nbcu_smil_and_extract_m3u8_url(tp_path, video_id, query) + + if mobj := re.fullmatch(self._M3U8_RE, m3u8_url): + query['formats'] = 'mpeg4' + m3u8_tmpl = self._download_nbcu_smil_and_extract_m3u8_url(tp_path, video_id, query) + # Example: https://vod-lf-oneapp-prd.akamaized.net/prod/video/{folders}master_hls.m3u8 + if '{folders}' in m3u8_tmpl: + self.write_debug('Found m3u8 URL template, formatting URL path') + m3u8_url = m3u8_tmpl.format(folders=mobj.group('folders')) + + if '/mpeg_cenc' in m3u8_url or '/mpeg_cbcs' in m3u8_url: + self.report_drm(video_id) + + return self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls') + + def _extract_nbcu_video(self, url, display_id, old_ie_key=None): + webpage = self._download_webpage(url, display_id) + settings = self._search_json( + r']+data-drupal-selector="drupal-settings-json"[^>]*>', + webpage, 'settings', display_id) + + query = {} + tve = extract_attributes(get_element_html_by_class('tve-video-deck-app', webpage) or '') + if tve: + account_pid = tve.get('data-mpx-media-account-pid') or tve['data-mpx-account-pid'] + account_id = tve['data-mpx-media-account-id'] + metadata = self._parse_json( + tve.get('data-normalized-video') or '', display_id, fatal=False, transform_source=unescapeHTML) + video_id = tve.get('data-guid') or metadata['guid'] + if tve.get('data-entitlement') == 'auth': + auth = settings['tve_adobe_auth'] + release_pid = tve['data-release-pid'] + resource = self._get_mvpd_resource( + tve.get('data-adobe-pass-resource-id') or auth['adobePassResourceId'], + tve['data-title'], release_pid, tve.get('data-rating')) + query['auth'] = self._extract_mvpd_auth( + url, release_pid, auth['adobePassRequestorId'], + resource, auth['adobePassSoftwareStatement']) + else: + ls_playlist = traverse_obj(settings, ( + 'ls_playlist', lambda _, v: v['defaultGuid'], any, {require('LS playlist')})) + video_id = ls_playlist['defaultGuid'] + account_pid = ls_playlist.get('mpxMediaAccountPid') or ls_playlist['mpxAccountPid'] + account_id = ls_playlist['mpxMediaAccountId'] + metadata = traverse_obj(ls_playlist, ('videos', lambda _, v: v['guid'] == video_id, any)) or {} + + tp_path = f'{account_pid}/media/guid/{account_id}/{video_id}' + formats, subtitles = self._extract_nbcu_formats_and_subtitles(tp_path, video_id, query) + tp_metadata = self._download_theplatform_metadata(tp_path, video_id, fatal=False) + parsed_info = self._parse_theplatform_metadata(tp_metadata) + self._merge_subtitles(parsed_info['subtitles'], target=subtitles) + + return { + **parsed_info, + **traverse_obj(metadata, { + 'title': ('title', {str}), + 'description': ('description', {str}), + 'duration': ('durationInSeconds', {int_or_none}), + 'timestamp': ('airDate', {parse_iso8601}), + 'thumbnail': ('thumbnailUrl', {url_or_none}), + 'season_number': ('seasonNumber', {int_or_none}), + 'episode_number': ('episodeNumber', {int_or_none}), + 'episode': ('episodeTitle', {str}), + 'series': ('show', {str}), + }), + 'id': video_id, + 'display_id': display_id, + 'formats': formats, + 'subtitles': subtitles, + '_old_archive_ids': [make_archive_id(old_ie_key, video_id)] if old_ie_key else None, + } + + +class NBCIE(NBCUniversalBaseIE): + _VALID_URL = r'https?(?P://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/?#]+/video/[^/?#]+/(?P\w+))' _TESTS = [ { 'url': 'http://www.nbc.com/the-tonight-show/video/jimmy-fallon-surprises-fans-at-ben-jerrys/2848237', @@ -49,47 +153,20 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE 'episode_number': 86, 'season': 'Season 2', 'season_number': 2, - 'series': 'Tonight Show: Jimmy Fallon', - 'duration': 237.0, - 'chapters': 'count:1', - 'tags': 'count:4', + 'series': 'Tonight', + 'duration': 236.504, + 'tags': 'count:2', 'thumbnail': r're:https?://.+\.jpg', 'categories': ['Series/The Tonight Show Starring Jimmy Fallon'], 'media_type': 'Full Episode', + 'age_limit': 14, + '_old_archive_ids': ['theplatform 2848237'], }, 'params': { 'skip_download': 'm3u8', }, }, { - 'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821', - 'info_dict': { - 'id': '2832821', - 'ext': 'mp4', - 'title': 'Star Wars Teaser', - 'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442', - 'timestamp': 1417852800, - 'upload_date': '20141206', - 'uploader': 'NBCU-COM', - }, - 'skip': 'page not found', - }, - { - # HLS streams requires the 'hdnea3' cookie - 'url': 'http://www.nbc.com/Kings/video/goliath/n1806', - 'info_dict': { - 'id': '101528f5a9e8127b107e98c5e6ce4638', - 'ext': 'mp4', - 'title': 'Goliath', - 'description': 'When an unknown soldier saves the life of the King\'s son in battle, he\'s thrust into the limelight and politics of the kingdom.', - 'timestamp': 1237100400, - 'upload_date': '20090315', - 'uploader': 'NBCU-COM', - }, - 'skip': 'page not found', - }, - { - # manifest url does not have extension 'url': 'https://www.nbc.com/the-golden-globe-awards/video/oprah-winfrey-receives-cecil-b-de-mille-award-at-the-2018-golden-globes/3646439', 'info_dict': { 'id': '3646439', @@ -99,48 +176,47 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE 'episode_number': 1, 'season': 'Season 75', 'season_number': 75, - 'series': 'The Golden Globe Awards', + 'series': 'Golden Globes', 'description': 'Oprah Winfrey receives the Cecil B. de Mille Award at the 75th Annual Golden Globe Awards.', 'uploader': 'NBCU-COM', 'upload_date': '20180107', 'timestamp': 1515312000, - 'duration': 570.0, + 'duration': 569.703, 'tags': 'count:8', 'thumbnail': r're:https?://.+\.jpg', - 'chapters': 'count:1', + 'media_type': 'Highlight', + 'age_limit': 0, + 'categories': ['Series/The Golden Globe Awards'], + '_old_archive_ids': ['theplatform 3646439'], }, 'params': { 'skip_download': 'm3u8', }, }, { - # new video_id format - 'url': 'https://www.nbc.com/quantum-leap/video/bens-first-leap-nbcs-quantum-leap/NBCE125189978', + # Needs to be extracted from webpage instead of GraphQL + 'url': 'https://www.nbc.com/paris2024/video/ali-truwit-found-purpose-pool-after-her-life-changed/para24_sww_alitruwittodayshow_240823', 'info_dict': { - 'id': 'NBCE125189978', + 'id': 'para24_sww_alitruwittodayshow_240823', 'ext': 'mp4', - 'title': 'Ben\'s First Leap | NBC\'s Quantum Leap', - 'description': 'md5:a82762449b7ec4bb83291a7b355ebf8e', - 'uploader': 'NBCU-COM', - 'series': 'Quantum Leap', - 'season': 'Season 1', - 'season_number': 1, - 'episode': 'Ben\'s First Leap | NBC\'s Quantum Leap', - 'episode_number': 1, - 'duration': 170.171, - 'chapters': [], - 'timestamp': 1663956155, - 'upload_date': '20220923', - 'tags': 'count:10', - 'age_limit': 0, + 'title': 'Ali Truwit found purpose in the pool after her life changed', + 'description': 'md5:c16d7489e1516593de1cc5d3f39b9bdb', + 'uploader': 'NBCU-SPORTS', + 'duration': 311.077, 'thumbnail': r're:https?://.+\.jpg', - 'categories': ['Series/Quantum Leap 2022'], - 'media_type': 'Highlight', + 'episode': 'Ali Truwit found purpose in the pool after her life changed', + 'timestamp': 1724435902.0, + 'upload_date': '20240823', + '_old_archive_ids': ['theplatform para24_sww_alitruwittodayshow_240823'], }, 'params': { 'skip_download': 'm3u8', }, }, + { + 'url': 'https://www.nbc.com/quantum-leap/video/bens-first-leap-nbcs-quantum-leap/NBCE125189978', + 'only_matching': True, + }, { 'url': 'https://www.nbc.com/classic-tv/charles-in-charge/video/charles-in-charge-pilot/n3310', 'only_matching': True, @@ -151,6 +227,7 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE 'only_matching': True, }, ] + _SOFTWARE_STATEMENT = 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI1Yzg2YjdkYy04NDI3LTRjNDUtOGQwZi1iNDkzYmE3MmQwYjQiLCJuYmYiOjE1Nzg3MDM2MzEsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTc4NzAzNjMxfQ.QQKIsBhAjGQTMdAqRTqhcz2Cddr4Y2hEjnSiOeKKki4nLrkDOsjQMmqeTR0hSRarraxH54wBgLvsxI7LHwKMvr7G8QpynNAxylHlQD3yhN9tFhxt4KR5wW3as02B-W2TznK9bhNWPKIyHND95Uo2Mi6rEQoq8tM9O09WPWaanE5BX_-r6Llr6dPq5F0Lpx2QOn2xYRb1T4nFxdFTNoss8GBds8OvChTiKpXMLHegLTc1OS4H_1a8tO_37jDwSdJuZ8iTyRLV4kZ2cpL6OL5JPMObD4-HQiec_dfcYgMKPiIfP9ZqdXpec2SVaCLsWEk86ZYvD97hLIQrK5rrKd1y-A' def _real_extract(self, url): permalink, video_id = self._match_valid_url(url).groups() @@ -196,62 +273,50 @@ def _real_extract(self, url): 'userId': '0', }), })['data']['bonanzaPage']['metadata'] - query = { - 'mbr': 'true', - 'manifest': 'm3u', - 'switch': 'HLSServiceSecure', - } + + if not video_data: + # Some videos are not available via GraphQL API + webpage = self._download_webpage(url, video_id) + video_data = self._search_json( + r'''', + { + 'thumbnails': [{'url': 'https://www.nobelprize.org/images/12693-landscape-medium-gallery.jpg'}], + }, + {}, + ), ] for html, expected_dict, search_json_ld_kwargs in _TESTS: expect_dict( diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index d5607296d..1174bd4f5 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -1675,9 +1675,9 @@ def extract_video_object(e): 'ext': mimetype2ext(e.get('encodingFormat')), 'title': unescapeHTML(e.get('name')), 'description': unescapeHTML(e.get('description')), - 'thumbnails': [{'url': unescapeHTML(url)} - for url in variadic(traverse_obj(e, 'thumbnailUrl', 'thumbnailURL')) - if url_or_none(url)], + 'thumbnails': traverse_obj(e, (('thumbnailUrl', 'thumbnailURL', 'thumbnail_url'), (None, ...), { + 'url': ({str}, {unescapeHTML}, {self._proto_relative_url}, {url_or_none}), + })), 'duration': parse_duration(e.get('duration')), 'timestamp': unified_timestamp(e.get('uploadDate')), # author can be an instance of 'Organization' or 'Person' types. From 148a1eb4c59e127965396c7a6e6acf1979de459e Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sun, 1 Jun 2025 18:18:24 -0500 Subject: [PATCH 47/67] [ie/odnoklassniki] Detect and raise when login is required (#13361) Closes #13360 Authored by: bashonly --- yt_dlp/extractor/odnoklassniki.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yt_dlp/extractor/odnoklassniki.py b/yt_dlp/extractor/odnoklassniki.py index d27d1c3f0..18eba42e6 100644 --- a/yt_dlp/extractor/odnoklassniki.py +++ b/yt_dlp/extractor/odnoklassniki.py @@ -273,6 +273,8 @@ def _extract_desktop(self, url): return self._extract_desktop(smuggle_url(url, {'referrer': 'https://boosty.to'})) elif error: raise ExtractorError(error, expected=True) + elif '>Access to this video is restricted

' in webpage: + self.raise_login_required() player = self._parse_json( unescapeHTML(self._search_regex( @@ -429,7 +431,7 @@ def _extract_mobile(self, url): video_id = self._match_id(url) webpage = self._download_webpage( - f'http://m.ok.ru/video/{video_id}', video_id, + f'https://m.ok.ru/video/{video_id}', video_id, note='Downloading mobile webpage') error = self._search_regex( From c723c4e5e78263df178dbe69844a3d05f3ef9e35 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sun, 1 Jun 2025 18:20:29 -0500 Subject: [PATCH 48/67] [ie/vimeo] Extract subtitles from player subdomain (#13350) Closes #12198 Authored by: bashonly --- yt_dlp/extractor/vimeo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py index 09497b699..b268fad56 100644 --- a/yt_dlp/extractor/vimeo.py +++ b/yt_dlp/extractor/vimeo.py @@ -236,7 +236,7 @@ def _parse_config(self, config, video_id): for tt in (request.get('text_tracks') or []): subtitles.setdefault(tt['lang'], []).append({ 'ext': 'vtt', - 'url': urljoin('https://vimeo.com', tt['url']), + 'url': urljoin('https://player.vimeo.com/', tt['url']), }) thumbnails = [] From e1b6062f8c4a3fa33c65269d48d09ec78de765a2 Mon Sep 17 00:00:00 2001 From: barsnick Date: Tue, 3 Jun 2025 04:29:03 +0200 Subject: [PATCH 49/67] [ie/svt:play] Fix extractor (#13329) Closes #13312 Authored by: barsnick, bashonly Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com> --- yt_dlp/extractor/_extractors.py | 1 - yt_dlp/extractor/svt.py | 134 +++++++++++--------------------- 2 files changed, 44 insertions(+), 91 deletions(-) diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index b0c52e0fc..34c98b537 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -2017,7 +2017,6 @@ SverigesRadioPublicationIE, ) from .svt import ( - SVTIE, SVTPageIE, SVTPlayIE, SVTSeriesIE, diff --git a/yt_dlp/extractor/svt.py b/yt_dlp/extractor/svt.py index 6a72f8d42..a48d7858d 100644 --- a/yt_dlp/extractor/svt.py +++ b/yt_dlp/extractor/svt.py @@ -6,10 +6,13 @@ determine_ext, dict_get, int_or_none, - traverse_obj, try_get, unified_timestamp, ) +from ..utils.traversal import ( + require, + traverse_obj, +) class SVTBaseIE(InfoExtractor): @@ -97,40 +100,8 @@ def _extract_video(self, video_info, video_id): } -class SVTIE(SVTBaseIE): - _VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.*?&)?widgetId=(?P\d+)&.*?\barticleId=(?P\d+)' - _EMBED_REGEX = [rf'(?: