and x other(s)")',
+ 'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
+ 'info_dict': {
+ 'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
+ 'modified_date': '20220407',
+ 'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
+ 'tags': [],
+ 'availability': 'unlisted',
+ 'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
+ 'channel': 'pukkandan',
+ 'description': 'Test for collaborative playlist',
+ 'title': 'yt-dlp test - collaborative playlist',
+ 'view_count': int,
+ 'uploader_url': 'https://www.youtube.com/@pukkandan',
+ 'uploader_id': '@pukkandan',
+ 'uploader': 'pukkandan',
+ },
+ 'playlist_mincount': 2,
+ }, {
+ 'note': 'translated tab name',
+ 'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
+ 'info_dict': {
+ 'id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'tags': [],
+ 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
+ 'description': 'test description',
+ 'title': 'cole-dlp-test-acc - 再生リスト',
+ 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'channel': 'cole-dlp-test-acc',
+ 'uploader_url': 'https://www.youtube.com/@coletdjnz',
+ 'uploader_id': '@coletdjnz',
+ 'uploader': 'cole-dlp-test-acc',
+ },
+ 'playlist_mincount': 1,
+ 'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
+ 'expected_warnings': ['Preferring "ja"'],
+ }, {
+ # XXX: this should really check flat playlist entries, but the test suite doesn't support that
+ 'note': 'preferred lang set with playlist with translated video titles',
+ 'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
+ 'info_dict': {
+ 'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
+ 'tags': [],
+ 'view_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
+ 'channel': 'cole-dlp-test-acc',
+ 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'description': 'test',
+ 'title': 'dlp test playlist',
+ 'availability': 'public',
+ 'uploader_url': 'https://www.youtube.com/@coletdjnz',
+ 'uploader_id': '@coletdjnz',
+ 'uploader': 'cole-dlp-test-acc',
+ },
+ 'playlist_mincount': 1,
+ 'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
+ 'expected_warnings': ['Preferring "ja"'],
+ }, {
+ # shorts audio pivot for 2GtVksBMYFM.
+ 'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
+ 'info_dict': {
+ 'id': 'sfv_audio_pivot',
+ 'title': 'sfv_audio_pivot',
+ 'tags': [],
+ },
+ 'playlist_mincount': 50,
+
+ }, {
+ # Channel with a real live tab (not to be mistaken with streams tab)
+ # Do not treat like it should redirect to live stream
+ 'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
+ 'info_dict': {
+ 'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
+ 'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
+ 'tags': [],
+ },
+ 'playlist_mincount': 20,
+ }, {
+ # Tab name is not the same as tab id
+ 'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
+ 'info_dict': {
+ 'id': 'UCQvWX73GQygcwXOTSf_VDVg',
+ 'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
+ 'tags': [],
+ },
+ 'playlist_mincount': 8,
+ }, {
+ # Home tab id is literally home. Not to get mistaken with featured
+ 'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
+ 'info_dict': {
+ 'id': 'UCQvWX73GQygcwXOTSf_VDVg',
+ 'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
+ 'tags': [],
+ },
+ 'playlist_mincount': 8,
+ }, {
+ # Should get three playlists for videos, shorts and streams tabs
+ 'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
+ 'info_dict': {
+ 'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
+ 'title': 'Polka Ch. 尾丸ポルカ',
+ 'channel_follower_count': int,
+ 'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
+ 'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
+ 'description': 'md5:49809d8bf9da539bc48ed5d1f83c33f2',
+ 'channel': 'Polka Ch. 尾丸ポルカ',
+ 'tags': 'count:35',
+ 'uploader_url': 'https://www.youtube.com/@OmaruPolka',
+ 'uploader': 'Polka Ch. 尾丸ポルカ',
+ 'uploader_id': '@OmaruPolka',
+ 'channel_is_verified': True,
+ },
+ 'playlist_count': 3,
+ }, {
+ # Shorts tab with channel with handle
+ # TODO: fix channel description
+ 'url': 'https://www.youtube.com/@NotJustBikes/shorts',
+ 'info_dict': {
+ 'id': 'UC0intLFzLaudFG-xAvUEO-A',
+ 'title': 'Not Just Bikes - Shorts',
+ 'tags': 'count:10',
+ 'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
+ 'description': 'md5:5e82545b3a041345927a92d0585df247',
+ 'channel_follower_count': int,
+ 'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
+ 'channel': 'Not Just Bikes',
+ 'uploader_url': 'https://www.youtube.com/@NotJustBikes',
+ 'uploader': 'Not Just Bikes',
+ 'uploader_id': '@NotJustBikes',
+ 'channel_is_verified': True,
+ },
+ 'playlist_mincount': 10,
+ }, {
+ # Streams tab
+ 'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
+ 'info_dict': {
+ 'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
+ 'title': '中村悠一 - Live',
+ 'tags': 'count:7',
+ 'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
+ 'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
+ 'channel': '中村悠一',
+ 'channel_follower_count': int,
+ 'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
+ 'uploader_url': 'https://www.youtube.com/@Yuichi-Nakamura',
+ 'uploader_id': '@Yuichi-Nakamura',
+ 'uploader': '中村悠一',
+ },
+ 'playlist_mincount': 60,
+ }, {
+ # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
+ # See test_youtube_lists
+ 'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
+ 'only_matching': True,
+ }, {
+ # No uploads and no UCID given. Should fail with no uploads error
+ # See test_youtube_lists
+ 'url': 'https://www.youtube.com/news',
+ 'only_matching': True,
+ }, {
+ # No videos tab but has a shorts tab
+ 'url': 'https://www.youtube.com/c/TKFShorts',
+ 'info_dict': {
+ 'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
+ 'title': 'Shorts Break - Shorts',
+ 'tags': 'count:48',
+ 'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
+ 'channel': 'Shorts Break',
+ 'description': 'md5:6de33c5e7ba686e5f3efd4e19c7ef499',
+ 'channel_follower_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
+ 'uploader_url': 'https://www.youtube.com/@ShortsBreak_Official',
+ 'uploader': 'Shorts Break',
+ 'uploader_id': '@ShortsBreak_Official',
+ },
+ 'playlist_mincount': 30,
+ }, {
+ # Trending Now Tab. tab id is empty
+ 'url': 'https://www.youtube.com/feed/trending',
+ 'info_dict': {
+ 'id': 'trending',
+ 'title': 'trending - Now',
+ 'tags': [],
+ },
+ 'playlist_mincount': 30,
+ }, {
+ # Trending Gaming Tab. tab id is empty
+ 'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
+ 'info_dict': {
+ 'id': 'trending',
+ 'title': 'trending - Gaming',
+ 'tags': [],
+ },
+ 'playlist_mincount': 30,
+ }, {
+ # Shorts url result in shorts tab
+ # TODO: Fix channel id extraction
+ 'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
+ 'info_dict': {
+ 'id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'title': 'cole-dlp-test-acc - Shorts',
+ 'channel': 'cole-dlp-test-acc',
+ 'description': 'test description',
+ 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
+ 'tags': [],
+ 'uploader_url': 'https://www.youtube.com/@coletdjnz',
+ 'uploader_id': '@coletdjnz',
+ 'uploader': 'cole-dlp-test-acc',
+ },
+ 'playlist': [{
+ 'info_dict': {
+ # Channel data is not currently available for short renderers (as of 2023-03-01)
+ '_type': 'url',
+ 'ie_key': 'Youtube',
+ 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
+ 'id': 'sSM9J5YH_60',
+ 'title': 'SHORT short',
+ 'view_count': int,
+ 'thumbnails': list,
+ },
+ }],
+ 'params': {'extract_flat': True},
+ }, {
+ # Live video status should be extracted
+ 'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
+ 'info_dict': {
+ 'id': 'UCQvWX73GQygcwXOTSf_VDVg',
+ 'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live', # TODO: should be Minecraft - Live or Minecraft - Topic - Live
+ 'tags': [],
+ },
+ 'playlist': [{
+ 'info_dict': {
+ '_type': 'url',
+ 'ie_key': 'Youtube',
+ 'url': 'startswith:https://www.youtube.com/watch?v=',
+ 'id': str,
+ 'title': str,
+ 'live_status': 'is_live',
+ 'channel_id': str,
+ 'channel_url': str,
+ 'concurrent_view_count': int,
+ 'channel': str,
+ 'uploader': str,
+ 'uploader_url': str,
+ 'uploader_id': str,
+ 'channel_is_verified': bool, # this will keep changing
+ },
+ }],
+ 'params': {'extract_flat': True, 'playlist_items': '1'},
+ 'playlist_mincount': 1,
+ }, {
+ # Channel renderer metadata. Contains number of videos on the channel
+ 'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
+ 'info_dict': {
+ 'id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'title': 'cole-dlp-test-acc - Channels',
+ 'channel': 'cole-dlp-test-acc',
+ 'description': 'test description',
+ 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
+ 'tags': [],
+ 'uploader_url': 'https://www.youtube.com/@coletdjnz',
+ 'uploader_id': '@coletdjnz',
+ 'uploader': 'cole-dlp-test-acc',
+ },
+ 'playlist': [{
+ 'info_dict': {
+ '_type': 'url',
+ 'ie_key': 'YoutubeTab',
+ 'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
+ 'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
+ 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
+ 'title': 'PewDiePie',
+ 'channel': 'PewDiePie',
+ 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
+ 'thumbnails': list,
+ 'channel_follower_count': int,
+ 'playlist_count': int,
+ 'uploader': 'PewDiePie',
+ 'uploader_url': 'https://www.youtube.com/@PewDiePie',
+ 'uploader_id': '@PewDiePie',
+ 'channel_is_verified': True,
+ },
+ }],
+ 'params': {'extract_flat': True},
+ }, {
+ 'url': 'https://www.youtube.com/@3blue1brown/about',
+ 'info_dict': {
+ 'id': '@3blue1brown',
+ 'tags': ['Mathematics'],
+ 'title': '3Blue1Brown',
+ 'channel_follower_count': int,
+ 'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
+ 'channel': '3Blue1Brown',
+ 'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
+ 'description': 'md5:4d1da95432004b7ba840ebc895b6b4c9',
+ 'uploader_url': 'https://www.youtube.com/@3blue1brown',
+ 'uploader_id': '@3blue1brown',
+ 'uploader': '3Blue1Brown',
+ 'channel_is_verified': True,
+ },
+ 'playlist_count': 0,
+ }, {
+ # Podcasts tab, with rich entry lockupViewModel
+ 'url': 'https://www.youtube.com/@99percentinvisiblepodcast/podcasts',
+ 'info_dict': {
+ 'id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+ 'channel_id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+ 'uploader_url': 'https://www.youtube.com/@99percentinvisiblepodcast',
+ 'description': 'md5:3a0ed38f1ad42a68ef0428c04a15695c',
+ 'title': '99% Invisible - Podcasts',
+ 'uploader': '99% Invisible',
+ 'channel_follower_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+ 'tags': [],
+ 'channel': '99% Invisible',
+ 'uploader_id': '@99percentinvisiblepodcast',
+ },
+ 'playlist_count': 5,
+ }, {
+ # Releases tab, with rich entry playlistRenderers (same as Podcasts tab)
+ 'url': 'https://www.youtube.com/@AHimitsu/releases',
+ 'info_dict': {
+ 'id': 'UCgFwu-j5-xNJml2FtTrrB3A',
+ 'channel': 'A Himitsu',
+ 'uploader_url': 'https://www.youtube.com/@AHimitsu',
+ 'title': 'A Himitsu - Releases',
+ 'uploader_id': '@AHimitsu',
+ 'uploader': 'A Himitsu',
+ 'channel_id': 'UCgFwu-j5-xNJml2FtTrrB3A',
+ 'tags': 'count:12',
+ 'description': 'I make music',
+ 'channel_url': 'https://www.youtube.com/channel/UCgFwu-j5-xNJml2FtTrrB3A',
+ 'channel_follower_count': int,
+ 'channel_is_verified': True,
+ },
+ 'playlist_mincount': 10,
+ }, {
+ # Playlist with only shorts, shown as reel renderers
+ # FIXME: future: YouTube currently doesn't give continuation for this,
+ # may do in future.
+ 'url': 'https://www.youtube.com/playlist?list=UUxqPAgubo4coVn9Lx1FuKcg',
+ 'info_dict': {
+ 'id': 'UUxqPAgubo4coVn9Lx1FuKcg',
+ 'channel_url': 'https://www.youtube.com/channel/UCxqPAgubo4coVn9Lx1FuKcg',
+ 'view_count': int,
+ 'uploader_id': '@BangyShorts',
+ 'description': '',
+ 'uploader_url': 'https://www.youtube.com/@BangyShorts',
+ 'channel_id': 'UCxqPAgubo4coVn9Lx1FuKcg',
+ 'channel': 'Bangy Shorts',
+ 'uploader': 'Bangy Shorts',
+ 'tags': [],
+ 'availability': 'public',
+ 'modified_date': r're:\d{8}',
+ 'title': 'Uploads from Bangy Shorts',
+ },
+ 'playlist_mincount': 100,
+ 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
+ }, {
+ 'note': 'Tags containing spaces',
+ 'url': 'https://www.youtube.com/channel/UC7_YxT-KID8kRbqZo7MyscQ',
+ 'playlist_count': 3,
+ 'info_dict': {
+ 'id': 'UC7_YxT-KID8kRbqZo7MyscQ',
+ 'channel': 'Markiplier',
+ 'channel_id': 'UC7_YxT-KID8kRbqZo7MyscQ',
+ 'title': 'Markiplier',
+ 'channel_follower_count': int,
+ 'description': 'md5:0c010910558658824402809750dc5d97',
+ 'uploader_id': '@markiplier',
+ 'uploader_url': 'https://www.youtube.com/@markiplier',
+ 'uploader': 'Markiplier',
+ 'channel_url': 'https://www.youtube.com/channel/UC7_YxT-KID8kRbqZo7MyscQ',
+ 'channel_is_verified': True,
+ 'tags': ['markiplier', 'comedy', 'gaming', 'funny videos', 'funny moments',
+ 'sketch comedy', 'laughing', 'lets play', 'challenge videos', 'hilarious',
+ 'challenges', 'sketches', 'scary games', 'funny games', 'rage games',
+ 'mark fischbach'],
+ },
+ }]
+
+ @classmethod
+ def suitable(cls, url):
+ return False if YoutubeIE.suitable(url) else super().suitable(url)
+
+ _URL_RE = re.compile(rf'(?P{_VALID_URL})(?(not_channel)|(?P/[^?#/]+))?(?P.*)$')
+
+ def _get_url_mobj(self, url):
+ mobj = self._URL_RE.match(url).groupdict()
+ mobj.update((k, '') for k, v in mobj.items() if v is None)
+ return mobj
+
+ def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
+ tab_name = (tab.get('title') or '').lower()
+ tab_url = urljoin(base_url, traverse_obj(
+ tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
+
+ tab_id = ((tab_url and self._get_url_mobj(tab_url)['tab'][1:])
+ or traverse_obj(tab, 'tabIdentifier', expected_type=str))
+ if tab_id:
+ return {
+ 'TAB_ID_SPONSORSHIPS': 'membership',
+ }.get(tab_id, tab_id), tab_name
+
+ # Fallback to tab name if we cannot get the tab id.
+ # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
+ # Note that in the case of translated tab name this may result in an empty string, which we don't want.
+ if tab_name:
+ self.write_debug(f'Falling back to selected tab name: {tab_name}')
+ return {
+ 'home': 'featured',
+ 'live': 'streams',
+ }.get(tab_name, tab_name), tab_name
+
+ def _has_tab(self, tabs, tab_id):
+ return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
+
+ def _empty_playlist(self, item_id, data):
+ return self.playlist_result([], item_id, **self._extract_metadata_from_tabs(item_id, data))
+
+ @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
+ def _real_extract(self, url, smuggled_data):
+ item_id = self._match_id(url)
+ url = urllib.parse.urlunparse(
+ urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
+ compat_opts = self.get_param('compat_opts', [])
+
+ mobj = self._get_url_mobj(url)
+ pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
+ if is_channel and smuggled_data.get('is_music_url'):
+ if item_id[:2] == 'VL': # Youtube music VL channels have an equivalent playlist
+ return self.url_result(
+ f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
+ elif item_id[:2] == 'MP': # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
+ mdata = self._extract_tab_endpoint(
+ f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
+ murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
+ get_all=False, expected_type=str)
+ if not murl:
+ raise ExtractorError('Failed to resolve album to playlist')
+ return self.url_result(murl, YoutubeTabIE)
+ elif mobj['channel_type'] == 'browse': # Youtube music /browse/ should be changed to /channel/
+ return self.url_result(
+ f'https://music.youtube.com/channel/{item_id}{tab}{post}', YoutubeTabIE, item_id)
+
+ original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
+ if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
+ url = f'{pre}/videos{post}'
+ if smuggled_data.get('is_music_url'):
+ self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}')
+
+ # Handle both video/playlist URLs
+ qs = parse_qs(url)
+ video_id, playlist_id = (traverse_obj(qs, (key, 0)) for key in ('v', 'list'))
+ if not video_id and mobj['not_channel'].startswith('watch'):
+ if not playlist_id:
+ # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
+ raise ExtractorError('A video URL was given without video ID', expected=True)
+ # Common mistake: https://www.youtube.com/watch?list=playlist_id
+ self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
+ return self.url_result(
+ f'https://www.youtube.com/playlist?list={playlist_id}', YoutubeTabIE, playlist_id)
+
+ if not self._yes_playlist(playlist_id, video_id):
+ return self.url_result(
+ f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
+
+ data, ytcfg = self._extract_data(url, display_id)
+
+ # YouTube may provide a non-standard redirect to the regional channel
+ # See: https://github.com/yt-dlp/yt-dlp/issues/2694
+ # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
+ redirect_url = traverse_obj(
+ data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
+ if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
+ redirect_url = ''.join((urljoin('https://www.youtube.com', redirect_url), tab, post))
+ self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
+ return self.url_result(redirect_url, YoutubeTabIE)
+
+ tabs, extra_tabs = self._extract_tab_renderers(data), []
+ if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
+ selected_tab = self._extract_selected_tab(tabs)
+ selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url) # NB: Name may be translated
+ self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
+
+ # /about is no longer a tab
+ if original_tab_id == 'about':
+ return self._empty_playlist(item_id, data)
+
+ if not original_tab_id and selected_tab_name:
+ self.to_screen('Downloading all uploads of the channel. '
+ 'To download only the videos in a specific tab, pass the tab\'s URL')
+ if self._has_tab(tabs, 'streams'):
+ extra_tabs.append(''.join((pre, '/streams', post)))
+ if self._has_tab(tabs, 'shorts'):
+ extra_tabs.append(''.join((pre, '/shorts', post)))
+ # XXX: Members-only tab should also be extracted
+
+ if not extra_tabs and selected_tab_id != 'videos':
+ # Channel does not have streams, shorts or videos tabs
+ if item_id[:2] != 'UC':
+ return self._empty_playlist(item_id, data)
+
+ # Topic channels don't have /videos. Use the equivalent playlist instead
+ pl_id = f'UU{item_id[2:]}'
+ pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
+ try:
+ data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
+ except ExtractorError:
+ return self._empty_playlist(item_id, data)
+ else:
+ item_id, url = pl_id, pl_url
+ self.to_screen(
+ f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
+
+ elif extra_tabs and selected_tab_id != 'videos':
+ # When there are shorts/live tabs but not videos tab
+ url, data = f'{pre}{post}', None
+
+ elif (original_tab_id or 'videos') != selected_tab_id:
+ if original_tab_id == 'live':
+ # Live tab should have redirected to the video
+ # Except in the case the channel has an actual live tab
+ # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
+ raise UserNotLive(video_id=item_id)
+ elif selected_tab_name:
+ raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
+
+ # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
+ url = f'{pre}{post}'
+
+ # YouTube sometimes provides a button to reload playlist with unavailable videos.
+ if 'no-youtube-unavailable-videos' not in compat_opts:
+ data = self._reload_with_unavailable_videos(display_id, data, ytcfg) or data
+ self._extract_and_report_alerts(data, only_once=True)
+
+ tabs, entries = self._extract_tab_renderers(data), []
+ if tabs:
+ entries = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
+ entries[0].update({
+ 'extractor_key': YoutubeTabIE.ie_key(),
+ 'extractor': YoutubeTabIE.IE_NAME,
+ 'webpage_url': url,
+ })
+ if self.get_param('playlist_items') == '0':
+ entries.extend(self.url_result(u, YoutubeTabIE) for u in extra_tabs)
+ else: # Users expect to get all `video_id`s even with `--flat-playlist`. So don't return `url_result`
+ entries.extend(map(self._real_extract, extra_tabs))
+
+ if len(entries) == 1:
+ return entries[0]
+ elif entries:
+ metadata = self._extract_metadata_from_tabs(item_id, data)
+ uploads_url = 'the Uploads (UU) playlist URL'
+ if try_get(metadata, lambda x: x['channel_id'].startswith('UC')):
+ uploads_url = f'https://www.youtube.com/playlist?list=UU{metadata["channel_id"][2:]}'
+ self.to_screen(
+ 'Downloading as multiple playlists, separated by tabs. '
+ f'To download as a single playlist instead, pass {uploads_url}')
+ return self.playlist_result(entries, item_id, **metadata)
+
+ # Inline playlist
+ playlist = traverse_obj(
+ data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
+ if playlist:
+ return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
+
+ video_id = traverse_obj(
+ data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
+ if video_id:
+ if tab != '/live': # live tab is expected to redirect to video
+ self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
+ return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
+
+ raise ExtractorError('Unable to recognize tab page')
+
+
+# xxx: This is tightly coupled to YoutubeTabBaseInfoExtractor. Should be decoupled at some point
+class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
+ IE_DESC = 'YouTube playlists'
+ _VALID_URL = r'''(?x)(?:
+ (?:https?://)?
+ (?:\w+\.)?
+ (?:
+ (?:
+ youtube(?:kids)?\.com|
+ {invidious}
+ )
+ /.*?\?.*?\blist=
+ )?
+ (?P{playlist_id})
+ )'''.format(
+ playlist_id=YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
+ invidious='|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
+ )
+ IE_NAME = 'youtube:playlist'
+ _TESTS = [{
+ 'note': 'issue #673',
+ 'url': 'PLBB231211A4F62143',
+ 'info_dict': {
+ 'title': '[OLD]Team Fortress 2 (Class-based LP)',
+ 'id': 'PLBB231211A4F62143',
+ 'uploader': 'Wickman',
+ 'uploader_id': '@WickmanVT',
+ 'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
+ 'view_count': int,
+ 'uploader_url': 'https://www.youtube.com/@WickmanVT',
+ 'modified_date': r're:\d{8}',
+ 'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
+ 'channel': 'Wickman',
+ 'tags': [],
+ 'channel_url': 'https://www.youtube.com/channel/UCKSpbfbl5kRQpTdL7kMc-1Q',
+ 'availability': 'public',
+ },
+ 'playlist_mincount': 29,
+ }, {
+ 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
+ 'info_dict': {
+ 'title': 'YDL_safe_search',
+ 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
+ },
+ 'playlist_count': 2,
+ 'skip': 'This playlist is private',
+ }, {
+ 'note': 'embedded',
+ 'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
+ 'playlist_count': 4,
+ 'info_dict': {
+ 'title': 'JODA15',
+ 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
+ 'uploader': 'milan',
+ 'uploader_id': '@milan5503',
+ 'description': '',
+ 'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
+ 'tags': [],
+ 'modified_date': '20140919',
+ 'view_count': int,
+ 'channel': 'milan',
+ 'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
+ 'uploader_url': 'https://www.youtube.com/@milan5503',
+ 'availability': 'public',
+ },
+ 'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden', 'Retrying', 'Giving up'],
+ }, {
+ 'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
+ 'playlist_mincount': 455,
+ 'info_dict': {
+ 'title': '2018 Chinese New Singles (11/6 updated)',
+ 'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
+ 'uploader': 'LBK',
+ 'uploader_id': '@music_king',
+ 'description': 'md5:da521864744d60a198e3a88af4db0d9d',
+ 'channel': 'LBK',
+ 'view_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UC21nz3_MesPLqtDqwdvnoxA',
+ 'tags': [],
+ 'uploader_url': 'https://www.youtube.com/@music_king',
+ 'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
+ 'modified_date': r're:\d{8}',
+ 'availability': 'public',
+ },
+ 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
+ }, {
+ 'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
+ 'only_matching': True,
+ }, {
+ # music album playlist
+ 'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
+ 'only_matching': True,
+ }]
+
+ @classmethod
+ def suitable(cls, url):
+ if YoutubeTabIE.suitable(url):
+ return False
+ from yt_dlp.utils import parse_qs
+ qs = parse_qs(url)
+ if qs.get('v', [None])[0]:
+ return False
+ return super().suitable(url)
+
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+ is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
+ url = update_url_query(
+ 'https://www.youtube.com/playlist',
+ parse_qs(url) or {'list': playlist_id})
+ if is_music_url:
+ url = smuggle_url(url, {'is_music_url': True})
+ return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube/_video.py
similarity index 50%
rename from yt_dlp/extractor/youtube.py
rename to yt_dlp/extractor/youtube/_video.py
index 87888aa61a..2fa8344c5c 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube/_video.py
@@ -1,35 +1,35 @@
import base64
import binascii
-import calendar
import collections
-import copy
import datetime as dt
-import enum
import functools
-import hashlib
import itertools
import json
import math
import os.path
import random
import re
-import shlex
import sys
import threading
import time
import traceback
import urllib.parse
-from .common import InfoExtractor, SearchInfoExtractor
-from ..jsinterp import JSInterpreter, PhantomJSwrapper
-from ..networking.exceptions import HTTPError, network_exceptions
-from ..utils import (
+from ._base import (
+ INNERTUBE_CLIENTS,
+ BadgeType,
+ YoutubeBaseInfoExtractor,
+ _PoTokenContext,
+ _split_innertube_client,
+ short_client_name,
+)
+from ...jsinterp import JSInterpreter, PhantomJSwrapper
+from ...networking.exceptions import HTTPError
+from ...utils import (
NO_DEFAULT,
ExtractorError,
LazyList,
- UserNotLive,
bug_reports_message,
- classproperty,
clean_html,
datetime_from_str,
filesize_from_tbr,
@@ -38,7 +38,6 @@
format_field,
get_first,
int_or_none,
- is_html,
join_nonempty,
js_to_json,
mimetype2ext,
@@ -60,7 +59,6 @@
try_get,
unescapeHTML,
unified_strdate,
- unified_timestamp,
unsmuggle_url,
update_url_query,
url_or_none,
@@ -73,1211 +71,6 @@
PO_TOKEN_GUIDE_URL = 'https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide'
-class _PoTokenContext(enum.Enum):
- PLAYER = 'player'
- GVS = 'gvs'
-
-
-# any clients starting with _ cannot be explicitly requested by the user
-INNERTUBE_CLIENTS = {
- 'web': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB',
- 'clientVersion': '2.20241126.01.00',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'SUPPORTS_COOKIES': True,
- },
- # Safari UA returns pre-merged video+audio 144p/240p/360p/720p/1080p HLS formats
- 'web_safari': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB',
- 'clientVersion': '2.20241126.01.00',
- 'userAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'SUPPORTS_COOKIES': True,
- },
- 'web_embedded': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB_EMBEDDED_PLAYER',
- 'clientVersion': '1.20241201.00.00',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 56,
- 'SUPPORTS_COOKIES': True,
- },
- 'web_music': {
- 'INNERTUBE_HOST': 'music.youtube.com',
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB_REMIX',
- 'clientVersion': '1.20241127.01.00',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'SUPPORTS_COOKIES': True,
- },
- # This client now requires sign-in for every video
- 'web_creator': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB_CREATOR',
- 'clientVersion': '1.20241203.01.00',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- 'SUPPORTS_COOKIES': True,
- },
- 'android': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'ANDROID',
- 'clientVersion': '19.44.38',
- 'androidSdkVersion': 30,
- 'userAgent': 'com.google.android.youtube/19.44.38 (Linux; U; Android 11) gzip',
- 'osName': 'Android',
- 'osVersion': '11',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- },
- # This client now requires sign-in for every video
- 'android_music': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'ANDROID_MUSIC',
- 'clientVersion': '7.27.52',
- 'androidSdkVersion': 30,
- 'userAgent': 'com.google.android.apps.youtube.music/7.27.52 (Linux; U; Android 11) gzip',
- 'osName': 'Android',
- 'osVersion': '11',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- },
- # This client now requires sign-in for every video
- 'android_creator': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'ANDROID_CREATOR',
- 'clientVersion': '24.45.100',
- 'androidSdkVersion': 30,
- 'userAgent': 'com.google.android.apps.youtube.creator/24.45.100 (Linux; U; Android 11) gzip',
- 'osName': 'Android',
- 'osVersion': '11',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- },
- # YouTube Kids videos aren't returned on this client for some reason
- 'android_vr': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'ANDROID_VR',
- 'clientVersion': '1.60.19',
- 'deviceMake': 'Oculus',
- 'deviceModel': 'Quest 3',
- 'androidSdkVersion': 32,
- 'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.60.19 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip',
- 'osName': 'Android',
- 'osVersion': '12L',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 28,
- 'REQUIRE_JS_PLAYER': False,
- },
- # iOS clients have HLS live streams. Setting device model to get 60fps formats.
- # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
- 'ios': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'IOS',
- 'clientVersion': '20.03.02',
- 'deviceMake': 'Apple',
- 'deviceModel': 'iPhone16,2',
- 'userAgent': 'com.google.ios.youtube/20.03.02 (iPhone16,2; U; CPU iOS 18_2_1 like Mac OS X;)',
- 'osName': 'iPhone',
- 'osVersion': '18.2.1.22C161',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_JS_PLAYER': False,
- },
- # This client now requires sign-in for every video
- 'ios_music': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'IOS_MUSIC',
- 'clientVersion': '7.27.0',
- 'deviceMake': 'Apple',
- 'deviceModel': 'iPhone16,2',
- 'userAgent': 'com.google.ios.youtubemusic/7.27.0 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
- 'osName': 'iPhone',
- 'osVersion': '18.1.0.22B83',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- },
- # This client now requires sign-in for every video
- 'ios_creator': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'IOS_CREATOR',
- 'clientVersion': '24.45.100',
- 'deviceMake': 'Apple',
- 'deviceModel': 'iPhone16,2',
- 'userAgent': 'com.google.ios.ytcreator/24.45.100 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
- 'osName': 'iPhone',
- 'osVersion': '18.1.0.22B83',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- },
- # mweb has 'ultralow' formats
- # See: https://github.com/yt-dlp/yt-dlp/pull/557
- 'mweb': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'MWEB',
- 'clientVersion': '2.20241202.07.00',
- # mweb previously did not require PO Token with this UA
- 'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 2,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'SUPPORTS_COOKIES': True,
- },
- 'tv': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'TVHTML5',
- 'clientVersion': '7.20250120.19.00',
- 'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
- 'SUPPORTS_COOKIES': True,
- },
- # This client now requires sign-in for every video
- # It was previously an age-gate workaround for videos that were `playable_in_embed`
- # It may still be useful if signed into an EU account that is not age-verified
- 'tv_embedded': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
- 'clientVersion': '2.0',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 85,
- 'REQUIRE_AUTH': True,
- 'SUPPORTS_COOKIES': True,
- },
-}
-
-
-def _split_innertube_client(client_name):
- variant, *base = client_name.rsplit('.', 1)
- if base:
- return variant, base[0], variant
- base, *variant = client_name.split('_', 1)
- return client_name, base, variant[0] if variant else None
-
-
-def short_client_name(client_name):
- main, *parts = _split_innertube_client(client_name)[0].split('_')
- return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
-
-
-def build_innertube_clients():
- THIRD_PARTY = {
- 'embedUrl': 'https://www.youtube.com/', # Can be any valid URL
- }
- BASE_CLIENTS = ('ios', 'web', 'tv', 'mweb', 'android')
- priority = qualities(BASE_CLIENTS[::-1])
-
- for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
- ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
- ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
- ytcfg.setdefault('PO_TOKEN_REQUIRED_CONTEXTS', [])
- ytcfg.setdefault('REQUIRE_AUTH', False)
- ytcfg.setdefault('SUPPORTS_COOKIES', False)
- ytcfg.setdefault('PLAYER_PARAMS', None)
- ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
-
- _, base_client, variant = _split_innertube_client(client)
- ytcfg['priority'] = 10 * priority(base_client)
-
- if variant == 'embedded':
- ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
- ytcfg['priority'] -= 2
- elif variant:
- ytcfg['priority'] -= 3
-
-
-build_innertube_clients()
-
-
-class BadgeType(enum.Enum):
- AVAILABILITY_UNLISTED = enum.auto()
- AVAILABILITY_PRIVATE = enum.auto()
- AVAILABILITY_PUBLIC = enum.auto()
- AVAILABILITY_PREMIUM = enum.auto()
- AVAILABILITY_SUBSCRIPTION = enum.auto()
- LIVE_NOW = enum.auto()
- VERIFIED = enum.auto()
-
-
-class YoutubeBaseInfoExtractor(InfoExtractor):
- """Provide base functions for Youtube extractors"""
-
- _RESERVED_NAMES = (
- r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
- r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
- r'browse|oembed|get_video_info|iframe_api|s/player|source|'
- r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
-
- _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
-
- # _NETRC_MACHINE = 'youtube'
-
- # If True it will raise an error if no login info is provided
- _LOGIN_REQUIRED = False
-
- _INVIDIOUS_SITES = (
- # invidious-redirect websites
- r'(?:www\.)?redirect\.invidious\.io',
- r'(?:(?:www|dev)\.)?invidio\.us',
- # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
- r'(?:www\.)?invidious\.pussthecat\.org',
- r'(?:www\.)?invidious\.zee\.li',
- r'(?:www\.)?invidious\.ethibox\.fr',
- r'(?:www\.)?iv\.ggtyler\.dev',
- r'(?:www\.)?inv\.vern\.i2p',
- r'(?:www\.)?am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd\.onion',
- r'(?:www\.)?inv\.riverside\.rocks',
- r'(?:www\.)?invidious\.silur\.me',
- r'(?:www\.)?inv\.bp\.projectsegfau\.lt',
- r'(?:www\.)?invidious\.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid\.onion',
- r'(?:www\.)?invidious\.slipfox\.xyz',
- r'(?:www\.)?invidious\.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd\.onion',
- r'(?:www\.)?inv\.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad\.onion',
- r'(?:www\.)?invidious\.tiekoetter\.com',
- r'(?:www\.)?iv\.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd\.onion',
- r'(?:www\.)?invidious\.nerdvpn\.de',
- r'(?:www\.)?invidious\.weblibre\.org',
- r'(?:www\.)?inv\.odyssey346\.dev',
- r'(?:www\.)?invidious\.dhusch\.de',
- r'(?:www\.)?iv\.melmac\.space',
- r'(?:www\.)?watch\.thekitty\.zone',
- r'(?:www\.)?invidious\.privacydev\.net',
- r'(?:www\.)?ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid\.onion',
- r'(?:www\.)?invidious\.drivet\.xyz',
- r'(?:www\.)?vid\.priv\.au',
- r'(?:www\.)?euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd\.onion',
- r'(?:www\.)?inv\.vern\.cc',
- r'(?:www\.)?invidious\.esmailelbob\.xyz',
- r'(?:www\.)?invidious\.sethforprivacy\.com',
- r'(?:www\.)?yt\.oelrichsgarcia\.de',
- r'(?:www\.)?yt\.artemislena\.eu',
- r'(?:www\.)?invidious\.flokinet\.to',
- r'(?:www\.)?invidious\.baczek\.me',
- r'(?:www\.)?y\.com\.sb',
- r'(?:www\.)?invidious\.epicsite\.xyz',
- r'(?:www\.)?invidious\.lidarshield\.cloud',
- r'(?:www\.)?yt\.funami\.tech',
- r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
- r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
- r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
- # youtube-dl invidious instances list
- r'(?:(?:www|no)\.)?invidiou\.sh',
- r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
- r'(?:www\.)?invidious\.kabi\.tk',
- r'(?:www\.)?invidious\.mastodon\.host',
- r'(?:www\.)?invidious\.zapashcanon\.fr',
- r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
- r'(?:www\.)?invidious\.tinfoil-hat\.net',
- r'(?:www\.)?invidious\.himiko\.cloud',
- r'(?:www\.)?invidious\.reallyancient\.tech',
- r'(?:www\.)?invidious\.tube',
- r'(?:www\.)?invidiou\.site',
- r'(?:www\.)?invidious\.site',
- r'(?:www\.)?invidious\.xyz',
- r'(?:www\.)?invidious\.nixnet\.xyz',
- r'(?:www\.)?invidious\.048596\.xyz',
- r'(?:www\.)?invidious\.drycat\.fr',
- r'(?:www\.)?inv\.skyn3t\.in',
- r'(?:www\.)?tube\.poal\.co',
- r'(?:www\.)?tube\.connect\.cafe',
- r'(?:www\.)?vid\.wxzm\.sx',
- r'(?:www\.)?vid\.mint\.lgbt',
- r'(?:www\.)?vid\.puffyan\.us',
- r'(?:www\.)?yewtu\.be',
- r'(?:www\.)?yt\.elukerio\.org',
- r'(?:www\.)?yt\.lelux\.fi',
- r'(?:www\.)?invidious\.ggc-project\.de',
- r'(?:www\.)?yt\.maisputain\.ovh',
- r'(?:www\.)?ytprivate\.com',
- r'(?:www\.)?invidious\.13ad\.de',
- r'(?:www\.)?invidious\.toot\.koeln',
- r'(?:www\.)?invidious\.fdn\.fr',
- r'(?:www\.)?watch\.nettohikari\.com',
- r'(?:www\.)?invidious\.namazso\.eu',
- r'(?:www\.)?invidious\.silkky\.cloud',
- r'(?:www\.)?invidious\.exonip\.de',
- r'(?:www\.)?invidious\.riverside\.rocks',
- r'(?:www\.)?invidious\.blamefran\.net',
- r'(?:www\.)?invidious\.moomoo\.de',
- r'(?:www\.)?ytb\.trom\.tf',
- r'(?:www\.)?yt\.cyberhost\.uk',
- r'(?:www\.)?kgg2m7yk5aybusll\.onion',
- r'(?:www\.)?qklhadlycap4cnod\.onion',
- r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
- r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
- r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
- r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
- r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
- r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
- r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
- r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
- r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
- r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
- # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
- r'(?:www\.)?piped\.kavin\.rocks',
- r'(?:www\.)?piped\.tokhmi\.xyz',
- r'(?:www\.)?piped\.syncpundit\.io',
- r'(?:www\.)?piped\.mha\.fi',
- r'(?:www\.)?watch\.whatever\.social',
- r'(?:www\.)?piped\.garudalinux\.org',
- r'(?:www\.)?piped\.rivo\.lol',
- r'(?:www\.)?piped-libre\.kavin\.rocks',
- r'(?:www\.)?yt\.jae\.fi',
- r'(?:www\.)?piped\.mint\.lgbt',
- r'(?:www\.)?il\.ax',
- r'(?:www\.)?piped\.esmailelbob\.xyz',
- r'(?:www\.)?piped\.projectsegfau\.lt',
- r'(?:www\.)?piped\.privacydev\.net',
- r'(?:www\.)?piped\.palveluntarjoaja\.eu',
- r'(?:www\.)?piped\.smnz\.de',
- r'(?:www\.)?piped\.adminforge\.de',
- r'(?:www\.)?watch\.whatevertinfoil\.de',
- r'(?:www\.)?piped\.qdi\.fi',
- r'(?:(?:www|cf)\.)?piped\.video',
- r'(?:www\.)?piped\.aeong\.one',
- r'(?:www\.)?piped\.moomoo\.me',
- r'(?:www\.)?piped\.chauvet\.pro',
- r'(?:www\.)?watch\.leptons\.xyz',
- r'(?:www\.)?pd\.vern\.cc',
- r'(?:www\.)?piped\.hostux\.net',
- r'(?:www\.)?piped\.lunar\.icu',
- # Hyperpipe instances from https://hyperpipe.codeberg.page/
- r'(?:www\.)?hyperpipe\.surge\.sh',
- r'(?:www\.)?hyperpipe\.esmailelbob\.xyz',
- r'(?:www\.)?listen\.whatever\.social',
- r'(?:www\.)?music\.adminforge\.de',
- )
-
- # extracted from account/account_menu ep
- # XXX: These are the supported YouTube UI and API languages,
- # which is slightly different from languages supported for translation in YouTube studio
- _SUPPORTED_LANG_CODES = [
- 'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
- 'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
- 'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
- 'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
- 'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
- 'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko',
- ]
-
- _IGNORED_WARNINGS = {
- 'Unavailable videos will be hidden during playback',
- 'Unavailable videos are hidden',
- }
-
- _YT_HANDLE_RE = r'@[\w.-]{3,30}' # https://support.google.com/youtube/answer/11585688?hl=en
- _YT_CHANNEL_UCID_RE = r'UC[\w-]{22}'
-
- _NETRC_MACHINE = 'youtube'
-
- def ucid_or_none(self, ucid):
- return self._search_regex(rf'^({self._YT_CHANNEL_UCID_RE})$', ucid, 'UC-id', default=None)
-
- def handle_or_none(self, handle):
- return self._search_regex(rf'^({self._YT_HANDLE_RE})$', urllib.parse.unquote(handle or ''),
- '@-handle', default=None)
-
- def handle_from_url(self, url):
- return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_HANDLE_RE})',
- urllib.parse.unquote(url or ''), 'channel handle', default=None)
-
- def ucid_from_url(self, url):
- return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_CHANNEL_UCID_RE})',
- url, 'channel id', default=None)
-
- @functools.cached_property
- def _preferred_lang(self):
- """
- Returns a language code supported by YouTube for the user preferred language.
- Returns None if no preferred language set.
- """
- preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
- if not preferred_lang:
- return
- if preferred_lang not in self._SUPPORTED_LANG_CODES:
- raise ExtractorError(
- f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
- expected=True)
- elif preferred_lang != 'en':
- self.report_warning(
- f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
- return preferred_lang
-
- def _initialize_consent(self):
- cookies = self._get_cookies('https://www.youtube.com/')
- if cookies.get('__Secure-3PSID'):
- return
- socs = cookies.get('SOCS')
- if socs and not socs.value.startswith('CAA'): # not consented
- return
- self._set_cookie('.youtube.com', 'SOCS', 'CAI', secure=True) # accept all (required for mixes)
-
- def _initialize_pref(self):
- cookies = self._get_cookies('https://www.youtube.com/')
- pref_cookie = cookies.get('PREF')
- pref = {}
- if pref_cookie:
- try:
- pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
- except ValueError:
- self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
- pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
- self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
-
- def _initialize_cookie_auth(self):
- yt_sapisid, yt_1psapisid, yt_3psapisid = self._get_sid_cookies()
- if yt_sapisid or yt_1psapisid or yt_3psapisid:
- self.write_debug('Found YouTube account cookies')
-
- def _real_initialize(self):
- self._initialize_pref()
- self._initialize_consent()
- self._initialize_cookie_auth()
- self._check_login_required()
-
- def _perform_login(self, username, password):
- if username.startswith('oauth'):
- raise ExtractorError(
- f'Login with OAuth is no longer supported. {self._youtube_login_hint}', expected=True)
-
- self.report_warning(
- f'Login with password is not supported for YouTube. {self._youtube_login_hint}')
-
- @property
- def _youtube_login_hint(self):
- return (f'{self._login_hint(method="cookies")}. Also see '
- 'https://github.com/yt-dlp/yt-dlp/wiki/Extractors#exporting-youtube-cookies '
- 'for tips on effectively exporting YouTube cookies')
-
- def _check_login_required(self):
- if self._LOGIN_REQUIRED and not self.is_authenticated:
- self.raise_login_required(
- f'Login details are needed to download this content. {self._youtube_login_hint}', method=None)
-
- _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
- _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
-
- def _get_default_ytcfg(self, client='web'):
- return copy.deepcopy(INNERTUBE_CLIENTS[client])
-
- def _get_innertube_host(self, client='web'):
- return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
-
- def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
- # try_get but with fallback to default ytcfg client values when present
- _func = lambda y: try_get(y, getter, expected_type)
- return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
-
- def _extract_client_name(self, ytcfg, default_client='web'):
- return self._ytcfg_get_safe(
- ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
- lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
-
- def _extract_client_version(self, ytcfg, default_client='web'):
- return self._ytcfg_get_safe(
- ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
- lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
-
- def _select_api_hostname(self, req_api_hostname, default_client=None):
- return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
- or req_api_hostname or self._get_innertube_host(default_client or 'web'))
-
- def _extract_context(self, ytcfg=None, default_client='web'):
- context = get_first(
- (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
- # Enforce language and tz for extraction
- client_context = traverse_obj(context, 'client', expected_type=dict, default={})
- client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
- return context
-
- @staticmethod
- def _make_sid_authorization(scheme, sid, origin, additional_parts):
- timestamp = str(round(time.time()))
-
- hash_parts = []
- if additional_parts:
- hash_parts.append(':'.join(additional_parts.values()))
- hash_parts.extend([timestamp, sid, origin])
- sidhash = hashlib.sha1(' '.join(hash_parts).encode()).hexdigest()
-
- parts = [timestamp, sidhash]
- if additional_parts:
- parts.append(''.join(additional_parts))
-
- return f'{scheme} {"_".join(parts)}'
-
- def _get_sid_cookies(self):
- """
- Get SAPISID, 1PSAPISID, 3PSAPISID cookie values
- @returns sapisid, 1psapisid, 3psapisid
- """
- yt_cookies = self._get_cookies('https://www.youtube.com')
- yt_sapisid = try_call(lambda: yt_cookies['SAPISID'].value)
- yt_3papisid = try_call(lambda: yt_cookies['__Secure-3PAPISID'].value)
- yt_1papisid = try_call(lambda: yt_cookies['__Secure-1PAPISID'].value)
-
- # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
- # YouTube also falls back to __Secure-3PAPISID if SAPISID is missing.
- # See: https://github.com/yt-dlp/yt-dlp/issues/393
-
- return yt_sapisid or yt_3papisid, yt_1papisid, yt_3papisid
-
- def _get_sid_authorization_header(self, origin='https://www.youtube.com', user_session_id=None):
- """
- Generate API Session ID Authorization for Innertube requests. Assumes all requests are secure (https).
- @param origin: Origin URL
- @param user_session_id: Optional User Session ID
- @return: Authorization header value
- """
-
- authorizations = []
- additional_parts = {}
- if user_session_id:
- additional_parts['u'] = user_session_id
-
- yt_sapisid, yt_1psapisid, yt_3psapisid = self._get_sid_cookies()
-
- for scheme, sid in (('SAPISIDHASH', yt_sapisid),
- ('SAPISID1PHASH', yt_1psapisid),
- ('SAPISID3PHASH', yt_3psapisid)):
- if sid:
- authorizations.append(self._make_sid_authorization(scheme, sid, origin, additional_parts))
-
- if not authorizations:
- return None
-
- return ' '.join(authorizations)
-
- def _call_api(self, ep, query, video_id, fatal=True, headers=None,
- note='Downloading API JSON', errnote='Unable to download API page',
- context=None, api_key=None, api_hostname=None, default_client='web'):
-
- data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
- data.update(query)
- real_headers = self.generate_api_headers(default_client=default_client)
- real_headers.update({'content-type': 'application/json'})
- if headers:
- real_headers.update(headers)
- return self._download_json(
- f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
- video_id=video_id, fatal=fatal, note=note, errnote=errnote,
- data=json.dumps(data).encode('utf8'), headers=real_headers,
- query=filter_dict({
- 'key': self._configuration_arg(
- 'innertube_key', [api_key], ie_key=YoutubeIE.ie_key(), casesense=True)[0],
- 'prettyPrint': 'false',
- }, cndn=lambda _, v: v))
-
- def extract_yt_initial_data(self, item_id, webpage, fatal=True):
- return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
-
- @staticmethod
- def _extract_session_index(*data):
- """
- Index of current account in account list.
- See: https://github.com/yt-dlp/yt-dlp/pull/519
- """
- for ytcfg in data:
- session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
- if session_index is not None:
- return session_index
-
- @staticmethod
- def _parse_data_sync_id(data_sync_id):
- """
- Parse data_sync_id into delegated_session_id and user_session_id.
-
- data_sync_id is of the form "delegated_session_id||user_session_id" for secondary channel
- and just "user_session_id||" for primary channel.
-
- @param data_sync_id: data_sync_id string
- @return: Tuple of (delegated_session_id, user_session_id)
- """
- if not data_sync_id:
- return None, None
- first, _, second = data_sync_id.partition('||')
- if second:
- return first, second
- return None, first
-
- def _extract_delegated_session_id(self, *args):
- """
- Extract current delegated session ID required to download private playlists of secondary channels
- @params response and/or ytcfg
- @return: delegated session ID
- """
- # ytcfg includes channel_syncid if on secondary channel
- if delegated_sid := traverse_obj(args, (..., 'DELEGATED_SESSION_ID', {str}, any)):
- return delegated_sid
-
- data_sync_id = self._extract_data_sync_id(*args)
- return self._parse_data_sync_id(data_sync_id)[0]
-
- def _extract_user_session_id(self, *args):
- """
- Extract current user session ID
- @params response and/or ytcfg
- @return: user session ID
- """
- if user_sid := traverse_obj(args, (..., 'USER_SESSION_ID', {str}, any)):
- return user_sid
-
- data_sync_id = self._extract_data_sync_id(*args)
- return self._parse_data_sync_id(data_sync_id)[1]
-
- def _extract_data_sync_id(self, *args):
- """
- Extract current account dataSyncId.
- In the format DELEGATED_SESSION_ID||USER_SESSION_ID or USER_SESSION_ID||
- @params response and/or ytcfg
- """
- if data_sync_id := self._configuration_arg('data_sync_id', [None], ie_key=YoutubeIE, casesense=True)[0]:
- return data_sync_id
-
- return traverse_obj(
- args, (..., ('DATASYNC_ID', ('responseContext', 'mainAppWebResponseContext', 'datasyncId')), {str}, any))
-
- def _extract_visitor_data(self, *args):
- """
- Extracts visitorData from an API response or ytcfg
- Appears to be used to track session state
- """
- if visitor_data := self._configuration_arg('visitor_data', [None], ie_key=YoutubeIE, casesense=True)[0]:
- return visitor_data
- return get_first(
- args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
- expected_type=str)
-
- @functools.cached_property
- def is_authenticated(self):
- return bool(self._get_sid_authorization_header())
-
- def extract_ytcfg(self, video_id, webpage):
- if not webpage:
- return {}
- return self._parse_json(
- self._search_regex(
- r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
- default='{}'), video_id, fatal=False) or {}
-
- def _generate_cookie_auth_headers(self, *, ytcfg=None, delegated_session_id=None, user_session_id=None, session_index=None, origin=None, **kwargs):
- headers = {}
- delegated_session_id = delegated_session_id or self._extract_delegated_session_id(ytcfg)
- if delegated_session_id:
- headers['X-Goog-PageId'] = delegated_session_id
- if session_index is None:
- session_index = self._extract_session_index(ytcfg)
- if delegated_session_id or session_index is not None:
- headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
-
- auth = self._get_sid_authorization_header(origin, user_session_id=user_session_id or self._extract_user_session_id(ytcfg))
- if auth is not None:
- headers['Authorization'] = auth
- headers['X-Origin'] = origin
-
- if traverse_obj(ytcfg, 'LOGGED_IN', expected_type=bool):
- headers['X-Youtube-Bootstrap-Logged-In'] = 'true'
-
- return headers
-
- def generate_api_headers(
- self, *, ytcfg=None, delegated_session_id=None, user_session_id=None, session_index=None,
- visitor_data=None, api_hostname=None, default_client='web', **kwargs):
-
- origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
- headers = {
- 'X-YouTube-Client-Name': str(
- self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
- 'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
- 'Origin': origin,
- 'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
- 'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client),
- **self._generate_cookie_auth_headers(
- ytcfg=ytcfg,
- delegated_session_id=delegated_session_id,
- user_session_id=user_session_id,
- session_index=session_index,
- origin=origin),
- }
- return filter_dict(headers)
-
- def _download_webpage_with_retries(self, *args, retry_fatal=False, retry_on_status=None, **kwargs):
- for retry in self.RetryManager(fatal=retry_fatal):
- try:
- return self._download_webpage(*args, **kwargs)
- except ExtractorError as e:
- if isinstance(e.cause, network_exceptions):
- if not isinstance(e.cause, HTTPError) or e.cause.status not in (retry_on_status or (403, 429)):
- retry.error = e
- continue
- self._error_or_warning(e, fatal=retry_fatal)
- break
-
- def _download_ytcfg(self, client, video_id):
- url = {
- 'web': 'https://www.youtube.com',
- 'web_music': 'https://music.youtube.com',
- 'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1',
- 'tv': 'https://www.youtube.com/tv',
- }.get(client)
- if not url:
- return {}
- webpage = self._download_webpage_with_retries(
- url, video_id, note=f'Downloading {client.replace("_", " ").strip()} client config',
- headers=traverse_obj(self._get_default_ytcfg(client), {
- 'User-Agent': ('INNERTUBE_CONTEXT', 'client', 'userAgent', {str}),
- }))
- return self.extract_ytcfg(video_id, webpage) or {}
-
- @staticmethod
- def _build_api_continuation_query(continuation, ctp=None):
- query = {
- 'continuation': continuation,
- }
- # TODO: Inconsistency with clickTrackingParams.
- # Currently we have a fixed ctp contained within context (from ytcfg)
- # and a ctp in root query for continuation.
- if ctp:
- query['clickTracking'] = {'clickTrackingParams': ctp}
- return query
-
- @classmethod
- def _extract_next_continuation_data(cls, renderer):
- next_continuation = try_get(
- renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
- lambda x: x['continuation']['reloadContinuationData']), dict)
- if not next_continuation:
- return
- continuation = next_continuation.get('continuation')
- if not continuation:
- return
- ctp = next_continuation.get('clickTrackingParams')
- return cls._build_api_continuation_query(continuation, ctp)
-
- @classmethod
- def _extract_continuation_ep_data(cls, continuation_ep: dict):
- if isinstance(continuation_ep, dict):
- continuation = try_get(
- continuation_ep, lambda x: x['continuationCommand']['token'], str)
- if not continuation:
- return
- ctp = continuation_ep.get('clickTrackingParams')
- return cls._build_api_continuation_query(continuation, ctp)
-
- @classmethod
- def _extract_continuation(cls, renderer):
- next_continuation = cls._extract_next_continuation_data(renderer)
- if next_continuation:
- return next_continuation
-
- return traverse_obj(renderer, (
- ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
- ('continuationEndpoint', ('button', 'buttonRenderer', 'command')),
- ), get_all=False, expected_type=cls._extract_continuation_ep_data)
-
- @classmethod
- def _extract_alerts(cls, data):
- for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
- if not isinstance(alert_dict, dict):
- continue
- for alert in alert_dict.values():
- alert_type = alert.get('type')
- if not alert_type:
- continue
- message = cls._get_text(alert, 'text')
- if message:
- yield alert_type, message
-
- def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
- errors, warnings = [], []
- for alert_type, alert_message in alerts:
- if alert_type.lower() == 'error' and fatal:
- errors.append([alert_type, alert_message])
- elif alert_message not in self._IGNORED_WARNINGS:
- warnings.append([alert_type, alert_message])
-
- for alert_type, alert_message in (warnings + errors[:-1]):
- self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
- if errors:
- raise ExtractorError(f'YouTube said: {errors[-1][1]}', expected=expected)
-
- def _extract_and_report_alerts(self, data, *args, **kwargs):
- return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
-
- def _extract_badges(self, badge_list: list):
- """
- Extract known BadgeType's from a list of badge renderers.
- @returns [{'type': BadgeType}]
- """
- icon_type_map = {
- 'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
- 'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
- 'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC,
- 'CHECK_CIRCLE_THICK': BadgeType.VERIFIED,
- 'OFFICIAL_ARTIST_BADGE': BadgeType.VERIFIED,
- 'CHECK': BadgeType.VERIFIED,
- }
-
- badge_style_map = {
- 'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
- 'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
- 'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW,
- 'BADGE_STYLE_TYPE_VERIFIED': BadgeType.VERIFIED,
- 'BADGE_STYLE_TYPE_VERIFIED_ARTIST': BadgeType.VERIFIED,
- }
-
- label_map = {
- 'unlisted': BadgeType.AVAILABILITY_UNLISTED,
- 'private': BadgeType.AVAILABILITY_PRIVATE,
- 'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
- 'live': BadgeType.LIVE_NOW,
- 'premium': BadgeType.AVAILABILITY_PREMIUM,
- 'verified': BadgeType.VERIFIED,
- 'official artist channel': BadgeType.VERIFIED,
- }
-
- badges = []
- for badge in traverse_obj(badge_list, (..., lambda key, _: re.search(r'[bB]adgeRenderer$', key))):
- badge_type = (
- icon_type_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
- or badge_style_map.get(traverse_obj(badge, 'style'))
- )
- if badge_type:
- badges.append({'type': badge_type})
- continue
-
- # fallback, won't work in some languages
- label = traverse_obj(
- badge, 'label', ('accessibilityData', 'label'), 'tooltip', 'iconTooltip', get_all=False, expected_type=str, default='')
- for match, label_badge_type in label_map.items():
- if match in label.lower():
- badges.append({'type': label_badge_type})
- break
-
- return badges
-
- @staticmethod
- def _has_badge(badges, badge_type):
- return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
-
- @staticmethod
- def _get_text(data, *path_list, max_runs=None):
- for path in path_list or [None]:
- if path is None:
- obj = [data]
- else:
- obj = traverse_obj(data, path, default=[])
- if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
- obj = [obj]
- for item in obj:
- text = try_get(item, lambda x: x['simpleText'], str)
- if text:
- return text
- runs = try_get(item, lambda x: x['runs'], list) or []
- if not runs and isinstance(item, list):
- runs = item
-
- runs = runs[:min(len(runs), max_runs or len(runs))]
- text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str))
- if text:
- return text
-
- def _get_count(self, data, *path_list):
- count_text = self._get_text(data, *path_list) or ''
- count = parse_count(count_text)
- if count is None:
- count = str_to_int(
- self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
- return count
-
- @staticmethod
- def _extract_thumbnails(data, *path_list, final_key='thumbnails'):
- """
- Extract thumbnails from thumbnails dict
- @param path_list: path list to level that contains 'thumbnails' key
- """
- thumbnails = []
- for path in path_list or [()]:
- for thumbnail in traverse_obj(data, (*variadic(path), final_key, ...)):
- thumbnail_url = url_or_none(thumbnail.get('url'))
- if not thumbnail_url:
- continue
- # Sometimes youtube gives a wrong thumbnail URL. See:
- # https://github.com/yt-dlp/yt-dlp/issues/233
- # https://github.com/ytdl-org/youtube-dl/issues/28023
- if 'maxresdefault' in thumbnail_url:
- thumbnail_url = thumbnail_url.split('?')[0]
- thumbnails.append({
- 'url': thumbnail_url,
- 'height': int_or_none(thumbnail.get('height')),
- 'width': int_or_none(thumbnail.get('width')),
- })
- return thumbnails
-
- @staticmethod
- def extract_relative_time(relative_time_text):
- """
- Extracts a relative time from string and converts to dt object
- e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today', '8 yr ago'
- """
-
- # XXX: this could be moved to a general function in utils/_utils.py
- # The relative time text strings are roughly the same as what
- # Javascript's Intl.RelativeTimeFormat function generates.
- # See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/RelativeTimeFormat
- mobj = re.search(
- r'(?Ptoday|yesterday|now)|(?P