diff --git a/yt_dlp/extractor/substack.py b/yt_dlp/extractor/substack.py index b70d40f2ca..8f8a0a547c 100644 --- a/yt_dlp/extractor/substack.py +++ b/yt_dlp/extractor/substack.py @@ -5,6 +5,7 @@ from ..networking import HEADRequest from ..utils import ( determine_ext, + float_or_none, js_to_json, str_or_none, ) @@ -62,6 +63,74 @@ class SubstackIE(InfoExtractor): 'uploader': 'Persuasion', 'uploader_id': '61579', }, + }, { + # An embedded public video on a regular newsletter page + 'url': 'https://jessica.substack.com/p/livestream-today-chat-with-me-and', + 'md5': '7466bdd760b8ef8a60a1336c4e86ce20', + 'info_dict': { + 'id': '86e0da19-8263-4376-a84a-76b501db12d0', + 'ext': 'mp4', + 'title': 'liz.mp4', + 'thumbnail': r're:https://substack.*\.com/video_upload/post/158453351/86e0da19-8263-4376-a84a-76b501db12d0/.*\.png', + 'duration': 3087.8333, + 'uploader_id': '535611', + }, + }, { + # An embedded public audio on a regular newsletter page + 'url': 'https://jessica.substack.com/p/audio-of-jd-vance-calling-rape-an?utm_source=publication-search', + 'md5': 'bd6c2a9e0c590844496db94ad7653781', + 'info_dict': { + 'id': '17e80ab3-d5d6-488e-a939-404f33c17ef5', + 'ext': 'mp3', + 'title': 'JD Vance rape inconvenient.m4a', + 'duration': 25.939592, + 'uploader_id': '535611', + }, + }, { + # A "podcast" page which contains embedded video + 'url': 'https://vigilantfox.substack.com/p/global-medical-tyranny-just-got-real?source=queue', + 'info_dict': { + 'id': '161501224', + 'title': 'Videos for Global Medical Tyranny Just Got Real | Daily Pulse', + }, + 'playlist': [{ + 'info_dict': { + 'id': '161501224', + 'ext': 'mp3', + 'title': 'Global Medical Tyranny Just Got Real | Daily Pulse', + 'uploader_id': '975571', + 'uploader': 'The Vigilant Fox', + 'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png', + 'description': 'The news you weren’t supposed to see.', + }, + }, { + 'info_dict': { + 'id': '95b4b5a7-3873-465d-a6f4-8d3c499f29f2', + 'ext': 'mp4', + 'title': 'WHO.mp4', + 'uploader_id': '82027648', + 'duration': 593.3333, + 'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png', + }, + }, { + 'info_dict': { + 'id': '235db8bf-c6e8-439a-b51a-eb7566fc7ac1', + 'ext': 'mp4', + 'title': 'The Vigilant Fox\'s Video - Apr 16, 2025-VEED.mp4', + 'uploader_id': '82027648', + 'duration': 498.86667, + 'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png', + }, + }, { + 'info_dict': { + 'id': 'ef3fe4c1-d748-4665-ba9c-33396008e75b', + 'ext': 'mp4', + 'title': 'The Vigilant Fox\'s Video - Apr 16, 2025-VEED (1).mp4', + 'uploader_id': '82027648', + 'duration': 410.53333, + 'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png', + }, + }], }] @classmethod @@ -75,10 +144,13 @@ def _extract_embed_urls(cls, url, webpage): yield parsed._replace(netloc=f'{mobj.group("subdomain")}.substack.com').geturl() raise cls.StopExtraction - def _extract_video_formats(self, video_id, url): + def _extract_video_formats(self, video_id, url, query_params={}): formats, subtitles = [], {} for video_format in ('hls', 'mp4'): - video_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}/src?type={video_format}') + q = {'type': video_format} + q.update(query_params) + query_string = '&'.join(f'{k}={v}' for k, v in q.items()) + video_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}/src?{query_string}') if video_format == 'hls': fmts, subs = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', fatal=False) @@ -106,25 +178,11 @@ def _real_extract(self, url): canonical_url = urllib.parse.urlparse(url)._replace(netloc=domain).geturl() post_type = webpage_info['post']['type'] + post_title = traverse_obj(webpage_info, ('post', 'title')) formats, subtitles = [], {} - if post_type == 'podcast': - fmt = {'url': webpage_info['post']['podcast_url']} - if not determine_ext(fmt['url'], default_ext=None): - # The redirected format URL expires but the original URL doesn't, - # so we only want to extract the extension from this request - fmt['ext'] = determine_ext(self._request_webpage( - HEADRequest(fmt['url']), display_id, - 'Resolving podcast file extension', - 'Podcast URL is invalid').url) - formats.append(fmt) - elif post_type == 'video': - formats, subtitles = self._extract_video_formats(webpage_info['post']['videoUpload']['id'], canonical_url) - else: - self.raise_no_formats(f'Page type "{post_type}" is not supported') - - return { + all_results = [] + result_info = { 'id': str(webpage_info['post']['id']), - 'formats': formats, 'subtitles': subtitles, 'title': traverse_obj(webpage_info, ('post', 'title')), 'description': traverse_obj(webpage_info, ('post', 'description')), @@ -133,3 +191,98 @@ def _real_extract(self, url): 'uploader_id': str_or_none(traverse_obj(webpage_info, ('post', 'publication_id'))), 'webpage_url': canonical_url, } + # handle specific post types which are based on media + if post_type == 'podcast': + podcast_result = dict(result_info) + fmt = {'url': webpage_info['post']['podcast_url']} + if not determine_ext(fmt['url'], default_ext=None): + # The redirected format URL expires but the original URL doesn't, + # so we only want to extract the extension from this request + fmt['ext'] = determine_ext(self._request_webpage( + HEADRequest(fmt['url']), display_id, + 'Resolving podcast file extension', + 'Podcast URL is invalid').url) + podcast_result['formats'] = [fmt] + all_results.append(podcast_result) + if post_type == 'video': + video_result = dict(result_info) + formats, subtitles = self._extract_video_formats( + webpage_info['post']['videoUpload']['id'], canonical_url) + video_result.update({ + 'formats': formats, + 'subtitles': subtitles, + }) + all_results.append(video_result) + + # search for embedded players on the page + found_items = [] + post_id = str(webpage_info['post']['id']) + assert result_info['uploader_id'] is not None, 'newsletter posted without user_id' + + video_players = re.finditer( + r']*data-component-name="VideoEmbedPlayer"[^>]*>', + webpage) + for vp in video_players: + video_id = self._search_regex(r'id="([^"]+)"', + vp.group(0), + 'video id', group=1).replace('media-', '') + video_metadata_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}') + json_vid_data = self._download_json(video_metadata_url, video_id) + assert video_id == json_vid_data['id'], 'unexpected json metadata retrieved' + json_vid_data['_type'] = 'video' + found_items.append(json_vid_data) + + audio_players = re.finditer( + r']*data-component-name="AudioEmbedPlayer"[^>]*>', + webpage) + for ap in audio_players: + video_uri = self._search_regex(r'src="(/api/v1/audio/upload/[^"]+)"', + webpage[ap.start():], + 'video uri', group=1) + video_metadata_url = urllib.parse.urljoin(url, video_uri.replace('/src', '')) + video_id = self._search_regex(r'upload/([^/]+)/src', + video_uri, + 'video id', group=1) + json_vid_data = self._download_json(video_metadata_url, video_id) + assert video_id == json_vid_data['id'], 'unexpected json metadata retrieved' + json_vid_data['_type'] = 'audio' + json_vid_data['_uri'] = video_uri + found_items.append(json_vid_data) + + for json_vid_data in found_items: + video_id = json_vid_data['id'] + if 'video' in json_vid_data['_type']: + formats, subtitles = self._extract_video_formats( + video_id, canonical_url, + query_params={'override_publication_id': result_info['uploader_id']}) + else: + fmt = {'url': urllib.parse.urljoin(url, json_vid_data['_uri'])} + if not determine_ext(fmt['url'], default_ext=None): + # The redirected format URL expires but the original URL doesn't, + # so we only want to extract the extension from this request + fmt['ext'] = determine_ext(self._request_webpage( + HEADRequest(fmt['url']), video_id, + 'Resolving audio file extension', + 'Embedded audio URL is invalid').url) + formats.append(fmt) + + new_result = dict(result_info) + new_result.update({ + 'formats': formats, + 'subtitles': subtitles, + 'id': video_id, + 'title': json_vid_data.get('name', None), + 'description': None, + 'thumbnail': json_vid_data.get('thumbnail_url', None), + 'duration': float_or_none(json_vid_data.get('duration', None)), + # videos can be cross-embedded, so the publication id should be of the uploader + 'uploader_id': str_or_none(json_vid_data.get('user_id', None)), + 'uploader': None, # video uploader username is not included + }) + all_results.append(new_result) + + if len(all_results) > 0: + playlist_title = f'Videos for {post_title}' + return self.playlist_result(all_results, post_id, playlist_title) + else: + self.raise_no_formats(f'Page type "{post_type}" contains no supported embeds')