mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2025-07-08 22:38:33 +00:00
Merge 2f2866a467 into 73bf102116
This commit is contained in:
jhwgh1968
GitHub
commit
33e640714d
@ -5,6 +5,7 @@
|
||||
from ..networking import HEADRequest
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
float_or_none,
|
||||
js_to_json,
|
||||
str_or_none,
|
||||
)
|
||||
@ -62,6 +63,74 @@ class SubstackIE(InfoExtractor):
|
||||
'uploader': 'Persuasion',
|
||||
'uploader_id': '61579',
|
||||
},
|
||||
}, {
|
||||
# An embedded public video on a regular newsletter page
|
||||
'url': 'https://jessica.substack.com/p/livestream-today-chat-with-me-and',
|
||||
'md5': '7466bdd760b8ef8a60a1336c4e86ce20',
|
||||
'info_dict': {
|
||||
'id': '86e0da19-8263-4376-a84a-76b501db12d0',
|
||||
'ext': 'mp4',
|
||||
'title': 'liz.mp4',
|
||||
'thumbnail': r're:https://substack.*\.com/video_upload/post/158453351/86e0da19-8263-4376-a84a-76b501db12d0/.*\.png',
|
||||
'duration': 3087.8333,
|
||||
'uploader_id': '535611',
|
||||
},
|
||||
}, {
|
||||
# An embedded public audio on a regular newsletter page
|
||||
'url': 'https://jessica.substack.com/p/audio-of-jd-vance-calling-rape-an?utm_source=publication-search',
|
||||
'md5': 'bd6c2a9e0c590844496db94ad7653781',
|
||||
'info_dict': {
|
||||
'id': '17e80ab3-d5d6-488e-a939-404f33c17ef5',
|
||||
'ext': 'mp3',
|
||||
'title': 'JD Vance rape inconvenient.m4a',
|
||||
'duration': 25.939592,
|
||||
'uploader_id': '535611',
|
||||
},
|
||||
}, {
|
||||
# A "podcast" page which contains embedded video
|
||||
'url': 'https://vigilantfox.substack.com/p/global-medical-tyranny-just-got-real?source=queue',
|
||||
'info_dict': {
|
||||
'id': '161501224',
|
||||
'title': 'Videos for Global Medical Tyranny Just Got Real | Daily Pulse',
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '161501224',
|
||||
'ext': 'mp3',
|
||||
'title': 'Global Medical Tyranny Just Got Real | Daily Pulse',
|
||||
'uploader_id': '975571',
|
||||
'uploader': 'The Vigilant Fox',
|
||||
'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png',
|
||||
'description': 'The news you weren’t supposed to see.',
|
||||
},
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '95b4b5a7-3873-465d-a6f4-8d3c499f29f2',
|
||||
'ext': 'mp4',
|
||||
'title': 'WHO.mp4',
|
||||
'uploader_id': '82027648',
|
||||
'duration': 593.3333,
|
||||
'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png',
|
||||
},
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '235db8bf-c6e8-439a-b51a-eb7566fc7ac1',
|
||||
'ext': 'mp4',
|
||||
'title': 'The Vigilant Fox\'s Video - Apr 16, 2025-VEED.mp4',
|
||||
'uploader_id': '82027648',
|
||||
'duration': 498.86667,
|
||||
'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png',
|
||||
},
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': 'ef3fe4c1-d748-4665-ba9c-33396008e75b',
|
||||
'ext': 'mp4',
|
||||
'title': 'The Vigilant Fox\'s Video - Apr 16, 2025-VEED (1).mp4',
|
||||
'uploader_id': '82027648',
|
||||
'duration': 410.53333,
|
||||
'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png',
|
||||
},
|
||||
}],
|
||||
}]
|
||||
|
||||
@classmethod
|
||||
@ -75,10 +144,13 @@ def _extract_embed_urls(cls, url, webpage):
|
||||
yield parsed._replace(netloc=f'{mobj.group("subdomain")}.substack.com').geturl()
|
||||
raise cls.StopExtraction
|
||||
|
||||
def _extract_video_formats(self, video_id, url):
|
||||
def _extract_video_formats(self, video_id, url, query_params={}):
|
||||
formats, subtitles = [], {}
|
||||
for video_format in ('hls', 'mp4'):
|
||||
video_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}/src?type={video_format}')
|
||||
q = {'type': video_format}
|
||||
q.update(query_params)
|
||||
query_string = '&'.join(f'{k}={v}' for k, v in q.items())
|
||||
video_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}/src?{query_string}')
|
||||
|
||||
if video_format == 'hls':
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', fatal=False)
|
||||
@ -106,25 +178,11 @@ def _real_extract(self, url):
|
||||
canonical_url = urllib.parse.urlparse(url)._replace(netloc=domain).geturl()
|
||||
|
||||
post_type = webpage_info['post']['type']
|
||||
post_title = traverse_obj(webpage_info, ('post', 'title'))
|
||||
formats, subtitles = [], {}
|
||||
if post_type == 'podcast':
|
||||
fmt = {'url': webpage_info['post']['podcast_url']}
|
||||
if not determine_ext(fmt['url'], default_ext=None):
|
||||
# The redirected format URL expires but the original URL doesn't,
|
||||
# so we only want to extract the extension from this request
|
||||
fmt['ext'] = determine_ext(self._request_webpage(
|
||||
HEADRequest(fmt['url']), display_id,
|
||||
'Resolving podcast file extension',
|
||||
'Podcast URL is invalid').url)
|
||||
formats.append(fmt)
|
||||
elif post_type == 'video':
|
||||
formats, subtitles = self._extract_video_formats(webpage_info['post']['videoUpload']['id'], canonical_url)
|
||||
else:
|
||||
self.raise_no_formats(f'Page type "{post_type}" is not supported')
|
||||
|
||||
return {
|
||||
all_results = []
|
||||
result_info = {
|
||||
'id': str(webpage_info['post']['id']),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'title': traverse_obj(webpage_info, ('post', 'title')),
|
||||
'description': traverse_obj(webpage_info, ('post', 'description')),
|
||||
@ -133,3 +191,98 @@ def _real_extract(self, url):
|
||||
'uploader_id': str_or_none(traverse_obj(webpage_info, ('post', 'publication_id'))),
|
||||
'webpage_url': canonical_url,
|
||||
}
|
||||
# handle specific post types which are based on media
|
||||
if post_type == 'podcast':
|
||||
podcast_result = dict(result_info)
|
||||
fmt = {'url': webpage_info['post']['podcast_url']}
|
||||
if not determine_ext(fmt['url'], default_ext=None):
|
||||
# The redirected format URL expires but the original URL doesn't,
|
||||
# so we only want to extract the extension from this request
|
||||
fmt['ext'] = determine_ext(self._request_webpage(
|
||||
HEADRequest(fmt['url']), display_id,
|
||||
'Resolving podcast file extension',
|
||||
'Podcast URL is invalid').url)
|
||||
podcast_result['formats'] = [fmt]
|
||||
all_results.append(podcast_result)
|
||||
if post_type == 'video':
|
||||
video_result = dict(result_info)
|
||||
formats, subtitles = self._extract_video_formats(
|
||||
webpage_info['post']['videoUpload']['id'], canonical_url)
|
||||
video_result.update({
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
})
|
||||
all_results.append(video_result)
|
||||
|
||||
# search for embedded players on the page
|
||||
found_items = []
|
||||
post_id = str(webpage_info['post']['id'])
|
||||
assert result_info['uploader_id'] is not None, 'newsletter posted without user_id'
|
||||
|
||||
video_players = re.finditer(
|
||||
r'<div[^>]*data-component-name="VideoEmbedPlayer"[^>]*>',
|
||||
webpage)
|
||||
for vp in video_players:
|
||||
video_id = self._search_regex(r'id="([^"]+)"',
|
||||
vp.group(0),
|
||||
'video id', group=1).replace('media-', '')
|
||||
video_metadata_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}')
|
||||
json_vid_data = self._download_json(video_metadata_url, video_id)
|
||||
assert video_id == json_vid_data['id'], 'unexpected json metadata retrieved'
|
||||
json_vid_data['_type'] = 'video'
|
||||
found_items.append(json_vid_data)
|
||||
|
||||
audio_players = re.finditer(
|
||||
r'<div[^>]*data-component-name="AudioEmbedPlayer"[^>]*>',
|
||||
webpage)
|
||||
for ap in audio_players:
|
||||
video_uri = self._search_regex(r'src="(/api/v1/audio/upload/[^"]+)"',
|
||||
webpage[ap.start():],
|
||||
'video uri', group=1)
|
||||
video_metadata_url = urllib.parse.urljoin(url, video_uri.replace('/src', ''))
|
||||
video_id = self._search_regex(r'upload/([^/]+)/src',
|
||||
video_uri,
|
||||
'video id', group=1)
|
||||
json_vid_data = self._download_json(video_metadata_url, video_id)
|
||||
assert video_id == json_vid_data['id'], 'unexpected json metadata retrieved'
|
||||
json_vid_data['_type'] = 'audio'
|
||||
json_vid_data['_uri'] = video_uri
|
||||
found_items.append(json_vid_data)
|
||||
|
||||
for json_vid_data in found_items:
|
||||
video_id = json_vid_data['id']
|
||||
if 'video' in json_vid_data['_type']:
|
||||
formats, subtitles = self._extract_video_formats(
|
||||
video_id, canonical_url,
|
||||
query_params={'override_publication_id': result_info['uploader_id']})
|
||||
else:
|
||||
fmt = {'url': urllib.parse.urljoin(url, json_vid_data['_uri'])}
|
||||
if not determine_ext(fmt['url'], default_ext=None):
|
||||
# The redirected format URL expires but the original URL doesn't,
|
||||
# so we only want to extract the extension from this request
|
||||
fmt['ext'] = determine_ext(self._request_webpage(
|
||||
HEADRequest(fmt['url']), video_id,
|
||||
'Resolving audio file extension',
|
||||
'Embedded audio URL is invalid').url)
|
||||
formats.append(fmt)
|
||||
|
||||
new_result = dict(result_info)
|
||||
new_result.update({
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'id': video_id,
|
||||
'title': json_vid_data.get('name', None),
|
||||
'description': None,
|
||||
'thumbnail': json_vid_data.get('thumbnail_url', None),
|
||||
'duration': float_or_none(json_vid_data.get('duration', None)),
|
||||
# videos can be cross-embedded, so the publication id should be of the uploader
|
||||
'uploader_id': str_or_none(json_vid_data.get('user_id', None)),
|
||||
'uploader': None, # video uploader username is not included
|
||||
})
|
||||
all_results.append(new_result)
|
||||
|
||||
if len(all_results) > 0:
|
||||
playlist_title = f'Videos for {post_title}'
|
||||
return self.playlist_result(all_results, post_id, playlist_title)
|
||||
else:
|
||||
self.raise_no_formats(f'Page type "{post_type}" contains no supported embeds')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user