From 8ea08ecc2fe2b893d4f9d200da624b8043b173f0 Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Sun, 18 May 2025 11:04:15 +0900 Subject: [PATCH 1/5] [ie/dw] Rework extractor --- yt_dlp/extractor/_extractors.py | 5 +- yt_dlp/extractor/dw.py | 198 ++++++++++++++++++++------------ 2 files changed, 124 insertions(+), 79 deletions(-) diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index e7dcb9853e..c5d21b1262 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -567,10 +567,7 @@ from .dumpert import DumpertIE from .duoplay import DuoplayIE from .dvtv import DVTVIE -from .dw import ( - DWIE, - DWArticleIE, -) +from .dw import DWIE from .eagleplatform import ( ClipYouEmbedIE, EaglePlatformIE, diff --git a/yt_dlp/extractor/dw.py b/yt_dlp/extractor/dw.py index 320e29bfdb..2fc8579f41 100644 --- a/yt_dlp/extractor/dw.py +++ b/yt_dlp/extractor/dw.py @@ -1,111 +1,159 @@ -import urllib.parse +import re from .common import InfoExtractor from ..utils import ( int_or_none, - unified_strdate, + parse_iso8601, url_or_none, + variadic, ) +from ..utils.traversal import traverse_obj class DWIE(InfoExtractor): - _WORKING = False - _ENABLED = None # XXX: pass through to GenericIE IE_NAME = 'dw' - _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+(?:av|e)-(?P\d+)' + IE_DESC = 'Deutsche Welle' + + _VALID_URL = r'https?://(?:(?:amp|www)\.)?dw\.com/(?P[^/]+)/[^/]+/(?Pa(?:udio)?|live|program|video)-(?P\d+)' _TESTS = [{ - # video - 'url': 'http://www.dw.com/en/intelligent-light/av-19112290', - 'md5': 'fb9dfd9520811d3ece80f04befd73428', + 'url': 'https://www.dw.com/en/intelligent-light/video-19112290', 'info_dict': { 'id': '19112290', 'ext': 'mp4', 'title': 'Intelligent light', + 'categories': ['Science'], 'description': 'md5:90e00d5881719f2a6a5827cb74985af1', - 'upload_date': '20160605', + 'duration': 194, + 'modified_date': '20160603', + 'modified_timestamp': 1464917222, + 'release_date': '20160605', + 'release_timestamp': 1465162200, + 'thumbnail': 'https://static.dw.com/image/18342580_605.webp', }, }, { - # audio - 'url': 'http://www.dw.com/en/worldlink-my-business/av-19111941', - 'md5': '2814c9a1321c3a51f8a7aeb067a360dd', + 'url': 'https://www.dw.com/en/30-years-since-the-fall-of-the-berlin-wall-what-happened-to-the-euphoria/audio-51155232', 'info_dict': { - 'id': '19111941', + 'id': '51155232', 'ext': 'mp3', - 'title': 'WorldLink: My business', - 'description': 'md5:bc9ca6e4e063361e21c920c53af12405', - 'upload_date': '20160311', + 'title': '30 years since the fall of the Berlin Wall: What happened to the euphoria?', + 'categories': ['Politics'], + 'description': 'md5:ed0686725ef2d8fa5230aa8171b6476b', + 'duration': 1561, + 'release_date': '20191107', + 'release_timestamp': 1573140547, + 'thumbnail': 'https://static.dw.com/image/51151661_605.webp', }, }, { - # DW documentaries, only last for one or two weeks - 'url': 'http://www.dw.com/en/documentaries-welcome-to-the-90s-2016-05-21/e-19220158-9798', - 'md5': '56b6214ef463bfb9a3b71aeb886f3cf1', + 'url': 'https://www.dw.com/en/10-years-since-higgs-boson-whats-next/a-62353123', 'info_dict': { - 'id': '19274438', - 'ext': 'mp4', - 'title': 'Welcome to the 90s – Hip Hop', - 'description': 'Welcome to the 90s - The Golden Decade of Hip Hop', - 'upload_date': '20160521', + 'id': '62353123', + 'title': '10 years since Higgs boson. What\'s next?', }, - 'skip': 'Video removed', + 'playlist_count': 1, + }, { + 'url': 'https://www.dw.com/am/31-%E1%8B%93%E1%88%98%E1%89%B5-%E1%8A%A0%E1%8C%88%E1%88%8D%E1%8C%8D%E1%88%88%E1%8B%8D-%E1%8A%A8100-%E1%8B%B6%E1%88%8B%E1%88%AD-%E1%8B%AB%E1%8A%90%E1%88%B0-%E1%88%B6%E1%88%9B%E1%88%8A%E1%88%8B%E1%8A%95%E1%8B%B5-4%E1%88%BA%E1%88%85-%E1%8B%B6%E1%88%8B%E1%88%AD-%E1%8B%A8%E1%8B%88%E1%88%AD-%E1%8A%AD%E1%8D%8D%E1%8B%AB-%E1%8B%B6-%E1%88%AD-%E1%8D%8D%E1%88%B5%E1%88%83/a-72565331', + 'info_dict': { + 'id': '72565331', + 'title': 'md5:b01a937d9023dc9d6e96f0c30e86816e', + }, + 'playlist_count': 1, + }, { + 'url': 'https://amp.dw.com/am/%E1%88%81%E1%88%88%E1%89%B5-%E1%8B%A8%E1%88%98%E1%8A%95%E1%8C%8D%E1%88%B5%E1%89%B5-%E1%8D%8B%E1%89%A5%E1%88%AA%E1%8A%AB%E1%8B%8E%E1%89%BD-%E1%8B%88%E1%8B%B0-%E1%8C%A5%E1%88%A8%E1%89%B5-%E1%88%8A%E1%8B%9B%E1%8B%88%E1%88%A9-%E1%8A%90%E1%8B%8D/a-37166407', + 'info_dict': { + 'id': '37166407', + 'title': 'md5:56fc6855d20de908fc5bd6cff226ddb4', + }, + 'playlist_count': 1, + }, { + 'url': 'https://www.dw.com/en/0/live-72516011', + 'info_dict': { + 'id': '72516011', + 'title': 'Germany updates: Afghan family sues government — report', + }, + 'playlist_count': 5, + }, { + 'url': 'https://www.dw.com/en/choices/program-68776240', + 'info_dict': { + 'id': '68776240', + 'title': 'CHOICES', + }, + 'playlist_count': 13, }] + def _entries(self, url, graph_api, media_type): + if path := { + 'a': 'videos', + 'live': ('posts', ..., 'videos'), + 'program': 'moreContentsFromUnifiedProgram', + }.get(media_type): + for dct in traverse_obj(graph_api, ( + *variadic(path), lambda _, v: v['namedUrl'] not in url, + )): + yield self.url_result( + f'https://www.dw.com{dct["namedUrl"]}', DWIE) + + if media_type == 'a': + for dct in traverse_obj(graph_api, ('audios', lambda _, v: v['mp3Src'])): + m3u8_url = traverse_obj(dct, 'mp3Src', {url_or_none}) + + yield from [{ + 'id': m3u8_url.rpartition('/')[2].removesuffix('.mp3'), + 'ext': 'mp3', + 'title': dct.get('title'), + 'url': m3u8_url, + }] + def _real_extract(self, url): - media_id = self._match_id(url) + lang, media_type, media_id = self._match_valid_url(url).groups() webpage = self._download_webpage(url, media_id) - hidden_inputs = self._hidden_inputs(webpage) - title = hidden_inputs['media_title'] - media_id = hidden_inputs.get('media_id') or media_id - direct_url = url_or_none(hidden_inputs.get('file_name')) - if direct_url: - formats = [{'url': hidden_inputs['file_name']}] + path = { + 'a': 'article', + 'audio': 'audio', + 'live': 'liveblog', + 'program': 'unified-program', + 'video': 'video', + }[media_type] + + if not (app_state := self._search_json( + r'window\.__APP_STATE__\s*=\s*', webpage, 'app state', media_id, default={}, + )): + title = self._html_search_meta('twitter:title', webpage) + pattern = re.compile(r']+src\s*=\s*(["\'])(?P.+?)\1') + entries = [{ + 'id': m.group('url').rpartition('/')[2].removesuffix('.mp3'), + 'ext': 'mp3', + 'title': title, + 'url': m.group('url'), + } for m in pattern.finditer(webpage)] + + return self.playlist_result(entries, media_id, title) else: - formats = self._extract_smil_formats( - f'http://www.dw.com/smil/v-{media_id}', media_id, - transform_source=lambda s: s.replace( - 'rtmp://tv-od.dw.de/flash/', - 'http://tv-download.dw.de/dwtv_video/flv/')) + graph_api = traverse_obj(app_state, ( + f'/graph-api/{lang}/content/{path}/{media_id}', 'data', 'content', {dict})) - upload_date = hidden_inputs.get('display_date') - if not upload_date: - upload_date = self._html_search_regex( - r']+class="date">([0-9.]+)\s*\|', webpage, - 'upload date', default=None) - upload_date = unified_strdate(upload_date) + if media_type in {'a', 'live', 'program'}: + return self.playlist_result( + self._entries(url, graph_api, media_type), media_id, graph_api.get('title')) + elif media_type == 'audio': + formats = [{ + 'ext': 'mp3', + 'url': traverse_obj(graph_api, 'mp3Src', {url_or_none}), + }] + else: + m3u8_url = traverse_obj(graph_api, 'hlsVideoSrc', {url_or_none}) + formats = self._extract_m3u8_formats(m3u8_url, media_id, 'mp4', m3u8_id='hls') return { 'id': media_id, - 'title': title, - 'description': self._og_search_description(webpage), - 'thumbnail': hidden_inputs.get('preview_image'), - 'duration': int_or_none(hidden_inputs.get('file_duration')), - 'upload_date': upload_date, 'formats': formats, + **traverse_obj(graph_api, { + 'title': ('title', {str}), + 'categories': ('thematicFocusCategory', 'name', {str}, all, filter), + 'description': ('teaser', {str}), + 'duration': ('duration', {int_or_none}), + 'modified_timestamp': ('lastModifiedDate', {parse_iso8601}), + 'thumbnail': ('posterImageUrl', {url_or_none}), + 'release_timestamp': ('contentDate', {parse_iso8601}), + }), } - - -class DWArticleIE(InfoExtractor): - _WORKING = False - _ENABLED = None # XXX: pass through to GenericIE - IE_NAME = 'dw:article' - _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+a-(?P\d+)' - _TEST = { - 'url': 'http://www.dw.com/en/no-hope-limited-options-for-refugees-in-idomeni/a-19111009', - 'md5': '8ca657f9d068bbef74d6fc38b97fc869', - 'info_dict': { - 'id': '19105868', - 'ext': 'mp4', - 'title': 'The harsh life of refugees in Idomeni', - 'description': 'md5:196015cc7e48ebf474db9399420043c7', - 'upload_date': '20160310', - }, - } - - def _real_extract(self, url): - article_id = self._match_id(url) - webpage = self._download_webpage(url, article_id) - hidden_inputs = self._hidden_inputs(webpage) - media_id = hidden_inputs['media_id'] - media_path = self._search_regex(rf'href="([^"]+av-{media_id})"\s+class="overlayLink"', webpage, 'media url') - media_url = urllib.parse.urljoin(url, media_path) - return self.url_result(media_url, 'DW', media_id) From d940f11ae5bae3668562e73da5763af82f0039bc Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Sun, 18 May 2025 13:00:56 +0900 Subject: [PATCH 2/5] fix --- yt_dlp/extractor/dw.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/dw.py b/yt_dlp/extractor/dw.py index 2fc8579f41..d5eaa1ed60 100644 --- a/yt_dlp/extractor/dw.py +++ b/yt_dlp/extractor/dw.py @@ -28,7 +28,7 @@ class DWIE(InfoExtractor): 'modified_timestamp': 1464917222, 'release_date': '20160605', 'release_timestamp': 1465162200, - 'thumbnail': 'https://static.dw.com/image/18342580_605.webp', + 'thumbnail': r're:https?://static\.dw\.com/image/.+\.webp', }, }, { 'url': 'https://www.dw.com/en/30-years-since-the-fall-of-the-berlin-wall-what-happened-to-the-euphoria/audio-51155232', @@ -41,7 +41,7 @@ class DWIE(InfoExtractor): 'duration': 1561, 'release_date': '20191107', 'release_timestamp': 1573140547, - 'thumbnail': 'https://static.dw.com/image/51151661_605.webp', + 'thumbnail': r're:https?://static\.dw\.com/image/.+\.webp', }, }, { 'url': 'https://www.dw.com/en/10-years-since-higgs-boson-whats-next/a-62353123', From 989dfdfbfcf6e41ba2f7559a2047d5d46fcd338a Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Tue, 27 May 2025 19:41:23 +0900 Subject: [PATCH 3/5] Apply suggestions --- yt_dlp/extractor/dw.py | 87 ++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/yt_dlp/extractor/dw.py b/yt_dlp/extractor/dw.py index d5eaa1ed60..3b361b785a 100644 --- a/yt_dlp/extractor/dw.py +++ b/yt_dlp/extractor/dw.py @@ -4,6 +4,7 @@ from ..utils import ( int_or_none, parse_iso8601, + url_basename, url_or_none, variadic, ) @@ -14,7 +15,19 @@ class DWIE(InfoExtractor): IE_NAME = 'dw' IE_DESC = 'Deutsche Welle' - _VALID_URL = r'https?://(?:(?:amp|www)\.)?dw\.com/(?P[^/]+)/[^/]+/(?Pa(?:udio)?|live|program|video)-(?P\d+)' + _ENTRIES_PATH_MAP = { + 'a': 'videos', + 'live': ('posts', ..., 'videos'), + 'program': 'moreContentsFromUnifiedProgram', + } + _PATH_MAP = { + 'a': 'article', + 'audio': 'audio', + 'live': 'liveblog', + 'program': 'unified-program', + 'video': 'video', + } + _VALID_URL = fr'https?://(?:(?:amp|www)\.)?dw\.com/(?P[^/?#]+)/[^/?#]+/(?P{"|".join(_PATH_MAP)})-(?P\d+)' _TESTS = [{ 'url': 'https://www.dw.com/en/intelligent-light/video-19112290', 'info_dict': { @@ -81,68 +94,60 @@ class DWIE(InfoExtractor): }] def _entries(self, url, graph_api, media_type): - if path := { - 'a': 'videos', - 'live': ('posts', ..., 'videos'), - 'program': 'moreContentsFromUnifiedProgram', - }.get(media_type): - for dct in traverse_obj(graph_api, ( - *variadic(path), lambda _, v: v['namedUrl'] not in url, - )): - yield self.url_result( - f'https://www.dw.com{dct["namedUrl"]}', DWIE) + path = self._ENTRIES_PATH_MAP[media_type] + for dct in traverse_obj(graph_api, ( + *variadic(path), lambda _, v: v['namedUrl'] not in url, + )): + yield self.url_result( + f'https://www.dw.com{dct["namedUrl"]}', DWIE) if media_type == 'a': for dct in traverse_obj(graph_api, ('audios', lambda _, v: v['mp3Src'])): - m3u8_url = traverse_obj(dct, 'mp3Src', {url_or_none}) + mp3_url = traverse_obj(dct, 'mp3Src', {url_or_none}) yield from [{ - 'id': m3u8_url.rpartition('/')[2].removesuffix('.mp3'), + 'id': url_basename(mp3_url).removesuffix('.mp3'), 'ext': 'mp3', 'title': dct.get('title'), - 'url': m3u8_url, + 'url': mp3_url, + 'vcodec': 'none', }] def _real_extract(self, url): - lang, media_type, media_id = self._match_valid_url(url).groups() + lang, media_type, media_id = self._match_valid_url(url).group('lang', 'type', 'id') webpage = self._download_webpage(url, media_id) - path = { - 'a': 'article', - 'audio': 'audio', - 'live': 'liveblog', - 'program': 'unified-program', - 'video': 'video', - }[media_type] - - if not (app_state := self._search_json( - r'window\.__APP_STATE__\s*=\s*', webpage, 'app state', media_id, default={}, - )): + app_state = self._search_json( + r'window\.__APP_STATE__\s*=\s*', webpage, 'app state', media_id, default={}) + if not app_state: title = self._html_search_meta('twitter:title', webpage) pattern = re.compile(r']+src\s*=\s*(["\'])(?P.+?)\1') entries = [{ - 'id': m.group('url').rpartition('/')[2].removesuffix('.mp3'), + 'id': url_basename(m.group('url')).removesuffix('.mp3'), 'ext': 'mp3', 'title': title, 'url': m.group('url'), + 'vcodec': 'none', } for m in pattern.finditer(webpage)] return self.playlist_result(entries, media_id, title) - else: - graph_api = traverse_obj(app_state, ( - f'/graph-api/{lang}/content/{path}/{media_id}', 'data', 'content', {dict})) - if media_type in {'a', 'live', 'program'}: - return self.playlist_result( - self._entries(url, graph_api, media_type), media_id, graph_api.get('title')) - elif media_type == 'audio': - formats = [{ - 'ext': 'mp3', - 'url': traverse_obj(graph_api, 'mp3Src', {url_or_none}), - }] - else: - m3u8_url = traverse_obj(graph_api, 'hlsVideoSrc', {url_or_none}) - formats = self._extract_m3u8_formats(m3u8_url, media_id, 'mp4', m3u8_id='hls') + path = self._PATH_MAP[media_type] + graph_api = traverse_obj(app_state, ( + f'/graph-api/{lang}/content/{path}/{media_id}', 'data', 'content', {dict})) + + if media_type in self._ENTRIES_PATH_MAP: + return self.playlist_result( + self._entries(url, graph_api, media_type), media_id, graph_api.get('title')) + elif media_type == 'audio': + formats = [{ + 'ext': 'mp3', + 'url': traverse_obj(graph_api, 'mp3Src', {url_or_none}), + 'vcodec': 'none', + }] + else: + m3u8_url = traverse_obj(graph_api, 'hlsVideoSrc', {url_or_none}) + formats = self._extract_m3u8_formats(m3u8_url, media_id, 'mp4', m3u8_id='hls') return { 'id': media_id, From 53a5f6d765278c125b9278cdd5120c7ecb20e486 Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Wed, 28 May 2025 03:06:03 +0900 Subject: [PATCH 4/5] Apply suggestions --- yt_dlp/extractor/dw.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/yt_dlp/extractor/dw.py b/yt_dlp/extractor/dw.py index 3b361b785a..2d92f94f8c 100644 --- a/yt_dlp/extractor/dw.py +++ b/yt_dlp/extractor/dw.py @@ -6,9 +6,9 @@ parse_iso8601, url_basename, url_or_none, - variadic, + urljoin, ) -from ..utils.traversal import traverse_obj +from ..utils.traversal import require, traverse_obj class DWIE(InfoExtractor): @@ -16,9 +16,9 @@ class DWIE(InfoExtractor): IE_DESC = 'Deutsche Welle' _ENTRIES_PATH_MAP = { - 'a': 'videos', + 'a': ('videos',), 'live': ('posts', ..., 'videos'), - 'program': 'moreContentsFromUnifiedProgram', + 'program': ('moreContentsFromUnifiedProgram',), } _PATH_MAP = { 'a': 'article', @@ -94,12 +94,11 @@ class DWIE(InfoExtractor): }] def _entries(self, url, graph_api, media_type): - path = self._ENTRIES_PATH_MAP[media_type] for dct in traverse_obj(graph_api, ( - *variadic(path), lambda _, v: v['namedUrl'] not in url, + *self._ENTRIES_PATH_MAP[media_type], lambda _, v: v['namedUrl'] not in url, )): yield self.url_result( - f'https://www.dw.com{dct["namedUrl"]}', DWIE) + urljoin('https://www.dw.com/', dct['namedUrl']), DWIE) if media_type == 'a': for dct in traverse_obj(graph_api, ('audios', lambda _, v: v['mp3Src'])): @@ -142,19 +141,19 @@ def _real_extract(self, url): elif media_type == 'audio': formats = [{ 'ext': 'mp3', - 'url': traverse_obj(graph_api, 'mp3Src', {url_or_none}), + 'url': traverse_obj(graph_api, ('mp3Src', {url_or_none}, {require('mp3 URL')})), 'vcodec': 'none', }] else: - m3u8_url = traverse_obj(graph_api, 'hlsVideoSrc', {url_or_none}) - formats = self._extract_m3u8_formats(m3u8_url, media_id, 'mp4', m3u8_id='hls') + formats = self._extract_m3u8_formats( + graph_api['hlsVideoSrc'], media_id, 'mp4', m3u8_id='hls') return { 'id': media_id, 'formats': formats, **traverse_obj(graph_api, { 'title': ('title', {str}), - 'categories': ('thematicFocusCategory', 'name', {str}, all, filter), + 'categories': ('thematicFocusCategory', 'name', {str}, filter, all, filter), 'description': ('teaser', {str}), 'duration': ('duration', {int_or_none}), 'modified_timestamp': ('lastModifiedDate', {parse_iso8601}), From a6a5d31c6dfeaf5c2e7c115eab931a2265a62411 Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Sun, 8 Jun 2025 15:05:59 +0900 Subject: [PATCH 5/5] Apply suggestions --- yt_dlp/extractor/dw.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/dw.py b/yt_dlp/extractor/dw.py index 2d92f94f8c..6a80f838e3 100644 --- a/yt_dlp/extractor/dw.py +++ b/yt_dlp/extractor/dw.py @@ -104,20 +104,20 @@ def _entries(self, url, graph_api, media_type): for dct in traverse_obj(graph_api, ('audios', lambda _, v: v['mp3Src'])): mp3_url = traverse_obj(dct, 'mp3Src', {url_or_none}) - yield from [{ + yield { 'id': url_basename(mp3_url).removesuffix('.mp3'), 'ext': 'mp3', 'title': dct.get('title'), 'url': mp3_url, 'vcodec': 'none', - }] + } def _real_extract(self, url): lang, media_type, media_id = self._match_valid_url(url).group('lang', 'type', 'id') webpage = self._download_webpage(url, media_id) app_state = self._search_json( - r'window\.__APP_STATE__\s*=\s*', webpage, 'app state', media_id, default={}) + r'window\.__APP_STATE__\s*=', webpage, 'app state', media_id, default={}) if not app_state: title = self._html_search_meta('twitter:title', webpage) pattern = re.compile(r']+src\s*=\s*(["\'])(?P.+?)\1')