From 5759e2604b23fd9f41af6dbd64862757e41449e2 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Tue, 17 Jun 2025 04:51:05 +0100 Subject: [PATCH 1/8] [ie/tbsjp] Migrate to StreaksIE --- yt_dlp/extractor/tbsjp.py | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py index 0d521f106..bf2b0012f 100644 --- a/yt_dlp/extractor/tbsjp.py +++ b/yt_dlp/extractor/tbsjp.py @@ -1,17 +1,15 @@ from .common import InfoExtractor -from ..networking.exceptions import HTTPError +from .streaks import StreaksBaseIE from ..utils import ( - ExtractorError, clean_html, int_or_none, str_or_none, unified_timestamp, - urljoin, ) from ..utils.traversal import find_element, traverse_obj -class TBSJPEpisodeIE(InfoExtractor): +class TBSJPEpisodeIE(StreaksBaseIE): _VALID_URL = r'https?://cu\.tbs\.co\.jp/episode/(?P[\d_]+)' _GEO_BYPASS = False _TESTS = [{ @@ -40,28 +38,8 @@ def _real_extract(self, url): meta = self._search_json(r'window\.app\s*=', webpage, 'episode info', video_id, fatal=False) episode = traverse_obj(meta, ('falcorCache', 'catalog', 'episode', video_id, 'value')) - tf_path = self._search_regex( - r']+src=["\'](/assets/tf\.[^"\']+\.js)["\']', webpage, 'stream API config') - tf_js = self._download_webpage(urljoin(url, tf_path), video_id, note='Downloading stream API config') - video_url = self._search_regex(r'videoPlaybackUrl:\s*[\'"]([^\'"]+)[\'"]', tf_js, 'stream API url') - api_key = self._search_regex(r'api_key:\s*[\'"]([^\'"]+)[\'"]', tf_js, 'stream API key') - - try: - source_meta = self._download_json(f'{video_url}ref:{video_id}', video_id, - headers={'X-Streaks-Api-Key': api_key}, - note='Downloading stream metadata') - except ExtractorError as e: - if isinstance(e.cause, HTTPError) and e.cause.status == 403: - self.raise_geo_restricted(countries=['JP']) - raise - - formats, subtitles = [], {} - for src in traverse_obj(source_meta, ('sources', ..., 'src')): - fmts, subs = self._extract_m3u8_formats_and_subtitles(src, video_id, fatal=False) - formats.extend(fmts) - self._merge_subtitles(subs, target=subtitles) - return { + **self._extract_from_streaks_api('tbs', f'ref:{video_id}', headers={'Referer': 'https://cu.tbs.co.jp/'}), 'title': traverse_obj(webpage, ({find_element(tag='h3')}, {clean_html})), 'id': video_id, **traverse_obj(episode, { @@ -75,8 +53,6 @@ def _real_extract(self, url): 'episode': ('title', lambda _, v: not v.get('is_phonetic'), 'value'), 'series': ('custom_data', 'program_name'), }, get_all=False), - 'formats': formats, - 'subtitles': subtitles, } From 2634cff1a5422a7868ed64da61d72a82976c1562 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Tue, 17 Jun 2025 05:02:44 +0100 Subject: [PATCH 2/8] [ie/tbsjp] update tests --- yt_dlp/extractor/tbsjp.py | 48 ++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py index bf2b0012f..735964cda 100644 --- a/yt_dlp/extractor/tbsjp.py +++ b/yt_dlp/extractor/tbsjp.py @@ -13,22 +13,28 @@ class TBSJPEpisodeIE(StreaksBaseIE): _VALID_URL = r'https?://cu\.tbs\.co\.jp/episode/(?P[\d_]+)' _GEO_BYPASS = False _TESTS = [{ - 'url': 'https://cu.tbs.co.jp/episode/23613_2044134_1000049010', - 'skip': 'streams geo-restricted, Japan only. Also, will likely expire eventually', + 'url': 'https://cu.tbs.co.jp/episode/14694_2090934_1000117476', + 'skip': 'geo-blocked to japan + 7-day expiry', 'info_dict': { - 'title': 'VIVANT 第三話 誤送金完結へ!絶体絶命の反撃開始', - 'id': '23613_2044134_1000049010', + 'title': '次世代リアクション王発掘トーナメント', + 'id': '14694_2090934_1000117476', 'ext': 'mp4', - 'upload_date': '20230728', - 'duration': 3517, - 'release_timestamp': 1691118230, - 'episode': '第三話 誤送金完結へ!絶体絶命の反撃開始', - 'release_date': '20230804', - 'categories': 'count:11', - 'episode_number': 3, - 'timestamp': 1690522538, - 'description': 'md5:2b796341af1ef772034133174ba4a895', - 'series': 'VIVANT', + 'display_id': 'ref:14694_2090934_1000117476', + 'description': 'md5:63a2f445387f9d8c50f4e76396df968f', + 'uploader_id': 'tbs', + 'duration': 2761, + 'thumbnail': 'md5:5e044cd3431b1301de1a25911a6a9a4e', + 'categories': ['エンタメ', '水曜日のダウンタウン', 'ダウンタウン', '浜田雅功', '松本人志', '水ダウ', 'バラエティ', '動画'], + 'series': '水曜日のダウンタウン', + 'episode': '次世代リアクション王発掘トーナメント', + 'episode_number': 335, + 'timestamp': 1749547434, + 'upload_date': '20250610', + 'release_timestamp': 1749646802, + 'release_date': '20250611', + 'modified_timestamp': 1749647146, + 'modified_date': '20250611', + 'live_status': 'not_live', }, }] @@ -59,14 +65,14 @@ def _real_extract(self, url): class TBSJPProgramIE(InfoExtractor): _VALID_URL = r'https?://cu\.tbs\.co\.jp/program/(?P\d+)' _TESTS = [{ - 'url': 'https://cu.tbs.co.jp/program/23601', - 'playlist_mincount': 4, + 'url': 'https://cu.tbs.co.jp/program/14694', + 'playlist_mincount': 1, 'info_dict': { - 'id': '23601', - 'categories': ['エンタメ', 'ミライカプセル', '会社', '働く', 'バラエティ', '動画'], - 'description': '幼少期の夢は大人になって、どう成長したのだろうか？\nそしてその夢は今後、どのように広がっていくのか？\nいま話題の会社で働く人の「夢の成長」を描く', - 'series': 'ミライカプセル　-I have a dream-', - 'title': 'ミライカプセル　-I have a dream-', + 'id': '14694', + 'title': '水曜日のダウンタウン', + 'description': 'md5:cf1d46c76c2755d7f87512498718b837', + 'categories': ['エンタメ', '水曜日のダウンタウン', 'ダウンタウン', '浜田雅功', '松本人志', '水ダウ', 'バラエティ', '動画'], + 'series': '水曜日のダウンタウン', }, }] From 92587555ee73dc5c5e511dd7a84183975483bbc5 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Tue, 17 Jun 2025 15:34:35 +0100 Subject: [PATCH 3/8] [ie/tbsjp] make BaseIE with window.app json function to avoid code duplication Co-Authored-By: doe1080 <98906116+doe1080@users.noreply.github.com> --- yt_dlp/extractor/tbsjp.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py index 735964cda..6810e6832 100644 --- a/yt_dlp/extractor/tbsjp.py +++ b/yt_dlp/extractor/tbsjp.py @@ -1,4 +1,3 @@ -from .common import InfoExtractor from .streaks import StreaksBaseIE from ..utils import ( clean_html, @@ -9,7 +8,12 @@ from ..utils.traversal import find_element, traverse_obj -class TBSJPEpisodeIE(StreaksBaseIE): +class TBSJPBaseIE(StreaksBaseIE): + def _window_app(self, webpage, name, item_id, fatal=True): + return self._search_json(r'window\.app\s*=', webpage, f'{name} info', item_id, fatal=fatal) + + +class TBSJPEpisodeIE(TBSJPBaseIE): _VALID_URL = r'https?://cu\.tbs\.co\.jp/episode/(?P[\d_]+)' _GEO_BYPASS = False _TESTS = [{ @@ -41,7 +45,7 @@ class TBSJPEpisodeIE(StreaksBaseIE): def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - meta = self._search_json(r'window\.app\s*=', webpage, 'episode info', video_id, fatal=False) + meta = self._window_app(webpage, 'episode', video_id) episode = traverse_obj(meta, ('falcorCache', 'catalog', 'episode', video_id, 'value')) return { @@ -62,7 +66,7 @@ def _real_extract(self, url): } -class TBSJPProgramIE(InfoExtractor): +class TBSJPProgramIE(TBSJPBaseIE): _VALID_URL = r'https?://cu\.tbs\.co\.jp/program/(?P\d+)' _TESTS = [{ 'url': 'https://cu.tbs.co.jp/program/14694', @@ -79,8 +83,7 @@ class TBSJPProgramIE(InfoExtractor): def _real_extract(self, url): programme_id = self._match_id(url) webpage = self._download_webpage(url, programme_id) - meta = self._search_json(r'window\.app\s*=', webpage, 'programme info', programme_id) - + meta = self._window_app(webpage, 'programme', programme_id) programme = traverse_obj(meta, ('falcorCache', 'catalog', 'program', programme_id, 'false', 'value')) return { @@ -98,7 +101,7 @@ def _real_extract(self, url): } -class TBSJPPlaylistIE(InfoExtractor): +class TBSJPPlaylistIE(TBSJPBaseIE): _VALID_URL = r'https?://cu\.tbs\.co\.jp/playlist/(?P[\da-f]+)' _TESTS = [{ 'url': 'https://cu.tbs.co.jp/playlist/184f9970e7ba48e4915f1b252c55015e', @@ -111,8 +114,8 @@ class TBSJPPlaylistIE(InfoExtractor): def _real_extract(self, url): playlist_id = self._match_id(url) - page = self._download_webpage(url, playlist_id) - meta = self._search_json(r'window\.app\s*=', page, 'playlist info', playlist_id) + webpage = self._download_webpage(url, playlist_id) + meta = self._window_app(webpage, 'playlist', playlist_id) playlist = traverse_obj(meta, ('falcorCache', 'playList', playlist_id)) def entries(): From 50c54eb9bc11d56bab689e441f8eb900ad8df6fb Mon Sep 17 00:00:00 2001 From: garret1317 Date: Tue, 17 Jun 2025 15:36:09 +0100 Subject: [PATCH 4/8] [ie/tbsjp] More precise metadata extraction Co-Authored-By: doe1080 <98906116+doe1080@users.noreply.github.com> --- yt_dlp/extractor/tbsjp.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py index 6810e6832..0d3eacf1d 100644 --- a/yt_dlp/extractor/tbsjp.py +++ b/yt_dlp/extractor/tbsjp.py @@ -4,8 +4,9 @@ int_or_none, str_or_none, unified_timestamp, + url_or_none, ) -from ..utils.traversal import find_element, traverse_obj +from ..utils.traversal import traverse_obj class TBSJPBaseIE(StreaksBaseIE): @@ -49,20 +50,31 @@ def _real_extract(self, url): episode = traverse_obj(meta, ('falcorCache', 'catalog', 'episode', video_id, 'value')) return { - **self._extract_from_streaks_api('tbs', f'ref:{video_id}', headers={'Referer': 'https://cu.tbs.co.jp/'}), - 'title': traverse_obj(webpage, ({find_element(tag='h3')}, {clean_html})), - 'id': video_id, + **self._extract_from_streaks_api( + 'tbs', f'ref:{video_id}', headers={'Referer': 'https://cu.tbs.co.jp/'}), **traverse_obj(episode, { - 'categories': ('keywords', {list}), - 'id': ('content_id', {str}), - 'description': ('description', 0, 'value'), - 'timestamp': ('created_at', {unified_timestamp}), - 'release_timestamp': ('pub_date', {unified_timestamp}), + 'title': ('title', ..., 'value', {str}, any), + 'cast': ('credit', ..., 'name', ..., 'value', {str}, any, {lambda x: x.split(',')}, filter), + 'categories': ('keywords', ..., {str}, filter, all, filter), + 'description': ('description', ..., 'value', {clean_html}, any), 'duration': ('tv_episode_info', 'duration', {int_or_none}), + 'episode': ('title', lambda _, v: not v.get('is_phonetic'), 'value', {str}, any), + 'episode_id': ('content_id', {str}), 'episode_number': ('tv_episode_info', 'episode_number', {int_or_none}), - 'episode': ('title', lambda _, v: not v.get('is_phonetic'), 'value'), - 'series': ('custom_data', 'program_name'), - }, get_all=False), + 'genres': ('genre', ..., {str}, filter, all, filter), + 'release_timestamp': ('pub_date', {unified_timestamp}), + 'series': ('custom_data', 'program_name', {str}), + 'tags': ('tags', ..., {str}, filter, all, filter), + 'thumbnail': ('artwork', ..., 'url', {url_or_none}, any), + 'timestamp': ('created_at', {unified_timestamp}), + 'uploader': ('tv_show_info', 'networks', ..., {str}, any), + }), + **traverse_obj(episode, ('tv_episode_info', { + 'duration': ('duration', {int_or_none}), + 'episode_number': ('episode_number', {int_or_none}), + 'series_id': ('show_content_id', {str}), + })), + 'id': video_id, } From 71ed78c1634cc31dafd90ed3f61aa6c0e017fa73 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Tue, 17 Jun 2025 15:36:42 +0100 Subject: [PATCH 5/8] [ie/tbsjp] remove _GEO_BYPASS as this is now handled by StreaksIE Co-Authored-By: doe1080 <98906116+doe1080@users.noreply.github.com> --- yt_dlp/extractor/tbsjp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py index 0d3eacf1d..bfa0cbcd1 100644 --- a/yt_dlp/extractor/tbsjp.py +++ b/yt_dlp/extractor/tbsjp.py @@ -16,7 +16,6 @@ def _window_app(self, webpage, name, item_id, fatal=True): class TBSJPEpisodeIE(TBSJPBaseIE): _VALID_URL = r'https?://cu\.tbs\.co\.jp/episode/(?P[\d_]+)' - _GEO_BYPASS = False _TESTS = [{ 'url': 'https://cu.tbs.co.jp/episode/14694_2090934_1000117476', 'skip': 'geo-blocked to japan + 7-day expiry', From 8e8cf2bc67765e9171e4a3841ec660918648151d Mon Sep 17 00:00:00 2001 From: garret1317 Date: Tue, 17 Jun 2025 15:37:18 +0100 Subject: [PATCH 6/8] [ie/tbsjp] update tests for new metadata --- yt_dlp/extractor/tbsjp.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py index bfa0cbcd1..5e17dda9b 100644 --- a/yt_dlp/extractor/tbsjp.py +++ b/yt_dlp/extractor/tbsjp.py @@ -24,14 +24,19 @@ class TBSJPEpisodeIE(TBSJPBaseIE): 'id': '14694_2090934_1000117476', 'ext': 'mp4', 'display_id': 'ref:14694_2090934_1000117476', - 'description': 'md5:63a2f445387f9d8c50f4e76396df968f', + 'description': 'md5:0f57448221519627dce7802432729159', + 'uploader': 'TBS', 'uploader_id': 'tbs', 'duration': 2761, - 'thumbnail': 'md5:5e044cd3431b1301de1a25911a6a9a4e', + 'thumbnail': 'md5:76882f287053dfec6a5adffad70ff1e3', 'categories': ['エンタメ', '水曜日のダウンタウン', 'ダウンタウン', '浜田雅功', '松本人志', '水ダウ', 'バラエティ', '動画'], + 'cast': ['浜田\u3000雅功', '千原\u3000ジュニア', 'くっきー！（野性爆弾）', '田中\u3000卓志', '菊地\u3000亜美', '池田\u3000美優', '服部\u3000潤'], + 'genres': ['variety'], 'series': '水曜日のダウンタウン', + 'series_id': '14694', 'episode': '次世代リアクション王発掘トーナメント', 'episode_number': 335, + 'episode_id': '14694_2090934_1000117476', 'timestamp': 1749547434, 'upload_date': '20250610', 'release_timestamp': 1749646802, From 5c6a350222316eaff858c750f6d480a2ce88ca99 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Tue, 17 Jun 2025 16:09:28 +0100 Subject: [PATCH 7/8] [ie/tbsjp] make window.app function non-fatal for EpisodeIE Co-authored-by: doe1080 <98906116+doe1080@users.noreply.github.com> --- yt_dlp/extractor/tbsjp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py index 5e17dda9b..655877c33 100644 --- a/yt_dlp/extractor/tbsjp.py +++ b/yt_dlp/extractor/tbsjp.py @@ -11,7 +11,7 @@ class TBSJPBaseIE(StreaksBaseIE): def _window_app(self, webpage, name, item_id, fatal=True): - return self._search_json(r'window\.app\s*=', webpage, f'{name} info', item_id, fatal=fatal) + return self._search_json(r'window\.app\s*=', webpage, f'{name} info', item_id, fatal=fatal, default={}) class TBSJPEpisodeIE(TBSJPBaseIE): @@ -50,7 +50,7 @@ class TBSJPEpisodeIE(TBSJPBaseIE): def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - meta = self._window_app(webpage, 'episode', video_id) + meta = self._window_app(webpage, 'episode', video_id, fatal=False) episode = traverse_obj(meta, ('falcorCache', 'catalog', 'episode', video_id, 'value')) return { From e0c568675f0a9045ed9b8f2c1c35ba6e27239869 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 19 Jun 2025 23:35:49 +0100 Subject: [PATCH 8/8] filter empty string from cast list Co-authored-by: doe1080 <98906116+doe1080@users.noreply.github.com> --- yt_dlp/extractor/tbsjp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py index 655877c33..dd96ece1a 100644 --- a/yt_dlp/extractor/tbsjp.py +++ b/yt_dlp/extractor/tbsjp.py @@ -58,7 +58,7 @@ def _real_extract(self, url): 'tbs', f'ref:{video_id}', headers={'Referer': 'https://cu.tbs.co.jp/'}), **traverse_obj(episode, { 'title': ('title', ..., 'value', {str}, any), - 'cast': ('credit', ..., 'name', ..., 'value', {str}, any, {lambda x: x.split(',')}, filter), + 'cast': ('credit', ..., 'name', ..., 'value', {clean_html}, any, {lambda x: x.split(',')}, ..., {str.strip}, filter, all, filter), 'categories': ('keywords', ..., {str}, filter, all, filter), 'description': ('description', ..., 'value', {clean_html}, any), 'duration': ('tv_episode_info', 'duration', {int_or_none}),