From 060092e5f36c109e5d1be1a92c7f05488e1e4546 Mon Sep 17 00:00:00 2001 From: Simone Dotto Date: Thu, 15 May 2025 23:43:58 +0200 Subject: [PATCH 1/3] [ie/IlPost] Fix extractor --- yt_dlp/extractor/_extractors.py | 5 +- yt_dlp/extractor/ilpost.py | 95 +++++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 29 deletions(-) diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index e7dcb9853..15966b6da 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -851,7 +851,10 @@ IHeartRadioIE, IHeartRadioPodcastIE, ) -from .ilpost import IlPostIE +from .ilpost import ( + IlPostIE, + IlPostPodcastIE, +) from .iltalehti import IltalehtiIE from .imdb import ( ImdbIE, diff --git a/yt_dlp/extractor/ilpost.py b/yt_dlp/extractor/ilpost.py index da203cf5f..a54cdd762 100644 --- a/yt_dlp/extractor/ilpost.py +++ b/yt_dlp/extractor/ilpost.py @@ -1,19 +1,19 @@ - from .common import InfoExtractor from ..utils import ( ExtractorError, + clean_html, float_or_none, int_or_none, + unescapeHTML, url_or_none, - urlencode_postdata, ) from ..utils.traversal import traverse_obj class IlPostIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?ilpost\.it/episodes/(?P[^/?#]+)' + _VALID_URL = r'https?://(?:www\.)?ilpost\.it/podcasts/[^/]+/(?P[^/?#]+)' _TESTS = [{ - 'url': 'https://www.ilpost.it/episodes/1-avis-akvasas-ka/', + 'url': 'https://www.ilpost.it/podcasts/l-invasione/1-avis-akvasas-ka/', 'md5': '43649f002d85e1c2f319bb478d479c40', 'info_dict': { 'id': '2972047', @@ -23,10 +23,11 @@ class IlPostIE(InfoExtractor): 'url': 'https://www.ilpost.it/wp-content/uploads/2023/12/28/1703781217-l-invasione-pt1-v6.mp3', 'timestamp': 1703835014, 'upload_date': '20231229', + 'description': 'md5:54c5f12fb5b90f6c7cca6476a0802a99', 'duration': 2495.0, 'availability': 'public', 'series_id': '235598', - 'description': '', + 'thumbnail': 'https://www.ilpost.it/wp-content/uploads/2023/12/22/1703238848-copertina500x500.jpg', }, }] @@ -34,35 +35,73 @@ def _real_extract(self, url): display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) - endpoint_metadata = self._search_json( - r'var\s+ilpostpodcast\s*=', webpage, 'metadata', display_id) - episode_id = endpoint_metadata['post_id'] - podcast_id = endpoint_metadata['podcast_id'] - podcast_metadata = self._download_json( - endpoint_metadata['ajax_url'], display_id, data=urlencode_postdata({ - 'action': 'checkpodcast', - 'cookie': endpoint_metadata['cookie'], - 'post_id': episode_id, - 'podcast_id': podcast_id, - })) - - episode = traverse_obj(podcast_metadata, ( - 'data', 'postcastList', lambda _, v: str(v['id']) == episode_id, {dict}), get_all=False) - if not episode: - raise ExtractorError('Episode could not be extracted') + try: + episode = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['data']['data']['episode']['data'][0] + episode_id = episode['id'] + except KeyError: + raise ExtractorError('Failed to extract episode') return { - 'id': episode_id, + 'id': str(episode_id), 'display_id': display_id, - 'series_id': podcast_id, - 'vcodec': 'none', **traverse_obj(episode, { - 'title': ('title', {str}), - 'description': ('description', {str}), - 'url': ('podcast_raw_url', {url_or_none}), + 'series_id': ('parent', 'id', {int_or_none}), + 'title': ('title', {unescapeHTML}), + 'description': ('content_html', {clean_html}), + 'url': ('episode_raw_url', {url_or_none}), 'thumbnail': ('image', {url_or_none}), 'timestamp': ('timestamp', {int_or_none}), 'duration': ('milliseconds', {float_or_none(scale=1000)}), - 'availability': ('free', {lambda v: 'public' if v else 'subscriber_only'}), + 'availability': ('access_level', {lambda v: 'public' if v == 'all' else 'subscriber_only'}), }), } + + +class IlPostPodcastIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?ilpost\.it/podcasts/(?P[\w\-]+)[/?#]?$' + _TESTS = [{ + 'url': 'https://www.ilpost.it/podcasts/morning/', + 'info_dict': { + 'id': 'morning', + 'display_id': 'morning', + 'title': 'Morning', + 'series': 'Morning', + }, + 'playlist_mincount': 20, + }, { + 'url': 'https://www.ilpost.it/podcasts/basaglia-e-i-suoi/', + 'info_dict': { + 'id': 'basaglia-e-i-suoi', + 'display_id': 'basaglia-e-i-suoi', + 'title': 'Basaglia e i suoi', + 'series': 'Basaglia e i suoi', + }, + 'playlist_mincount': 5, + }] + + def _real_extract(self, url): + display_id = self._match_valid_url(url).group('id') + data = self._download_json(f'https://api-prod.ilpost.it/podcast/v1/podcast/{display_id}?hits=20', display_id) + + try: + podcast = data['data'][0]['parent'] + except KeyError: + raise ExtractorError('Failed to extract series') + + entries = [{ + '_type': 'url', + 'ie_key': 'IlPost', + 'url': episode['url'], + 'episode_id': episode['id'], + 'title': unescapeHTML(episode.get('title')), + 'description': clean_html(episode.get('content_html')), + 'series': unescapeHTML(podcast.get('title')), + } for episode in traverse_obj(data, ('data'))] + + return { + '_type': 'playlist', + 'id': podcast.get('id'), + 'display_id': display_id, + 'title': unescapeHTML(podcast.get('title')), + 'entries': entries, + } From f736c9627f91ce8852907c631ab1c7c5e9739971 Mon Sep 17 00:00:00 2001 From: Simone Dotto Date: Fri, 30 May 2025 00:56:16 +0200 Subject: [PATCH 2/3] [ie/IlPost] Fix extractor - merge review --- yt_dlp/extractor/ilpost.py | 139 ++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 65 deletions(-) diff --git a/yt_dlp/extractor/ilpost.py b/yt_dlp/extractor/ilpost.py index a54cdd762..5f7a59fc0 100644 --- a/yt_dlp/extractor/ilpost.py +++ b/yt_dlp/extractor/ilpost.py @@ -1,52 +1,67 @@ from .common import InfoExtractor from ..utils import ( - ExtractorError, clean_html, float_or_none, int_or_none, - unescapeHTML, + str_or_none, url_or_none, ) from ..utils.traversal import traverse_obj class IlPostIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?ilpost\.it/podcasts/[^/]+/(?P[^/?#]+)' - _TESTS = [{ - 'url': 'https://www.ilpost.it/podcasts/l-invasione/1-avis-akvasas-ka/', - 'md5': '43649f002d85e1c2f319bb478d479c40', - 'info_dict': { - 'id': '2972047', - 'ext': 'mp3', - 'display_id': '1-avis-akvasas-ka', - 'title': '1. Avis akvasas ka', - 'url': 'https://www.ilpost.it/wp-content/uploads/2023/12/28/1703781217-l-invasione-pt1-v6.mp3', - 'timestamp': 1703835014, - 'upload_date': '20231229', - 'description': 'md5:54c5f12fb5b90f6c7cca6476a0802a99', - 'duration': 2495.0, - 'availability': 'public', - 'series_id': '235598', - 'thumbnail': 'https://www.ilpost.it/wp-content/uploads/2023/12/22/1703238848-copertina500x500.jpg', + _VALID_URL = r'https?://(?:www\.)?ilpost\.it/podcasts/[^/?#]+/(?P[^/?#]+)' + _TESTS = [ + { + 'url': 'https://www.ilpost.it/podcasts/timbuctu/ep-323-lanno-record-della-pena-di-morte/', + 'md5': '55d88cc23bcab991639ebcbf1b4c0aa1', + 'info_dict': { + 'id': '3326553', + 'ext': 'mp3', + 'display_id': 'ep-323-lanno-record-della-pena-di-morte', + 'title': 'Ep. 323 – L’anno record della pena di morte', + 'url': 'https://static-prod.cdnilpost.com/wp-content/uploads/2025/05/25/1748196012-timbuctu_250526_v1_-16lufs.mp3', + 'timestamp': 1748235641, + 'upload_date': '20250526', + 'description': 'md5:331514a14779fab06e902160ec8c89ba', + 'duration': 751, + 'availability': 'public', + 'series_id': '233679', + 'thumbnail': 'https://www.ilpost.it/wp-content/uploads/2023/05/19/1684536738-copertina500x500.jpg', + }, }, - }] + { + 'url': 'https://www.ilpost.it/podcasts/l-invasione/1-avis-akvasas-ka/', + 'md5': '43649f002d85e1c2f319bb478d479c40', + 'info_dict': { + 'id': '2972047', + 'ext': 'mp3', + 'display_id': '1-avis-akvasas-ka', + 'title': '1. Avis akvasas ka', + 'url': 'https://www.ilpost.it/wp-content/uploads/2023/12/28/1703781217-l-invasione-pt1-v6.mp3', + 'timestamp': 1703835014, + 'upload_date': '20231229', + 'description': 'md5:57d147951b522c92095f64e28570cf4a', + 'duration': 2495.0, + 'availability': 'public', + 'series_id': '235598', + 'thumbnail': 'https://www.ilpost.it/wp-content/uploads/2023/12/22/1703238848-copertina500x500.jpg', + }, + }, + ] def _real_extract(self, url): display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) - try: - episode = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['data']['data']['episode']['data'][0] - episode_id = episode['id'] - except KeyError: - raise ExtractorError('Failed to extract episode') + episode = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['data']['data']['episode']['data'][0] return { - 'id': str(episode_id), + 'id': str(episode['id']), 'display_id': display_id, **traverse_obj(episode, { - 'series_id': ('parent', 'id', {int_or_none}), - 'title': ('title', {unescapeHTML}), + 'series_id': ('parent', 'id', {str_or_none}), + 'title': ('title', {clean_html}), 'description': ('content_html', {clean_html}), 'url': ('episode_raw_url', {url_or_none}), 'thumbnail': ('image', {url_or_none}), @@ -58,50 +73,44 @@ def _real_extract(self, url): class IlPostPodcastIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?ilpost\.it/podcasts/(?P[\w\-]+)[/?#]?$' - _TESTS = [{ - 'url': 'https://www.ilpost.it/podcasts/morning/', - 'info_dict': { - 'id': 'morning', - 'display_id': 'morning', - 'title': 'Morning', - 'series': 'Morning', + _VALID_URL = r'https?://(?:www\.)?ilpost\.it/podcasts/(?P[^/?#]+)/?(?:[?#]|$)' + _TESTS = [ + { + 'url': 'https://www.ilpost.it/podcasts/basaglia-e-i-suoi/', + 'info_dict': { + 'id': '239295', + 'title': 'Basaglia e i suoi', + }, + 'playlist_mincount': 5, }, - 'playlist_mincount': 20, - }, { - 'url': 'https://www.ilpost.it/podcasts/basaglia-e-i-suoi/', - 'info_dict': { - 'id': 'basaglia-e-i-suoi', - 'display_id': 'basaglia-e-i-suoi', - 'title': 'Basaglia e i suoi', - 'series': 'Basaglia e i suoi', + { + 'url': 'https://www.ilpost.it/podcasts/morning/', + 'info_dict': { + 'id': '227474', + 'title': 'Morning', + }, + 'playlist_mincount': 20, }, - 'playlist_mincount': 5, - }] + ] def _real_extract(self, url): - display_id = self._match_valid_url(url).group('id') - data = self._download_json(f'https://api-prod.ilpost.it/podcast/v1/podcast/{display_id}?hits=20', display_id) + display_id = self._match_id(url) - try: - podcast = data['data'][0]['parent'] - except KeyError: - raise ExtractorError('Failed to extract series') + data = self._download_json(f'https://api-prod.ilpost.it/podcast/v1/podcast/{display_id}', display_id, query={'hits': '20'}) + data = self._download_json(f'https://api-prod.ilpost.it/podcast/v1/podcast/{display_id}', display_id, query={'hits': data['head']['data']['total']}) + + podcast = data['data'][0]['parent'] entries = [{ '_type': 'url', - 'ie_key': 'IlPost', + 'ie_key': IlPostIE.ie_key(), 'url': episode['url'], - 'episode_id': episode['id'], - 'title': unescapeHTML(episode.get('title')), - 'description': clean_html(episode.get('content_html')), - 'series': unescapeHTML(podcast.get('title')), - } for episode in traverse_obj(data, ('data'))] + **traverse_obj(episode, { + 'episode_id': ('id', {str_or_none}), + 'title': ('title', {clean_html}), + 'description': ('content_html', {clean_html}), + }), + } for episode in traverse_obj(data, ('data', lambda _, v: url_or_none(v['url'])))] - return { - '_type': 'playlist', - 'id': podcast.get('id'), - 'display_id': display_id, - 'title': unescapeHTML(podcast.get('title')), - 'entries': entries, - } + return self.playlist_result(entries, + str(podcast['id']), clean_html(podcast.get('title'))) From 1b4110c8deabf91f8f1010c084a9548ac6b21795 Mon Sep 17 00:00:00 2001 From: Simone Dotto Date: Sun, 15 Jun 2025 16:16:48 +0200 Subject: [PATCH 3/3] [ie/IlPost] Fix extractor - merge review pt2 --- yt_dlp/extractor/ilpost.py | 103 ++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/yt_dlp/extractor/ilpost.py b/yt_dlp/extractor/ilpost.py index 5f7a59fc0..66b46c027 100644 --- a/yt_dlp/extractor/ilpost.py +++ b/yt_dlp/extractor/ilpost.py @@ -1,3 +1,5 @@ +import itertools + from .common import InfoExtractor from ..utils import ( clean_html, @@ -11,25 +13,24 @@ class IlPostIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?ilpost\.it/podcasts/[^/?#]+/(?P[^/?#]+)' - _TESTS = [ - { - 'url': 'https://www.ilpost.it/podcasts/timbuctu/ep-323-lanno-record-della-pena-di-morte/', - 'md5': '55d88cc23bcab991639ebcbf1b4c0aa1', - 'info_dict': { - 'id': '3326553', - 'ext': 'mp3', - 'display_id': 'ep-323-lanno-record-della-pena-di-morte', - 'title': 'Ep. 323 – L’anno record della pena di morte', - 'url': 'https://static-prod.cdnilpost.com/wp-content/uploads/2025/05/25/1748196012-timbuctu_250526_v1_-16lufs.mp3', - 'timestamp': 1748235641, - 'upload_date': '20250526', - 'description': 'md5:331514a14779fab06e902160ec8c89ba', - 'duration': 751, - 'availability': 'public', - 'series_id': '233679', - 'thumbnail': 'https://www.ilpost.it/wp-content/uploads/2023/05/19/1684536738-copertina500x500.jpg', - }, + _TESTS = [{ + 'url': 'https://www.ilpost.it/podcasts/timbuctu/ep-323-lanno-record-della-pena-di-morte/', + 'md5': '55d88cc23bcab991639ebcbf1b4c0aa1', + 'info_dict': { + 'id': '3326553', + 'ext': 'mp3', + 'display_id': 'ep-323-lanno-record-della-pena-di-morte', + 'title': 'Ep. 323 – L’anno record della pena di morte', + 'url': 'https://static-prod.cdnilpost.com/wp-content/uploads/2025/05/25/1748196012-timbuctu_250526_v1_-16lufs.mp3', + 'timestamp': 1748235641, + 'upload_date': '20250526', + 'description': 'md5:331514a14779fab06e902160ec8c89ba', + 'duration': 751, + 'availability': 'public', + 'series_id': '233679', + 'thumbnail': 'https://www.ilpost.it/wp-content/uploads/2023/05/19/1684536738-copertina500x500.jpg', }, + }, { 'url': 'https://www.ilpost.it/podcasts/l-invasione/1-avis-akvasas-ka/', 'md5': '43649f002d85e1c2f319bb478d479c40', @@ -47,14 +48,14 @@ class IlPostIE(InfoExtractor): 'series_id': '235598', 'thumbnail': 'https://www.ilpost.it/wp-content/uploads/2023/12/22/1703238848-copertina500x500.jpg', }, - }, - ] + }] def _real_extract(self, url): display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) - episode = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['data']['data']['episode']['data'][0] + episode = self._search_nextjs_data( + webpage, display_id)['props']['pageProps']['data']['data']['episode']['data'][0] return { 'id': str(episode['id']), @@ -74,15 +75,14 @@ def _real_extract(self, url): class IlPostPodcastIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?ilpost\.it/podcasts/(?P[^/?#]+)/?(?:[?#]|$)' - _TESTS = [ - { - 'url': 'https://www.ilpost.it/podcasts/basaglia-e-i-suoi/', - 'info_dict': { - 'id': '239295', - 'title': 'Basaglia e i suoi', - }, - 'playlist_mincount': 5, + _TESTS = [{ + 'url': 'https://www.ilpost.it/podcasts/basaglia-e-i-suoi/', + 'info_dict': { + 'id': '239295', + 'title': 'Basaglia e i suoi', }, + 'playlist_mincount': 5, + }, { 'url': 'https://www.ilpost.it/podcasts/morning/', 'info_dict': { @@ -90,27 +90,38 @@ class IlPostPodcastIE(InfoExtractor): 'title': 'Morning', }, 'playlist_mincount': 20, - }, - ] + }] def _real_extract(self, url): display_id = self._match_id(url) + entries = [] + podcast = None - data = self._download_json(f'https://api-prod.ilpost.it/podcast/v1/podcast/{display_id}', display_id, query={'hits': '20'}) - data = self._download_json(f'https://api-prod.ilpost.it/podcast/v1/podcast/{display_id}', display_id, query={'hits': data['head']['data']['total']}) + max_hits = 10000 # found experimentally - podcast = data['data'][0]['parent'] + for page in itertools.count(1): + data = self._download_json( + f'https://api-prod.ilpost.it/podcast/v1/podcast/{display_id}', + display_id, + query={'hits': max_hits, 'pg': page}, + expected_status=500, + ) - entries = [{ - '_type': 'url', - 'ie_key': IlPostIE.ie_key(), - 'url': episode['url'], - **traverse_obj(episode, { - 'episode_id': ('id', {str_or_none}), - 'title': ('title', {clean_html}), - 'description': ('content_html', {clean_html}), - }), - } for episode in traverse_obj(data, ('data', lambda _, v: url_or_none(v['url'])))] + if podcast is None: + podcast = traverse_obj(data, ('data', 0, 'parent')) - return self.playlist_result(entries, - str(podcast['id']), clean_html(podcast.get('title'))) + if data.get('data') is None: + break + + entries += [{ + '_type': 'url', + 'ie_key': IlPostIE.ie_key(), + 'url': episode['url'], + **traverse_obj(episode, { + 'episode_id': ('id', {str_or_none}), + 'title': ('title', {clean_html}), + 'description': ('content_html', {clean_html}), + }), + } for episode in traverse_obj(data, ('data', lambda _, v: url_or_none(v['url'])))] + + return self.playlist_result(entries, str_or_none(podcast.get('id')), clean_html(podcast.get('title')))