From 451eca31ecd0a0e93c184d8dd0adf1d0fd688c10 Mon Sep 17 00:00:00 2001 From: DTrombett Date: Sun, 25 May 2025 20:00:44 +0200 Subject: [PATCH 1/3] [ie/gedidigital] Fix extractor --- yt_dlp/extractor/gedidigital.py | 171 +++++++++++++------------------- 1 file changed, 67 insertions(+), 104 deletions(-) diff --git a/yt_dlp/extractor/gedidigital.py b/yt_dlp/extractor/gedidigital.py index 2ffa66030..648c32aed 100644 --- a/yt_dlp/extractor/gedidigital.py +++ b/yt_dlp/extractor/gedidigital.py @@ -1,20 +1,14 @@ -import re +import json from .common import InfoExtractor -from ..utils import ( - base_url, - determine_ext, - int_or_none, - url_basename, - urljoin, -) +from ..utils import int_or_none, js_to_json class GediDigitalIE(InfoExtractor): - _VALID_URL = r'''(?x:(?P(?:https?:)//video\. + _VALID_URL = r'''(?x:(?:https?:)//(?:\w+\.)? (?: (?: - (?:espresso\.)?repubblica + repubblica |lastampa |ilsecoloxix |huffingtonpost @@ -33,24 +27,35 @@ class GediDigitalIE(InfoExtractor): |corrierealpi |lasentinella )\.gelocal - )\.it(?:/[^/]+){2,4}/(?P\d+))(?:$|[?&].*))''' - _EMBED_REGEX = [rf'''(?x) - (?: - data-frame-src=| - {_VALID_URL})\1'''] + )\.it/[^?]+/video/(?P[a-z0-9_-]+)-(?P\d+))''' _TESTS = [{ 'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683', - 'md5': '84658d7fb9e55a6e57ecc77b73137494', + 'md5': '6d1238ab5f4753b6f3d9eb396bff8ea3', 'info_dict': { - 'id': '121683', + 'id': '375544', 'ext': 'mp4', 'title': 'Il paradosso delle Regionali: ecco perché la Lega vince ma sembra aver perso', - 'description': 'md5:de7f4d6eaaaf36c153b599b10f8ce7ca', + 'description': 'md5:fad65b086b8b23fd4db66dc1f7a530f9', 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-full-.+?\.jpg$', 'duration': 125, + 'display_id': 'il_paradosso_delle_regionali_ecco_perche_la_lega_vince_ma_sembra_aver_perso', + 'uploader_id': '6210505280001', }, + }, { + 'url': 'https://www.repubblica.it/video/tv/2023/10/25/video/maurizio_molinari_israele_sotto_shock_per_le_parole_del_segretario_generale_dellonu-422547542', + 'md5': '3cdb25ee59373cb326dd402c7e18490f', + 'info_dict': { + 'id': '422547542', + 'ext': 'mp4', + 'display_id': 'maurizio_molinari_israele_sotto_shock_per_le_parole_del_segretario_generale_dellonu', + 'title': 'Maurizio Molinari: "Israele sotto shock per le parole del segretario generale dell\'Onu"', + 'uploader_id': '6210505280001', + 'duration': 89, + 'thumbnail': 'https://www.repstatic.it/video/photo/2023/10/25/918897/918897-thumb-full-720-molinari_porta_a_porta_24_10_23.jpg', + }, + }, { + 'url': 'https://roma.repubblica.it/cronaca/2025/05/22/video/giornata_biodiversita_mattarella_si_improvvisa_giardiniere_annaffia_una_piantina_a_castelporziano-424598388/', + 'only_matching': True, }, { 'url': 'https://video.huffingtonpost.it/embed/politica/cotticelli-non-so-cosa-mi-sia-successo-sto-cercando-di-capire-se-ho-avuto-un-malore/29312/29276?responsive=true&el=video971040871621586700', 'only_matching': True, @@ -104,95 +109,53 @@ class GediDigitalIE(InfoExtractor): 'only_matching': True, }] - @staticmethod - def _sanitize_urls(urls): - # add protocol if missing - for i, e in enumerate(urls): - if e.startswith('//'): - urls[i] = f'https:{e}' - # clean iframes urls - for i, e in enumerate(urls): - urls[i] = urljoin(base_url(e), url_basename(e)) - return urls - - @classmethod - def _extract_embed_urls(cls, url, webpage): - return cls._sanitize_urls(tuple(super()._extract_embed_urls(url, webpage))) - - @staticmethod - def _clean_formats(formats): - format_urls = set() - clean_formats = [] - for f in formats: - if f['url'] not in format_urls: - if f.get('audio_ext') != 'none' and not f.get('acodec'): - continue - format_urls.add(f['url']) - clean_formats.append(f) - formats[:] = clean_formats - def _real_extract(self, url): - video_id, url = self._match_valid_url(url).group('id', 'base_url') + slug, video_id = self._match_valid_url(url).group('slug', 'id') webpage = self._download_webpage(url, video_id) - title = self._html_search_meta( - ['twitter:title', 'og:title'], webpage, fatal=True) - player_data = re.findall( - r"PlayerFactory\.setParam\('(?Pformat|param)',\s*'(?P[^']+)',\s*'(?P[^']+)'\);", - webpage) - + data = self._search_json( + r'BrightcoveVideoPlayerOptions\s*=', webpage, 'Brightcove video player options', video_id, transform_source=js_to_json, + ) + streams = json.loads(data['videoSrc']) formats = [] - duration = thumb = None - for t, n, v in player_data: - if t == 'format': - if n in ('video-hds-vod-ec', 'video-hls-vod-ec', 'video-viralize', 'video-youtube-pfp'): - continue - elif n.endswith('-vod-ak'): - formats.extend(self._extract_akamai_formats( - v, video_id, {'http': 'media.gedidigital.it'})) - else: - ext = determine_ext(v) - if ext == 'm3u8': - formats.extend(self._extract_m3u8_formats( - v, video_id, 'mp4', 'm3u8_native', m3u8_id=n, fatal=False)) - continue - f = { - 'format_id': n, - 'url': v, - } - if ext == 'mp3': - abr = int_or_none(self._search_regex( - r'-mp3-audio-(\d+)', v, 'abr', default=None)) - f.update({ - 'abr': abr, - 'tbr': abr, - 'acodec': ext, - 'vcodec': 'none', - }) - else: - mobj = re.match(r'^video-rrtv-(\d+)(?:-(\d+))?$', n) - if mobj: - f.update({ - 'height': int(mobj.group(1)), - 'vbr': int_or_none(mobj.group(2)), - }) - if not f.get('vbr'): - f['vbr'] = int_or_none(self._search_regex( - r'-video-rrtv-(\d+)', v, 'abr', default=None)) - formats.append(f) - elif t == 'param': - if n in ['image_full', 'image']: - thumb = v - elif n == 'videoDuration': - duration = int_or_none(v) - - self._clean_formats(formats) + for stream in streams: + if stream['type'] == 'video/mp4': + bitrate = self._search_regex(r'video-rrtv-(\d+)', stream['src'], 'vbr', None) + formats.append({ + 'format_id': f'mp4-{bitrate}' if bitrate else 'mp4', + 'url': stream['src'], + 'ext': 'mp4', + 'vcodec': 'avc1', + 'acodec': 'mp4a', + 'vbr': int_or_none(bitrate), + }) + elif stream['type'] == 'application/x-mpegURL': + new_formats = self._extract_m3u8_formats(stream['src'], video_id) + for fmt in new_formats: + fmt.setdefault('vbr', int_or_none(self._search_regex(r'/hls/_(\d+)/', fmt['url'], 'vbr', None))) + formats.extend(new_formats) + elif stream['type'] == 'audio/mp3': + bitrate = self._search_regex(r'mp3-audio-(\d+)', stream['src'], 'vbr', None) + formats.append({ + 'format_id': f'mp3-{bitrate}' if bitrate else 'mp3', + 'url': stream['src'], + 'ext': 'mp3', + 'acodec': 'mp3', + 'abr': int_or_none(bitrate), + 'tbr': int_or_none(bitrate), + }) + else: + formats.append({ + 'format_id': stream['type'], + 'url': stream['src'], + }) return { 'id': video_id, - 'title': title, - 'description': self._html_search_meta( - ['twitter:description', 'og:description', 'description'], webpage), - 'thumbnail': thumb or self._og_search_thumbnail(webpage), + 'display_id': slug, + 'title': data.get('videoTitle') or self._html_search_meta(['twitter:title', 'og:title'], webpage), + 'description': self._html_search_meta(['twitter:description', 'og:description', 'description'], webpage, default=None), 'formats': formats, - 'duration': duration, + 'thumbnail': data.get('posterSrc') or self._og_search_thumbnail(webpage), + 'uploader_id': data.get('accountId'), + 'duration': int_or_none(data.get('videoLenght')), } From cb60ff3a8d881ba2c5e12ac077c0113ac3ddb3d5 Mon Sep 17 00:00:00 2001 From: DTrombett Date: Sun, 25 May 2025 20:40:48 +0200 Subject: [PATCH 2/3] fix: support all types of url --- yt_dlp/extractor/gedidigital.py | 51 ++++----------------------------- 1 file changed, 5 insertions(+), 46 deletions(-) diff --git a/yt_dlp/extractor/gedidigital.py b/yt_dlp/extractor/gedidigital.py index 648c32aed..5da9e59fa 100644 --- a/yt_dlp/extractor/gedidigital.py +++ b/yt_dlp/extractor/gedidigital.py @@ -27,18 +27,17 @@ class GediDigitalIE(InfoExtractor): |corrierealpi |lasentinella )\.gelocal - )\.it/[^?]+/video/(?P[a-z0-9_-]+)-(?P\d+))''' + )\.it/[^?]+(?:/video/(?P[a-z0-9_-]+)-|/)(?P\d+))''' _TESTS = [{ 'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683', 'md5': '6d1238ab5f4753b6f3d9eb396bff8ea3', 'info_dict': { - 'id': '375544', + 'id': '121683', 'ext': 'mp4', 'title': 'Il paradosso delle Regionali: ecco perché la Lega vince ma sembra aver perso', 'description': 'md5:fad65b086b8b23fd4db66dc1f7a530f9', 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-full-.+?\.jpg$', 'duration': 125, - 'display_id': 'il_paradosso_delle_regionali_ecco_perche_la_lega_vince_ma_sembra_aver_perso', 'uploader_id': '6210505280001', }, }, { @@ -59,58 +58,16 @@ class GediDigitalIE(InfoExtractor): }, { 'url': 'https://video.huffingtonpost.it/embed/politica/cotticelli-non-so-cosa-mi-sia-successo-sto-cercando-di-capire-se-ho-avuto-un-malore/29312/29276?responsive=true&el=video971040871621586700', 'only_matching': True, - }, { - 'url': 'https://video.espresso.repubblica.it/embed/tutti-i-video/01-ted-villa/14772/14870&width=640&height=360', - 'only_matching': True, }, { 'url': 'https://video.repubblica.it/motori/record-della-pista-a-spa-francorchamps-la-pagani-huayra-roadster-bc-stupisce/367415/367963', 'only_matching': True, }, { 'url': 'https://video.ilsecoloxix.it/sport/cassani-e-i-brividi-azzurri-ai-mondiali-di-imola-qui-mi-sono-innamorato-del-ciclismo-da-ragazzino-incredibile-tornarci-da-ct/66184/66267', 'only_matching': True, - }, { - 'url': 'https://video.iltirreno.gelocal.it/sport/dentro-la-notizia-ferrari-cosa-succede-a-maranello/141059/142723', - 'only_matching': True, - }, { - 'url': 'https://video.messaggeroveneto.gelocal.it/locale/maria-giovanna-elmi-covid-vaccino/138155/139268', - 'only_matching': True, - }, { - 'url': 'https://video.ilpiccolo.gelocal.it/dossier/big-john/dinosauro-big-john-al-via-le-visite-guidate-a-trieste/135226/135751', - 'only_matching': True, - }, { - 'url': 'https://video.gazzettadimantova.gelocal.it/locale/dal-ponte-visconteo-di-valeggio-l-and-8217sos-dei-ristoratori-aprire-anche-a-cena/137310/137818', - 'only_matching': True, - }, { - 'url': 'https://video.mattinopadova.gelocal.it/dossier/coronavirus-in-veneto/covid-a-vo-un-anno-dopo-un-cuore-tricolore-per-non-dimenticare/138402/138964', - 'only_matching': True, - }, { - 'url': 'https://video.laprovinciapavese.gelocal.it/locale/mede-zona-rossa-via-alle-vaccinazioni-per-gli-over-80/137545/138120', - 'only_matching': True, - }, { - 'url': 'https://video.tribunatreviso.gelocal.it/dossier/coronavirus-in-veneto/ecco-le-prima-vaccinazioni-di-massa-nella-marca/134485/135024', - 'only_matching': True, - }, { - 'url': 'https://video.nuovavenezia.gelocal.it/locale/camion-troppo-alto-per-il-ponte-ferroviario-perde-il-carico/135734/136266', - 'only_matching': True, - }, { - 'url': 'https://video.gazzettadimodena.gelocal.it/locale/modena-scoperta-la-proteina-che-predice-il-livello-di-gravita-del-covid/139109/139796', - 'only_matching': True, - }, { - 'url': 'https://video.lanuovaferrara.gelocal.it/locale/due-bombole-di-gpl-aperte-e-abbandonate-i-vigili-bruciano-il-gas/134391/134957', - 'only_matching': True, - }, { - 'url': 'https://video.corrierealpi.gelocal.it/dossier/cortina-2021-i-mondiali-di-sci-alpino/mondiali-di-sci-il-timelapse-sulla-splendida-olympia/133760/134331', - 'only_matching': True, - }, { - 'url': 'https://video.lasentinella.gelocal.it/locale/vestigne-centra-un-auto-e-si-ribalta/138931/139466', - 'only_matching': True, - }, { - 'url': 'https://video.espresso.repubblica.it/tutti-i-video/01-ted-villa/14772', - 'only_matching': True, }] def _real_extract(self, url): - slug, video_id = self._match_valid_url(url).group('slug', 'id') + video_id, slug = self._match_valid_url(url).group('id', 'slug') webpage = self._download_webpage(url, video_id) data = self._search_json( r'BrightcoveVideoPlayerOptions\s*=', webpage, 'Brightcove video player options', video_id, transform_source=js_to_json, @@ -118,6 +75,8 @@ def _real_extract(self, url): streams = json.loads(data['videoSrc']) formats = [] for stream in streams: + if isinstance(stream, str): + continue if stream['type'] == 'video/mp4': bitrate = self._search_regex(r'video-rrtv-(\d+)', stream['src'], 'vbr', None) formats.append({ From eb475eee282a44a04c38298beea1317c7a2d41c2 Mon Sep 17 00:00:00 2001 From: DTrombett Date: Sun, 25 May 2025 20:48:02 +0200 Subject: [PATCH 3/3] fix: remove old base urls --- yt_dlp/extractor/gedidigital.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/yt_dlp/extractor/gedidigital.py b/yt_dlp/extractor/gedidigital.py index 5da9e59fa..3cd4f80db 100644 --- a/yt_dlp/extractor/gedidigital.py +++ b/yt_dlp/extractor/gedidigital.py @@ -7,26 +7,10 @@ class GediDigitalIE(InfoExtractor): _VALID_URL = r'''(?x:(?:https?:)//(?:\w+\.)? (?: - (?: - repubblica - |lastampa - |ilsecoloxix - |huffingtonpost - )| - (?: - iltirreno - |messaggeroveneto - |ilpiccolo - |gazzettadimantova - |mattinopadova - |laprovinciapavese - |tribunatreviso - |nuovavenezia - |gazzettadimodena - |lanuovaferrara - |corrierealpi - |lasentinella - )\.gelocal + repubblica + |lastampa + |ilsecoloxix + |huffingtonpost )\.it/[^?]+(?:/video/(?P[a-z0-9_-]+)-|/)(?P\d+))''' _TESTS = [{ 'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683',