Merge 386132ba25 into 73bf102116

README.md

@@ -2349,7 +2349,7 @@ These options are not intended to be used by the end-user
     --test                           Download only part of video for testing extractors
     --load-pages                     Load pages dumped by --write-pages
     --youtube-print-sig-code         For testing youtube signatures
-    --allow-unplayable-formats       List unplayable formats also
+    --allow-unplayable-formats       List unplayable formats. Implies `--simulate` and `--list-formats`.
     --no-allow-unplayable-formats    Default
 
 #### Old aliases

yt_dlp/YoutubeDL.py

@@ -248,7 +248,7 @@ class YoutubeDL:
                        You can also pass a function. The function takes 'ctx' as
                        argument and returns the formats to download.
                        See "build_format_selector" for an implementation
-    allow_unplayable_formats:   Allow unplayable formats to be extracted and downloaded.
+    allow_unplayable_formats:   Allow unplayable formats to be extracted.
     ignore_no_formats_error: Ignore "No video formats" error. Usefull for
                        extracting metadata even if the video is not actually
                        available for download (experimental)
@@ -702,11 +702,14 @@ class YoutubeDL:
             self.deprecated_feature(system_deprecation.replace('\n', '\n                    '))
 
         if self.params.get('allow_unplayable_formats'):
+            from . import _IN_CLI
+
+            switch = '--allow-unplayable-formats' if _IN_CLI else 'allow_unplayable_formats'
             self.report_warning(
-                f'You have asked for {self._format_err("UNPLAYABLE", self.Styles.EMPHASIS)} formats to be listed/downloaded. '
+                f'{switch} is a {self._format_err("developer option", self.Styles.EMPHASIS)} intended for {self._format_err("debugging", self.Styles.EMPHASIS)}. \n'
-                'This is a developer option intended for debugging. \n'
+                f'         If you experience issues {self._format_err("DO NOT", self.Styles.ERROR)} open a bug report.')
-                '         If you experience any issues while using this option, '
+            self.params['listformats'] = True
-                f'{self._format_err("DO NOT", self.Styles.ERROR)} open a bug report')
+            self.params['simulate'] = True
 
         if self.params.get('bidi_workaround', False):
             try:
@@ -2849,6 +2852,7 @@ class YoutubeDL:
         info_dict['_has_drm'] = any(  # or None ensures --clean-infojson removes it
             f.get('has_drm') and f['has_drm'] != 'maybe' for f in formats) or None
         if not self.params.get('allow_unplayable_formats'):
+            # Allow bypassing flaky `has_drm` detection
             formats = [f for f in formats if not f.get('has_drm') or f['has_drm'] == 'maybe']
 
         if formats and all(f.get('acodec') == f.get('vcodec') == 'none' for f in formats):
@@ -3464,12 +3468,7 @@ class YoutubeDL:
                         success, real_download = self.dl(temp_filename, info_dict)
                         info_dict['__real_download'] = real_download
                     else:
-                        if self.params.get('allow_unplayable_formats'):
+                        if not merger.available:
-                            self.report_warning(
-                                'You have requested merging of multiple formats '
-                                'while also allowing unplayable formats to be downloaded. '
-                                'The formats won\'t be merged to prevent data corruption.')
-                        elif not merger.available:
                             msg = 'You have requested merging of multiple formats but ffmpeg is not installed'
                             if not self.params.get('ignoreerrors'):
                                 self.report_error(f'{msg}. Aborting due to --abort-on-error')
@@ -3500,7 +3499,7 @@ class YoutubeDL:
                             info_dict['__real_download'] = info_dict['__real_download'] or real_download
                             success = success and partial_success
 
-                    if downloaded and merger.available and not self.params.get('allow_unplayable_formats'):
+                    if downloaded and merger.available:
                         info_dict['__postprocessors'].append(merger)
                         info_dict['__files_to_merge'] = downloaded
                         # Even if there were no downloads, it is being merged only now

yt_dlp/__init__.py

@@ -511,8 +511,7 @@ def validate_options(opts):
         opts.postprocessor_args['default'] = opts.postprocessor_args.pop('default-compat')
         opts.postprocessor_args.setdefault('sponskrub', [])
 
-    def report_conflict(arg1, opt1, arg2='--allow-unplayable-formats', opt2='allow_unplayable_formats',
+    def report_conflict(arg1, opt1, arg2, opt2, val1=NO_DEFAULT, val2=NO_DEFAULT, default=False):
-                        val1=NO_DEFAULT, val2=NO_DEFAULT, default=False):
         if val2 is NO_DEFAULT:
             val2 = getattr(opts, opt2)
         if not val2:
@@ -540,21 +539,6 @@ def validate_options(opts):
     report_conflict('--sponskrub-cut', 'sponskrub_cut', '--split-chapter', 'split_chapters',
                     val1=opts.sponskrub and opts.sponskrub_cut)
 
-    # Conflicts with --allow-unplayable-formats
-    report_conflict('--embed-metadata', 'addmetadata')
-    report_conflict('--embed-chapters', 'addchapters')
-    report_conflict('--embed-info-json', 'embed_infojson')
-    report_conflict('--embed-subs', 'embedsubtitles')
-    report_conflict('--embed-thumbnail', 'embedthumbnail')
-    report_conflict('--extract-audio', 'extractaudio')
-    report_conflict('--fixup', 'fixup', val1=opts.fixup not in (None, 'never', 'ignore'), default='never')
-    report_conflict('--recode-video', 'recodevideo')
-    report_conflict('--remove-chapters', 'remove_chapters', default=[])
-    report_conflict('--remux-video', 'remuxvideo')
-    report_conflict('--sponskrub', 'sponskrub')
-    report_conflict('--sponsorblock-remove', 'sponsorblock_remove', default=set())
-    report_conflict('--xattrs', 'xattrs')
-
     # Fully deprecated options
     def report_deprecation(val, old, new=None):
         if not val:

yt_dlp/downloader/external.py

@@ -468,7 +468,6 @@ class FFmpegFD(ExternalFD):
         return (
             info_dict.get('requested_formats')
             and info_dict.get('protocol')
-            and not params.get('allow_unplayable_formats')
             and 'no-direct-merge' not in params.get('compat_opts', [])
             and cls.can_download(info_dict))
 

yt_dlp/downloader/f4m.py

@@ -256,14 +256,13 @@ class F4mFD(FragmentFD):
         media = doc.findall(_add_ns('media'))
         if not media:
             self.report_error('No media found')
-        if not self.params.get('allow_unplayable_formats'):
+        for e in (doc.findall(_add_ns('drmAdditionalHeader'))
-            for e in (doc.findall(_add_ns('drmAdditionalHeader'))
+                  + doc.findall(_add_ns('drmAdditionalHeaderSet'))):
-                      + doc.findall(_add_ns('drmAdditionalHeaderSet'))):
+            # If id attribute is missing it's valid for all media nodes
-                # If id attribute is missing it's valid for all media nodes
+            # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
-                # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
+            if 'id' not in e.attrib:
-                if 'id' not in e.attrib:
+                self.report_error('Missing ID in f4m DRM')
-                    self.report_error('Missing ID in f4m DRM')
+        media = remove_encrypted_media(media)
-            media = remove_encrypted_media(media)
         if not media:
             self.report_error('Unsupported DRM')
         return media

yt_dlp/downloader/hls.py

@@ -15,6 +15,7 @@ from ..utils import (
     traverse_obj,
     update_url_query,
     urljoin,
+    deprecation_warning,
 )
 from ..utils._utils import _request_dump_filename
 
@@ -39,6 +40,8 @@ class HlsFD(FragmentFD):
 
     @classmethod
     def can_download(cls, manifest, info_dict, allow_unplayable_formats=False):
+        if allow_unplayable_formats:
+            deprecation_warning('allow_unplayable_formats is not supported', stacklevel=1)
         UNSUPPORTED_FEATURES = [
             # r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
 
@@ -58,17 +61,15 @@ class HlsFD(FragmentFD):
             # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
             # 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5
         ]
-        if not allow_unplayable_formats:
+        UNSUPPORTED_FEATURES += [
-            UNSUPPORTED_FEATURES += [
+            r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1], but not necessarily DRM
-                r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1], but not necessarily DRM
+        ]
-            ]
 
         def check_results():
             yield not info_dict.get('is_live')
             for feature in UNSUPPORTED_FEATURES:
                 yield not re.search(feature, manifest)
-            if not allow_unplayable_formats:
+            yield not cls._has_drm(manifest)
-                yield not cls._has_drm(manifest)
         return all(check_results())
 
     def real_download(self, filename, info_dict):
@@ -91,7 +92,7 @@ class HlsFD(FragmentFD):
                     outf.write(s_bytes)
             s = s_bytes.decode('utf-8', 'ignore')
 
-        can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
+        can_download, message = self.can_download(s, info_dict), None
         if can_download:
             has_ffmpeg = FFmpegFD.available()
             no_crypto = not Cryptodome.AES and '#EXT-X-KEY:METHOD=AES-128' in s
@@ -105,7 +106,7 @@ class HlsFD(FragmentFD):
                 message = ('Live HLS streams are not supported by the native downloader. If this is a livestream, '
                            f'please {install_ffmpeg}add "--downloader ffmpeg --hls-use-mpegts" to your command')
         if not can_download:
-            if self._has_drm(s) and not self.params.get('allow_unplayable_formats'):
+            if self._has_drm(s):
                 if info_dict.get('has_drm') and self.params.get('test'):
                     self.to_screen(f'[{self.FD_NAME}] This format is DRM protected', skip_eol=True)
                 else:

yt_dlp/extractor/hotstar.py

@@ -236,7 +236,7 @@ class HotStarIE(HotStarBaseIE):
             self._call_api_v1(
                 f'{video_type}/detail', video_id, fatal=False, query={'tas': 10000, 'contentId': video_id}),
             ('body', 'results', 'item', {dict})) or {}
-        if not self.get_param('allow_unplayable_formats') and video_data.get('drmProtected'):
+        if video_data.get('drmProtected'):
             self.report_drm(video_id)
 
         # See https://github.com/yt-dlp/yt-dlp/issues/396

yt_dlp/extractor/ivi.py

@@ -140,13 +140,14 @@ class IviIE(InfoExtractor):
         quality = qualities(self._KNOWN_FORMATS)
 
         formats = []
+        has_drm = False
         for f in result.get('files', []):
             f_url = f.get('url')
             content_format = f.get('content_format')
             if not f_url:
                 continue
-            if (not self.get_param('allow_unplayable_formats')
+            if '-MDRM-' in content_format or '-FPS-' in content_format:
-                    and ('-MDRM-' in content_format or '-FPS-' in content_format)):
+                has_drm = True
                 continue
             formats.append({
                 'url': f_url,
@@ -154,6 +155,8 @@ class IviIE(InfoExtractor):
                 'quality': quality(content_format),
                 'filesize': int_or_none(f.get('size_in_bytes')),
             })
+        if not formats and has_drm:
+            self.report_drm(video_id)
 
         compilation = result.get('compilation')
         episode = title if compilation else None

yt_dlp/extractor/kaltura.py

@@ -464,6 +464,7 @@ class KalturaIE(InfoExtractor):
 
         formats = []
         subtitles = {}
+        has_drm = False
         for f in flavor_assets:
             # Continue if asset is not ready
             if f.get('status') != 2:
@@ -473,7 +474,8 @@ class KalturaIE(InfoExtractor):
             if f.get('fileExt') == 'chun':
                 continue
             # DRM-protected video, cannot be decrypted
-            if not self.get_param('allow_unplayable_formats') and f.get('fileExt') == 'wvm':
+            if f.get('fileExt') == 'wvm':
+                has_drm = True
                 continue
             if not f.get('fileExt'):
                 # QT indicates QuickTime; some videos have broken fileExt
@@ -513,6 +515,9 @@ class KalturaIE(InfoExtractor):
             formats.extend(fmts)
             self._merge_subtitles(subs, target=subtitles)
 
+        if not formats and has_drm:
+            self.report_drm(entry_id)
+
         if captions:
             for caption in captions.get('objects', []):
                 # Continue if caption is not ready

yt_dlp/extractor/limelight.py

@@ -91,11 +91,13 @@ class LimelightBaseIE(InfoExtractor):
 
         formats = []
         urls = []
+        has_drm = False
         for stream in pc_item.get('streams', []):
             stream_url = stream.get('url')
             if not stream_url or stream_url in urls:
                 continue
-            if not self.get_param('allow_unplayable_formats') and stream.get('drmProtected'):
+            if stream.get('drmProtected'):
+                has_drm = True
                 continue
             urls.append(stream_url)
             ext = determine_ext(stream_url)
@@ -159,8 +161,8 @@ class LimelightBaseIE(InfoExtractor):
             format_id = mobile_url.get('targetMediaPlatform')
             if not media_url or media_url in urls:
                 continue
-            if (format_id in ('Widevine', 'SmoothStreaming')
+            if format_id in ('Widevine', 'SmoothStreaming'):
-                    and not self.get_param('allow_unplayable_formats', False)):
+                has_drm = True
                 continue
             urls.append(media_url)
             ext = determine_ext(media_url)
@@ -179,6 +181,9 @@ class LimelightBaseIE(InfoExtractor):
                     'ext': ext,
                 })
 
+        if not formats and has_drm:
+            self.report_drm(video_id)
+
         subtitles = {}
         for flag in mobile_item.get('flags'):
             if flag == 'ClosedCaptions':

yt_dlp/extractor/ninecninemedia.py

@@ -29,8 +29,7 @@ class NineCNineMediaIE(InfoExtractor):
                 '$include': '[HasClosedCaptions]',
             })
 
-        if (not self.get_param('allow_unplayable_formats')
+        if try_get(content_package, lambda x: x['Constraints']['Security']['Type']):
-                and try_get(content_package, lambda x: x['Constraints']['Security']['Type'])):
             self.report_drm(content_id)
 
         manifest_base_url = content_package_url + 'manifest.'

yt_dlp/extractor/nova.py

@@ -65,8 +65,7 @@ class NovaEmbedIE(InfoExtractor):
             for format_dict in format_list:
                 if not isinstance(format_dict, dict):
                     continue
-                if (not self.get_param('allow_unplayable_formats')
+                if traverse_obj(format_dict, ('drm', 'keySystem')):
-                        and traverse_obj(format_dict, ('drm', 'keySystem'))):
                     has_drm = True
                     continue
                 format_url = url_or_none(format_dict.get('src'))

yt_dlp/extractor/npo.py

@@ -275,9 +275,8 @@ class NPOIE(InfoExtractor):
                     'url': stream_url,
                 })
 
-        if not formats:
+        if not formats and drm:
-            if not self.get_param('allow_unplayable_formats') and drm:
+            self.report_drm(video_id)
-                self.report_drm(video_id)
 
         info = {
             'id': video_id,

yt_dlp/extractor/prosiebensat1.py

@@ -31,7 +31,7 @@ class ProSiebenSat1BaseIE(InfoExtractor):
                 'ids': clip_id,
             })[0]
 
-        if not self.get_param('allow_unplayable_formats') and video.get('is_protected') is True:
+        if video.get('is_protected') is True:
             self.report_drm(clip_id)
 
         formats = []

yt_dlp/extractor/redbee.py

@@ -337,12 +337,16 @@ class RTBFIE(RedBeeBaseIE):
                     'height': height,
                 })
 
+        has_drm = False
         mpd_url = None if data.get('isLive') else data.get('urlDash')
-        if mpd_url and (self.get_param('allow_unplayable_formats') or not data.get('drm')):
+        if mpd_url:
-            fmts, subs = self._extract_mpd_formats_and_subtitles(
+            if data.get('drm'):
-                mpd_url, media_id, mpd_id='dash', fatal=False)
+                has_drm = True
-            formats.extend(fmts)
+            else:
-            self._merge_subtitles(subs, target=subtitles)
+                fmts, subs = self._extract_mpd_formats_and_subtitles(
+                    mpd_url, media_id, mpd_id='dash', fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
 
         audio_url = data.get('urlAudio')
         if audio_url:
@@ -365,6 +369,9 @@ class RTBFIE(RedBeeBaseIE):
             formats.extend(fmts)
             self._merge_subtitles(subs, target=subtitles)
 
+        if not formats and has_drm:
+            self.report_drm(media_id)
+
         return {
             'id': media_id,
             'formats': formats,

yt_dlp/extractor/ruutu.py

@@ -237,8 +237,7 @@ class RuutuIE(InfoExtractor):
                 return value or None
 
         if not formats:
-            if (not self.get_param('allow_unplayable_formats')
+            if xpath_text(video_xml, './Clip/DRM', default=None):
-                    and xpath_text(video_xml, './Clip/DRM', default=None)):
                 self.report_drm(video_id)
             ns_st_cds = pv('ns_st_cds')
             if ns_st_cds != 'free':

yt_dlp/extractor/shahid.py

@@ -111,7 +111,7 @@ class ShahidIE(ShahidBaseIE):
         playout = self._call_api(
             'playout/new/url/' + video_id, video_id)['playout']
 
-        if not self.get_param('allow_unplayable_formats') and playout.get('drm'):
+        if playout.get('drm'):
             self.report_drm(video_id)
 
         formats = self._extract_m3u8_formats(re.sub(

yt_dlp/extractor/sonyliv.py

@@ -142,7 +142,7 @@ class SonyLIVIE(InfoExtractor):
         video_id = self._match_id(url)
         content = self._call_api(
             '1.5', 'IN/CONTENT/VIDEOURL/VOD/' + video_id, video_id)
-        if not self.get_param('allow_unplayable_formats') and content.get('isEncrypted'):
+        if content.get('isEncrypted'):
             self.report_drm(video_id)
         dash_url = content['videoURL']
         headers = {

yt_dlp/extractor/toggle.py

@@ -126,8 +126,7 @@ class ToggleIE(InfoExtractor):
                 })
         if not formats:
             for meta in (info.get('Metas') or []):
-                if (not self.get_param('allow_unplayable_formats')
+                if meta.get('Key') == 'Encryption' and meta.get('Value') == '1':
-                        and meta.get('Key') == 'Encryption' and meta.get('Value') == '1'):
                     self.report_drm(video_id)
             # Most likely because geo-blocked if no formats and no DRM
 

yt_dlp/extractor/toutv.py

@@ -68,7 +68,7 @@ class TouTvIE(RadioCanadaIE):  # XXX: Do not subclass from concrete IE
             })
         # IsDrm does not necessarily mean the video is DRM protected (see
         # https://github.com/ytdl-org/youtube-dl/issues/13994).
-        if not self.get_param('allow_unplayable_formats') and metadata.get('IsDrm'):
+        if metadata.get('IsDrm'):
             self.report_warning('This video is probably DRM protected.', path)
         video_id = metadata['IdMedia']
         details = metadata['Details']

yt_dlp/extractor/vidio.py

@@ -261,8 +261,7 @@ class VidioLiveIE(VidioBaseIE):
 
         formats = []
         if stream_meta.get('is_drm'):
-            if not self.get_param('allow_unplayable_formats'):
+            self.report_drm(video_id)
-                self.report_drm(video_id)
         if stream_meta.get('is_premium'):
             sources = self._download_json(
                 f'https://www.vidio.com/interactions_stream.json?video_id={video_id}&type=livestreamings',