[ie/soop] Support subscription-only VODs (#15523)

* Add custom downloader SoopVodFD

Closes #13636
Authored by: thematuu

yt_dlp/downloader/__init__.py

@@ -36,6 +36,7 @@ from .rtsp import RtspFD
 from .websocket import WebSocketFragmentFD
 from .youtube_live_chat import YoutubeLiveChatFD
 from .bunnycdn import BunnyCdnFD
+from .soop import SoopVodFD
 
 PROTOCOL_MAP = {
     'rtmp': RtmpFD,
@@ -56,6 +57,7 @@ PROTOCOL_MAP = {
     'youtube_live_chat': YoutubeLiveChatFD,
     'youtube_live_chat_replay': YoutubeLiveChatFD,
     'bunnycdn': BunnyCdnFD,
+    'soopvod': SoopVodFD,
 }
 
 

yt_dlp/downloader/soop.py

@@ -0,0 +1,61 @@
+import threading
+import time
+
+from .common import FileDownloader
+from . import HlsFD
+from ..extractor.afreecatv import _cloudfront_auth_request
+from ..networking.exceptions import network_exceptions
+
+
+class SoopVodFD(FileDownloader):
+    """
+    Downloads Soop subscription VODs with required cookie refresh requests
+    Note, this is not a part of public API, and will be removed without notice.
+    DO NOT USE
+    """
+
+    def real_download(self, filename, info_dict):
+        self.to_screen(f'[{self.FD_NAME}] Downloading Soop subscription VOD HLS')
+        fd = HlsFD(self.ydl, self.params)
+        refresh_params = info_dict['_cookie_refresh_params']
+        referer_url = info_dict['webpage_url']
+
+        stop_event = threading.Event()
+        refresh_thread = threading.Thread(
+            target=self._cookie_refresh_thread,
+            args=(stop_event, refresh_params, referer_url),
+        )
+        refresh_thread.start()
+
+        try:
+            return fd.real_download(filename, info_dict)
+        finally:
+            stop_event.set()
+
+    def _cookie_refresh_thread(self, stop_event, refresh_params, referer_url):
+        m3u8_url = refresh_params['m3u8_url']
+        strm_id = refresh_params['strm_id']
+        video_id = refresh_params['video_id']
+
+        def _get_cloudfront_cookie_expiration(m3u8_url):
+            cookies = self.ydl.cookiejar.get_cookies_for_url(m3u8_url)
+            return min((cookie.expires for cookie in cookies if 'CloudFront' in cookie.name and cookie.expires), default=0)
+
+        while not stop_event.wait(5):
+            current_time = time.time()
+            expiration_time = _get_cloudfront_cookie_expiration(m3u8_url)
+            last_refresh_check = refresh_params.get('_last_refresh', 0)
+
+            # Cookie TTL is 90 seconds, but let's give ourselves a 15-second cushion
+            should_refresh = (
+                (expiration_time and current_time >= expiration_time - 15)
+                or (not expiration_time and current_time - last_refresh_check >= 75)
+            )
+
+            if should_refresh:
+                try:
+                    self.ydl.urlopen(_cloudfront_auth_request(
+                        m3u8_url, strm_id, video_id, referer_url)).read()
+                    refresh_params['_last_refresh'] = current_time
+                except network_exceptions as e:
+                    self.to_screen(f'[{self.FD_NAME}] Cookie refresh attempt failed: {e}')

yt_dlp/extractor/afreecatv.py

@@ -1,5 +1,6 @@
 import datetime as dt
 import functools
+import time
 
 from .common import InfoExtractor
 from ..networking import Request
@@ -16,7 +17,23 @@ from ..utils import (
     urlencode_postdata,
     urljoin,
 )
-from ..utils.traversal import traverse_obj
+from ..utils.traversal import require, traverse_obj
+
+
+def _cloudfront_auth_request(m3u8_url, strm_id, video_id, referer_url):
+    return Request(
+        'https://live.sooplive.co.kr/api/private_auth.php',
+        method='POST',
+        headers={
+            'Referer': referer_url,
+            'Origin': 'https://vod.sooplive.co.kr',
+        },
+        data=urlencode_postdata({
+            'type': 'vod',
+            'strm_id': strm_id,
+            'title_no': video_id,
+            'url': m3u8_url,
+        }))
 
 
 class AfreecaTVBaseIE(InfoExtractor):
@@ -153,6 +170,13 @@ class AfreecaTVIE(AfreecaTVBaseIE):
                 'nApiLevel': 10,
             }))['data']
 
+        initial_refresh_time = 0
+        strm_id = None
+        # For subscriber-only VODs, we need to call private_auth.php to get CloudFront cookies
+        needs_private_auth = traverse_obj(data, ('sub_upload_type', {str}))
+        if needs_private_auth:
+            strm_id = traverse_obj(data, ('bj_id', {str}, {require('stream ID')}))
+
         error_code = traverse_obj(data, ('code', {int}))
         if error_code == -6221:
             raise ExtractorError('The VOD does not exist', expected=True)
@@ -172,9 +196,23 @@ class AfreecaTVIE(AfreecaTVBaseIE):
                 traverse_obj(data, ('files', lambda _, v: url_or_none(v['file']))), start=1):
             file_url = file_element['file']
             if determine_ext(file_url) == 'm3u8':
+                if needs_private_auth:
+                    self._request_webpage(
+                        _cloudfront_auth_request(file_url, strm_id, video_id, url),
+                        video_id, 'Requesting CloudFront cookies', 'Failed to get CloudFront cookies')
+                    initial_refresh_time = time.time()
                 formats = self._extract_m3u8_formats(
                     file_url, video_id, 'mp4', m3u8_id='hls',
                     note=f'Downloading part {file_num} m3u8 information')
+                if needs_private_auth:
+                    for fmt in formats:
+                        fmt['protocol'] = 'soopvod'
+                        fmt['_cookie_refresh_params'] = {
+                            'm3u8_url': file_url,
+                            'strm_id': strm_id,
+                            'video_id': video_id,
+                            '_last_refresh': initial_refresh_time,
+                        }
             else:
                 formats = [{
                     'url': file_url,

yt_dlp/extractor/whyp.py

@@ -2,6 +2,7 @@ from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
+    parse_iso8601,
     str_or_none,
     traverse_obj,
     url_or_none,
@@ -16,9 +17,12 @@ class WhypIE(InfoExtractor):
         'info_dict': {
             'id': '18337',
             'title': 'Example Track',
+            'display_id': 'example-track',
             'description': 'md5:e0b1bcf1d267dc1a0f15efff09c8f297',
             'ext': 'flac',
             'duration': 135.63,
+            'timestamp': 1643216583,
+            'upload_date': '20220126',
             'uploader': 'Brad',
             'uploader_id': '1',
             'thumbnail': 'https://cdn.whyp.it/6ad0bbd9-577d-42bb-9b61-2a4f57f647eb.jpg',
@@ -44,10 +48,12 @@ class WhypIE(InfoExtractor):
                 'http_headers': {'Referer': 'https://whyp.it/'},
             } for prefix in ('audio', 'lossy', 'lossless') if url_or_none(data.get(f'{prefix}_url'))],
             **traverse_obj(data, {
-                'title': 'title',
+                'title': ('title', {str}),
+                'display_id': ('slug', {str}),
                 'description': 'description',
                 'duration': ('duration', {float_or_none}),
-                'uploader': ('user', 'username'),
+                'timestamp': ('created_at', {parse_iso8601}),
+                'uploader': ('user', 'username', {str}),
                 'uploader_id': ('user', 'id', {str_or_none}),
                 'thumbnail': ('artwork_url', {url_or_none}),
             }),