From 2d1617a46a3191093b5d9fdeb170d2281d993dda Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 22 Sep 2023 10:34:26 -0400 Subject: [PATCH 1/9] Add option for HLS quality in user preferences --- config/config.example.yml | 2 +- locales/en-US.json | 1 + src/invidious/routes/api/manifest.cr | 2 +- src/invidious/routes/watch.cr | 6 ++++- src/invidious/videos.cr | 12 ++++----- src/invidious/videos/parser.cr | 33 ++++++++++++++++------- src/invidious/views/components/player.ecr | 4 ++- src/invidious/views/user/preferences.ecr | 2 +- 8 files changed, 41 insertions(+), 21 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index a3a2eeb7..bb8b5ff2 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -796,7 +796,7 @@ default_user_preferences: ## ## Default video quality. ## - ## Accepted values: dash, hd720, medium, small + ## Accepted values: hls, dash, hd720, medium, small ## Default: hd720 ## #quality: hd720 diff --git a/locales/en-US.json b/locales/en-US.json index c23f6bc3..ae3b4f50 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -81,6 +81,7 @@ "preferences_speed_label": "Default speed: ", "preferences_quality_label": "Preferred video quality: ", "preferences_quality_option_dash": "DASH (adaptive quality)", + "preferences_quality_option_hls": "HLS", "preferences_quality_option_hd720": "HD720", "preferences_quality_option_medium": "Medium", "preferences_quality_option_small": "Small", diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index d89e752c..41aa434e 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -177,7 +177,7 @@ module Invidious::Routes::API::Manifest manifest = response.body if local - manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match| + manifest = manifest.gsub(/^(https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*)|(https:\/\/\w+---.{11}\.googlevideo\.com[^\n]*)/m) do |match| path = URI.parse(match).path path = path.lchop("/videoplayback/") diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index aabe8dfc..634b4cee 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -52,7 +52,11 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - video = get_video(id, region: params.region) + if params.quality == "hls" + video = get_video(id, region: params.region, force_hls: true) + else + video = get_video(id, region: params.region) + end rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index ae09e736..e8165f84 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -294,8 +294,8 @@ struct Video predicate_bool upcoming, isUpcoming end -def get_video(id, refresh = true, region = nil, force_refresh = false) - if (video = Invidious::Database::Videos.select(id)) && !region +def get_video(id, refresh = true, region = nil, force_hls = false, force_refresh = false) + if (video = Invidious::Database::Videos.select(id)) && !region && !force_hls # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) if (refresh && @@ -312,8 +312,8 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) end end else - video = fetch_video(id, region) - Invidious::Database::Videos.insert(video) if !region + video = fetch_video(id, region, force_hls) + Invidious::Database::Videos.insert(video) if !region && !force_hls end return video @@ -323,8 +323,8 @@ rescue DB::Error return fetch_video(id, region) end -def fetch_video(id, region) - info = extract_video_info(video_id: id) +def fetch_video(id, region, force_hls = false) + info = extract_video_info(video_id: id, force_hls: force_hls) if reason = info["reason"]? if reason == "Video unavailable" diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 915c9baf..7b6605f3 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -50,7 +50,7 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? } end -def extract_video_info(video_id : String) +def extract_video_info(video_id : String, force_hls : Bool = false) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new @@ -101,16 +101,29 @@ def extract_video_info(video_id : String) params["reason"] = JSON::Any.new(reason) if reason new_player_response = nil - - # Don't use Android test suite client if po_token is passed because po_token doesn't - # work for Android test suite client. - if reason.nil? && CONFIG.po_token.nil? - # Fetch the video streams using an Android client in order to get the - # decrypted URLs and maybe fix throttling issues (#2194). See the - # following issue for an explanation about decrypted URLs: - # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite + if force_hls + client_config.client_type = YoutubeAPI::ClientType::IOS new_player_response = try_fetch_streaming_data(video_id, client_config) + else + # Don't use Android test suite client if po_token is passed because po_token doesn't + # work for Android test suite client. + if reason.nil? && CONFIG.po_token.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite + new_player_response = try_fetch_streaming_data(video_id, client_config) + else + if reason.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite + new_player_response = try_fetch_streaming_data(video_id, client_config) + end + end end # Replace player response and reset reason diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 5c28358b..b9b29601 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -4,8 +4,10 @@ <% if params.autoplay %>autoplay<% end %> <% if params.video_loop %>loop<% end %> <% if params.controls %>controls<% end %>> - <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %> + <% if (hlsvp = video.hls_manifest_url) && video.live_now && !CONFIG.disabled?("livestreams") %> + <% elsif (hlsvp = video.hls_manifest_url) && params.quality == "hls" %> + <% else %> <% if params.listen %> <% # default to 128k m4a stream diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index cf8b5593..11f7890b 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -54,7 +54,7 @@
<% {"hls", "dash", "hd720", "medium", "small"}.each do |option| %> - <% if !(option == "dash" && CONFIG.disabled?("dash")) %> - - <% end %> + <% next if (option == "dash" && CONFIG.disabled?("dash"))%> + <% next if (option == "hls" && CONFIG.disabled?("hls"))%> + <% end %>
From b34c79a6ea98e3557d7e5e1a7ecc4c93f41aa132 Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Tue, 14 Nov 2023 20:34:08 +0000 Subject: [PATCH 4/9] Fix typo in disabled quality fallback Co-authored-by: Samantaz Fox --- src/invidious/videos/video_preferences.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr index b8f8acf9..79b510d8 100644 --- a/src/invidious/videos/video_preferences.cr +++ b/src/invidious/videos/video_preferences.cr @@ -107,7 +107,7 @@ def process_video_params(query, preferences) # Force set quality to "high" if dash or hls has been disabled by the server {"dash", "hls"}.each do |disabled_quality| - if CONFIG.disabled?("dash") && quality == "dash" + if CONFIG.disabled?(disabled_quality) && quality == disabled_quality quality = "high" break end From 136bea11df668d2f39d68977ddde6e239c232812 Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Tue, 14 Nov 2023 20:35:28 +0000 Subject: [PATCH 5/9] Update src/invidious/routes/watch.cr Co-authored-by: Samantaz Fox --- src/invidious/routes/watch.cr | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 634b4cee..00069545 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -52,11 +52,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - if params.quality == "hls" - video = get_video(id, region: params.region, force_hls: true) - else - video = get_video(id, region: params.region) - end + video = get_video(id, region: params.region, force_hls: (params.quality == "hls")) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) From 978e9f36efc273e370f1cae15635fc65686d0d7b Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 14 Nov 2023 12:49:53 -0800 Subject: [PATCH 6/9] Add translations for video quality labels --- locales/en-US.json | 8 +++++++- src/invidious/views/components/player.ecr | 8 ++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/locales/en-US.json b/locales/en-US.json index ae3b4f50..82f933bb 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -499,5 +499,11 @@ "toggle_theme": "Toggle Theme", "carousel_slide": "Slide {{current}} of {{total}}", "carousel_skip": "Skip the Carousel", - "carousel_go_to": "Go to slide `x`" + "carousel_go_to": "Go to slide `x`", + "video_quality_livestream_label": "Livestreams", + "video_quality_hls_label": "HLS", + "video_quality_dash_label": "DASH", + "video_quality_hd720_label": "hd720", + "video_quality_medium_label": "medium", + "video_quality_small_label": "small" } diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 4f1ab518..d75202cb 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -5,9 +5,9 @@ <% if params.video_loop %>loop<% end %> <% if params.controls %>controls<% end %>> <% if (hlsvp = video.hls_manifest_url) && video.live_now && !CONFIG.disabled?("livestreams") %> - + "> <% elsif (hlsvp = video.hls_manifest_url) && params.quality == "hls" && !CONFIG.disabled?("hls") %> - + "> <% else %> <% if params.listen %> <% # default to 128k m4a stream @@ -37,7 +37,7 @@ <% end %> <% else %> <% if params.quality == "dash" %> - + "> <% end %> <% @@ -52,7 +52,7 @@ selected = params.quality ? (params.quality == quality) : (i == 0) %> - + " selected="<%= selected %>"> <% if !params.local && !CONFIG.disabled?("local") %> <% end %> From fea391ddf1565a01de183bc2d5eb7961b734b5f5 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 14 Nov 2023 12:55:15 -0800 Subject: [PATCH 7/9] typo --- locales/en-US.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en-US.json b/locales/en-US.json index 82f933bb..3693f150 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -500,7 +500,7 @@ "carousel_slide": "Slide {{current}} of {{total}}", "carousel_skip": "Skip the Carousel", "carousel_go_to": "Go to slide `x`", - "video_quality_livestream_label": "Livestreams", + "video_quality_livestream_label": "Livestream", "video_quality_hls_label": "HLS", "video_quality_dash_label": "DASH", "video_quality_hd720_label": "hd720", From 122c8598bac003afd5993ae51953912ecb0314e4 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 18 Feb 2024 17:55:31 -0800 Subject: [PATCH 8/9] VideoJS: Don't override native player on Safari --- assets/js/player.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/js/player.js b/assets/js/player.js index 353a5296..3f6d8ff3 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -25,7 +25,7 @@ var options = { html5: { preloadTextTracks: false, vhs: { - overrideNative: true + overrideNative: !videojs.browser.IS_ANY_SAFARI } } }; From ffc44b0cff507206880365b17cf09e308f3f18ae Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 10 Nov 2024 23:19:11 -0800 Subject: [PATCH 9/9] Add support for caching IOS client player requests When a client requests HLS streams, Invidious will first check the database to see if the cached video has any HLS streams. If not we request the IOS client and update the streamingData field with the now gotten HLS manifest data. Afterwards, we update the cached video in the database. --- src/invidious/routes/watch.cr | 2 +- src/invidious/videos.cr | 25 ++++++++++++++----- src/invidious/videos/parser.cr | 44 ++++++++++++++++++++++------------ 3 files changed, 49 insertions(+), 22 deletions(-) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 00069545..d0fcf662 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -52,7 +52,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - video = get_video(id, region: params.region, force_hls: (params.quality == "hls")) + video = get_video(id, region: params.region, get_hls: (params.quality == "hls")) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index e8165f84..6787b957 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -294,8 +294,8 @@ struct Video predicate_bool upcoming, isUpcoming end -def get_video(id, refresh = true, region = nil, force_hls = false, force_refresh = false) - if (video = Invidious::Database::Videos.select(id)) && !region && !force_hls +def get_video(id, refresh = true, region = nil, get_hls = false, force_refresh = false) + if (video = Invidious::Database::Videos.select(id)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) if (refresh && @@ -312,8 +312,21 @@ def get_video(id, refresh = true, region = nil, force_hls = false, force_refresh end end else - video = fetch_video(id, region, force_hls) - Invidious::Database::Videos.insert(video) if !region && !force_hls + video = fetch_video(id, region) + Invidious::Database::Videos.insert(video) if !region + end + + # The video object we got above could be from a previous request that was not + # done through the IOS client. If the users wants HLS we should check if + # a manifest exists in the data returned. If not we will rerequest one. + if get_hls && !video.hls_manifest_url + begin + video_with_hls_data = update_video_object_with_hls_data(id, video) + return video if !video_with_hls_data + Invidious::Database::Videos.update(video_with_hls_data) if !region + rescue ex + # Use old database video if IOS client request fails + end end return video @@ -323,8 +336,8 @@ rescue DB::Error return fetch_video(id, region) end -def fetch_video(id, region, force_hls = false) - info = extract_video_info(video_id: id, force_hls: force_hls) +def fetch_video(id, region) + info = extract_video_info(video_id: id) if reason = info["reason"]? if reason == "Video unavailable" diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 7b6605f3..1dba5440 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -50,7 +50,7 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? } end -def extract_video_info(video_id : String, force_hls : Bool = false) +def extract_video_info(video_id : String) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new @@ -101,28 +101,24 @@ def extract_video_info(video_id : String, force_hls : Bool = false) params["reason"] = JSON::Any.new(reason) if reason new_player_response = nil - if force_hls - client_config.client_type = YoutubeAPI::ClientType::IOS + + # Don't use Android test suite client if po_token is passed because po_token doesn't + # work for Android test suite client. + if reason.nil? && CONFIG.po_token.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite new_player_response = try_fetch_streaming_data(video_id, client_config) else - # Don't use Android test suite client if po_token is passed because po_token doesn't - # work for Android test suite client. - if reason.nil? && CONFIG.po_token.nil? + if reason.nil? # Fetch the video streams using an Android client in order to get the # decrypted URLs and maybe fix throttling issues (#2194). See the # following issue for an explanation about decrypted URLs: # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite new_player_response = try_fetch_streaming_data(video_id, client_config) - else - if reason.nil? - # Fetch the video streams using an Android client in order to get the - # decrypted URLs and maybe fix throttling issues (#2194). See the - # following issue for an explanation about decrypted URLs: - # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite - new_player_response = try_fetch_streaming_data(video_id, client_config) - end end end @@ -157,6 +153,24 @@ def extract_video_info(video_id : String, force_hls : Bool = false) return params end +def update_video_object_with_hls_data(id : String, video : Video) + client_config = YoutubeAPI::ClientConfig.new(client_type: YoutubeAPI::ClientType::IOS) + + new_player_response = try_fetch_streaming_data(id, client_config) + current_streaming_data = video.info["streamingData"].try &.as_h + + return nil if !new_player_response + + if current_streaming_data && (manifest = new_player_response.dig?("streamingData", "hlsManifestUrl")) + current_streaming_data["hlsManifestUrl"] = JSON::Any.new(manifest.as_s) + video.info["streamingData"] = JSON::Any.new(current_streaming_data) + + return video + end + + return nil +end + def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)