Merge 93c4a6bf26 into 1ae0f45b0e

spec/streaming_fallback_spec.cr

@@ -0,0 +1,94 @@
+require "spectator"
+
+# Bring in the helper under test
+require "../src/invidious/videos/parser.cr"
+
+Spectator.describe Invidious::Videos::ParserHelpers do
+  def json_any_hash(h : Hash(String, JSON::Any))
+    h
+  end
+
+  def json_any_array(a : Array(JSON::Any))
+    JSON::Any.new(a)
+  end
+
+  def json_any_str(s : String)
+    JSON::Any.new(s)
+  end
+
+  def json_any_obj(h : Hash(String, JSON::Any))
+    JSON::Any.new(h)
+  end
+
+  it "patches formats when primary missing and fallback has usable formats" do
+    primary_sd = {
+      "formats"         => JSON::Any.new([] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([] of JSON::Any),
+    } of String => JSON::Any
+
+    fallback_sd = {
+      "formats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://example.com/video.mp4")}),
+      ] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://example.com/audio.m4a")}),
+      ] of JSON::Any),
+    } of String => JSON::Any
+
+    res = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(primary_sd, fallback_sd)
+
+    expect(res[:patched_formats]).to be_true
+    expect(res[:patched_adaptive]).to be_true
+
+    # Ensure formats now have a non-empty URL
+    first_fmt = primary_sd["formats"].as_a[0].as_h
+    expect(first_fmt["url"].as_s).to_not be_empty
+  end
+
+  it "does not overwrite valid primary data" do
+    primary_sd = {
+      "formats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://primary/video.mp4")}),
+      ] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://primary/audio.m4a")}),
+      ] of JSON::Any),
+    } of String => JSON::Any
+
+    fallback_sd = {
+      "formats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://fallback/video.mp4")}),
+      ] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://fallback/audio.m4a")}),
+      ] of JSON::Any),
+    } of String => JSON::Any
+
+    res = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(primary_sd, fallback_sd)
+
+    expect(res[:patched_formats]).to be_false
+    expect(res[:patched_adaptive]).to be_false
+
+    # Primary values should remain
+    expect(primary_sd["formats"].as_a[0].as_h["url"].as_s).to eq("https://primary/video.mp4")
+    expect(primary_sd["adaptiveFormats"].as_a[0].as_h["url"].as_s).to eq("https://primary/audio.m4a")
+  end
+
+  it "handles fallback without formats gracefully" do
+    primary_sd = {
+      "formats"         => JSON::Any.new([] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([] of JSON::Any),
+    } of String => JSON::Any
+
+    fallback_sd = {
+      "adaptiveFormats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://example.com/audio.m4a")}),
+      ] of JSON::Any),
+    } of String => JSON::Any
+
+    res = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(primary_sd, fallback_sd)
+
+    expect(res[:patched_adaptive]).to be_true
+    expect(res[:patched_formats]).to be_false
+  end
+end

src/invidious/routes/video_playback.cr

@@ -299,6 +299,8 @@ module Invidious::Routes::VideoPlayback
     url = fmt.try &.["url"]?.try &.as_s
 
     if !url
+      # Extra context for debugging playback errors
+      LOGGER.warn("playback_404: no URL for id=#{id} itag=#{itag.inspect} fmt=#{video.fmt_stream.size} adaptive=#{video.adaptive_fmts.size}")
       haltf env, status_code: 404
     end
 

src/invidious/videos/parser.cr

@@ -58,6 +58,59 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
   }
 end
 
+# Small helper utilities extracted for testability and clarity (per LLVM policy)
+module Invidious::Videos::ParserHelpers
+  extend self
+
+  # True if any entry has a usable URL or signatureCipher/cipher that can be converted to a URL
+  def has_usable_stream?(container : JSON::Any?) : Bool
+    return false unless container
+    arr = container.as_a?
+    return false unless arr && arr.size > 0
+    arr.each do |entry|
+      obj = entry.as_h?
+      next unless obj
+      # Accept direct URL
+      if (u = obj["url"]?.try &.as_s?) && !u.empty?
+        return true
+      end
+      # Accept ciphered URL that convert_url can handle
+      if (sc = obj["signatureCipher"]?.try &.as_s?) && !sc.empty?
+        return true
+      end
+      if (c = obj["cipher"]?.try &.as_s?) && !c.empty?
+        return true
+      end
+    end
+    false
+  end
+
+  # Mutates streaming_data by patching only the sections missing a usable URL
+  # Returns flags indicating what was patched.
+  def patch_streaming_data_if_missing!(streaming_data : Hash(String, JSON::Any), fallback_sd : Hash(String, JSON::Any)) : NamedTuple(patched_formats: Bool, patched_adaptive: Bool)
+    patched_formats = false
+    patched_adaptive = false
+
+    # Adaptive formats
+    unless has_usable_stream?(streaming_data["adaptiveFormats"]?)
+      if has_usable_stream?(fallback_sd["adaptiveFormats"]?)
+        streaming_data["adaptiveFormats"] = fallback_sd["adaptiveFormats"]
+        patched_adaptive = true
+      end
+    end
+
+    # Progressive formats
+    unless has_usable_stream?(streaming_data["formats"]?)
+      if has_usable_stream?(fallback_sd["formats"]?)
+        streaming_data["formats"] = fallback_sd["formats"]
+        patched_formats = true
+      end
+    end
+
+    {patched_formats: patched_formats, patched_adaptive: patched_adaptive}
+  end
+end
+
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
@@ -109,8 +162,14 @@ def extract_video_info(video_id : String)
   params["reason"] = JSON::Any.new(reason) if reason
 
   if !CONFIG.invidious_companion.present?
-    if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
-      LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
+    # Fix for issue #5420: /api/v1/videos/<video_id> endpoint has formatStreams blank
+    # Determine if we are missing URLs for either adaptive or progressive formats
+    need_adaptive = player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
+    need_formats = player_response.dig?("streamingData", "formats", 0, "url").nil?
+
+    if need_adaptive || need_formats
+      missing = [need_adaptive ? "adaptiveFormats" : nil, need_formats ? "formats" : nil].compact.join(", ")
+      LOGGER.warn("Missing URLs for #{missing}, falling back to other YT clients.")
       players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile}
 
       players_fallback.each do |player_fallback|
@@ -118,9 +177,13 @@ def extract_video_info(video_id : String)
 
         next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))
 
-        if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
-          streaming_data = player_response["streamingData"].as_h
-          streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
+        # Only patch what is actually missing to avoid downgrading good data
+        streaming_data = player_response["streamingData"].as_h
+        fallback_sd = player_fallback_response["streamingData"].as_h
+        patched = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(streaming_data, fallback_sd)
+
+        if patched[:patched_adaptive] || patched[:patched_formats]
+          LOGGER.debug("fallback_patched: client=#{player_fallback} video=#{video_id} patched_adaptive=#{patched[:patched_adaptive]} patched_formats=#{patched[:patched_formats]}")
           player_response["streamingData"] = JSON::Any.new(streaming_data)
           break
         end
@@ -479,14 +542,25 @@ private def convert_url(fmt)
 
     LOGGER.debug("convert_url: Decoding '#{cfr}'")
 
-    unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
+    # When using Invidious Companion, streaming URLs should already be usable.
+    # Skip inv-sig-helper based signature decryption in that case.
+    unsig = if CONFIG.invidious_companion.present?
+              nil
+            else
+              DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
+            end
     params[sp] = unsig if unsig
   else
     url = URI.parse(fmt["url"].as_s)
     params = url.query_params
   end
 
-  n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
+  # Skip nsig decryption when using Invidious Companion
+  n = if CONFIG.invidious_companion.present?
+        nil
+      else
+        DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
+      end
   params["n"] = n if n
 
   if token = CONFIG.po_token

src/invidious/yt_backend/youtube_api.cr

@@ -473,8 +473,12 @@ module YoutubeAPI
     } of String => String | Int64
 
     if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s }
-      if sts = DECRYPT_FUNCTION.try &.get_sts
-        playback_ctx["signatureTimestamp"] = sts.to_i64
+      # When using Invidious Companion, player requests are proxied there,
+      # so we skip fetching signatureTimestamp via inv-sig-helper.
+      if !CONFIG.invidious_companion.present?
+        if sts = DECRYPT_FUNCTION.try &.get_sts
+          playback_ctx["signatureTimestamp"] = sts.to_i64
+        end
       end
     end