Merge remote-tracking branch 'upstream/master'

src/invidious.cr

@@ -75,14 +75,19 @@ end
 
 HMAC_KEY = CONFIG.hmac_key
 
-PG_DB = DB.open CONFIG.database_url
-
-ARCHIVE_URL     = URI.parse("https://archive.org")
-PUBSUB_URL      = URI.parse("https://pubsubhubbub.appspot.com")
-REDDIT_URL      = URI.parse("https://www.reddit.com")
-YT_URL          = URI.parse("https://www.youtube.com")
+PG_DB = begin
+  DB.open CONFIG.database_url
+rescue ex
+  puts "Failed to connect to PostgreSQL database: #{ex.cause.try &.message}"
+  puts "Check your 'config.yml' database settings or PostgreSQL settings."
+  exit(1)
+end
+ARCHIVE_URL = URI.parse("https://archive.org")
+PUBSUB_URL  = URI.parse("https://pubsubhubbub.appspot.com")
+REDDIT_URL  = URI.parse("https://www.reddit.com")
+YT_URL      = URI.parse("https://www.youtube.com")
 PUBSUB_HOST_URL = CONFIG.pubsub_domain
-HOST_URL        = make_host_url(Kemal.config)
+HOST_URL    = make_host_url(Kemal.config)
 
 CHARS_SAFE         = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
 TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@@ -249,8 +254,8 @@ error 404 do |env|
   Invidious::Routes::ErrorRoutes.error_404(env)
 end
 
-error 500 do |env, ex|
-  error_template(500, ex)
+error 500 do |env, exception|
+  error_template(500, exception)
 end
 
 static_headers do |env|

src/invidious/channels/community.cr

@@ -3,8 +3,8 @@ private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000}
 # TODO: Add "sort_by"
 def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
   if cursor.nil?
-    # Egljb21tdW5pdHk%3D is the protobuf object to load "community"
-    initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D")
+    # EgVwb3N0c_IGBAoCSgA%3D is the protobuf object to load "posts"
+    initial_data = YoutubeAPI.browse(ucid, params: "EgVwb3N0c_IGBAoCSgA%3D")
 
     items = [] of JSON::Any
     extract_items(initial_data) do |item|
@@ -24,15 +24,21 @@ def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
   return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode)
 end
 
+def decode_ucid_from_post_protobuf(params)
+  decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
+    .try { |i| Base64.decode(i) }
+    .try { |i| IO::Memory.new(i) }
+    .try { |i| Protodec::Any.parse(i) }
+
+  return decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s)
+end
+
 def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode)
   object = {
-    "2:string"    => "community",
-    "25:embedded" => {
-      "22:string" => post_id.to_s,
-    },
-    "45:embedded" => {
-      "2:varint" => 1_i64,
-      "3:varint" => 1_i64,
+    "56:embedded" => {
+      "2:string"  => ucid,
+      "3:string"  => post_id.to_s,
+      "11:string" => ucid,
     },
   }
   params = object.try { |i| Protodec::Any.cast_json(i) }
@@ -40,7 +46,7 @@ def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode)
     .try { |i| Base64.urlsafe_encode(i) }
     .try { |i| URI.encode_www_form(i) }
 
-  initial_data = YoutubeAPI.browse(ucid, params: params)
+  initial_data = YoutubeAPI.browse("FEpost_detail", params: params)
 
   items = [] of JSON::Any
   extract_items(initial_data) do |item|

src/invidious/channels/playlists.cr

@@ -6,19 +6,19 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
       case sort_by
       when "last", "last_added"
         # Equivalent to "&sort=lad"
-        # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1}
-        "EglwbGF5bGlzdHMYBCABMAE%3D"
+        # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
+        "EglwbGF5bGlzdHMYBCABMAHyBgQKAkIA"
       when "oldest", "oldest_created"
         # formerly "&sort=da"
         # Not available anymore :c or maybe ??
-        # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1}
-        "EglwbGF5bGlzdHMYAiABMAE%3D"
+        # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
+        "EglwbGF5bGlzdHMYAiABMAHyBgQKAkIA"
         # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
         # "EglwbGF5bGlzdHMYASABMAE%3D"
       when "newest", "newest_created"
         # Formerly "&sort=dd"
-        # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1}
-        "EglwbGF5bGlzdHMYAyABMAE%3D"
+        # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
+        "EglwbGF5bGlzdHMYAyABMAHyBgQKAkIA"
       end
 
     initial_data = YoutubeAPI.browse(ucid, params: params || "")

src/invidious/comments/youtube.cr

@@ -16,23 +16,27 @@ module Invidious::Comments
     return parse_youtube(id, response, format, locale, thin_mode, sort_by)
   end
 
-  def fetch_community_post_comments(ucid, post_id)
+  def fetch_community_post_comments(ucid, post_id, sort_by = "top")
+    case sort_by
+    when "top"
+      sort_by_val = 0_i64
+    when "new", "newest"
+      sort_by_val = 1_i64
+    else # top
+      sort_by_val = 0_i64
+    end
+
     object = {
-      "2:string"    => "community",
-      "25:embedded" => {
-        "22:string" => post_id,
-      },
-      "45:embedded" => {
-        "2:varint" => 1_i64,
-        "3:varint" => 1_i64,
-      },
+      "2:string"    => "posts",
       "53:embedded" => {
         "4:embedded" => {
-          "6:varint"  => 0_i64,
-          "27:varint" => 1_i64,
+          "6:varint"  => sort_by_val,
+          "15:varint" => 2_i64,
+          "25:varint" => 0_i64,
           "29:string" => post_id,
           "30:string" => ucid,
         },
+        "7:varint" => 0_i64,
         "8:string" => "comments-section",
       },
     }
@@ -43,7 +47,7 @@ module Invidious::Comments
 
     object2 = {
       "80226972:embedded" => {
-        "2:string" => ucid,
+        "2:string" => "FEcomment_post_detail_page_web_top_level",
         "3:string" => object_parsed,
       },
     }
@@ -320,6 +324,15 @@ module Invidious::Comments
   end
 
   def produce_continuation(video_id, cursor = "", sort_by = "top")
+    case sort_by
+    when "top"
+      sort_by_val = 0_i64
+    when "new", "newest"
+      sort_by_val = 1_i64
+    else # top
+      sort_by_val = 0_i64
+    end
+
     object = {
       "2:embedded" => {
         "2:string"    => video_id,
@@ -340,21 +353,12 @@ module Invidious::Comments
         "1:string"   => cursor,
         "4:embedded" => {
           "4:string" => video_id,
-          "6:varint" => 0_i64,
+          "6:varint" => sort_by_val,
         },
         "5:varint" => 20_i64,
       },
     }
 
-    case sort_by
-    when "top"
-      object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
-    when "new", "newest"
-      object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64
-    else # top
-      object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
-    end
-
     continuation = object.try { |i| Protodec::Any.cast_json(i) }
       .try { |i| Protodec::Any.from_json(i) }
       .try { |i| Base64.urlsafe_encode(i) }

src/invidious/frontend/watch_page.cr

@@ -35,7 +35,7 @@ module Invidious::Frontend::WatchPage
       str << " class=\"pure-form pure-form-stacked\""
       str << " action='#{url}'"
       str << " method='post'"
-      str << " rel='noopener'"
+      str << " rel='noopener noreferrer'"
       str << " target='_blank'>"
       str << '\n'
 

src/invidious/routes/api/v1/channels.cr

@@ -436,7 +436,7 @@ module Invidious::Routes::API::V1::Channels
     if ucid.nil?
       response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
       return error_json(400, "Invalid post ID") if response["error"]?
-      ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+      ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s)
     else
       ucid = ucid.to_s
     end
@@ -460,13 +460,15 @@ module Invidious::Routes::API::V1::Channels
 
     format = env.params.query["format"]?
     format ||= "json"
+    sort_by = env.params.query["sort_by"]?.try &.downcase
+    sort_by ||= "top"
 
     continuation = env.params.query["continuation"]?
 
     case continuation
     when nil, ""
       ucid = env.params.query["ucid"]
-      comments = Comments.fetch_community_post_comments(ucid, id)
+      comments = Comments.fetch_community_post_comments(ucid, id, sort_by: sort_by)
     else
       comments = YoutubeAPI.browse(continuation: continuation)
     end

src/invidious/routes/api/v1/misc.cr

@@ -190,15 +190,30 @@ module Invidious::Routes::API::V1::Misc
 
       sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint
       params = sub_endpoint.try &.dig?("params")
+
+      if sub_endpoint["browseId"]?.try &.as_s == "FEpost_detail"
+        decoded_protobuf = params.try &.as_s.try { |i| URI.decode_www_form(i) }
+          .try { |i| Base64.decode(i) }
+          .try { |i| IO::Memory.new(i) }
+          .try { |i| Protodec::Any.parse(i) }
+
+        ucid = decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s)
+        post_id = decoded_protobuf.try(&.["56:0:embedded"]["3:1:string"].as_s)
+      else
+        ucid = sub_endpoint["browseId"]? if sub_endpoint["browseId"]? && sub_endpoint["browseId"]?.try &.as_s.starts_with? "UC"
+        post_id = nil
+      end
     rescue ex
       return error_json(500, ex)
     end
     JSON.build do |json|
       json.object do
-        json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
+        json.field "browseId", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
+        json.field "ucid", ucid if ucid != nil
         json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]?
         json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
         json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
+        json.field "postId", post_id if post_id != nil
         json.field "params", params.try &.as_s
         json.field "pageType", page_type
       end

src/invidious/routes/channels.cr

@@ -285,7 +285,7 @@ module Invidious::Routes::Channels
       response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
       return error_template(400, "Invalid post ID") if response["error"]?
 
-      ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+      ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s)
       post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
     end
 

src/invidious/routes/embed.cr

@@ -20,7 +20,7 @@ module Invidious::Routes::Embed
         return error_template(500, ex)
       end
 
-      url = "/embed/#{first_playlist_video}?#{env.params.query}"
+      url = "/embed/#{first_playlist_video.id}?#{env.params.query}"
 
       if env.params.query.size > 0
         url += "?#{env.params.query}"

src/invidious/videos/parser.cr

@@ -82,7 +82,7 @@ def extract_video_info(video_id : String, env : HTTP::Server::Context | Nil = ni
         "reason"  => JSON::Any.new(reason),
       }
     end
-  elsif video_id != player_response.dig("videoDetails", "videoId")
+  elsif video_id != player_response.dig?("videoDetails", "videoId")
     # YouTube may return a different video player response than expected.
     # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
     # Line to be reverted if one day we solve the video not available issue.
@@ -109,21 +109,34 @@ def extract_video_info(video_id : String, env : HTTP::Server::Context | Nil = ni
   params["reason"] = JSON::Any.new(reason) if reason
 
   if !CONFIG.invidious_companion.present?
-    if player_response["streamingData"]? && player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
+    if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
       LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
-      players_fallback = [YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile]
+      players_fallback = {YoutubeAPI::ClientType::TvSimply, YoutubeAPI::ClientType::WebMobile}
+
       players_fallback.each do |player_fallback|
         client_config.client_type = player_fallback
-        player_fallback_response = try_fetch_streaming_data(video_id, client_config, env)
-        if player_fallback_response && player_fallback_response["streamingData"]? &&
-           player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
+
+        next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config, env))
+
+        adaptive_formats = player_fallback_response.dig?("streamingData", "adaptiveFormats")
+        if adaptive_formats && (adaptive_formats.dig?(0, "url") || adaptive_formats.dig?(0, "signatureCipher"))
           streaming_data = player_response["streamingData"].as_h
-          streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
+          streaming_data["adaptiveFormats"] = adaptive_formats
           player_response["streamingData"] = JSON::Any.new(streaming_data)
           break
         end
+      rescue InfoException
+        next LOGGER.warn("Failed to fetch streams with #{player_fallback}")
       end
     end
+
+    # Seems like video page can still render even without playable streams.
+    # its better than nothing.
+    #
+    # # Were we able to find playable video streams?
+    # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
+    #   # No :(
+    # end
   end
 
   {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
@@ -134,7 +147,11 @@ def extract_video_info(video_id : String, env : HTTP::Server::Context | Nil = ni
   if streaming_data = player_response["streamingData"]?
     %w[formats adaptiveFormats].each do |key|
       streaming_data.as_h[key]?.try &.as_a.each do |format|
-        format.as_h["url"] = JSON::Any.new(convert_url(format))
+        format = format.as_h
+        if format["url"]?.nil?
+          format["url"] = format["signatureCipher"]
+        end
+        format["url"] = JSON::Any.new(convert_url(format))
       end
     end
 
@@ -154,7 +171,7 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf
   playability_status = response["playabilityStatus"]["status"]
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
 
-  if id != response.dig("videoDetails", "videoId")
+  if id != response.dig?("videoDetails", "videoId")
     # YouTube may return a different video player response than expected.
     # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
     raise InfoException.new(

src/invidious/views/user/data_control.ecr

@@ -14,7 +14,7 @@
 
             <div class="pure-control-group">
                 <label for="import_youtube">
-                    <a rel="noopener" target="_blank" href="https://github.com/iv-org/documentation/blob/master/docs/export-youtube-subscriptions.md">
+                    <a rel="noopener noreferrer" target="_blank" href="https://github.com/iv-org/documentation/blob/master/docs/export-youtube-subscriptions.md">
                         <%= translate(locale, "Import YouTube subscriptions") %>
                     </a>
                 </label>

src/invidious/yt_backend/youtube_api.cr

@@ -6,10 +6,10 @@ module YoutubeAPI
   extend self
 
   # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history
-  private ANDROID_APP_VERSION = "19.32.34"
-  private ANDROID_VERSION     = "12"
-  private ANDROID_USER_AGENT  = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; US) gzip"
-  private ANDROID_SDK_VERSION = 31_i64
+  private ANDROID_APP_VERSION = "19.35.36"
+  private ANDROID_VERSION     = "13"
+  private ANDROID_USER_AGENT  = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; en_US; SM-S908E Build/TP1A.220624.014) gzip"
+  private ANDROID_SDK_VERSION = 33_i64
 
   private ANDROID_TS_APP_VERSION = "1.9"
   private ANDROID_TS_USER_AGENT  = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip"
@@ -17,9 +17,9 @@ module YoutubeAPI
   # For Apple device names, see https://gist.github.com/adamawolf/3048717
   # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases,
   # then go to the dedicated article of the major version you want.
-  private IOS_APP_VERSION = "19.32.8"
-  private IOS_USER_AGENT  = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)"
-  private IOS_VERSION     = "17.6.1.21G93" # Major.Minor.Patch.Build
+  private IOS_APP_VERSION = "20.11.6"
+  private IOS_USER_AGENT  = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 18_5 like Mac OS X;)"
+  private IOS_VERSION     = "18.5.0.22F76" # Major.Minor.Patch.Build
 
   private WINDOWS_VERSION = "10.0"
 
@@ -29,6 +29,7 @@ module YoutubeAPI
     WebEmbeddedPlayer
     WebMobile
     WebScreenEmbed
+    WebCreator
 
     Android
     AndroidEmbeddedPlayer
@@ -41,6 +42,7 @@ module YoutubeAPI
 
     TvHtml5
     TvHtml5ScreenEmbed
+    TvSimply
   end
 
   # List of hard-coded values used by the different clients
@@ -48,7 +50,7 @@ module YoutubeAPI
     ClientType::Web => {
       name:       "WEB",
       name_proto: "1",
-      version:    "2.20240814.00.00",
+      version:    "2.20250222.10.00",
       screen:     "WATCH_FULL_SCREEN",
       os_name:    "Windows",
       os_version: WINDOWS_VERSION,
@@ -57,7 +59,7 @@ module YoutubeAPI
     ClientType::WebEmbeddedPlayer => {
       name:       "WEB_EMBEDDED_PLAYER",
       name_proto: "56",
-      version:    "1.20240812.01.00",
+      version:    "1.20250219.01.00",
       screen:     "EMBED",
       os_name:    "Windows",
       os_version: WINDOWS_VERSION,
@@ -66,7 +68,7 @@ module YoutubeAPI
     ClientType::WebMobile => {
       name:       "MWEB",
       name_proto: "2",
-      version:    "2.20240813.02.00",
+      version:    "2.20250224.01.00",
       os_name:    "Android",
       os_version: ANDROID_VERSION,
       platform:   "MOBILE",
@@ -74,12 +76,20 @@ module YoutubeAPI
     ClientType::WebScreenEmbed => {
       name:       "WEB",
       name_proto: "1",
-      version:    "2.20240814.00.00",
+      version:    "2.20250222.10.00",
       screen:     "EMBED",
       os_name:    "Windows",
       os_version: WINDOWS_VERSION,
       platform:   "DESKTOP",
     },
+    ClientType::WebCreator => {
+      name:       "WEB_CREATOR",
+      name_proto: "62",
+      version:    "1.20241203.01.00",
+      os_name:    "Windows",
+      os_version: WINDOWS_VERSION,
+      platform:   "DESKTOP",
+    },
 
     # Android
 
@@ -161,7 +171,7 @@ module YoutubeAPI
     ClientType::TvHtml5 => {
       name:       "TVHTML5",
       name_proto: "7",
-      version:    "7.20240813.07.00",
+      version:    "7.20250219.14.00",
     },
     ClientType::TvHtml5ScreenEmbed => {
       name:       "TVHTML5_SIMPLY_EMBEDDED_PLAYER",
@@ -169,6 +179,11 @@ module YoutubeAPI
       version:    "2.0",
       screen:     "EMBED",
     },
+    ClientType::TvSimply => {
+      name:       "TVHTML5_SIMPLY",
+      name_proto: "74",
+      version:    "1.0",
+    },
   }
 
   ####################################################################