Merge pull request #2 from ashley0143/ashley0143-patch-1

fix channel pages

src/invidious/channels/videos.cr

@@ -1,106 +1,3 @@
-def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
-  object_inner_2 = {
-    "2:0:embedded" => {
-      "1:0:varint" => 0_i64,
-    },
-    "5:varint"  => 50_i64,
-    "6:varint"  => 1_i64,
-    "7:varint"  => (page * 30).to_i64,
-    "9:varint"  => 1_i64,
-    "10:varint" => 0_i64,
-  }
-
-  object_inner_2_encoded = object_inner_2
-    .try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  content_type_numerical =
-    case content_type
-    when "videos"      then 15
-    when "livestreams" then 14
-    else                    15 # Fallback to "videos"
-    end
-
-  if content_type == "livestreams"
-    sort_by_numerical =
-      case sort_by
-      when "newest"  then 12_i64
-      when "popular" then 14_i64
-      when "oldest"  then 13_i64
-      else                12_i64 # Fallback to "newest"
-      end
-  else
-    sort_by_numerical =
-      case sort_by
-      when "newest"  then 1_i64
-      when "popular" then 2_i64
-      when "oldest"  then 4_i64
-      else                1_i64 # Fallback to "newest"
-      end
-  end
-
-  if content_type == "livestreams"
-    object_inner_1 = {
-      "110:embedded" => {
-        "3:embedded" => {
-          "#{content_type_numerical}:embedded" => {
-            "1:embedded" => {
-              "1:string" => object_inner_2_encoded,
-            },
-            "2:embedded" => {
-              "1:string" => "00000000-0000-0000-0000-000000000000",
-            },
-            "5:varint" => sort_by_numerical,
-          },
-        },
-      },
-    }
-  else
-    object_inner_1 = {
-      "110:embedded" => {
-        "3:embedded" => {
-          "#{content_type_numerical}:embedded" => {
-            "1:embedded" => {
-              "1:string" => object_inner_2_encoded,
-            },
-            "2:embedded" => {
-              "1:string" => "00000000-0000-0000-0000-000000000000",
-            },
-            "3:varint" => sort_by_numerical,
-          },
-        },
-      },
-    }
-  end
-
-  object_inner_1_encoded = object_inner_1
-    .try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  object = {
-    "80226972:embedded" => {
-      "2:string"  => ucid,
-      "3:string"  => object_inner_1_encoded,
-      "35:string" => "browse-feed#{ucid}videos102",
-    },
-  }
-
-  continuation = object.try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  return continuation
-end
-
-def make_initial_content_ctoken(ucid, content_type, sort_by) : String
-  return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by)
-end
-
 module Invidious::Channel::Tabs
   extend self
 
@@ -129,7 +26,7 @@ module Invidious::Channel::Tabs
   end
 
   def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by)
+    continuation ||= make_initial_videos_ctoken(ucid, sort_by)
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, author, ucid)
@@ -158,14 +55,10 @@ module Invidious::Channel::Tabs
   #  Shorts
   # -------------------
 
-  def get_shorts(channel : AboutChannel, continuation : String? = nil)
+  def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
-    if continuation.nil?
+    continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by)
-      # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
+    initial_data = YoutubeAPI.browse(continuation: continuation)
-      # TODO: try to extract the continuation tokens that allows other sorting options
+
-      initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
-    else
-      initial_data = YoutubeAPI.browse(continuation: continuation)
-    end
     return extract_items(initial_data, channel.author, channel.ucid)
   end
 
@@ -173,9 +66,8 @@ module Invidious::Channel::Tabs
   #  Livestreams
   # -------------------
 
-  def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest")
+  def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by)
+    continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by)
-
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, channel.author, channel.ucid)
@@ -199,4 +91,102 @@ module Invidious::Channel::Tabs
 
     return items, next_continuation
   end
+
+  # -------------------
+  #  C-tokens
+  # -------------------
+
+  private def sort_options_videos_short(sort_by : String)
+    case sort_by
+    when "newest"  then return 4_i64
+    when "popular" then return 2_i64
+    when "oldest"  then return 5_i64
+    else                return 4_i64 # Fallback to "newest"
+    end
+  end
+
+  # Generate the initial "continuation token" to get the first page of the
+  # "videos" tab. The following page requires the ctoken provided in that
+  # first page, and so on.
+  private def make_initial_videos_ctoken(ucid : String, sort_by = "newest")
+    object = {
+      "15:embedded" => {
+        "2:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+        },
+        "4:varint" => sort_options_videos_short(sort_by),
+      },
+    }
+
+    return channel_ctoken_wrap(ucid, object)
+  end
+
+  # Generate the initial "continuation token" to get the first page of the
+  # "shorts" tab. The following page requires the ctoken provided in that
+  # first page, and so on.
+  private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest")
+    object = {
+      "10:embedded" => {
+        "2:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+        },
+        "4:varint" => sort_options_videos_short(sort_by),
+      },
+    }
+
+    return channel_ctoken_wrap(ucid, object)
+  end
+
+  # Generate the initial "continuation token" to get the first page of the
+  # "livestreams" tab. The following page requires the ctoken provided in that
+  # first page, and so on.
+  private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest")
+    sort_by_numerical =
+      case sort_by
+      when "newest"  then 12_i64
+      when "popular" then 14_i64
+      when "oldest"  then 13_i64
+      else                12_i64 # Fallback to "newest"
+      end
+
+    object = {
+      "14:embedded" => {
+        "2:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+        },
+        "5:varint" => sort_by_numerical,
+      },
+    }
+
+    return channel_ctoken_wrap(ucid, object)
+  end
+
+  # The protobuf structure common between videos/shorts/livestreams
+  private def channel_ctoken_wrap(ucid : String, object)
+    object_inner = {
+      "110:embedded" => {
+        "3:embedded" => object,
+      },
+    }
+
+    object_inner_encoded = object_inner
+      .try { |i| Protodec::Any.cast_json(i) }
+      .try { |i| Protodec::Any.from_json(i) }
+      .try { |i| Base64.urlsafe_encode(i) }
+      .try { |i| URI.encode_www_form(i) }
+
+    object = {
+      "80226972:embedded" => {
+        "2:string" => ucid,
+        "3:string" => object_inner_encoded,
+      },
+    }
+
+    continuation = object.try { |i| Protodec::Any.cast_json(i) }
+      .try { |i| Protodec::Any.from_json(i) }
+      .try { |i| Base64.urlsafe_encode(i) }
+      .try { |i| URI.encode_www_form(i) }
+
+    return continuation
+  end
 end

src/invidious/routes/channels.cr

@@ -20,10 +20,11 @@ module Invidious::Routes::Channels
     sort_by = env.params.query["sort_by"]?.try &.downcase
 
     if channel.auto_generated
+      sort_by ||= "last"
       sort_options = {"last", "oldest", "newest"}
 
       items, next_continuation = fetch_channel_playlists(
-        channel.ucid, channel.author, continuation, (sort_by || "last")
+        channel.ucid, channel.author, continuation, sort_by
       )
 
       items.uniq! do |item|
@@ -49,9 +50,11 @@ module Invidious::Routes::Channels
         end
         next_continuation = nil
       else
+        sort_by ||= "newest"
         sort_options = {"newest", "oldest", "popular"}
-        items, next_continuation = Channel::Tabs.get_videos(
+
-          channel, continuation: continuation, sort_by: (sort_by || "newest")
+        items, next_continuation = Channel::Tabs.get_60_videos(
+          channel, continuation: continuation, sort_by: sort_by
         )
       end
     end
@@ -82,13 +85,12 @@ module Invidious::Routes::Channels
       end
       next_continuation = nil
     else
-      # TODO: support sort option for shorts
+      sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
-      sort_by = ""
+      sort_options = {"newest", "oldest", "popular"}
-      sort_options = [] of String
 
       # Fetch items and continuation token
       items, next_continuation = Channel::Tabs.get_shorts(
-        channel, continuation: continuation
+        channel, continuation: continuation, sort_by: sort_by
       )
     end
 

src/invidious/yt_backend/extractors.cr

@@ -21,6 +21,7 @@ private ITEM_PARSERS = {
   Parsers::ItemSectionRendererParser,
   Parsers::ContinuationItemRendererParser,
   Parsers::HashtagRendererParser,
+  Parsers::LockupViewModelParser,
 }
 
 private alias InitialData = Hash(String, JSON::Any)
@@ -108,21 +109,30 @@ private module Parsers
         length_seconds = 0
       end
 
-      live_now = false
-      premium = false
-
       premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) }
-
+      badges = VideoBadges::None
       item_contents["badges"]?.try &.as_a.each do |badge|
         b = badge["metadataBadgeRenderer"]
         case b["label"].as_s
-        when "LIVE NOW"
+        when "LIVE"
-          live_now = true
+          badges |= VideoBadges::LiveNow
-        when "New", "4K", "CC"
+        when "New"
-          # TODO
+          badges |= VideoBadges::New
+        when "4K"
+          badges |= VideoBadges::FourK
+        when "8K"
+          badges |= VideoBadges::EightK
+        when "VR180"
+          badges |= VideoBadges::VR180
+        when "360°"
+          badges |= VideoBadges::VR360
+        when "3D"
+          badges |= VideoBadges::ThreeD
+        when "CC"
+          badges |= VideoBadges::ClosedCaptions
         when "Premium"
           # TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"]
-          premium = true
+          badges |= VideoBadges::Premium
         else nil # Ignore
         end
       end
@@ -136,10 +146,9 @@ private module Parsers
         views:              view_count,
         description_html:   description_html,
         length_seconds:     length_seconds,
-        live_now:           live_now,
-        premium:            premium,
         premiere_timestamp: premiere_timestamp,
         author_verified:    author_verified,
+        badges:             badges,
       })
     end
 
@@ -459,9 +468,9 @@ private module Parsers
   # Parses an InnerTube richItemRenderer into a SearchVideo.
   # Returns nil when the given object isn't a RichItemRenderer
   #
-  # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used
+  # A richItemRenderer seems to be a simple wrapper for a various other types,
-  # by the result page for hashtags and for the podcast tab on channels.
+  # used on the hashtags result page and the channel podcast tab. It is located
-  # It is located inside a continuationItems container for hashtags.
+  # itself inside a richGridRenderer container.
   #
   module RichItemRendererParser
     def self.process(item : JSON::Any, author_fallback : AuthorFallback)
@@ -474,6 +483,8 @@ private module Parsers
       child = VideoRendererParser.process(item_contents, author_fallback)
       child ||= ReelItemRendererParser.process(item_contents, author_fallback)
       child ||= PlaylistRendererParser.process(item_contents, author_fallback)
+      child ||= LockupViewModelParser.process(item_contents, author_fallback)
+      child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback)
       return child
     end
 
@@ -488,6 +499,9 @@ private module Parsers
   # reelItemRenderer items are used in the new (2022) channel layout,
   # in the "shorts" tab.
   #
+  # NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel
+  # TODO: Confirm that hypothesis
+  #
   module ReelItemRendererParser
     def self.process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["reelItemRenderer"]?
@@ -563,10 +577,138 @@ private module Parsers
         views:              view_count,
         description_html:   "",
         length_seconds:     duration,
-        live_now:           false,
-        premium:            false,
         premiere_timestamp: Time.unix(0),
         author_verified:    false,
+        badges:             VideoBadges::None,
+      })
+    end
+
+    def self.parser_name
+      return {{@type.name}}
+    end
+  end
+
+  # Parses an InnerTube lockupViewModel into a SearchPlaylist.
+  # Returns nil when the given object is not a lockupViewModel.
+  #
+  # This structure is present since November 2024 on the "podcasts" and
+  # "playlists" tabs of the channel page. It is usually encapsulated in either
+  # a richItemRenderer or a richGridRenderer.
+  #
+  module LockupViewModelParser
+    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+      if item_contents = item["lockupViewModel"]?
+        return self.parse(item_contents, author_fallback)
+      end
+    end
+
+    private def self.parse(item_contents, author_fallback)
+      playlist_id = item_contents["contentId"].as_s
+
+      thumbnail_view_model = item_contents.dig(
+        "contentImage", "collectionThumbnailViewModel",
+        "primaryThumbnail", "thumbnailViewModel"
+      )
+
+      thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s
+
+      # This complicated sequences tries to extract the following data structure:
+      # "overlays": [{
+      #   "thumbnailOverlayBadgeViewModel": {
+      #     "thumbnailBadges": [{
+      #       "thumbnailBadgeViewModel": {
+      #         "text": "430 episodes",
+      #         "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT"
+      #       }
+      #     }]
+      #   }
+      # }]
+      #
+      # NOTE: this simplistic `.to_i` conversion might not work on larger
+      # playlists and hasn't been tested.
+      video_count = thumbnail_view_model.dig("overlays").as_a
+        .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a)
+        .flatten
+        .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node|
+          {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) }
+        })
+        .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false)
+
+      metadata = item_contents.dig("metadata", "lockupMetadataViewModel")
+      title = metadata.dig("title", "content").as_s
+
+      # TODO: Retrieve "updated" info from metadata parts
+      # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a
+      # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s)
+      # One of these parts should contain a string like: "Updated 2 days ago"
+
+      # TODO: Maybe add a button to access the first video of the playlist?
+      # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
+      # Available fields: "videoId", "playlistId", "params"
+
+      return SearchPlaylist.new({
+        title:           title,
+        id:              playlist_id,
+        author:          author_fallback.name,
+        ucid:            author_fallback.id,
+        video_count:     video_count || -1,
+        videos:          [] of SearchPlaylistVideo,
+        thumbnail:       thumbnail,
+        author_verified: false,
+      })
+    end
+
+    def self.parser_name
+      return {{@type.name}}
+    end
+  end
+
+  # Parses an InnerTube shortsLockupViewModel into a SearchVideo.
+  # Returns nil when the given object is not a shortsLockupViewModel.
+  #
+  # This structure is present since around October 2024 on the "shorts" tab of
+  # the channel page and likely replaces the reelItemRenderer structure. It is
+  # usually (always?) encapsulated in a richItemRenderer.
+  #
+  module ShortsLockupViewModelParser
+    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+      if item_contents = item["shortsLockupViewModel"]?
+        return self.parse(item_contents, author_fallback)
+      end
+    end
+
+    private def self.parse(item_contents, author_fallback)
+      # TODO: Maybe add support for "oardefault.jpg" thumbnails?
+      # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s
+      # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?...
+
+      video_id = item_contents.dig(
+        "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId"
+      ).as_s
+
+      title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s
+
+      view_count = short_text_to_number(
+        item_contents.dig("overlayMetadata", "secondaryText", "content").as_s
+      )
+
+      # Approximate to one minute, as "shorts" generally don't exceed that.
+      # NOTE: The actual duration is not provided by Youtube anymore.
+      # TODO: Maybe use -1 as an error value and handle that on the frontend?
+      duration = 60_i32
+
+      SearchVideo.new({
+        title:              title,
+        id:                 video_id,
+        author:             author_fallback.name,
+        ucid:               author_fallback.id,
+        published:          Time.unix(0),
+        views:              view_count,
+        description_html:   "",
+        length_seconds:     duration,
+        premiere_timestamp: Time.unix(0),
+        author_verified:    false,
+        badges:             VideoBadges::None,
       })
     end