mirrors/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2025-07-15 01:48:33 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #2 from ashley0143/ashley0143-patch-1

Browse Source 
fix channel pages
This commit is contained in:
Tyler Flowers 2024-11-12 14:21:11 -05:00 committed by GitHub
commit 3def03b87a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 272 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

220
src/invidious/channels/videos.cr
View File

@ -1,106 +1,3 @@
def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
object_inner_2 = {
"2:0:embedded" => {
"1:0:varint" => 0_i64,
},
"5:varint" => 50_i64,
"6:varint" => 1_i64,
"7:varint" => (page * 30).to_i64,
"9:varint" => 1_i64,
"10:varint" => 0_i64,
}
object_inner_2_encoded = object_inner_2
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
content_type_numerical =
case content_type
when "videos" then 15
when "livestreams" then 14
else 15 # Fallback to "videos"
end
if content_type == "livestreams"
sort_by_numerical =
case sort_by
when "newest" then 12_i64
when "popular" then 14_i64
when "oldest" then 13_i64
else 12_i64 # Fallback to "newest"
end
else
sort_by_numerical =
case sort_by
when "newest" then 1_i64
when "popular" then 2_i64
when "oldest" then 4_i64
else 1_i64 # Fallback to "newest"
end
end
if content_type == "livestreams"
object_inner_1 = {
"110:embedded" => {
"3:embedded" => {
"#{content_type_numerical}:embedded" => {
"1:embedded" => {
"1:string" => object_inner_2_encoded,
},
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"5:varint" => sort_by_numerical,
},
},
},
}
else
object_inner_1 = {
"110:embedded" => {
"3:embedded" => {
"#{content_type_numerical}:embedded" => {
"1:embedded" => {
"1:string" => object_inner_2_encoded,
},
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"3:varint" => sort_by_numerical,
},
},
},
}
end
object_inner_1_encoded = object_inner_1
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => object_inner_1_encoded,
"35:string" => "browse-feed#{ucid}videos102",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def make_initial_content_ctoken(ucid, content_type, sort_by) : String
return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by)
end
module Invidious::Channel::Tabs
extend self
@ -129,7 +26,7 @@ module Invidious::Channel::Tabs
end
def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by)
continuation ||= make_initial_videos_ctoken(ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, author, ucid)
@ -158,14 +55,10 @@ module Invidious::Channel::Tabs
# Shorts
# -------------------
def get_shorts(channel : AboutChannel, continuation : String? = nil)
if continuation.nil?
# EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
# TODO: try to extract the continuation tokens that allows other sorting options
initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation: continuation)
end
def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, channel.author, channel.ucid)
end
@ -173,9 +66,8 @@ module Invidious::Channel::Tabs
# Livestreams
# -------------------
def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by)
def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, channel.author, channel.ucid)
@ -199,4 +91,102 @@ module Invidious::Channel::Tabs
return items, next_continuation
end
# -------------------
# C-tokens
# -------------------
private def sort_options_videos_short(sort_by : String)
case sort_by
when "newest" then return 4_i64
when "popular" then return 2_i64
when "oldest" then return 5_i64
else return 4_i64 # Fallback to "newest"
end
end
# Generate the initial "continuation token" to get the first page of the
# "videos" tab. The following page requires the ctoken provided in that
# first page, and so on.
private def make_initial_videos_ctoken(ucid : String, sort_by = "newest")
object = {
"15:embedded" => {
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"4:varint" => sort_options_videos_short(sort_by),
},
}
return channel_ctoken_wrap(ucid, object)
end
# Generate the initial "continuation token" to get the first page of the
# "shorts" tab. The following page requires the ctoken provided in that
# first page, and so on.
private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest")
object = {
"10:embedded" => {
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"4:varint" => sort_options_videos_short(sort_by),
},
}
return channel_ctoken_wrap(ucid, object)
end
# Generate the initial "continuation token" to get the first page of the
# "livestreams" tab. The following page requires the ctoken provided in that
# first page, and so on.
private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest")
sort_by_numerical =
case sort_by
when "newest" then 12_i64
when "popular" then 14_i64
when "oldest" then 13_i64
else 12_i64 # Fallback to "newest"
end
object = {
"14:embedded" => {
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"5:varint" => sort_by_numerical,
},
}
return channel_ctoken_wrap(ucid, object)
end
# The protobuf structure common between videos/shorts/livestreams
private def channel_ctoken_wrap(ucid : String, object)
object_inner = {
"110:embedded" => {
"3:embedded" => object,
},
}
object_inner_encoded = object_inner
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => object_inner_encoded,
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
end

16
src/invidious/routes/channels.cr
View File

@ -20,10 +20,11 @@ module Invidious::Routes::Channels
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
sort_by ||= "last"
sort_options = {"last", "oldest", "newest"}
items, next_continuation = fetch_channel_playlists(
channel.ucid, channel.author, continuation, (sort_by || "last")
channel.ucid, channel.author, continuation, sort_by
)
items.uniq! do |item|
@ -49,9 +50,11 @@ module Invidious::Routes::Channels
end
next_continuation = nil
else
sort_by ||= "newest"
sort_options = {"newest", "oldest", "popular"}
items, next_continuation = Channel::Tabs.get_videos(
channel, continuation: continuation, sort_by: (sort_by || "newest")
items, next_continuation = Channel::Tabs.get_60_videos(
channel, continuation: continuation, sort_by: sort_by
)
end
end
@ -82,13 +85,12 @@ module Invidious::Routes::Channels
end
next_continuation = nil
else
# TODO: support sort option for shorts
sort_by = ""
sort_options = [] of String
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
sort_options = {"newest", "oldest", "popular"}
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_shorts(
channel, continuation: continuation
channel, continuation: continuation, sort_by: sort_by
)
end

174
src/invidious/yt_backend/extractors.cr
View File

@ -21,6 +21,7 @@ private ITEM_PARSERS = {
Parsers::ItemSectionRendererParser,
Parsers::ContinuationItemRendererParser,
Parsers::HashtagRendererParser,
Parsers::LockupViewModelParser,
}
private alias InitialData = Hash(String, JSON::Any)
@ -108,21 +109,30 @@ private module Parsers
length_seconds = 0
end
live_now = false
premium = false
premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) }
badges = VideoBadges::None
item_contents["badges"]?.try &.as_a.each do |badge|
b = badge["metadataBadgeRenderer"]
case b["label"].as_s
when "LIVE NOW"
live_now = true
when "New", "4K", "CC"
# TODO
when "LIVE"
badges |= VideoBadges::LiveNow
when "New"
badges |= VideoBadges::New
when "4K"
badges |= VideoBadges::FourK
when "8K"
badges |= VideoBadges::EightK
when "VR180"
badges |= VideoBadges::VR180
when "360°"
badges |= VideoBadges::VR360
when "3D"
badges |= VideoBadges::ThreeD
when "CC"
badges |= VideoBadges::ClosedCaptions
when "Premium"
# TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"]
premium = true
badges |= VideoBadges::Premium
else nil # Ignore
end
end
@ -136,10 +146,9 @@ private module Parsers
views: view_count,
description_html: description_html,
length_seconds: length_seconds,
live_now: live_now,
premium: premium,
premiere_timestamp: premiere_timestamp,
author_verified: author_verified,
badges: badges,
})
end
@ -459,9 +468,9 @@ private module Parsers
# Parses an InnerTube richItemRenderer into a SearchVideo.
# Returns nil when the given object isn't a RichItemRenderer
#
# A richItemRenderer seems to be a simple wrapper for a videoRenderer, used
# by the result page for hashtags and for the podcast tab on channels.
# It is located inside a continuationItems container for hashtags.
# A richItemRenderer seems to be a simple wrapper for a various other types,
# used on the hashtags result page and the channel podcast tab. It is located
# itself inside a richGridRenderer container.
#
module RichItemRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
@ -474,6 +483,8 @@ private module Parsers
child = VideoRendererParser.process(item_contents, author_fallback)
child ||= ReelItemRendererParser.process(item_contents, author_fallback)
child ||= PlaylistRendererParser.process(item_contents, author_fallback)
child ||= LockupViewModelParser.process(item_contents, author_fallback)
child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback)
return child
end
@ -488,6 +499,9 @@ private module Parsers
# reelItemRenderer items are used in the new (2022) channel layout,
# in the "shorts" tab.
#
# NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel
# TODO: Confirm that hypothesis
#
module ReelItemRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["reelItemRenderer"]?
@ -563,10 +577,138 @@ private module Parsers
views: view_count,
description_html: "",
length_seconds: duration,
live_now: false,
premium: false,
premiere_timestamp: Time.unix(0),
author_verified: false,
badges: VideoBadges::None,
})
end
def self.parser_name
return {{@type.name}}
end
end
# Parses an InnerTube lockupViewModel into a SearchPlaylist.
# Returns nil when the given object is not a lockupViewModel.
#
# This structure is present since November 2024 on the "podcasts" and
# "playlists" tabs of the channel page. It is usually encapsulated in either
# a richItemRenderer or a richGridRenderer.
#
module LockupViewModelParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["lockupViewModel"]?
return self.parse(item_contents, author_fallback)
end
end
private def self.parse(item_contents, author_fallback)
playlist_id = item_contents["contentId"].as_s
thumbnail_view_model = item_contents.dig(
"contentImage", "collectionThumbnailViewModel",
"primaryThumbnail", "thumbnailViewModel"
)
thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s
# This complicated sequences tries to extract the following data structure:
# "overlays": [{
# "thumbnailOverlayBadgeViewModel": {
# "thumbnailBadges": [{
# "thumbnailBadgeViewModel": {
# "text": "430 episodes",
# "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT"
# }
# }]
# }
# }]
#
# NOTE: this simplistic `.to_i` conversion might not work on larger
# playlists and hasn't been tested.
video_count = thumbnail_view_model.dig("overlays").as_a
.compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a)
.flatten
.find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node|
{"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) }
})
.try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false)
metadata = item_contents.dig("metadata", "lockupMetadataViewModel")
title = metadata.dig("title", "content").as_s
# TODO: Retrieve "updated" info from metadata parts
# rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a
# parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s)
# One of these parts should contain a string like: "Updated 2 days ago"
# TODO: Maybe add a button to access the first video of the playlist?
# item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
# Available fields: "videoId", "playlistId", "params"
return SearchPlaylist.new({
title: title,
id: playlist_id,
author: author_fallback.name,
ucid: author_fallback.id,
video_count: video_count || -1,
videos: [] of SearchPlaylistVideo,
thumbnail: thumbnail,
author_verified: false,
})
end
def self.parser_name
return {{@type.name}}
end
end
# Parses an InnerTube shortsLockupViewModel into a SearchVideo.
# Returns nil when the given object is not a shortsLockupViewModel.
#
# This structure is present since around October 2024 on the "shorts" tab of
# the channel page and likely replaces the reelItemRenderer structure. It is
# usually (always?) encapsulated in a richItemRenderer.
#
module ShortsLockupViewModelParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["shortsLockupViewModel"]?
return self.parse(item_contents, author_fallback)
end
end
private def self.parse(item_contents, author_fallback)
# TODO: Maybe add support for "oardefault.jpg" thumbnails?
# thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s
# Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?...
video_id = item_contents.dig(
"onTap", "innertubeCommand", "reelWatchEndpoint", "videoId"
).as_s
title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s
view_count = short_text_to_number(
item_contents.dig("overlayMetadata", "secondaryText", "content").as_s
)
# Approximate to one minute, as "shorts" generally don't exceed that.
# NOTE: The actual duration is not provided by Youtube anymore.
# TODO: Maybe use -1 as an error value and handle that on the frontend?
duration = 60_i32
SearchVideo.new({
title: title,
id: video_id,
author: author_fallback.name,
ucid: author_fallback.id,
published: Time.unix(0),
views: view_count,
description_html: "",
length_seconds: duration,
premiere_timestamp: Time.unix(0),
author_verified: false,
badges: VideoBadges::None,
})
end