diff --git a/config/sql/channel_continuations.sql b/config/sql/channel_continuations.sql new file mode 100644 index 00000000..5ac2facf --- /dev/null +++ b/config/sql/channel_continuations.sql @@ -0,0 +1,23 @@ +-- Table: public.channel_continuations + +-- DROP TABLE public.channel_continuations; + +CREATE TABLE IF NOT EXISTS public.channel_continuations +( + id text NOT NULL, + page integer NOT NULL, + sort_by text NOT NULL, + continuation text, + CONSTRAINT channel_continuations_id_page_sort_by_key UNIQUE (id, page, sort_by) +); + +GRANT ALL ON TABLE public.channel_continuations TO default_user; + +-- Index: public.channel_continuations_id_idx + +-- DROP INDEX public.channel_continuations_id_idx; + +CREATE INDEX IF NOT EXISTS channel_continuations_id_idx + ON public.channel_continuations + USING btree + (id COLLATE pg_catalog."default"); diff --git a/docker/init-invidious-db.sh b/docker/init-invidious-db.sh index 22b4cc5f..a1e0b51e 100755 --- a/docker/init-invidious-db.sh +++ b/docker/init-invidious-db.sh @@ -3,6 +3,7 @@ set -eou pipefail psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/channels.sql psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/videos.sql +psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/channel_continuations.sql psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/channel_videos.sql psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/users.sql psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/session_ids.sql diff --git a/src/invidious.cr b/src/invidious.cr index ade13608..a5283e35 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -115,6 +115,7 @@ if CONFIG.check_tables check_enum(PG_DB, "privacy", PlaylistPrivacy) check_table(PG_DB, "channels", InvidiousChannel) + check_table(PG_DB, "channel_continuations", ChannelContinuation) check_table(PG_DB, "channel_videos", ChannelVideo) check_table(PG_DB, "playlists", InvidiousPlaylist) check_table(PG_DB, "playlist_videos", PlaylistVideo) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 827b6534..a27829db 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -8,6 +8,23 @@ struct InvidiousChannel property subscribed : Time? end +struct ChannelContinuation + include DB::Serializable + + property id : String + property page : Int32 + property sort_by : String + property continuation : String + + def to_tuple + {% begin %} + { + {{*@type.instance_vars.map(&.name)}} + } + {% end %} + end +end + struct ChannelVideo include DB::Serializable diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 48453bb7..88669754 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -58,10 +58,79 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so end def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") - continuation = produce_channel_videos_continuation(ucid, page, - auto_generated: auto_generated, sort_by: sort_by, v2: true) + continuation = "" + initial_data = Hash(String, JSON::Any).new - return YoutubeAPI.browse(continuation) + # Manually generating the continuation works correctly for both 'newest' and 'popular' sort modes, + # and for page 1 when sorting by 'oldest'. So only fallback to using the db if not in either of these states. + if sort_by != "oldest" || page == 1 + continuation = produce_channel_videos_continuation(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) + elsif channel_continuation = PG_DB.query_one?("SELECT * FROM channel_continuations WHERE id = $1 AND page = $2 AND sort_by = $3", ucid, page, sort_by, as: ChannelContinuation) + continuation = channel_continuation.continuation + else + # This branch should not be needed in normal operation (navigating via the previous/next page buttons). + # This is just here as a fallback in case someone requests, for example, page 3 without previously requesting page 2. + + # Iterate backwards from the wanted page to page 2 to find a stored continuation. + start = 1 + ((page - 1)..2).each do |i| + if channel_continuation = PG_DB.query_one?("SELECT * FROM channel_continuations WHERE id = $1 AND page = $2 AND sort_by = $3", ucid, i, sort_by, as: ChannelContinuation) + start = i + continuation = channel_continuation.continuation + break + end + end + + # If a continuation hasn't been found after getting to page 2, manually create the continuation for page 1. + if start == 1 + continuation = produce_channel_videos_continuation(ucid, 1, auto_generated: auto_generated, sort_by: sort_by, v2: true) + end + + # Iterate from the found/created continuation until we have the continuation for the wanted page or there are no more pages. + # Store the returned continuation each time so that it can be found in the db next time the current page is wanted. + (start..(page - 1)).each do |i| + initial_data = YoutubeAPI.browse(continuation) + continuation = fetch_continuation_token(initial_data) + + break if continuation.nil? || continuation.empty? + + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: i, + sort_by: sort_by, + continuation: continuation, + }) + PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + end + end + + # If we reached the channel's last page in the else loop above return an empty hash. + if continuation.nil? || continuation.empty? + initial_data.clear + else + # Get the wanted page and store the returned continuation for the next page, + # if there is one, so that it can be used the next time this function is called requesting that page. + initial_data = YoutubeAPI.browse(continuation) + + # Only get the continuation and store it if the sort mode is 'oldest'. + if sort_by == "oldest" + continuation = fetch_continuation_token(initial_data) + + if !continuation.nil? && !continuation.empty? + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: page + 1, + sort_by: sort_by, + continuation: continuation, + }) + PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + end + end + end + + return initial_data end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr index add5f488..be992ecd 100644 --- a/src/invidious/yt_backend/extractors_utils.cr +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -55,13 +55,20 @@ def fetch_continuation_token(items : Array(JSON::Any)) end def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) + continuation = "" # Fetches the continuation token from initial data if initial_data["onResponseReceivedActions"]? continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] - else + elsif initial_data["contents"]? tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] + elsif initial_data["continuationContents"]["gridContinuation"]["continuations"]? + continuation = initial_data["continuationContents"]["gridContinuation"]["continuations"][0]["nextContinuationData"]["continuation"].as_s end - return fetch_continuation_token(continuation_items.as_a) + if !continuation_items.nil? + continuation = fetch_continuation_token(continuation_items.as_a) + end + + return continuation end