From 35219a34a723610ffd871b9f3d77bda55a300790 Mon Sep 17 00:00:00 2001 From: Andrew Zhao Date: Sat, 20 Mar 2021 01:25:44 -0400 Subject: [PATCH] actual fix large playlists --- spec/helpers_spec.cr | 14 +++++++------- src/invidious/channels.cr | 32 ++++++++++++++++++-------------- src/invidious/playlists.cr | 25 ++++++++++--------------- 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/spec/helpers_spec.cr b/spec/helpers_spec.cr index 073d2700..b168946b 100644 --- a/spec/helpers_spec.cr +++ b/spec/helpers_spec.cr @@ -47,19 +47,19 @@ describe "Helper" do end end - describe "#produce_playlist_url" do + describe "#produce_playlist_continuation" do it "correctly produces url for requesting index `x` of a playlist" do - produce_playlist_url("UUCla9fZca4I7KagBtgRGnOw", 0).should eq("/browse_ajax?continuation=4qmFsgIqEhpWTFVVQ2xhOWZaY2E0STdLYWdCdGdSR25PdxoMZWdaUVZEcERRVUU9&gl=US&hl=en") + produce_playlist_continuation("UUCla9fZca4I7KagBtgRGnOw", 0).should eq("4qmFsgIqEhpWTFVVQ2xhOWZaY2E0STdLYWdCdGdSR25PdxoMZWdaUVZEcERRVUU9") - produce_playlist_url("UCCla9fZca4I7KagBtgRGnOw", 0).should eq("/browse_ajax?continuation=4qmFsgIqEhpWTFVVQ2xhOWZaY2E0STdLYWdCdGdSR25PdxoMZWdaUVZEcERRVUU9&gl=US&hl=en") + produce_playlist_continuation("UCCla9fZca4I7KagBtgRGnOw", 0).should eq("4qmFsgIqEhpWTFVVQ2xhOWZaY2E0STdLYWdCdGdSR25PdxoMZWdaUVZEcERRVUU9") - produce_playlist_url("PLt5AfwLFPxWLNVixpe1w3fi6lE2OTq0ET", 0).should eq("/browse_ajax?continuation=4qmFsgI0EiRWTFBMdDVBZndMRlB4V0xOVml4cGUxdzNmaTZsRTJPVHEwRVQaDGVnWlFWRHBEUVVFPQ%3D%3D&gl=US&hl=en") + produce_playlist_continuation("PLt5AfwLFPxWLNVixpe1w3fi6lE2OTq0ET", 0).should eq("4qmFsgI0EiRWTFBMdDVBZndMRlB4V0xOVml4cGUxdzNmaTZsRTJPVHEwRVQaDGVnWlFWRHBEUVVFPQ%3D%3D") - produce_playlist_url("PLt5AfwLFPxWLNVixpe1w3fi6lE2OTq0ET", 10000).should eq("/browse_ajax?continuation=4qmFsgI0EiRWTFBMdDVBZndMRlB4V0xOVml4cGUxdzNmaTZsRTJPVHEwRVQaDGVnZFFWRHBEU2tKUA%3D%3D&gl=US&hl=en") + produce_playlist_continuation("PLt5AfwLFPxWLNVixpe1w3fi6lE2OTq0ET", 10000).should eq("4qmFsgI0EiRWTFBMdDVBZndMRlB4V0xOVml4cGUxdzNmaTZsRTJPVHEwRVQaDGVnZFFWRHBEU2tKUA%3D%3D") - produce_playlist_url("PL55713C70BA91BD6E", 0).should eq("/browse_ajax?continuation=4qmFsgIkEhRWTFBMNTU3MTNDNzBCQTkxQkQ2RRoMZWdaUVZEcERRVUU9&gl=US&hl=en") + produce_playlist_continuation("PL55713C70BA91BD6E", 0).should eq("4qmFsgIkEhRWTFBMNTU3MTNDNzBCQTkxQkQ2RRoMZWdaUVZEcERRVUU9") - produce_playlist_url("PL55713C70BA91BD6E", 10000).should eq("/browse_ajax?continuation=4qmFsgIkEhRWTFBMNTU3MTNDNzBCQTkxQkQ2RRoMZWdkUVZEcERTa0pQ&gl=US&hl=en") + produce_playlist_continuation("PL55713C70BA91BD6E", 10000).should eq("4qmFsgIkEhRWTFBMNTU3MTNDNzBCQTkxQkQ2RRoMZWdkUVZEcERTa0pQ") end end diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index b9808d98..f909b5dc 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -937,23 +937,27 @@ def get_about_info(ucid, locale) }) end +def fetch_youtubei(continuation) + data = { + "context": { + "client": { + "clientName": "WEB", + "clientVersion": "2.20201021.03.00", + }, + }, + "continuation": continuation, + }.to_json + return YT_POOL.client &.post( + "/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + headers: HTTP::Headers{"content-type" => "application/json"}, + body: data + ) +end + def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest", youtubei_browse = true) if youtubei_browse continuation = produce_channel_videos_continuation(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) - data = { - "context": { - "client": { - "clientName": "WEB", - "clientVersion": "2.20201021.03.00", - }, - }, - "continuation": continuation, - }.to_json - return YT_POOL.client &.post( - "/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", - headers: HTTP::Headers{"content-type" => "application/json"}, - body: data - ) + return fetch_youtubei(continuation) else url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) return YT_POOL.client &.get(url) diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 0251a69c..f9b0d8db 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -307,7 +307,7 @@ def subscribe_playlist(db, user, playlist) return playlist end -def produce_playlist_url(id, index) +def produce_playlist_continuation(id, index) if id.starts_with? "UC" id = "UU" + id.lchop("UC") end @@ -332,7 +332,7 @@ def produce_playlist_url(id, index) .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + return continuation end def get_playlist(db, plid, locale, refresh = true, force_refresh = false) @@ -435,17 +435,12 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) end def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil) - if continuation - response = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en") - initial_data = extract_initial_data(response.body) - offset = initial_data["currentVideoEndpoint"]?.try &.["watchEndpoint"]?.try &.["index"]?.try &.as_i64 || offset + if !continuation && video_count > 100 + continuation = produce_playlist_continuation(plid, offset) end - - if video_count > 100 - url = produce_playlist_url(plid, offset) - - response = YT_POOL.client &.get(url) - initial_data = JSON.parse(response.body).as_a.find(&.as_h.["response"]?).try &.as_h + if continuation + response = fetch_youtubei(continuation) + initial_data = JSON.parse(response.body).as_h elsif offset > 100 return [] of PlaylistVideo else # Extract first page of videos @@ -465,9 +460,9 @@ end def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) videos = [] of PlaylistVideo - - (initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"].as_a || - initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a).try &.each do |item| + (initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]["content"]?.try &.["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"].as_a || + initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a || + initial_data["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a).try &.each do |item| if i = item["playlistVideoRenderer"]? video_id = i["navigationEndpoint"]["watchEndpoint"]["videoId"].as_s plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s