From 68cf24d100d55a87d57650716c589a8bdf9a2434 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 8 Sep 2019 12:08:59 -0400 Subject: [PATCH 01/20] Add support for channel redirects --- src/invidious.cr | 48 +++++++++++++++++++++++++++++++-------- src/invidious/channels.cr | 11 +++++++++ src/invidious/videos.cr | 6 ++++- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 8f7e1a63..58ba981d 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -391,7 +391,7 @@ get "/watch" do |env| begin video = get_video(id, PG_DB, region: params.region) rescue ex : VideoRedirect - next env.redirect "/watch?v=#{ex.message}" + next env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -668,7 +668,7 @@ get "/embed/:id" do |env| begin video = get_video(id, PG_DB, region: params.region) rescue ex : VideoRedirect - next env.redirect "/embed/#{ex.message}" + next env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -2632,6 +2632,8 @@ get "/feed/channel/:ucid" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -3035,6 +3037,8 @@ get "/channel/:ucid" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -3102,6 +3106,8 @@ get "/channel/:ucid/playlists" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -3140,6 +3146,8 @@ get "/channel/:ucid/community" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -3195,7 +3203,10 @@ get "/api/v1/storyboards/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - next env.redirect "/api/v1/storyboards/#{ex.message}" + error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + next error_message rescue ex env.response.status_code = 500 next @@ -3280,7 +3291,10 @@ get "/api/v1/captions/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - next env.redirect "/api/v1/captions/#{ex.message}" + error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + next error_message rescue ex env.response.status_code = 500 next @@ -3620,7 +3634,10 @@ get "/api/v1/videos/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - next env.redirect "/api/v1/videos/#{ex.message}" + error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + next error_message rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -3723,6 +3740,11 @@ get "/api/v1/channels/:ucid" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + next error_message rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -3853,6 +3875,11 @@ end begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + next error_message rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -3917,6 +3944,11 @@ end begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + next error_message rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -4501,11 +4533,7 @@ get "/api/manifest/dash/id/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - url = "/api/manifest/dash/id/#{ex.message}" - if env.params.query - url += "?#{env.params.query}" - end - next env.redirect url + next env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex env.response.status_code = 403 next diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 00eac902..3291efbd 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -127,6 +127,13 @@ struct AboutChannel }) end +class ChannelRedirect < Exception + property channel_id : String + + def initialize(@channel_id) + end +end + def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) finished_channel = Channel(String | Nil).new @@ -927,6 +934,10 @@ def get_about_info(ucid, locale) about = client.get("/user/#{ucid}/about?disable_polymer=1&gl=US&hl=en") end + if md = about.headers["location"]?.try &.match(/\/channel\/(?UC[a-zA-Z0-9_-]{22})/) + raise ChannelRedirect.new(channel_id: md["ucid"]) + end + about = XML.parse_html(about.body) if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")])) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 7b68656e..3cda835b 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -882,6 +882,10 @@ struct CaptionName end class VideoRedirect < Exception + property video_id : String + + def initialize(@video_id) + end end def get_video(id, db, refresh = true, region = nil, force_refresh = false) @@ -1149,7 +1153,7 @@ def fetch_video(id, region) response = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") if md = response.headers["location"]?.try &.match(/v=(?[a-zA-Z0-9_-]{11})/) - raise VideoRedirect.new(md["id"]) + raise VideoRedirect.new(video_id: md["id"]) end html = XML.parse_html(response.body) From 1f37faad42edb7fd73f3c850f47c4135c7ad9d71 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Mon, 9 Sep 2019 18:09:21 -0400 Subject: [PATCH 02/20] Fix plurilzation regex --- locales/el.json | 30 +++++++++++++++--------------- locales/en-US.json | 30 +++++++++++++++--------------- locales/it.json | 32 ++++++++++++++++---------------- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/locales/el.json b/locales/el.json index 32f154a1..222b7d0a 100644 --- a/locales/el.json +++ b/locales/el.json @@ -1,10 +1,10 @@ { "`x` subscribers": { - "(\\D|^)1(\\D|$)": "`x` συνδρομητής", + "([^0-9]|^)1([^,0-9]|$)": "`x` συνδρομητής", "": "`x` συνδρομητές" }, "`x` videos": { - "(\\D|^)1(\\D|$)": "`x` βίντεο", + "([^0-9]|^)1([^,0-9]|$)": "`x` βίντεο", "": "`x` βίντεο" }, "LIVE": "ΖΩΝΤΑΝΑ", @@ -119,11 +119,11 @@ "Token manager": "Διαχειριστής διασυνδέσεων", "Token": "Διασύνδεση", "`x` subscriptions": { - "(\\D|^)1(\\D|$)": "`x` συνδρομή", + "([^0-9]|^)1([^,0-9]|$)": "`x` συνδρομή", "": "`x` συνδρομές" }, "`x` tokens": { - "(\\D|^)1(\\D|$)": "`x` διασύνδεση", + "([^0-9]|^)1([^,0-9]|$)": "`x` διασύνδεση", "": "`x` διασυνδέσεις" }, "Import/export": "Εισαγωγή/εξαγωγή", @@ -131,7 +131,7 @@ "revoke": "ανάκληση", "Subscriptions": "Συνδρομές", "`x` unseen notifications": { - "(\\D|^)1(\\D|$)": "`x` καινούρια ειδοποίηση", + "([^0-9]|^)1([^,0-9]|$)": "`x` καινούρια ειδοποίηση", "": "`x` καινούριες ειδοποιήσεις" }, "search": "αναζήτηση", @@ -154,7 +154,7 @@ "Blacklisted regions: ": "Μη-επιτρεπτές περιοχές: ", "Shared `x`": "Μοιράστηκε το `x`", "`x` views": { - "(\\D|^)1(\\D|$)": "`x` προβολή", + "([^0-9]|^)1([^,0-9]|$)": "`x` προβολή", "": "`x` προβολές" }, "Premieres in `x`": "Πρώτη προβολή σε `x`", @@ -188,13 +188,13 @@ "Could not get channel info.": "Αδύναμια εύρεσης πληροφοριών καναλιού.", "Could not fetch comments": "Αδυναμία λήψης σχολίων", "View `x` replies": { - "(\\D|^)1(\\D|$)": "Προβολή `x` απάντησης", + "([^0-9]|^)1([^,0-9]|$)": "Προβολή `x` απάντησης", "": "Προβολή `x` απαντήσεων" }, "`x` ago": "Πριν `x`", "Load more": "Φόρτωση περισσότερων", "`x` points": { - "(\\D|^)1(\\D|$)": "`x` βαθμός", + "([^0-9]|^)1([^,0-9]|$)": "`x` βαθμός", "": "`x` βαθμοί" }, "Could not create mix.": "Αδυναμία δημιουργίας μίξης.", @@ -315,31 +315,31 @@ "Yoruba": "Γιορούμπα", "Zulu": "Ζουλού", "`x` years": { - "(\\D|^)1(\\D|$)": "`x` χρόνο", + "([^0-9]|^)1([^,0-9]|$)": "`x` χρόνο", "": "`x` χρόνια" }, "`x` months": { - "(\\D|^)1(\\D|$)": "`x` μήνα", + "([^0-9]|^)1([^,0-9]|$)": "`x` μήνα", "": "`x` μήνες" }, "`x` weeks": { - "(\\D|^)1(\\D|$)": "`x` εβδομάδα", + "([^0-9]|^)1([^,0-9]|$)": "`x` εβδομάδα", "": "`x` εβδομάδες" }, "`x` days": { - "(\\D|^)1(\\D|$)": "`x` ημέρα", + "([^0-9]|^)1([^,0-9]|$)": "`x` ημέρα", "": "`x` ημέρες" }, "`x` hours": { - "(\\D|^)1(\\D|$)": "`x` ώρα", + "([^0-9]|^)1([^,0-9]|$)": "`x` ώρα", "": "`x` ώρες" }, "`x` minutes": { - "(\\D|^)1(\\D|$)": "`x` λεπτό", + "([^0-9]|^)1([^,0-9]|$)": "`x` λεπτό", "": "`x` λεπτά" }, "`x` seconds": { - "(\\D|^)1(\\D|$)": "`x` δευτερόλεπτο", + "([^0-9]|^)1([^,0-9]|$)": "`x` δευτερόλεπτο", "": "`x` δευτερόλεπτα" }, "Fallback comments: ": "Εναλλακτικά σχόλια: ", diff --git a/locales/en-US.json b/locales/en-US.json index 580d9ead..8aaeee48 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -1,10 +1,10 @@ { "`x` subscribers": { - "(\\D|^)1(\\D|$)": "`x` subscriber", + "([^0-9]|^)1([^,0-9]|$)": "`x` subscriber", "": "`x` subscribers" }, "`x` videos": { - "(\\D|^)1(\\D|$)": "`x` video", + "([^0-9]|^)1([^,0-9]|$)": "`x` video", "": "`x` videos" }, "LIVE": "LIVE", @@ -119,11 +119,11 @@ "Token manager": "Token manager", "Token": "Token", "`x` subscriptions": { - "(\\D|^)1(\\D|$)": "`x` subscription", + "([^0-9]|^)1([^,0-9]|$)": "`x` subscription", "": "`x` subscriptions" }, "`x` tokens": { - "(\\D|^)1(\\D|$)": "`x` token", + "([^0-9]|^)1([^,0-9]|$)": "`x` token", "": "`x` tokens" }, "Import/export": "Import/export", @@ -131,7 +131,7 @@ "revoke": "revoke", "Subscriptions": "Subscriptions", "`x` unseen notifications": { - "(\\D|^)1(\\D|$)": "`x` unseen notification", + "([^0-9]|^)1([^,0-9]|$)": "`x` unseen notification", "": "`x` unseen notifications" }, "search": "search", @@ -154,7 +154,7 @@ "Blacklisted regions: ": "Blacklisted regions: ", "Shared `x`": "Shared `x`", "`x` views": { - "(\\D|^)1(\\D|$)": "`x` views", + "([^0-9]|^)1([^,0-9]|$)": "`x` views", "": "`x` views" }, "Premieres in `x`": "Premieres in `x`", @@ -188,13 +188,13 @@ "Could not get channel info.": "Could not get channel info.", "Could not fetch comments": "Could not fetch comments", "View `x` replies": { - "(\\D|^)1(\\D|$)": "View `x` reply", + "([^0-9]|^)1([^,0-9]|$)": "View `x` reply", "": "View `x` replies" }, "`x` ago": "`x` ago", "Load more": "Load more", "`x` points": { - "(\\D|^)1(\\D|$)": "`x` point", + "([^0-9]|^)1([^,0-9]|$)": "`x` point", "": "`x` points" }, "Could not create mix.": "Could not create mix.", @@ -315,31 +315,31 @@ "Yoruba": "Yoruba", "Zulu": "Zulu", "`x` years": { - "(\\D|^)1(\\D|$)": "`x` year", + "([^0-9]|^)1([^,0-9]|$)": "`x` year", "": "`x` years" }, "`x` months": { - "(\\D|^)1(\\D|$)": "`x` month", + "([^0-9]|^)1([^,0-9]|$)": "`x` month", "": "`x` months" }, "`x` weeks": { - "(\\D|^)1(\\D|$)": "`x` week", + "([^0-9]|^)1([^,0-9]|$)": "`x` week", "": "`x` weeks" }, "`x` days": { - "(\\D|^)1(\\D|$)": "`x` day", + "([^0-9]|^)1([^,0-9]|$)": "`x` day", "": "`x` days" }, "`x` hours": { - "(\\D|^)1(\\D|$)": "`x` hour", + "([^0-9]|^)1([^,0-9]|$)": "`x` hour", "": "`x` hours" }, "`x` minutes": { - "(\\D|^)1(\\D|$)": "`x` minute", + "([^0-9]|^)1([^,0-9]|$)": "`x` minute", "": "`x` minutes" }, "`x` seconds": { - "(\\D|^)1(\\D|$)": "`x` second", + "([^0-9]|^)1([^,0-9]|$)": "`x` second", "": "`x` seconds" }, "Fallback comments: ": "Fallback comments: ", diff --git a/locales/it.json b/locales/it.json index 74d42675..c2cd5d30 100644 --- a/locales/it.json +++ b/locales/it.json @@ -1,10 +1,10 @@ { "`x` subscribers": { - "(\\D|^)1(\\D|$)": "`x` iscritto", + "([^0-9]|^)1([^,0-9]|$)": "`x` iscritto", "": "`x` iscritti" }, "`x` videos": { - "(\\D|^)1(\\D|$)": "`x` video", + "([^0-9]|^)1([^,0-9]|$)": "`x` video", "": "`x` video" }, "LIVE": "IN DIRETTA", @@ -119,11 +119,11 @@ "Token manager": "Gestione dei gettoni", "Token": "Gettone", "`x` subscriptions": { - "(\\D|^)1(\\D|$)": "`x` iscrizione", + "([^0-9]|^)1([^,0-9]|$)": "`x` iscrizione", "": "`x` iscrizioni" }, "`x` tokens": { - "(\\D|^)1(\\D|$)": "`x` gettone", + "([^0-9]|^)1([^,0-9]|$)": "`x` gettone", "": "`x` gettoni" }, "Import/export": "Importa/esporta", @@ -131,7 +131,7 @@ "revoke": "revoca", "Subscriptions": "Iscrizioni", "`x` unseen notifications": { - "(\\D|^)1(\\D|$)": "`x` notifica non visualizzata", + "([^0-9]|^)1([^,0-9]|$)": "`x` notifica non visualizzata", "": "`x` notifiche non visualizzate" }, "search": "Cerca", @@ -154,7 +154,7 @@ "Blacklisted regions: ": "Regioni in lista nera: ", "Shared `x`": "Condiviso `x`", "`x` views": { - "(\\D|^)1(\\D|$)": "`x` visualizzazione", + "([^0-9]|^)1([^,0-9]|$)": "`x` visualizzazione", "": "`x` visualizzazioni" }, "Premieres in `x`": "", @@ -188,13 +188,13 @@ "Could not get channel info.": "Impossibile ottenere le informazioni del canale.", "Could not fetch comments": "Impossibile recuperare i commenti", "View `x` replies": { - "(\\D|^)1(\\D|$)": "Visualizza `x` risposta", + "([^0-9]|^)1([^,0-9]|$)": "Visualizza `x` risposta", "": "Visualizza `x` risposte" }, "`x` ago": "`x` fa", "Load more": "Carica altro", "`x` points": { - "(\\D|^)1(\\D|$)": "`x` punto", + "([^0-9]|^)1([^,0-9]|$)": "`x` punto", "": "`x` punti" }, "Could not create mix.": "Impossibile creare il mix.", @@ -315,31 +315,31 @@ "Yoruba": "Yoruba", "Zulu": "Zulu", "`x` years": { - "(\\D|^)1(\\D|$)": "`x` anno", + "([^0-9]|^)1([^,0-9]|$)": "`x` anno", "": "`x` anni" }, "`x` months": { - "(\\D|^)1(\\D|$)": "`x` mese", + "([^0-9]|^)1([^,0-9]|$)": "`x` mese", "": "`x` mesi" }, "`x` weeks": { - "(\\D|^)1(\\D|$)": "`x` settimana", + "([^0-9]|^)1([^,0-9]|$)": "`x` settimana", "": "`x` settimane" }, "`x` days": { - "(\\D|^)1(\\D|$)": "`x` giorno", + "([^0-9]|^)1([^,0-9]|$)": "`x` giorno", "": "`x` giorni" }, "`x` hours": { - "(\\D|^)1(\\D|$)": "`x` ora", + "([^0-9]|^)1([^,0-9]|$)": "`x` ora", "": "`x` ore" }, "`x` minutes": { - "(\\D|^)1(\\D|$)": "`x` minuto", + "([^0-9]|^)1([^,0-9]|$)": "`x` minuto", "": "`x` minuti" }, "`x` seconds": { - "(\\D|^)1(\\D|$)": "`x` secondo", + "([^0-9]|^)1([^,0-9]|$)": "`x` secondo", "": "`x` secondi" }, "Fallback comments: ": "Commenti alternativi: ", @@ -367,4 +367,4 @@ "Playlists": "Playlist", "Community": "Comunità", "Current version: ": "Versione attuale: " -} +} \ No newline at end of file From 7002a316fd29c68e05fe28dbf359f5612351ba69 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 12 Sep 2019 13:06:10 -0400 Subject: [PATCH 03/20] Filter movies from recommended videos --- src/invidious/videos.cr | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 3cda835b..40bcc513 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -935,6 +935,9 @@ def extract_recommended(recommended_videos) recommended_video = HTTP::Params.new recommended_video["id"] = video_renderer["videoId"].as_s recommended_video["title"] = video_renderer["title"]["simpleText"].as_s + + next if !video_renderer["shortBylineText"]? + recommended_video["author"] = video_renderer["shortBylineText"]["runs"].as_a[0]["text"].as_s recommended_video["ucid"] = video_renderer["shortBylineText"]["runs"].as_a[0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s recommended_video["author_thumbnail"] = video_renderer["channelThumbnail"]["thumbnails"][0]["url"].as_s From 34c43b83496a525560b589754a386905d9371782 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 12 Sep 2019 13:06:27 -0400 Subject: [PATCH 04/20] Add support for abbreviated sub count in search --- src/invidious/helpers/helpers.cr | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 331f6360..bd9d0b4e 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -415,7 +415,12 @@ def extract_items(nodeset, ucid = nil, author_name = nil) author_thumbnail ||= "" - subscriber_count = node.xpath_node(%q(.//span[contains(@class, "yt-subscriber-count")])).try &.["title"].gsub(/\D/, "").to_i? + subscriber_count_text = node.xpath_node(%q(.//span[contains(@class, "yt-subscriber-count")])).try &.["title"] + begin + subscriber_count = subscriber_count_text.try { |text| short_text_to_number(text) } + rescue ex + subscriber_count = subscriber_count_text.try &.gsub(/\D/, "").to_i? + end subscriber_count ||= 0 video_count = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li)).try &.content.split(" ")[0].gsub(/\D/, "").to_i? From 50d793e49b8e366f7b6c4bb39620035eb2ae2f09 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 12 Sep 2019 13:11:21 -0400 Subject: [PATCH 05/20] Hide video count for auto-generated channels --- src/invidious/helpers/helpers.cr | 6 +++--- src/invidious/search.cr | 3 +++ src/invidious/views/components/item.ecr | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index bd9d0b4e..43ccf0c5 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -424,15 +424,15 @@ def extract_items(nodeset, ucid = nil, author_name = nil) subscriber_count ||= 0 video_count = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li)).try &.content.split(" ")[0].gsub(/\D/, "").to_i? - video_count ||= 0 items << SearchChannel.new( author: author, ucid: ucid, author_thumbnail: author_thumbnail, subscriber_count: subscriber_count, - video_count: video_count, - description_html: description_html + video_count: video_count || 0, + description_html: description_html, + auto_generated: video_count ? false : true, ) else id = id.lchop("/watch?v=") diff --git a/src/invidious/search.cr b/src/invidious/search.cr index a55bb216..71e1f737 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -185,8 +185,10 @@ struct SearchChannel end end + json.field "autoGenerated", self.auto_generated json.field "subCount", self.subscriber_count json.field "videoCount", self.video_count + json.field "description", html_to_content(self.description_html) json.field "descriptionHtml", self.description_html end @@ -209,6 +211,7 @@ struct SearchChannel subscriber_count: Int32, video_count: Int32, description_html: String, + auto_generated: Bool, }) end diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 71ae70df..c722d0bc 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -11,7 +11,7 @@

<%= item.author %>

<%= translate(locale, "`x` subscribers", number_with_separator(item.subscriber_count)) %>

-

<%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %>

+ <% if !item.auto_generated %>

<%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %>

<% end %>
<%= item.description_html %>
<% when SearchPlaylist %> <% if item.id.starts_with? "RD" %> From b1fc80b79ae498f7bfd6ad7a32ebee98fabdaf4a Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 12 Sep 2019 21:09:23 -0400 Subject: [PATCH 06/20] Update sub_count extractor --- src/invidious/channels.cr | 31 +++++++++---------------------- src/invidious/helpers/helpers.cr | 9 ++------- src/invidious/helpers/utils.cr | 2 +- src/invidious/videos.cr | 2 +- src/invidious/views/channel.ecr | 2 +- src/invidious/views/community.ecr | 2 +- src/invidious/views/playlists.ecr | 2 +- 7 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 3291efbd..d88a8f71 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -118,7 +118,7 @@ struct AboutChannel description_html: String, paid: Bool, total_views: Int64, - sub_count: Int64, + sub_count: Int32, joined: Time, is_family_friendly: Bool, allowed_regions: Array(String), @@ -951,12 +951,6 @@ def get_about_info(ucid, locale) raise error_message end - sub_count = about.xpath_node(%q(//span[contains(text(), "subscribers")])) - if sub_count - sub_count = sub_count.content.delete(", subscribers").to_i? - end - sub_count ||= 0 - author = about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).not_nil!.content author_url = about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).not_nil!["href"] author_thumbnail = about.xpath_node(%q(//img[@class="channel-header-profile-image"])).not_nil!["src"] @@ -1000,21 +994,14 @@ def get_about_info(ucid, locale) ) end - total_views = 0_i64 - sub_count = 0_i64 + joined = about.xpath_node(%q(//span[contains(., "Joined")])) + .try &.content.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - joined = Time.unix(0) - metadata = about.xpath_nodes(%q(//span[@class="about-stat"])) - metadata.each do |item| - case item.content - when .includes? "views" - total_views = item.content.gsub(/\D/, "").to_i64 - when .includes? "subscribers" - sub_count = item.content.delete("subscribers").gsub(/\D/, "").to_i64 - when .includes? "Joined" - joined = Time.parse(item.content.lchop("Joined "), "%b %-d, %Y", Time::Location.local) - end - end + total_views = about.xpath_node(%q(//span[contains(., "views")]/b)) + .try &.content.try &.gsub(/\D/, "").to_i64? || 0_i64 + + sub_count = about.xpath_node(%q(.//span[contains(@class, "subscriber-count")])) + .try &.["title"].try { |text| short_text_to_number(text) } || 0 # Auto-generated channels # https://support.google.com/youtube/answer/2579942 @@ -1026,7 +1013,7 @@ def get_about_info(ucid, locale) tabs = about.xpath_nodes(%q(//ul[@id="channel-navigation-menu"]/li/a/span)).map { |node| node.content.downcase } - return AboutChannel.new( + AboutChannel.new( ucid: ucid, author: author, auto_generated: auto_generated, diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 43ccf0c5..0ec117c1 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -415,13 +415,8 @@ def extract_items(nodeset, ucid = nil, author_name = nil) author_thumbnail ||= "" - subscriber_count_text = node.xpath_node(%q(.//span[contains(@class, "yt-subscriber-count")])).try &.["title"] - begin - subscriber_count = subscriber_count_text.try { |text| short_text_to_number(text) } - rescue ex - subscriber_count = subscriber_count_text.try &.gsub(/\D/, "").to_i? - end - subscriber_count ||= 0 + subscriber_count = node.xpath_node(%q(.//span[contains(@class, "subscriber-count")])) + .try &.["title"].try { |text| short_text_to_number(text) } || 0 video_count = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li)).try &.content.split(" ")[0].gsub(/\D/, "").to_i? diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 5a813486..e17d58e2 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -157,7 +157,7 @@ def number_with_separator(number) number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse end -def short_text_to_number(short_text) +def short_text_to_number(short_text : String) : Int32 case short_text when .ends_with? "M" number = short_text.rstrip(" mM").to_f diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 40bcc513..829b384b 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -1262,7 +1262,7 @@ def fetch_video(id, region) end license = html.xpath_node(%q(//h4[contains(text(),"License")]/parent::*/ul/li)).try &.content || "" - sub_count_text = html.xpath_node(%q(//span[contains(@class, "yt-subscriber-count")])).try &.["title"]? || "0" + sub_count_text = html.xpath_node(%q(//span[contains(@class, "subscriber-count")])).try &.["title"]? || "0" author_thumbnail = html.xpath_node(%(//span[@class="yt-thumb-clip"]/img)).try &.["data-thumb"]?.try &.gsub(/^\/\//, "https://") || "" video = Video.new(id, info, Time.utc, title, views, likes, dislikes, wilson_score, published, description_html, diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 1074598d..b5eb46ea 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -34,7 +34,7 @@
<% ucid = channel.ucid %> <% author = channel.author %> - <% sub_count_text = channel.sub_count.format %> + <% sub_count_text = number_to_short_text(channel.sub_count) %> <%= rendered "components/subscribe_widget" %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 9d086b5d..218cc2d4 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -33,7 +33,7 @@
<% ucid = channel.ucid %> <% author = channel.author %> - <% sub_count_text = channel.sub_count.format %> + <% sub_count_text = number_to_short_text(channel.sub_count) %> <%= rendered "components/subscribe_widget" %>
diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr index 400922ff..a32192b5 100644 --- a/src/invidious/views/playlists.ecr +++ b/src/invidious/views/playlists.ecr @@ -33,7 +33,7 @@
<% ucid = channel.ucid %> <% author = channel.author %> - <% sub_count_text = channel.sub_count.format %> + <% sub_count_text = number_to_short_text(channel.sub_count) %> <%= rendered "components/subscribe_widget" %>
From 3c40c0be6b84e81f6984b6a297a49e70c4e8810b Mon Sep 17 00:00:00 2001 From: gnomus <43684+gnomus@users.noreply.github.com> Date: Fri, 13 Sep 2019 14:54:31 +0200 Subject: [PATCH 07/20] Update Package Repository for Install --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f29de7e1..224c0bf2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:edge AS builder -RUN apk add -u crystal shards libc-dev \ +RUN apk add --no-cache crystal shards libc-dev \ yaml-dev libxml2-dev sqlite-dev zlib-dev openssl-dev \ sqlite-static zlib-static openssl-libs-static WORKDIR /invidious @@ -15,7 +15,7 @@ RUN crystal build --static --release \ ./src/invidious.cr FROM alpine:latest -RUN apk add -u imagemagick ttf-opensans +RUN apk add --no-cache imagemagick ttf-opensans WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious From a006963fb8a03fcbc1358eb3292242cd73ba0d4a Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 17 Sep 2019 17:03:07 -0400 Subject: [PATCH 08/20] Update Google login --- src/invidious.cr | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 58ba981d..4016d07c 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1018,6 +1018,7 @@ post "/login" do |env| headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8" headers["Google-Accounts-XSRF"] = "1" + headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36" response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req)) lookup_results = JSON.parse(response.body[5..-1]) @@ -1057,6 +1058,13 @@ post "/login" do |env| next templated "error" end + # TODO: Handle Google's CAPTCHA + if captcha = challenge_results[0][-1]?.try &.[-1]?.try &.["5001"]?.try &.[-1].as_a? + error_message = "Unhandled CAPTCHA. Please try again later." + env.response.status_code = 401 + next templated "error" + end + if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" error_message = translate(locale, "Incorrect password") env.response.status_code = 401 @@ -1074,7 +1082,7 @@ post "/login" do |env| end # Prefer Authenticator app and SMS over unsupported protocols - if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8]) && prompt_type == 4 + if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2 tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0] traceback << "Selecting challenge #{tfa[8]}..." @@ -1182,8 +1190,12 @@ post "/login" do |env| break end - # TODO: Occasionally there will be a second page after login confirming - # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently choke on. + # Occasionally there will be a second page after login confirming + # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle. + + if location.includes? "/b/0/SmsAuthInterstitial" + traceback << "Unhandled dialog /b/0/SmsAuthInterstitial." + end login = client.get(location, headers) headers = login.cookies.add_request_headers(headers) From cfb68e3bff72cf431db3b6f8a89aa99cf837d35f Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 21 Sep 2019 19:58:10 -0400 Subject: [PATCH 09/20] Add additional handling for unplayable videos --- src/invidious/videos.cr | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 829b384b..4d662e9a 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -316,10 +316,10 @@ struct Video json.field "premiereTimestamp", self.premiere_timestamp.not_nil!.to_unix end - if self.player_response["streamingData"]?.try &.["hlsManifestUrl"]? + if player_response["streamingData"]?.try &.["hlsManifestUrl"]? host_url = make_host_url(config, kemal_config) - hlsvp = self.player_response["streamingData"]["hlsManifestUrl"].as_s + hlsvp = player_response["streamingData"]["hlsManifestUrl"].as_s hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", host_url) json.field "hlsUrl", hlsvp @@ -489,7 +489,7 @@ struct Video end def live_now - live_now = self.player_response["videoDetails"]?.try &.["isLive"]?.try &.as_bool + live_now = player_response["videoDetails"]?.try &.["isLive"]?.try &.as_bool if live_now.nil? return false @@ -536,7 +536,7 @@ struct Video end def keywords - keywords = self.player_response["videoDetails"]?.try &.["keywords"]?.try &.as_a + keywords = player_response["videoDetails"]?.try &.["keywords"]?.try &.as_a keywords ||= [] of String return keywords @@ -545,7 +545,7 @@ struct Video def fmt_stream(decrypt_function) streams = [] of HTTP::Params - if fmt_streams = self.player_response["streamingData"]?.try &.["formats"]? + if fmt_streams = player_response["streamingData"]?.try &.["formats"]? fmt_streams.as_a.each do |fmt_stream| if !fmt_stream.as_h? next @@ -619,7 +619,7 @@ struct Video def adaptive_fmts(decrypt_function) adaptive_fmts = [] of HTTP::Params - if fmts = self.player_response["streamingData"]?.try &.["adaptiveFormats"]? + if fmts = player_response["streamingData"]?.try &.["adaptiveFormats"]? fmts.as_a.each do |adaptive_fmt| if !adaptive_fmt.as_h? next @@ -712,12 +712,12 @@ struct Video end def storyboards - storyboards = self.player_response["storyboards"]? + storyboards = player_response["storyboards"]? .try &.as_h .try &.["playerStoryboardSpecRenderer"]? if !storyboards - storyboards = self.player_response["storyboards"]? + storyboards = player_response["storyboards"]? .try &.as_h .try &.["playerLiveStoryboardSpecRenderer"]? @@ -784,13 +784,8 @@ struct Video end def paid - reason = self.player_response["playabilityStatus"]?.try &.["reason"]? - - if reason == "This video requires payment to watch." - paid = true - else - paid = false - end + reason = player_response["playabilityStatus"]?.try &.["reason"]? + paid = reason == "This video requires payment to watch." ? true : false return paid end @@ -836,7 +831,7 @@ struct Video end def length_seconds - self.player_response["videoDetails"]["lengthSeconds"].as_s.to_i + player_response["videoDetails"]["lengthSeconds"].as_s.to_i end db_mapping({ @@ -1210,6 +1205,11 @@ def fetch_video(id, region) player_json = JSON.parse(info["player_response"]) + reason = player_json["playabilityStatus"]?.try &.["reason"]?.try &.as_s + if reason == "This video is not available." + raise "This video is not available." + end + title = player_json["videoDetails"]["title"].as_s author = player_json["videoDetails"]["author"]?.try &.as_s || "" ucid = player_json["videoDetails"]["channelId"]?.try &.as_s || "" From 9766322e99b772cea70099034b12fcb90fbc60d6 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 21 Sep 2019 22:20:36 -0400 Subject: [PATCH 10/20] Update videojs-quality-selector --- assets/js/silvermine-videojs-quality-selector.min.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/assets/js/silvermine-videojs-quality-selector.min.js b/assets/js/silvermine-videojs-quality-selector.min.js index 9d27202e..22387d04 100644 --- a/assets/js/silvermine-videojs-quality-selector.min.js +++ b/assets/js/silvermine-videojs-quality-selector.min.js @@ -1,4 +1,3 @@ -/*! silvermine-videojs-quality-selector 2018-01-09 v1.1.2 */ +/*! @silvermine/videojs-quality-selector 2019-09-21 v1.2.2-4-gc134430-dirty */ -!function n(t,e,r){function i(o,c){if(!e[o]){if(!t[o]){var a="function"==typeof require&&require;if(!c&&a)return a(o,!0);if(u)return u(o,!0);var s=new Error("Cannot find module '"+o+"'");throw s.code="MODULE_NOT_FOUND",s}var l=e[o]={exports:{}};t[o][0].call(l.exports,function(n){var e=t[o][1][n];return i(e||n)},l,l.exports,n,t,e,r)}return e[o].exports}for(var u="function"==typeof require&&require,o=0;o=0&&u0?0:c-1;return arguments.length<3&&(i=e[o?o[a]:a],a+=n),t(e,r,i,o,a,c)}}function r(n){return function(t,e,r){e=x(e,r);for(var i=T(t),u=n>0?0:i-1;u>=0&&u0?o=u>=0?u:Math.max(u+c,o):c=u>=0?Math.min(u+1,c):u+c+1;else if(e&&u&&c)return u=e(r,i),r[u]===i?u:-1;if(i!==i)return(u=t(p.call(r,o,c),m.isNaN))>=0?u+o:-1;for(u=n>0?o:c-1;u>=0&&u=0&&t<=E};m.each=m.forEach=function(n,t,e){t=b(t,e);var r,i;if(A(n))for(r=0,i=n.length;r=0},m.invoke=function(n,t){var e=p.call(arguments,2),r=m.isFunction(t);return m.map(n,function(n){var i=r?t:n[t];return null==i?i:i.apply(n,e)})},m.pluck=function(n,t){return m.map(n,m.property(t))},m.where=function(n,t){return m.filter(n,m.matcher(t))},m.findWhere=function(n,t){return m.find(n,m.matcher(t))},m.max=function(n,t,e){var r,i,u=-1/0,o=-1/0;if(null==t&&null!=n)for(var c=0,a=(n=A(n)?n:m.values(n)).length;cu&&(u=r);else t=x(t,e),m.each(n,function(n,e,r){((i=t(n,e,r))>o||i===-1/0&&u===-1/0)&&(u=n,o=i)});return u},m.min=function(n,t,e){var r,i,u=1/0,o=1/0;if(null==t&&null!=n)for(var c=0,a=(n=A(n)?n:m.values(n)).length;cr||void 0===e)return 1;if(et?(o&&(clearTimeout(o),o=null),c=s,u=n.apply(r,i),o||(r=i=null)):o||!1===e.trailing||(o=setTimeout(a,l)),u}},m.debounce=function(n,t,e){var r,i,u,o,c,a=function(){var s=m.now()-o;s=0?r=setTimeout(a,t-s):(r=null,e||(c=n.apply(u,i),r||(u=i=null)))};return function(){u=this,i=arguments,o=m.now();var s=e&&!r;return r||(r=setTimeout(a,t)),s&&(c=n.apply(u,i),u=i=null),c}},m.wrap=function(n,t){return m.partial(t,n)},m.negate=function(n){return function(){return!n.apply(this,arguments)}},m.compose=function(){var n=arguments,t=n.length-1;return function(){for(var e=t,r=n[t].apply(this,arguments);e--;)r=n[e].call(this,r);return r}},m.after=function(n,t){return function(){if(--n<1)return t.apply(this,arguments)}},m.before=function(n,t){var e;return function(){return--n>0&&(e=t.apply(this,arguments)),n<=1&&(t=null),e}},m.once=m.partial(m.before,2);var F=!{toString:null}.propertyIsEnumerable("toString"),q=["valueOf","isPrototypeOf","toString","propertyIsEnumerable","hasOwnProperty","toLocaleString"];m.keys=function(n){if(!m.isObject(n))return[];if(y)return y(n);var t=[];for(var e in n)m.has(n,e)&&t.push(e);return F&&u(n,t),t},m.allKeys=function(n){if(!m.isObject(n))return[];var t=[];for(var e in n)t.push(e);return F&&u(n,t),t},m.values=function(n){for(var t=m.keys(n),e=t.length,r=Array(e),i=0;i":">",'"':""","'":"'","`":"`"},N=m.invert(M),L=function(n){var t=function(t){return n[t]},e="(?:"+m.keys(n).join("|")+")",r=RegExp(e),i=RegExp(e,"g");return function(n){return n=null==n?"":""+n,r.test(n)?n.replace(i,t):n}};m.escape=L(M),m.unescape=L(N),m.result=function(n,t,e){var r=null==n?void 0:n[t];return void 0===r&&(r=e),m.isFunction(r)?r.call(n):r};var U=0;m.uniqueId=function(n){var t=++U+"";return n?n+t:t},m.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g};var D=/(.)^/,R={"'":"'","\\":"\\","\r":"r","\n":"n","\u2028":"u2028","\u2029":"u2029"},W=/\\|'|\r|\n|\u2028|\u2029/g,P=function(n){return"\\"+R[n]};m.template=function(n,t,e){!t&&e&&(t=e),t=m.defaults({},t,m.templateSettings);var r=RegExp([(t.escape||D).source,(t.interpolate||D).source,(t.evaluate||D).source].join("|")+"|$","g"),i=0,u="__p+='";n.replace(r,function(t,e,r,o,c){return u+=n.slice(i,c).replace(W,P),i=c+t.length,e?u+="'+\n((__t=("+e+"))==null?'':_.escape(__t))+\n'":r?u+="'+\n((__t=("+r+"))==null?'':__t)+\n'":o&&(u+="';\n"+o+"\n__p+='"),t}),u+="';\n",t.variable||(u="with(obj||{}){\n"+u+"}\n"),u="var __t,__p='',__j=Array.prototype.join,print=function(){__p+=__j.call(arguments,'');};\n"+u+"return __p;\n";try{var o=new Function(t.variable||"obj","_",u)}catch(n){throw n.source=u,n}var c=function(n){return o.call(this,n,m)},a=t.variable||"obj";return c.source="function("+a+"){\n"+u+"}",c},m.chain=function(n){var t=m(n);return t._chain=!0,t};var B=function(n,t){return n._chain?m(t).chain():t};m.mixin=function(n){m.each(m.functions(n),function(t){var e=m[t]=n[t];m.prototype[t]=function(){var n=[this._wrapped];return f.apply(n,arguments),B(this,e.apply(m,n))}})},m.mixin(m),m.each(["pop","push","reverse","shift","sort","splice","unshift"],function(n){var t=a[n];m.prototype[n]=function(){var e=this._wrapped;return t.apply(e,arguments),"shift"!==n&&"splice"!==n||0!==e.length||delete e[0],B(this,e)}}),m.each(["concat","join","slice"],function(n){var t=a[n];m.prototype[n]=function(){return B(this,t.apply(this._wrapped,arguments))}}),m.prototype.value=function(){return this._wrapped},m.prototype.valueOf=m.prototype.toJSON=m.prototype.value,m.prototype.toString=function(){return""+this._wrapped},"function"==typeof define&&define.amd&&define("underscore",[],function(){return m})}).call(this)},{}],3:[function(n,t,e){"use strict";var r=n("underscore"),i=n("../events");t.exports=function(n){var t=n.getComponent("MenuItem");return n.extend(t,{constructor:function(n,e){var i=e.source;if(!r.isObject(i))throw new Error('was not provided a "source" object, but rather: '+typeof i);e=r.extend({selectable:!0,label:i.label},e),t.call(this,n,e),this.source=i},handleClick:function(n){t.prototype.handleClick.call(this,n),this.player().trigger(i.QUALITY_REQUESTED,this.source)}})}},{"../events":5,underscore:2}],4:[function(n,t,e){"use strict";var r=n("underscore"),i=n("../events"),u=n("./QualityOption");t.exports=function(n){var t,e=n.getComponent("MenuButton"),o=u(n);return t=n.extend(e,{constructor:function(n,t){e.call(this,n,t),n.on(i.QUALITY_REQUESTED,function(t,e){this.setSelectedSource(e),n.addClass("vjs-quality-changing"),n.one("loadeddata",function(){n.removeClass("vjs-quality-changing")})}.bind(this)),n.on(i.QUALITY_SELECTED,function(n,t){this.setSelectedSource(t)}.bind(this)),n.one("ready",function(){this.selectedSrc=n.src(),this.update()}.bind(this)),this.controlText("Open quality selector menu")},setSelectedSource:function(n){var t=n?n.src:void 0;this.selectedSrc!==t&&(this.selectedSrc=t,this.update())},createItems:function(){var n=this.player(),t=n.currentSources();return!t||t.length<2?[]:r.map(t,function(t){return new o(n,{source:t,selected:t.src===this.selectedSrc})}.bind(this))},buildWrapperCSSClass:function(){return"vjs-quality-selector "+e.prototype.buildWrapperCSSClass.call(this)}}),n.registerComponent("QualitySelector",t),t}},{"../events":5,"./QualityOption":3,underscore:2}],5:[function(n,t,e){"use strict";t.exports={QUALITY_REQUESTED:"qualityRequested",QUALITY_SELECTED:"qualitySelected"}},{}],6:[function(n,t,e){"use strict";var r=n("underscore"),i=n("./events"),u=n("./components/QualitySelector"),o=n("./middleware/SourceInterceptor"),c=n("./util/SafeSeek");t.exports=function(n){n=n||window.videojs,u(n),o(n),n.hook("setup",function(n){n.on(i.QUALITY_REQUESTED,function(t,e){var i=n.currentSources(),u=n.currentTime(),o=n.paused();r.each(i,function(n){n.selected=!1}),r.findWhere(i,{src:e.src}).selected=!0,n._qualitySelectorSafeSeek&&n._qualitySelectorSafeSeek.onQualitySelectionChange(),n.src(i),n.ready(function(){n._qualitySelectorSafeSeek&&!n._qualitySelectorSafeSeek.hasFinished()||(n._qualitySelectorSafeSeek=new c(n,u)),o||n.play()})})})},t.exports.EVENTS=i},{"./components/QualitySelector":4,"./events":5,"./middleware/SourceInterceptor":7,"./util/SafeSeek":9,underscore:2}],7:[function(n,t,e){"use strict";var r=n("underscore"),i=n("../events");t.exports=function(n){n.use("*",function(n){return{setSource:function(t,e){var u,o=n.currentSources();n._qualitySelectorSafeSeek&&n._qualitySelectorSafeSeek.onPlayerSourcesChange(),u=r.find(o,function(n){return!0===n.selected||"true"===n.selected})||t,n.trigger(i.QUALITY_SELECTED,u),e(null,u)}}})}},{"../events":5,underscore:2}],8:[function(n,t,e){"use strict";n("./index")()},{"./index":6}],9:[function(n,t,e){"use strict";var r=n("class.extend");t.exports=r.extend({init:function(n,t){this._player=n,this._seekToTime=t,this._hasFinished=!1,this._keepThisInstanceWhenPlayerSourcesChange=!1,this._seekWhenSafe()},_seekWhenSafe:function(){this._player.readyState()<3?(this._seekFn=this._seek.bind(this),this._player.one("canplay",this._seekFn)):this._seek()},onPlayerSourcesChange:function(){this._keepThisInstanceWhenPlayerSourcesChange?this._keepThisInstanceWhenPlayerSourcesChange=!1:this.cancel()},onQualitySelectionChange:function(){this.hasFinished()||(this._keepThisInstanceWhenPlayerSourcesChange=!0)},_seek:function(){this._player.currentTime(this._seekToTime),this._keepThisInstanceWhenPlayerSourcesChange=!1,this._hasFinished=!0},hasFinished:function(){return this._hasFinished},cancel:function(){this._player.off("canplay",this._seekFn),this._keepThisInstanceWhenPlayerSourcesChange=!1,this._hasFinished=!0}})},{"class.extend":1}]},{},[8]); -//# sourceMappingURL=silvermine-videojs-quality-selector.min.js.map \ No newline at end of file +!function u(o,c,a){function l(e,n){if(!c[e]){if(!o[e]){var t="function"==typeof require&&require;if(!n&&t)return t(e,!0);if(s)return s(e,!0);var r=new Error("Cannot find module '"+e+"'");throw r.code="MODULE_NOT_FOUND",r}var i=c[e]={exports:{}};o[e][0].call(i.exports,function(n){return l(o[e][1][n]||n)},i,i.exports,u,o,c,a)}return c[e].exports}for(var s="function"==typeof require&&require,n=0;n":">",'"':""","'":"'","`":"`"},B=h.invert(D);h.escape=W(D),h.unescape=W(B),h.result=function(n,e,t){h.isArray(e)||(e=[e]);var r=e.length;if(!r)return h.isFunction(t)?t.call(n):t;for(var i=0;i/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g};function z(n){return"\\"+K[n]}var Y=/(.)^/,K={"'":"'","\\":"\\","\r":"r","\n":"n","\u2028":"u2028","\u2029":"u2029"},V=/\\|'|\r|\n|\u2028|\u2029/g;h.template=function(u,n,e){!n&&e&&(n=e),n=h.defaults({},n,h.templateSettings);var t,r=RegExp([(n.escape||Y).source,(n.interpolate||Y).source,(n.evaluate||Y).source].join("|")+"|$","g"),o=0,c="__p+='";u.replace(r,function(n,e,t,r,i){return c+=u.slice(o,i).replace(V,z),o=i+n.length,e?c+="'+\n((__t=("+e+"))==null?'':_.escape(__t))+\n'":t?c+="'+\n((__t=("+t+"))==null?'':__t)+\n'":r&&(c+="';\n"+r+"\n__p+='"),n}),c+="';\n",n.variable||(c="with(obj||{}){\n"+c+"}\n"),c="var __t,__p='',__j=Array.prototype.join,print=function(){__p+=__j.call(arguments,'');};\n"+c+"return __p;\n";try{t=new Function(n.variable||"obj","_",c)}catch(n){throw n.source=c,n}function i(n){return t.call(this,n,h)}var a=n.variable||"obj";return i.source="function("+a+"){\n"+c+"}",i},h.chain=function(n){var e=h(n);return e._chain=!0,e};function J(n,e){return n._chain?h(e).chain():e}h.mixin=function(t){return h.each(h.functions(t),function(n){var e=h[n]=t[n];h.prototype[n]=function(){var n=[this._wrapped];return i.apply(n,arguments),J(this,e.apply(h,n))}}),h},h.mixin(h),h.each(["pop","push","reverse","shift","sort","splice","unshift"],function(e){var t=r[e];h.prototype[e]=function(){var n=this._wrapped;return t.apply(n,arguments),"shift"!==e&&"splice"!==e||0!==n.length||delete n[0],J(this,n)}}),h.each(["concat","join","slice"],function(n){var e=r[n];h.prototype[n]=function(){return J(this,e.apply(this._wrapped,arguments))}}),h.prototype.value=function(){return this._wrapped},h.prototype.valueOf=h.prototype.toJSON=h.prototype.value,h.prototype.toString=function(){return String(this._wrapped)},"function"==typeof define&&define.amd&&define("underscore",[],function(){return h})}()}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{}],3:[function(n,e,t){"use strict";var i=n("underscore"),u=n("../events");e.exports=function(n){var r=n.getComponent("MenuItem");return n.extend(r,{constructor:function(n,e){var t=e.source;if(!i.isObject(t))throw new Error('was not provided a "source" object, but rather: '+typeof t);e=i.extend({selectable:!0,label:t.label},e),r.call(this,n,e),this.source=t},handleClick:function(n){r.prototype.handleClick.call(this,n),this.player().trigger(u.QUALITY_REQUESTED,this.source)}})}},{"../events":5,underscore:2}],4:[function(n,e,t){"use strict";var i=n("underscore"),u=n("../events"),o=n("./QualityOption"),c="vjs-quality-changing";e.exports=function(n){var e,r=n.getComponent("MenuButton"),t=o(n);return e=n.extend(r,{constructor:function(t,n){r.call(this,t,n),t.on(u.QUALITY_REQUESTED,function(n,e){this.setSelectedSource(e),t.addClass(c),t.one("loadeddata",function(){t.removeClass(c)})}.bind(this)),t.on(u.QUALITY_SELECTED,function(n,e){this.setSelectedSource(e)}.bind(this)),t.one("ready",function(){this.selectedSrc=t.src(),this.update()}.bind(this)),this.controlText("Open quality selector menu")},setSelectedSource:function(n){var e=n?n.src:void 0;this.selectedSrc!==e&&(this.selectedSrc=e,i.each(this.items,function(n){n.selected(n.source.src===e)}))},createItems:function(){var e=this.player(),n=e.currentSources();return i.map(n,function(n){return new t(e,{source:n,selected:n.src===this.selectedSrc})}.bind(this))},buildWrapperCSSClass:function(){return"vjs-quality-selector "+r.prototype.buildWrapperCSSClass.call(this)}}),n.registerComponent("QualitySelector",e),e}},{"../events":5,"./QualityOption":3,underscore:2}],5:[function(n,e,t){"use strict";e.exports={QUALITY_REQUESTED:"qualityRequested",QUALITY_SELECTED:"qualitySelected"}},{}],6:[function(n,e,t){"use strict";var o=n("underscore"),r=n("./events"),i=n("./components/QualitySelector"),u=n("./middleware/SourceInterceptor"),c=n("./util/SafeSeek");e.exports=function(n){n=n||window.videojs,i(n),u(n),n.hook("setup",function(u){u.on(r.QUALITY_REQUESTED,function(n,e){var t=u.currentSources(),r=u.currentTime(),i=(u.playbackRate(),u.paused());o.each(t,function(n){n.selected=!1}),o.findWhere(t,{src:e.src}).selected=!0,u._qualitySelectorSafeSeek&&u._qualitySelectorSafeSeek.onQualitySelectionChange(),u.src(t),u.ready(function(){u._qualitySelectorSafeSeek&&!u._qualitySelectorSafeSeek.hasFinished()||(u._qualitySelectorSafeSeek=new c(u,r),u.playbackRate=playbackRate),i||u.play()})})})},e.exports.EVENTS=r},{"./components/QualitySelector":4,"./events":5,"./middleware/SourceInterceptor":7,"./util/SafeSeek":9,underscore:2}],7:[function(n,e,t){"use strict";var u=n("underscore"),o=n("../events");e.exports=function(n){n.use("*",function(i){return{setSource:function(n,e){var t,r=i.currentSources();i._qualitySelectorSafeSeek&&i._qualitySelectorSafeSeek.onPlayerSourcesChange(),t=u.find(r,function(n){return!0===n.selected||"true"===n.selected})||n,i.trigger(o.QUALITY_SELECTED,t),e(null,t)}}})}},{"../events":5,underscore:2}],8:[function(n,e,t){"use strict";n("./index")()},{"./index":6}],9:[function(n,e,t){"use strict";var r=n("class.extend");e.exports=r.extend({init:function(n,e){this._player=n,this._seekToTime=e,this._hasFinished=!1,this._keepThisInstanceWhenPlayerSourcesChange=!1,this._seekWhenSafe()},_seekWhenSafe:function(){this._player.readyState()<3?(this._seekFn=this._seek.bind(this),this._player.one("canplay",this._seekFn)):this._seek()},onPlayerSourcesChange:function(){this._keepThisInstanceWhenPlayerSourcesChange?this._keepThisInstanceWhenPlayerSourcesChange=!1:this.cancel()},onQualitySelectionChange:function(){this.hasFinished()||(this._keepThisInstanceWhenPlayerSourcesChange=!0)},_seek:function(){this._player.currentTime(this._seekToTime),this._keepThisInstanceWhenPlayerSourcesChange=!1,this._hasFinished=!0},hasFinished:function(){return this._hasFinished},cancel:function(){this._player.off("canplay",this._seekFn),this._keepThisInstanceWhenPlayerSourcesChange=!1,this._hasFinished=!0}})},{"class.extend":1}]},{},[8]); \ No newline at end of file From 1085ca4a2df7565f559512cddc2203d1fbad041e Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 22 Sep 2019 09:54:54 -0400 Subject: [PATCH 11/20] Fix typo in Google login --- src/invidious.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index 4016d07c..b97aa6b2 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1059,7 +1059,7 @@ post "/login" do |env| end # TODO: Handle Google's CAPTCHA - if captcha = challenge_results[0][-1]?.try &.[-1]?.try &.["5001"]?.try &.[-1].as_a? + if captcha = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a? error_message = "Unhandled CAPTCHA. Please try again later." env.response.status_code = 401 next templated "error" From 1aefc5b5405de4cccd735891b3fd2b2c3e2d239c Mon Sep 17 00:00:00 2001 From: leonklingele Date: Tue, 24 Sep 2019 19:31:33 +0200 Subject: [PATCH 12/20] Update to Crystal 0.31.0, resolve compiler deprecation warnings, update dependencies (#764) * shard: update to crystal 0.31.0 Additionally, no longer use the Crystal "markdown" library which has been removed from the Crystal stdlib in version 0.31.0. See https://github.com/crystal-lang/crystal/pull/8115. Also fix some deprecation warnings using the following commands: find . \( -type d -name .git -prune \) -o -type f -exec sed -i 's/URI\.escape/URI\.encode_www_form/g' "{}" \; find . \( -type d -name .git -prune \) -o -type f -exec sed -i 's/URI\.unescape/URI\.decode_www_form/g' "{}" \; sed -i 's/while \%pull\.kind \!\= \:end_object/until \%pull\.kind\.end_object\?/g' src/invidious/helpers/patch_mapping.cr --- shard.yml | 6 +- src/invidious.cr | 25 ++-- src/invidious/channels.cr | 22 ++-- src/invidious/comments.cr | 6 +- src/invidious/helpers/handlers.cr | 4 +- src/invidious/helpers/patch_mapping.cr | 2 +- src/invidious/helpers/static_file_handler.cr | 2 +- src/invidious/helpers/tokens.cr | 2 +- src/invidious/helpers/utils.cr | 2 +- src/invidious/playlists.cr | 4 +- src/invidious/search.cr | 8 +- src/invidious/trending.cr | 2 +- src/invidious/videos.cr | 2 +- src/invidious/views/authorize_token.ecr | 2 +- src/invidious/views/change_password.ecr | 4 +- src/invidious/views/clear_watch_history.ecr | 6 +- src/invidious/views/components/item.ecr | 2 +- .../views/components/subscribe_widget.ecr | 6 +- src/invidious/views/data_control.ecr | 2 +- src/invidious/views/delete_account.ecr | 6 +- src/invidious/views/history.ecr | 4 +- src/invidious/views/login.ecr | 8 +- src/invidious/views/preferences.ecr | 10 +- src/invidious/views/privacy.ecr | 110 +++++++----------- src/invidious/views/subscription_manager.ecr | 6 +- src/invidious/views/subscriptions.ecr | 2 +- src/invidious/views/template.ecr | 2 +- src/invidious/views/token_manager.ecr | 6 +- src/invidious/views/watch.ecr | 8 +- 29 files changed, 125 insertions(+), 146 deletions(-) diff --git a/shard.yml b/shard.yml index 0f9beaf2..3980201d 100644 --- a/shard.yml +++ b/shard.yml @@ -11,14 +11,14 @@ targets: dependencies: pg: github: will/crystal-pg - version: ~> 0.18.1 + version: ~> 0.19.0 sqlite3: github: crystal-lang/crystal-sqlite3 - version: ~> 0.13.0 + version: ~> 0.14.0 kemal: github: kemalcr/kemal version: ~> 0.26.0 -crystal: 0.30.1 +crystal: 0.31.0 license: AGPLv3 diff --git a/src/invidious.cr b/src/invidious.cr index b97aa6b2..167050fd 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -17,7 +17,6 @@ require "digest/md5" require "file_utils" require "kemal" -require "markdown" require "openssl/hmac" require "option_parser" require "pg" @@ -296,7 +295,7 @@ before_all do |env| current_page += "?#{query}" end - env.set "current_page", URI.escape(current_page) + env.set "current_page", URI.encode_www_form(current_page) end get "/" do |env| @@ -841,7 +840,7 @@ get "/results" do |env| page ||= 1 if query - env.redirect "/search?q=#{URI.escape(query)}&page=#{page}" + env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}" else env.redirect "/" end @@ -1050,7 +1049,7 @@ post "/login" do |env| traceback << "done, returned #{response.status_code}.
" - headers["Cookie"] = URI.unescape(headers["Cookie"]) + headers["Cookie"] = URI.decode_www_form(headers["Cookie"]) if challenge_results[0][3]?.try &.== 7 error_message = translate(locale, "Account has temporarily been disabled") @@ -2423,7 +2422,7 @@ post "/authorize_token" do |env| access_token = generate_token(user.email, scopes, expire, HMAC_KEY, PG_DB) if callback_url - access_token = URI.escape(access_token) + access_token = URI.encode_www_form(access_token) url = URI.parse(callback_url) if url.query @@ -3327,7 +3326,7 @@ get "/api/v1/captions/:id" do |env| json.object do json.field "label", caption.name.simpleText json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.escape(caption.name.simpleText)}" + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" end end end @@ -3406,7 +3405,7 @@ get "/api/v1/captions/:id" do |env| if title = env.params.query["title"]? # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.escape(title)}\"; filename*=UTF-8''#{URI.escape(title)}" + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" end webvtt @@ -3594,7 +3593,7 @@ get "/api/v1/annotations/:id" do |env| id = id.sub(/^-/, 'A') end - file = URI.escape("#{id[0, 3]}/#{id}.xml") + file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") client = make_client(ARCHIVE_URL) location = client.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}") @@ -4093,7 +4092,7 @@ get "/api/v1/search/suggestions" do |env| begin client = make_client(URI.parse("https://suggestqueries.google.com")) - response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.escape(query)}&callback=suggestCallback").body + response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback").body body = response[35..-2] body = JSON.parse(body).as_a @@ -4477,7 +4476,7 @@ post "/api/v1/auth/tokens/register" do |env| access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB) if callback_url - access_token = URI.escape(access_token) + access_token = URI.encode_www_form(access_token) if query = callback_url.query query = HTTP::Params.parse(query.not_nil!) @@ -4712,7 +4711,7 @@ get "/api/manifest/hls_playlist/*" do |env| raw_params = {} of String => Array(String) path.each_slice(2) do |pair| key, value = pair - value = URI.unescape(value) + value = URI.decode_www_form(value) if raw_params[key]? raw_params[key] << value @@ -4837,7 +4836,7 @@ get "/videoplayback/*" do |env| raw_params = {} of String => Array(String) path.each_slice(2) do |pair| key, value = pair - value = URI.unescape(value) + value = URI.decode_www_form(value) if raw_params[key]? raw_params[key] << value @@ -5011,7 +5010,7 @@ get "/videoplayback" do |env| if title = query_params["title"]? # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.escape(title)}\"; filename*=UTF-8''#{URI.escape(title)}" + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" end if !response.headers.includes_word?("Transfer-Encoding", "chunked") diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index d88a8f71..7b7d3724 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -476,7 +476,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " end data = Base64.urlsafe_encode(data) - cursor = URI.escape(data) + cursor = URI.encode_www_form(data) data = IO::Memory.new @@ -497,7 +497,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" @@ -547,7 +547,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated data.rewind data = Base64.urlsafe_encode(data) - continuation = URI.escape(data) + continuation = URI.encode_www_form(data) data = IO::Memory.new @@ -568,7 +568,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" @@ -578,7 +578,7 @@ end def extract_channel_playlists_cursor(url, auto_generated) continuation = HTTP::Params.parse(URI.parse(url).query.not_nil!)["continuation"] - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0xe2 0xa9 0x85 0xb2 0x02 @@ -597,7 +597,7 @@ def extract_channel_playlists_cursor(url, auto_generated) data.read inner_continuation continuation = String.new(inner_continuation) - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0x12 0x09 playlists @@ -614,7 +614,7 @@ def extract_channel_playlists_cursor(url, auto_generated) cursor = String.new(cursor) if !auto_generated - cursor = URI.unescape(cursor) + cursor = URI.decode_www_form(cursor) cursor = Base64.decode_string(cursor) end @@ -877,7 +877,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo end def produce_channel_community_continuation(ucid, cursor) - cursor = URI.escape(cursor) + cursor = URI.encode_www_form(cursor) data = IO::Memory.new @@ -898,13 +898,13 @@ def produce_channel_community_continuation(ucid, cursor) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) return continuation end def extract_channel_community_cursor(continuation) - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0xe2 0xa9 0x85 0xb2 0x02 @@ -923,7 +923,7 @@ def extract_channel_community_cursor(continuation) data.read_byte end - return URI.unescape(data.gets_to_end) + return URI.decode_www_form(data.gets_to_end) end def get_about_info(ucid, locale) diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 04ba6f5d..740449d3 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -573,7 +573,7 @@ def content_to_comment_html(content) end def extract_comment_cursor(continuation) - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0x12 0x26 @@ -653,7 +653,7 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top") end continuation = Base64.urlsafe_encode(data) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) return continuation end @@ -695,7 +695,7 @@ def produce_comment_reply_continuation(video_id, ucid, comment_id) data.write(Bytes[0x48, 0x0a]) continuation = Base64.urlsafe_encode(data.to_slice) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) return continuation end diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 56c1c488..c8d4da4c 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -95,8 +95,8 @@ class AuthHandler < Kemal::Handler begin if token = env.request.headers["Authorization"]? - token = JSON.parse(URI.unescape(token.lchop("Bearer "))) - session = URI.unescape(token["session"].as_s) + token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) + session = URI.decode_www_form(token["session"].as_s) scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil) if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", session, as: String) diff --git a/src/invidious/helpers/patch_mapping.cr b/src/invidious/helpers/patch_mapping.cr index e138aa1c..19bd8ca1 100644 --- a/src/invidious/helpers/patch_mapping.cr +++ b/src/invidious/helpers/patch_mapping.cr @@ -50,7 +50,7 @@ macro patched_json_mapping(_properties_, strict = false) rescue exc : ::JSON::ParseException raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc) end - while %pull.kind != :end_object + until %pull.kind.end_object? %key_location = %pull.location key = %pull.read_object_key case key diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr index 87edbcd3..20d92b9c 100644 --- a/src/invidious/helpers/static_file_handler.cr +++ b/src/invidious/helpers/static_file_handler.cr @@ -119,7 +119,7 @@ module Kemal config = Kemal.config.serve_static original_path = context.request.path.not_nil! - request_path = URI.unescape(original_path) + request_path = URI.decode_www_form(original_path) # File path cannot contains '\0' (NUL) because all filesystem I know # don't accept '\0' character as file name. diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index f946fc2c..30f7d4f4 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -69,7 +69,7 @@ end def validate_request(token, session, request, key, db, locale = nil) case token when String - token = JSON.parse(URI.unescape(token)).as_h + token = JSON.parse(URI.decode_www_form(token)).as_h when JSON::Any token = token.as_h when Nil diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index e17d58e2..ed55dc9c 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -246,7 +246,7 @@ def get_referer(env, fallback = "/", unroll = true) if referer.query params = HTTP::Params.parse(referer.query.not_nil!) if params["referer"]? - referer = URI.parse(URI.unescape(params["referer"])) + referer = URI.parse(URI.decode_www_form(params["referer"])) else break end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 7965d990..a5383daf 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -172,7 +172,7 @@ def produce_playlist_url(id, index) continuation.print data data = Base64.urlsafe_encode(continuation) - cursor = URI.escape(data) + cursor = URI.encode_www_form(data) data = IO::Memory.new @@ -193,7 +193,7 @@ def produce_playlist_url(id, index) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 71e1f737..10475a8f 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -266,7 +266,7 @@ def search(query, page = 1, search_params = produce_search_params(content_type: return {0, [] of SearchItem} end - html = client.get("/results?q=#{URI.escape(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body + html = client.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body if html.empty? return {0, [] of SearchItem} end @@ -371,7 +371,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte end token = Base64.urlsafe_encode(token.to_slice) - token = URI.escape(token) + token = URI.encode_www_form(token) return token end @@ -396,7 +396,7 @@ def produce_channel_search_url(ucid, query, page) data.rewind data = Base64.urlsafe_encode(data) - continuation = URI.escape(data) + continuation = URI.encode_www_form(data) data = IO::Memory.new @@ -421,7 +421,7 @@ def produce_channel_search_url(ucid, query, page) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 32908157..26db51ea 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -42,7 +42,7 @@ end def extract_plid(url) wrapper = HTTP::Params.parse(URI.parse(url).query.not_nil!)["bp"] - wrapper = URI.unescape(wrapper) + wrapper = URI.decode_www_form(wrapper) wrapper = Base64.decode(wrapper) # 0xe2 0x02 0x2e diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 4d662e9a..424d1e50 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -408,7 +408,7 @@ struct Video json.object do json.field "label", caption.name.simpleText json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.escape(caption.name.simpleText)}" + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" end end end diff --git a/src/invidious/views/authorize_token.ecr b/src/invidious/views/authorize_token.ecr index 53b8f001..8ea99010 100644 --- a/src/invidious/views/authorize_token.ecr +++ b/src/invidious/views/authorize_token.ecr @@ -72,7 +72,7 @@ <% end %> - + <% end %> diff --git a/src/invidious/views/change_password.ecr b/src/invidious/views/change_password.ecr index 2e68556b..fb558f1d 100644 --- a/src/invidious/views/change_password.ecr +++ b/src/invidious/views/change_password.ecr @@ -6,7 +6,7 @@
-
+ <%= translate(locale, "Change password") %>
@@ -23,7 +23,7 @@ <%= translate(locale, "Change password") %> - +
diff --git a/src/invidious/views/clear_watch_history.ecr b/src/invidious/views/clear_watch_history.ecr index 2bb9884c..5f9d1032 100644 --- a/src/invidious/views/clear_watch_history.ecr +++ b/src/invidious/views/clear_watch_history.ecr @@ -3,7 +3,7 @@ <% end %>
-
+ <%= translate(locale, "Clear watch history?") %>
@@ -13,12 +13,12 @@
- + <%= translate(locale, "No") %>
- +
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index c722d0bc..d78d8c4b 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -91,7 +91,7 @@ <% if env.get? "show_watched" %>
" method="post"> - "> + ">

@@ -11,7 +11,7 @@ <% else %>

" method="post"> - "> + "> @@ -24,7 +24,7 @@ ucid: '<%= ucid %>', author: '<%= HTML.escape(author) %>', sub_count_text: '<%= HTML.escape(sub_count_text) %>', - csrf_token: '<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>', + csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', subscribe_text: '<%= HTML.escape(translate(locale, "Subscribe")) %>', unsubscribe_text: '<%= HTML.escape(translate(locale, "Unsubscribe")) %>' } diff --git a/src/invidious/views/data_control.ecr b/src/invidious/views/data_control.ecr index 463d5fd4..e3edb9ea 100644 --- a/src/invidious/views/data_control.ecr +++ b/src/invidious/views/data_control.ecr @@ -3,7 +3,7 @@ <% end %>

- +
<%= translate(locale, "Import") %> diff --git a/src/invidious/views/delete_account.ecr b/src/invidious/views/delete_account.ecr index 0fa1e77c..9103d5b7 100644 --- a/src/invidious/views/delete_account.ecr +++ b/src/invidious/views/delete_account.ecr @@ -3,7 +3,7 @@ <% end %>
- + <%= translate(locale, "Delete account?") %>
@@ -13,12 +13,12 @@
- + <%= translate(locale, "No") %>
- +
diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr index e5154560..7d7ded2c 100644 --- a/src/invidious/views/history.ecr +++ b/src/invidious/views/history.ecr @@ -20,7 +20,7 @@ @@ -35,7 +35,7 @@ var watched_data = {
" method="post"> - "> + ">