From aeaeacbf8da7d4d76e25f27d49b1141633dbf51e Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 20 Nov 2018 10:07:50 -0600 Subject: [PATCH 01/20] Refactor geo-bypass --- src/invidious.cr | 54 +++++++++++++-------------------------- src/invidious/comments.cr | 44 ++++++++++--------------------- src/invidious/videos.cr | 47 ++++++++-------------------------- 3 files changed, 43 insertions(+), 102 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index e7f28927..50b3092d 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -694,7 +694,9 @@ post "/login" do |env| challenge_req = { user_hash, nil, 1, nil, - {1, nil, nil, nil, {password, nil, true}}, + {1, nil, nil, nil, + {password, nil, true}, + }, {nil, nil, {2, 1, nil, 1, "https://accounts.google.com/ServiceLogin?passive=1209600&continue=https%3A%2F%2Faccounts.google.com%2FManageAccount&followup=https%3A%2F%2Faccounts.google.com%2FManageAccount", nil, [] of String, 4, [] of String}, 1, @@ -3342,45 +3344,24 @@ get "/videoplayback" do |env| host = "https://r#{fvip}---#{mn}.googlevideo.com" url = "/videoplayback?#{query_params.to_s}" - if query_params["region"]? - client = make_client(URI.parse(host)) - response = HTTP::Client::Response.new(status_code: 403) - - if !proxies[query_params["region"]]? - halt env, status_code: 403 - end - - proxies[query_params["region"]].each do |proxy| - begin - client = HTTPClient.new(URI.parse(host)) - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - - response = client.head(url) - if response.status_code == 200 - # For whatever reason the proxy needs to be set again - client.set_proxy(proxy) - break - end - rescue ex - end - end - else - client = make_client(URI.parse(host)) - response = client.head(url) - end - - if response.status_code != 200 - halt env, status_code: 403 - end + region = query_params["region"]? + client = make_client(URI.parse(host), proxies, region) + response = client.head(url) if response.headers["Location"]? url = URI.parse(response.headers["Location"]) env.response.headers["Access-Control-Allow-Origin"] = "*" - next env.redirect url.full_path + + url = url.full_path + if region + url += "®ion=#{region}" + end + + next env.redirect url + end + + if response.status_code >= 400 + halt env, status_code: 403 end headers = env.request.headers @@ -3389,6 +3370,7 @@ get "/videoplayback" do |env| headers.delete("User-Agent") headers.delete("Referer") + client = make_client(URI.parse(host), proxies, region) client.get(url, headers) do |response| env.response.status_code = response.status_code diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 70d365a1..3735aa44 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -70,34 +70,18 @@ def fetch_youtube_comments(id, continuation, proxies, format) if body.match(//) bypass_channel = Channel({String, HTTPClient, HTTP::Headers} | Nil).new - proxies.each do |region, list| + proxies.each do |proxy_region, list| spawn do - proxy_html = %() + proxy_client = make_client(YT_URL, proxies, proxy_region) - list.each do |proxy| - begin - proxy_client = HTTPClient.new(YT_URL) - proxy_client.read_timeout = 10.seconds - proxy_client.connect_timeout = 10.seconds + response = proxy_client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") + proxy_headers = HTTP::Headers.new + proxy_headers["Cookie"] = response.cookies.add_request_headers(headers)["cookie"] + proxy_html = response.body - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - proxy_client.set_proxy(proxy) - - response = proxy_client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") - proxy_headers = HTTP::Headers.new - proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] - proxy_html = response.body - - if !proxy_html.match(//) - bypass_channel.send({proxy_html, proxy_client, proxy_headers}) - break - end - rescue ex - end - end - - # If none of the proxies we tried returned a valid response - if proxy_html.match(//) + if !proxy_html.match(//) + bypass_channel.send({proxy_html, proxy_client, proxy_headers}) + else bypass_channel.send(nil) end end @@ -106,12 +90,12 @@ def fetch_youtube_comments(id, continuation, proxies, format) proxies.size.times do response = bypass_channel.receive if response - session_token = response[0].match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/).not_nil!["session_token"] - itct = response[0].match(/itct=(?[^"]+)"/).not_nil!["itct"] - ctoken = response[0].match(/'COMMENTS_TOKEN': "(?[^"]+)"/) + html, client, headers = response + + session_token = html.match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/).not_nil!["session_token"] + itct = html.match(/itct=(?[^"]+)"/).not_nil!["itct"] + ctoken = html.match(/'COMMENTS_TOKEN': "(?[^"]+)"/) - client = response[1] - headers = response[2] break end end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 89d4cfb1..3934a66f 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -578,47 +578,26 @@ def fetch_video(id, proxies, region) info = info_channel.receive if info["reason"]? && info["reason"].includes? "your country" - bypass_channel = Channel(HTTPProxy | Nil).new + bypass_channel = Channel({HTTPClient, String} | Nil).new - proxies.each do |region, list| + proxies.each do |proxy_region, list| spawn do - info = HTTP::Params.new({ - "reason" => [info["reason"]], - }) + client = make_client(YT_URL, proxies, proxy_region) - list.each do |proxy| - begin - client = HTTPClient.new(YT_URL) - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - - info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) - if !info["reason"]? - bypass_channel.send(proxy) - break - end - rescue ex - end - end - - # If none of the proxies we tried returned a valid response - if info["reason"]? + info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) + if !info["reason"]? + bypass_channel.send({client, proxy_region}) + else bypass_channel.send(nil) end end end proxies.size.times do - proxy = bypass_channel.receive - if proxy + response = bypass_channel.receive + if response begin - client = HTTPClient.new(YT_URL) - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - client.set_proxy(proxy) + client, proxy_region = response html = XML.parse_html(client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999").body) info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) @@ -627,11 +606,7 @@ def fetch_video(id, proxies, region) info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) end - proxy = {ip: proxy.proxy_host, port: proxy.proxy_port} - region_proxies = proxies.select { |region, list| list.includes? proxy } - if !region_proxies.empty? - info["region"] = region_proxies.keys[0] - end + info["region"] = proxy_region break rescue ex From 2e996421731c6024b6d3308ccfb0e0071bc4407b Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 20 Nov 2018 11:18:12 -0600 Subject: [PATCH 02/20] Add /feed/trending --- src/invidious.cr | 32 +++++++++++++++++++++---- src/invidious/trending.cr | 41 ++++++++++++++++++++++++++++++++ src/invidious/views/trending.ecr | 11 +++++++++ 3 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 src/invidious/trending.cr create mode 100644 src/invidious/views/trending.ecr diff --git a/src/invidious.cr b/src/invidious.cr index 50b3092d..b08cbc9e 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1590,6 +1590,20 @@ end # Feeds +get "/feed/trending" do |env| + trending_type = env.params.query["type"]? + region = env.params.query["region"]? + + begin + trending = fetch_trending(trending_type, proxies, region) + rescue ex + error_message = "#{ex.message}" + next templated "error" + end + + templated "trending" +end + get "/feed/subscriptions" do |env| user = env.get? "user" referer = get_referer(env) @@ -2467,14 +2481,19 @@ get "/api/v1/videos/:id" do |env| end get "/api/v1/trending" do |env| - client = make_client(YT_URL) - trending = client.get("/feed/trending?disable_polymer=1").body + region = env.params.query["region"]? + trending_type = env.params.query["type"]? + + begin + trending = fetch_trending(trending_type, proxies, region) + rescue ex + error_message = {"error" => ex.message}.to_json + halt env, status_code: 500, response: error_message + end - trending = XML.parse_html(trending) videos = JSON.build do |json| json.array do - nodeset = trending.xpath_nodes(%q(//ul/li[@class="expanded-shelf-content-item-wrapper"])) - extract_videos(nodeset).each do |video| + trending.each do |video| json.object do json.field "title", video.title json.field "videoId", video.id @@ -2493,6 +2512,9 @@ get "/api/v1/trending" do |env| json.field "publishedText", "#{recode_date(video.published)} ago" json.field "description", video.description json.field "descriptionHtml", video.description_html + json.field "liveNow", video.live_now + json.field "paid", video.paid + json.field "premium", video.premium end end end diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr new file mode 100644 index 00000000..b8ef8186 --- /dev/null +++ b/src/invidious/trending.cr @@ -0,0 +1,41 @@ +def fetch_trending(trending_type, proxies, region) + client = make_client(YT_URL) + headers = HTTP::Headers.new + headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" + + region ||= "US" + region = region.upcase + + trending = "" + if trending_type + trending_type = trending_type.downcase.capitalize + + response = client.get("/feed/trending?gl=#{region}&hl=en", headers).body + + yt_data = response.match(/window\["ytInitialData"\] = (?.*);/) + if yt_data + yt_data = JSON.parse(yt_data["data"].rchop(";")) + else + raise "Could not pull trending pages." + end + + tabs = yt_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][0]["tabRenderer"]["content"]["sectionListRenderer"]["subMenu"]["channelListSubMenuRenderer"]["contents"].as_a + url = tabs.select { |tab| tab["channelListSubMenuAvatarRenderer"]["title"]["simpleText"] == trending_type }[0]? + + if url + url = url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s + url += "&disable_polymer=1&gl=#{region}&hl=en" + trending = client.get(url).body + else + trending = client.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body + end + else + trending = client.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body + end + + trending = XML.parse_html(trending) + nodeset = trending.xpath_nodes(%q(//ul/li[@class="expanded-shelf-content-item-wrapper"])) + trending = extract_videos(nodeset) + + return trending +end diff --git a/src/invidious/views/trending.ecr b/src/invidious/views/trending.ecr new file mode 100644 index 00000000..b3de793c --- /dev/null +++ b/src/invidious/views/trending.ecr @@ -0,0 +1,11 @@ +<% content_for "header" do %> +Trending - Invidious +<% end %> + +
+<% trending.each_slice(4) do |slice| %> + <% slice.each do |item| %> + <%= rendered "components/item" %> + <% end %> +<% end %> +
From 2be43c17aba4d1bbc2cd88f9ad5adadebe7bb309 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 20 Nov 2018 11:18:48 -0600 Subject: [PATCH 03/20] Sample proxies to avoid overloading single proxy --- src/invidious/helpers/utils.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index f1339749..496c3e0d 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -30,7 +30,7 @@ def make_client(url, proxies = {} of String => Array({ip: String, port: Int32}), client.connect_timeout = 10.seconds if region - proxies[region]?.try &.each do |proxy| + proxies[region]?.try &.sample(40).each do |proxy| begin proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) client.set_proxy(proxy) From e5730f4cbcb72d0a51c2dc48a6981318f1cc6195 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 20 Nov 2018 11:19:04 -0600 Subject: [PATCH 04/20] Use 'ion-ios-trash' for /feed/history --- src/invidious/views/history.ecr | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr index 34ba3289..07fbb326 100644 --- a/src/invidious/views/history.ecr +++ b/src/invidious/views/history.ecr @@ -18,11 +18,7 @@ onmouseenter='this["href"]="javascript:void(0)"' href="/mark_unwatched?id=<%= item %>"' > - - +

From a242390fc189f2268ee284eb25bda370e1c437e1 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 20 Nov 2018 13:14:13 -0600 Subject: [PATCH 05/20] Fix typo in nonces.sql --- config/sql/nonces.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sql/nonces.sql b/config/sql/nonces.sql index 9693b936..d2ec5be6 100644 --- a/config/sql/nonces.sql +++ b/config/sql/nonces.sql @@ -5,7 +5,7 @@ CREATE TABLE public.nonces ( nonce text, - expire timestamp with time zone, + expire timestamp with time zone ) WITH ( OIDS=FALSE From 60c6778344c647ef0ebec58624f03926fdaf25fc Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 20 Nov 2018 22:57:51 -0600 Subject: [PATCH 06/20] Make 'watched' icon smaller --- assets/css/default.css | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assets/css/default.css b/assets/css/default.css index 4ec94a68..26558383 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -62,10 +62,10 @@ img.thumbnail { color: #fff; border-radius: 2px; padding: 4px 8px 4px 8px; - font-size: 25px; + font-size: 16px; font-family: sans-serif; - left: 0.25em; - top: -0.75em; + left: 0.20em; + top: -0.70em; } /* From f01cfd0226d31db548c9438db0800e14132b86d0 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 20 Nov 2018 22:58:04 -0600 Subject: [PATCH 07/20] Use material style for trash icon --- src/invidious/views/history.ecr | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr index 07fbb326..90bea14c 100644 --- a/src/invidious/views/history.ecr +++ b/src/invidious/views/history.ecr @@ -16,9 +16,8 @@ - + href="/mark_unwatched?id=<%= item %>"> +

From d6d73bd33630add1ff0d114fd468685ce88c3795 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 20 Nov 2018 22:58:30 -0600 Subject: [PATCH 08/20] Fix clickable titles in subscription feed --- src/invidious/views/components/item.ecr | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 5940ae8a..b38e7a5d 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -69,9 +69,9 @@
Shared <%= recode_date(item.published) %> ago
<% end %> <% else %> + <% if env.get?("user") && env.get("user").as(User).preferences.thin_mode %> + <% else %> - <% if env.get?("user") && env.get("user").as(User).preferences.thin_mode %> - <% else %>
<% if env.get? "show_watched" %> @@ -79,21 +79,19 @@ + href="/mark_watched?id=<%= item.id %>"> + class="icon ion-ios-eye">

<% end %>

<%= recode_length_seconds(item.length_seconds) %>

- <% end %> -

<%= item.title %>

+ <% end %> +

<%= item.title %>

<% if item.responds_to?(:live_now) && item.live_now %>

LIVE

<% end %> From 588f9b9bd68c1bde28324918f6123d4b5254213c Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 08:25:21 -0600 Subject: [PATCH 09/20] Fix 'order' expression --- src/invidious.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index b08cbc9e..520e4ebf 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1680,7 +1680,7 @@ get "/feed/subscriptions" do |env| user.watched, as: ChannelVideo) else videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} \ - ORDER BY published, ucid #{sort}", as: ChannelVideo) + ORDER BY ucid, published #{sort}", as: ChannelVideo) end videos.sort_by! { |video| video.published }.reverse! From 2ce038fb7aa0b089a16fc5dc430d7ac4ea2f63b1 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 13:06:29 -0600 Subject: [PATCH 10/20] Only show toggle watched button when relevant --- src/invidious.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index 520e4ebf..d0ef43c1 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1611,7 +1611,10 @@ get "/feed/subscriptions" do |env| if user user = user.as(User) preferences = user.preferences - env.set "show_watched", true + + if preferences.unseen_only + env.set "show_watched", true + end # Refresh account headers = HTTP::Headers.new From a15463cf377d55ee14269acd64db8fd2b242289a Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 13:10:09 -0600 Subject: [PATCH 11/20] Clarify options in preferences --- src/invidious/views/preferences.ecr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index f34c40aa..cad858c7 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -24,7 +24,7 @@ function update_value(element) {
- + checked<% end %>>
@@ -143,7 +143,7 @@ function update_value(element) {
- + checked<% end %>>
From c656a7cb9e12c6169aec5f58901f47d9b00d2c4a Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 13:10:56 -0600 Subject: [PATCH 12/20] Add link to watch history in preferences --- src/invidious/views/preferences.ecr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index cad858c7..3557f31f 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -166,6 +166,10 @@ function update_value(element) { Manage subscriptions +
+ Watch history +
+
Delete account
From e80884cfce3ec21d6e3bb08a682607d3952eb954 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 13:18:33 -0600 Subject: [PATCH 13/20] Remove unnecessary request header --- src/invidious/views/history.ecr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr index 90bea14c..691201cd 100644 --- a/src/invidious/views/history.ecr +++ b/src/invidious/views/history.ecr @@ -40,7 +40,6 @@ function mark_unwatched(target) { xhr.responseType = "json"; xhr.timeout = 20000; xhr.open("GET", url, true); - xhr.setRequestHeader("Redirect", "false"); xhr.send(); xhr.onreadystatechange = function() { From cdd916f51d68e9982ab3e98cad20c998504146cb Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 13:35:37 -0600 Subject: [PATCH 14/20] Add async for manage_subscriptions --- src/invidious.cr | 17 +++++++++- src/invidious/views/subscription_manager.ecr | 33 ++++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index d0ef43c1..34cf318b 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1438,6 +1438,12 @@ get "/subscription_ajax" do |env| user = env.get? "user" referer = get_referer(env) + redirect = env.params.query["redirect"]? + redirect ||= "false" + redirect = redirect == "true" + + count_text = "" + if user user = user.as(User) @@ -1500,9 +1506,18 @@ get "/subscription_ajax" do |env| PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE id = $2", channel_id, sid) end end + + count_text = PG_DB.query_one?("SELECT cardinality(subscriptions) FROM users WHERE id = $1", [sid], as: Int64) + count_text ||= 0 + count_text = "#{number_with_separator(count_text)} subscriptions" end - env.redirect referer + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + {"countText" => count_text}.to_json + end end get "/delete_account" do |env| diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/subscription_manager.ecr index b942e686..d420fbf8 100644 --- a/src/invidious/views/subscription_manager.ecr +++ b/src/invidious/views/subscription_manager.ecr @@ -4,7 +4,7 @@
-

<%= subscriptions.size %> subscriptions

+

<%= subscriptions.size %> subscriptions

@@ -24,7 +24,12 @@

- unsubscribe + + unsubscribe +

@@ -34,3 +39,27 @@ <% end %>
<% end %> + + \ No newline at end of file From fd7aa59e0fdc50ca886c2b3f243e444dd80fdeb6 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 17:12:13 -0600 Subject: [PATCH 15/20] Properly parse NewPipe imports --- shard.yml | 4 +++- src/invidious.cr | 60 +++++++++++++++++++++++++----------------------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/shard.yml b/shard.yml index 02777233..f848b239 100644 --- a/shard.yml +++ b/shard.yml @@ -16,7 +16,9 @@ dependencies: commit: afd17fc pg: github: will/crystal-pg + sqlite3: + github: crystal-lang/crystal-sqlite3 -crystal: 0.26.1 +crystal: 0.27.0 license: AGPLv3 diff --git a/src/invidious.cr b/src/invidious.cr index 34cf318b..dd134a72 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -20,6 +20,7 @@ require "kemal" require "openssl/hmac" require "option_parser" require "pg" +require "sqlite3" require "xml" require "yaml" require "zip" @@ -1142,7 +1143,7 @@ get "/mark_unwatched" do |env| if user user = user.as(User) - PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE id = $2", id, user.id) + PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email) end if redirect @@ -1320,12 +1321,13 @@ post "/data_control" do |env| user.subscriptions += body["subscriptions"].as_a.map { |a| a.as_s } user.subscriptions.uniq! - user.subscriptions.each do |ucid| + user.subscriptions.select! do |ucid| begin client = make_client(YT_URL) get_channel(ucid, client, PG_DB, false, false) + true rescue ex - next + false end end @@ -1349,12 +1351,13 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions.each do |ucid| + user.subscriptions.select! do |ucid| begin client = make_client(YT_URL) get_channel(ucid, client, PG_DB, false, false) + true rescue ex - next + false end end @@ -1365,12 +1368,13 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions.each do |ucid| + user.subscriptions.select! do |ucid| begin client = make_client(YT_URL) get_channel(ucid, client, PG_DB, false, false) + true rescue ex - next + false end end @@ -1396,34 +1400,32 @@ post "/data_control" do |env| Zip::Reader.open(IO::Memory.new(body)) do |file| file.each_entry do |entry| if entry.filename == "newpipe.db" - # We do this because the SQLite driver cannot parse a database from an IO - # Currently: channel URLs can **only** be subscriptions, and - # video URLs can **only** be watch history, so this works okay for now. + tempfile = File.tempfile(".db") + File.write(tempfile.path, entry.io.gets_to_end) + db = DB.open("sqlite3://" + tempfile.path) - db = entry.io.gets_to_end - - user.watched += db.scan(/youtube\.com\/watch\?v\=(?[a-zA-Z0-9_-]{11})/).map do |md| - md["id"] - end + user.watched += db.query_all("SELECT url FROM streams", as: String).map { |url| url.lchop("https://www.youtube.com/watch?v=") } user.watched.uniq! PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email) - user.subscriptions += db.scan(/youtube\.com\/channel\/(?[a-zA-Z0-9_-]{22})/).map do |md| - md["ucid"] - end + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map { |url| url.lchop("https://www.youtube.com/channel/") } user.subscriptions.uniq! - user.subscriptions.each do |ucid| + user.subscriptions.select! do |ucid| begin client = make_client(YT_URL) get_channel(ucid, client, PG_DB, false, false) + true rescue ex - next + false end end PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + + db.close + tempfile.delete end end end @@ -1482,32 +1484,32 @@ get "/subscription_ajax" do |env| # Update user if client.post(post_url, headers, post_req).status_code == 200 - sid = user.id + email = user.email case action when .starts_with? "action_create" - PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", channel_id, sid) + PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", channel_id, email) when .starts_with? "action_remove" - PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE id = $2", channel_id, sid) + PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE email = $2", channel_id, email) end end else - sid = user.id + email = user.email case action when .starts_with? "action_create" if !user.subscriptions.includes? channel_id - PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", channel_id, sid) + PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", channel_id, email) client = make_client(YT_URL) get_channel(channel_id, client, PG_DB, false, false) end when .starts_with? "action_remove" - PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE id = $2", channel_id, sid) + PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE email = $2", channel_id, email) end end - count_text = PG_DB.query_one?("SELECT cardinality(subscriptions) FROM users WHERE id = $1", [sid], as: Int64) + count_text = PG_DB.query_one?("SELECT cardinality(subscriptions) FROM users WHERE email = $1", email, as: Int32) count_text ||= 0 count_text = "#{number_with_separator(count_text)} subscriptions" end @@ -1742,8 +1744,8 @@ get "/feed/subscriptions" do |env| videos = videos[0..max_results] end - PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE id = $3", [] of String, Time.now, - user.id) + PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.now, + user.email) user.notifications = [] of String env.set "user", user From 95ebfd34c5b5484e9fc3d9932e3e8b61c0985603 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 19:26:55 -0600 Subject: [PATCH 16/20] Don't wait on server for subscription count --- src/invidious.cr | 8 +------- src/invidious/views/subscription_manager.ecr | 8 ++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index dd134a72..74ae35db 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1444,8 +1444,6 @@ get "/subscription_ajax" do |env| redirect ||= "false" redirect = redirect == "true" - count_text = "" - if user user = user.as(User) @@ -1508,17 +1506,13 @@ get "/subscription_ajax" do |env| PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE email = $2", channel_id, email) end end - - count_text = PG_DB.query_one?("SELECT cardinality(subscriptions) FROM users WHERE email = $1", email, as: Int32) - count_text ||= 0 - count_text = "#{number_with_separator(count_text)} subscriptions" end if redirect env.redirect referer else env.response.content_type = "application/json" - {"countText" => count_text}.to_json + "{}" end end diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/subscription_manager.ecr index d420fbf8..ac1c0cdd 100644 --- a/src/invidious/views/subscription_manager.ecr +++ b/src/invidious/views/subscription_manager.ecr @@ -4,7 +4,7 @@
-

<%= subscriptions.size %> subscriptions

+

<%= subscriptions.size %> subscriptions

@@ -44,6 +44,8 @@ function remove_subscription(target) { var row = target.parentNode.parentNode.parentNode.parentNode; row.style.display = "none"; + var count = document.getElementById("count") + count.innerText = count.innerText - 1; var url = "/subscription_ajax?action_remove_subscriptions=1&redirect=false&c=" + target.getAttribute("data-id"); var xhr = new XMLHttpRequest(); @@ -54,9 +56,7 @@ function remove_subscription(target) { xhr.onreadystatechange = function() { if (xhr.readyState == 4) { - if (xhr.status == 200) { - document.getElementById("count").innerHTML = xhr.response.countText; - } else { + if (xhr.status != 200) { row.style.display = ""; } } From 941a773b7d931ce097db11de8d3a09ade0c2071c Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 20:00:17 -0600 Subject: [PATCH 17/20] Add opensearch.xml --- src/invidious.cr | 15 +++++++++++++++ src/invidious/views/template.ecr | 1 + 2 files changed, 16 insertions(+) diff --git a/src/invidious.cr b/src/invidious.cr index 74ae35db..de40698f 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -512,6 +512,21 @@ end # Search +get "/opensearch.xml" do |env| + env.response.content_type = "application/opensearchdescription+xml" + + XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("OpenSearchDescription", xmlns: "http://a9.com/-/spec/opensearch/1.1/") do + xml.element("ShortName") { xml.text "Invidious" } + xml.element("LongName") { xml.text "Invidious Search" } + xml.element("Description") { xml.text "Search for videos, channels, and playlists on Invidious" } + xml.element("InputEncoding") { xml.text "UTF-8" } + xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "/favicon.ico" } + xml.element("Url", type: "text/html", method: "get", template: "/search?q={searchTerms}") + end + end +end + get "/results" do |env| query = env.params.query["search_query"]? query ||= env.params.query["q"]? diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 748a691f..4cfb41df 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -13,6 +13,7 @@ + From 568e55dfa6af08880fa445fc8cac0fa45c0f029e Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 20:00:33 -0600 Subject: [PATCH 18/20] Add description for home page --- src/invidious/views/index.ecr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/invidious/views/index.ecr b/src/invidious/views/index.ecr index 46d07a9a..cd88d540 100644 --- a/src/invidious/views/index.ecr +++ b/src/invidious/views/index.ecr @@ -1,4 +1,5 @@ <% content_for "header" do %> + Invidious <% end %> From ca4e8b800c62e4747a61c7358322e2fbf1fcb5cb Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 21 Nov 2018 20:49:14 -0600 Subject: [PATCH 19/20] Use absolute paths in /opensearch.xml --- src/invidious.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index de40698f..76d772dc 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -521,8 +521,8 @@ get "/opensearch.xml" do |env| xml.element("LongName") { xml.text "Invidious Search" } xml.element("Description") { xml.text "Search for videos, channels, and playlists on Invidious" } xml.element("InputEncoding") { xml.text "UTF-8" } - xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "/favicon.ico" } - xml.element("Url", type: "text/html", method: "get", template: "/search?q={searchTerms}") + xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "https://invidio.us/favicon.ico" } + xml.element("Url", type: "text/html", method: "get", template: "https://invidio.us/search?q={searchTerms}") end end end From 26eb59e00d9596adf69e3a3d5f8cb899f8a1da0a Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 22 Nov 2018 13:26:08 -0600 Subject: [PATCH 20/20] Add text CAPTCHA --- src/invidious.cr | 87 +++++++++++++++++++++++++++-------- src/invidious/views/login.ecr | 20 +++++++- 2 files changed, 85 insertions(+), 22 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 76d772dc..cbba17c1 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -16,6 +16,7 @@ require "crypto/bcrypt/password" require "detect_language" +require "digest/md5" require "kemal" require "openssl/hmac" require "option_parser" @@ -82,10 +83,11 @@ PG_URL = URI.new( path: CONFIG.db[:dbname], ) -PG_DB = DB.open PG_URL -YT_URL = URI.parse("https://www.youtube.com") -REDDIT_URL = URI.parse("https://www.reddit.com") -LOGIN_URL = URI.parse("https://accounts.google.com") +PG_DB = DB.open PG_URL +YT_URL = URI.parse("https://www.youtube.com") +REDDIT_URL = URI.parse("https://www.reddit.com") +LOGIN_URL = URI.parse("https://accounts.google.com") +TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com/omarroth@hotmail.com.json") crawl_threads.times do spawn do @@ -632,8 +634,25 @@ get "/login" do |env| account_type = env.params.query["type"]? account_type ||= "invidious" + captcha_type = env.params.query["captcha"]? + captcha_type ||= "image" + if account_type == "invidious" - captcha = generate_captcha(HMAC_KEY, PG_DB) + if captcha_type == "image" + captcha = generate_captcha(HMAC_KEY, PG_DB) + else + response = HTTP::Client.get(TEXTCAPTCHA_URL).body + response = JSON.parse(response) + + tokens = response["a"].as_a.map do |answer| + create_response(answer.as_s, "sign_in", HMAC_KEY, PG_DB) + end + + text_captcha = { + question: response["q"].as_s, + tokens: tokens, + } + end end tfa = env.params.query["tfa"]? @@ -827,27 +846,55 @@ post "/login" do |env| end elsif account_type == "invidious" answer = env.params.body["answer"]? + text_answer = env.params.body["text_answer"]? - if !answer - error_message = "CAPTCHA is a required field" - next templated "error" - end + if answer + answer = answer.lstrip('0') + answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) - answer = answer.lstrip('0') - answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) + challenge = env.params.body["challenge"]? + token = env.params.body["token"]? - challenge = env.params.body["challenge"]? - token = env.params.body["token"]? + begin + validate_response(challenge, token, answer, "sign_in", HMAC_KEY, PG_DB) + rescue ex + if ex.message == "Invalid user" + error_message = "Invalid answer" + else + error_message = ex.message + end - begin - validate_response(challenge, token, answer, "sign_in", HMAC_KEY, PG_DB) - rescue ex - if ex.message && ex.message == "Invalid user" - error_message = "Invalid CAPTCHA response" - else - error_message = ex.message + next templated "error" + end + elsif text_answer + text_answer = Digest::MD5.hexdigest(text_answer.downcase.strip) + + challenges = env.params.body.select { |k, v| k.match(/text_challenge\d+/) } + tokens = env.params.body.select { |k, v| k.match(/text_token\d+/) } + + found_valid_captcha = false + + error_message = "Invalid CAPTCHA" + challenges.each_with_index do |challenge, i| + begin + challenge = challenge[1] + token = tokens[i][1] + validate_response(challenge, token, text_answer, "sign_in", HMAC_KEY, PG_DB) + found_valid_captcha = true + rescue ex + if ex.message == "Invalid user" + error_message = "Invalid answer" + else + error_message = ex.message + end + end end + if !found_valid_captcha + next templated "error" + end + else + error_message = "CAPTCHA is a required field" next templated "error" end diff --git a/src/invidious/views/login.ecr b/src/invidious/views/login.ecr index 0243d900..69f04ed2 100644 --- a/src/invidious/views/login.ecr +++ b/src/invidious/views/login.ecr @@ -24,11 +24,27 @@ + <% if captcha_type == "image" %> - - + + + + <% else %> + <% text_captcha.not_nil![:tokens].each_with_index do |token, i| %> + + + <% end %> + + + + + <% end %>