diff --git a/README.md b/README.md index 24df6fa4..5176e13c 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ $ ./sentry - [Alternate Tube Redirector](https://addons.mozilla.org/en-US/firefox/addon/alternate-tube-redirector/): Automatically open Youtube Videos on alternate sites like Invidious or Hooktube. - [Invidious Redirect](https://greasyfork.org/en/scripts/370461-invidious-redirect): Redirects Youtube URLs to Invidio.us (userscript) - [Invidio.us embed](https://greasyfork.org/en/scripts/370442-invidious-embed): Replaces YouTube embeds with Invidio.us embeds (userscript) +- [Invidious Downloader](https://github.com/erupete/InvidiousDownloader): Tampermonkey userscript for downloading videos or audio on Invidious (userscript) ## Contributing diff --git a/shard.yml b/shard.yml index 8172b9fe..02777233 100644 --- a/shard.yml +++ b/shard.yml @@ -13,7 +13,7 @@ dependencies: github: detectlanguage/detectlanguage-crystal kemal: github: kemalcr/kemal - commit: b389022 + commit: afd17fc pg: github: will/crystal-pg diff --git a/src/invidious.cr b/src/invidious.cr index 73bfd405..b98c6fd6 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -216,26 +216,7 @@ get "/api/v1/comments/:id" do |env| halt env, status_code: 500, response: error_message end - if format == "json" - next comments - else - comments = JSON.parse(comments) - content_html = template_youtube_comments(comments) - - response = JSON.build do |json| - json.object do - json.field "contentHtml", content_html - - if comments["commentCount"]? - json.field "commentCount", comments["commentCount"] - else - json.field "commentCount", 0 - end - end - end - - next response - end + next comments elsif source == "reddit" begin comments, reddit_thread = fetch_reddit_comments(id) @@ -598,7 +579,12 @@ get "/api/v1/channels/:ucid" do |env| end page = 1 - videos, count = get_60_videos(ucid, page, auto_generated) + begin + videos, count = get_60_videos(ucid, page, auto_generated) + rescue ex + error_message = {"error" => ex.message}.to_json + halt env, status_code: 500, response: error_message + end client = make_client(YT_URL) channel_html = client.get("/channel/#{ucid}/about?disable_polymer=1").body @@ -711,6 +697,7 @@ get "/api/v1/channels/:ucid" do |env| json.field "published", video.published.to_unix json.field "publishedText", "#{recode_date(video.published)} ago" json.field "lengthSeconds", video.length_seconds + json.field "liveNow", video.live_now json.field "paid", video.paid json.field "premium", video.premium end @@ -738,7 +725,12 @@ end halt env, status_code: 500, response: error_message end - videos, count = get_60_videos(ucid, page, auto_generated) + begin + videos, count = get_60_videos(ucid, page, auto_generated) + rescue ex + error_message = {"error" => ex.message}.to_json + halt env, status_code: 500, response: error_message + end result = JSON.build do |json| json.array do @@ -768,6 +760,7 @@ end json.field "published", video.published.to_unix json.field "publishedText", "#{recode_date(video.published)} ago" json.field "lengthSeconds", video.length_seconds + json.field "liveNow", video.live_now json.field "paid", video.paid json.field "premium", video.premium end diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index a699aaac..f4398b58 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -58,7 +58,7 @@ end def fetch_youtube_comments(id, continuation, proxies, format) client = make_client(YT_URL) - html = client.get("/watch?v=#{id}&bpctr=#{Time.new.to_unix + 2000}&gl=US&hl=en&disable_polymer=1") + html = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") headers = HTTP::Headers.new headers["cookie"] = html.cookies.add_request_headers(headers)["cookie"] body = html.body @@ -83,7 +83,7 @@ def fetch_youtube_comments(id, continuation, proxies, format) proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) proxy_client.set_proxy(proxy) - response = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.to_unix + 2000}&gl=US&hl=en&disable_polymer=1") + response = proxy_client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") proxy_headers = HTTP::Headers.new proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] proxy_html = response.body @@ -140,8 +140,8 @@ def fetch_youtube_comments(id, continuation, proxies, format) headers["content-type"] = "application/x-www-form-urlencoded" headers["x-client-data"] = "CIi2yQEIpbbJAQipncoBCNedygEIqKPKAQ==" - headers["x-spf-previous"] = "https://www.youtube.com/watch?v=#{id}&bpctr=#{Time.new.to_unix + 2000}&gl=US&hl=en&disable_polymer=1" - headers["x-spf-referer"] = "https://www.youtube.com/watch?v=#{id}&bpctr=#{Time.new.to_unix + 2000}&gl=US&hl=en&disable_polymer=1" + headers["x-spf-previous"] = "https://www.youtube.com/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999" + headers["x-spf-referer"] = "https://www.youtube.com/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999" headers["x-youtube-client-name"] = "1" headers["x-youtube-client-version"] = "2.20180719" @@ -264,6 +264,23 @@ def fetch_youtube_comments(id, continuation, proxies, format) end end + if format == "html" + comments = JSON.parse(comments) + content_html = template_youtube_comments(comments) + + comments = JSON.build do |json| + json.object do + json.field "contentHtml", content_html + + if comments["commentCount"]? + json.field "commentCount", comments["commentCount"] + else + json.field "commentCount", 0 + end + end + end + end + return comments end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 7fd33c25..240d5438 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -296,3 +296,55 @@ def extract_items(nodeset, ucid = nil) return items end + +def create_response(user_id, operation, key, expire = 6.hours) + expire = Time.now + expire + nonce = Random::Secure.hex(4) + + challenge = "#{expire.to_unix}-#{nonce}-#{user_id}-#{operation}" + token = OpenSSL::HMAC.digest(:sha256, key, challenge) + + challenge = Base64.urlsafe_encode(challenge) + token = Base64.urlsafe_encode(token) + + return challenge, token +end + +def validate_response(challenge, token, user_id, operation, key) + if !challenge + raise "Hidden field \"challenge\" is a required field" + end + + if !token + raise "Hidden field \"token\" is a required field" + end + + challenge = Base64.decode_string(challenge) + if challenge.split("-").size == 4 + expire, nonce, challenge_user_id, challenge_operation = challenge.split("-") + + expire = expire.to_i? + expire ||= 0 + else + raise "Invalid challenge" + end + + challenge = OpenSSL::HMAC.digest(:sha256, HMAC_KEY, challenge) + challenge = Base64.urlsafe_encode(challenge) + + if challenge != token + raise "Invalid token" + end + + if challenge_operation != operation + raise "Invalid token" + end + + if challenge_user_id != user_id + raise "Invalid token" + end + + if expire < Time.now.to_unix + raise "Token is expired, please try again" + end +end diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 66f7371d..688a8622 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -26,7 +26,7 @@ def fetch_mix(rdid, video_id, cookies = nil) if cookies headers = cookies.add_request_headers(headers) end - response = client.get("/watch?v=#{video_id}&list=#{rdid}&bpctr=#{Time.new.to_unix + 2000}&gl=US&hl=en", headers) + response = client.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers) yt_data = response.body.match(/window\["ytInitialData"\] = (?.*);/) if yt_data diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index d85084eb..c1adb1d9 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -30,7 +30,7 @@ def fetch_playlist_videos(plid, page, video_count, continuation = nil) client = make_client(YT_URL) if continuation - html = client.get("/watch?v=#{continuation}&list=#{plid}&bpctr=#{Time.new.to_unix + 2000}&gl=US&hl=en&disable_polymer=1") + html = client.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") html = XML.parse_html(html.body) index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i? @@ -187,7 +187,7 @@ def fetch_playlist(plid) author = anchor.xpath_node(%q(.//li[1]/a)).not_nil!.content author_thumbnail = document.xpath_node(%q(//img[@class="channel-header-profile-image"])).try &.["src"] author_thumbnail ||= "" - ucid = anchor.xpath_node(%q(.//li[1]/a)).not_nil!["href"].split("/")[2] + ucid = anchor.xpath_node(%q(.//li[1]/a)).not_nil!["href"].split("/")[-1] video_count = anchor.xpath_node(%q(.//li[2])).not_nil!.content.delete("videos, ").to_i views = anchor.xpath_node(%q(.//li[3])).not_nil!.content.delete("No views, ") diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 25ee2770..a0fb7f22 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -546,7 +546,7 @@ def fetch_video(id, proxies) spawn do client = make_client(YT_URL) - html = client.get("/watch?v=#{id}&bpctr=#{Time.new.to_unix + 2000}&gl=US&hl=en&disable_polymer=1") + html = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") if md = html.headers["location"]?.try &.match(/v=(?[a-zA-Z0-9_-]{11})/) next html_channel.send(md["id"]) @@ -620,7 +620,7 @@ def fetch_video(id, proxies) client.connect_timeout = 10.seconds client.set_proxy(proxy) - html = XML.parse_html(client.get("/watch?v=#{id}&bpctr=#{Time.new.to_unix + 2000}&gl=US&hl=en&disable_polymer=1").body) + html = XML.parse_html(client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999").body) info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) if info["reason"]? @@ -641,7 +641,19 @@ def fetch_video(id, proxies) end if info["reason"]? - raise info["reason"] + html_info = html.to_s.match(/ytplayer\.config = (?.*?);ytplayer\.load/).try &.["info"] + if html_info + html_info = JSON.parse(html_info)["args"].as_h + info.delete("reason") + + html_info.each do |k, v| + info[k] = v.to_s + end + end + + if info["reason"]? + raise info["reason"] + end end title = info["title"] @@ -699,7 +711,7 @@ def fetch_video(id, proxies) sub_count_text = "0" end - author_thumbnail = html.xpath_node(%(//img[@alt="#{author}"])) + author_thumbnail = html.xpath_node(%(//span[@class="yt-thumb-clip"]/img)) if author_thumbnail author_thumbnail = author_thumbnail["data-thumb"] else