diff --git a/README.md b/README.md index 5176e13c..233baf54 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ - Support for Reddit comments in place of YT comments - Import/Export subscriptions, watch history, preferences - Does not use any of the official YouTube APIs +- Developer [API](https://github.com/omarroth/invidious/wiki/API) Liberapay: https://liberapay.com/omarroth Patreon: https://patreon.com/omarroth diff --git a/src/invidious.cr b/src/invidious.cr index b98c6fd6..055812c6 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -100,10 +100,11 @@ get "/api/v1/captions/:id" do |env| env.response.content_type = "application/json" id = env.params.url["id"] + region = env.params.query["region"]? client = make_client(YT_URL) begin - video = fetch_video(id, proxies) + video = fetch_video(id, proxies, region) rescue ex : VideoRedirect next env.redirect "/api/v1/captions/#{ex.message}" rescue ex @@ -332,9 +333,10 @@ get "/api/v1/videos/:id" do |env| env.response.content_type = "application/json" id = env.params.url["id"] + region = env.params.query["region"]? begin - video = fetch_video(id, proxies) + video = fetch_video(id, proxies, region) rescue ex : VideoRedirect next env.redirect "/api/v1/videos/#{ex.message}" rescue ex @@ -570,6 +572,8 @@ get "/api/v1/channels/:ucid" do |env| env.response.content_type = "application/json" ucid = env.params.url["ucid"] + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by ||= "newest" begin author, ucid, auto_generated = get_about_info(ucid) @@ -580,7 +584,7 @@ get "/api/v1/channels/:ucid" do |env| page = 1 begin - videos, count = get_60_videos(ucid, page, auto_generated) + videos, count = get_60_videos(ucid, page, auto_generated, sort_by) rescue ex error_message = {"error" => ex.message}.to_json halt env, status_code: 500, response: error_message @@ -717,6 +721,8 @@ end ucid = env.params.url["ucid"] page = env.params.query["page"]?.try &.to_i? page ||= 1 + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by ||= "newest" begin author, ucid, auto_generated = get_about_info(ucid) @@ -726,7 +732,7 @@ end end begin - videos, count = get_60_videos(ucid, page, auto_generated) + videos, count = get_60_videos(ucid, page, auto_generated, sort_by) rescue ex error_message = {"error" => ex.message}.to_json halt env, status_code: 500, response: error_message @@ -1178,10 +1184,11 @@ get "/api/manifest/dash/id/:id" do |env| local = env.params.query["local"]?.try &.== "true" id = env.params.url["id"] + region = env.params.query["region"]? client = make_client(YT_URL) begin - video = fetch_video(id, proxies) + video = fetch_video(id, proxies, region) rescue ex : VideoRedirect next env.redirect "/api/manifest/dash/id/#{ex.message}" rescue ex diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index dcab5e29..f713d97b 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -163,7 +163,7 @@ def fetch_channel(ucid, client, db, pull_all_videos = true) return channel end -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil) +def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest") if auto_generated seed = Time.unix(1525757349) @@ -190,6 +190,16 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil) meta += page.size.to_u8.unsafe_chr meta += page + case sort_by + when "newest" + # Empty tags can be omitted + # meta += "\x18\x00" + when "popular" + meta += "\x18\x01" + when "oldest" + meta += "\x18\x02" + end + meta = Base64.urlsafe_encode(meta) meta = URI.escape(meta) @@ -254,14 +264,14 @@ def get_about_info(ucid) return {author, ucid, auto_generated, sub_count} end -def get_60_videos(ucid, page, auto_generated) +def get_60_videos(ucid, page, auto_generated, sort_by = "newest") count = 0 videos = [] of SearchVideo client = make_client(YT_URL) 2.times do |i| - url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated) + url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) response = client.get(url) json = JSON.parse(response.body) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 240d5438..4bcfdfb6 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -16,6 +16,7 @@ class Config hmac_key: String?, full_refresh: Bool, geo_bypass: Bool, + domain: String?, }) end @@ -36,55 +37,6 @@ def login_req(login_form, f_req) return HTTP::Params.encode(data) end -def generate_captcha(key) - minute = Random::Secure.rand(12) - minute_angle = minute * 30 - minute = minute * 5 - - hour = Random::Secure.rand(12) - hour_angle = hour * 30 + minute_angle.to_f / 12 - if hour == 0 - hour = 12 - end - - clock_svg = <<-END_SVG - - - - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 - 12 - - - - - - END_SVG - - challenge = "" - convert = Process.run(%(convert -density 1200 -resize 400x400 -background none svg:- png:-), shell: true, - input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe) do |proc| - challenge = proc.output.gets_to_end - challenge = Base64.strict_encode(challenge) - challenge = "data:image/png;base64,#{challenge}" - end - - answer = "#{hour}:#{minute.to_s.rjust(2, '0')}" - token = OpenSSL::HMAC.digest(:sha256, key, answer) - token = Base64.urlsafe_encode(token) - - return {challenge: challenge, token: token} -end - def html_to_content(description_html) if !description_html description = "" @@ -296,55 +248,3 @@ def extract_items(nodeset, ucid = nil) return items end - -def create_response(user_id, operation, key, expire = 6.hours) - expire = Time.now + expire - nonce = Random::Secure.hex(4) - - challenge = "#{expire.to_unix}-#{nonce}-#{user_id}-#{operation}" - token = OpenSSL::HMAC.digest(:sha256, key, challenge) - - challenge = Base64.urlsafe_encode(challenge) - token = Base64.urlsafe_encode(token) - - return challenge, token -end - -def validate_response(challenge, token, user_id, operation, key) - if !challenge - raise "Hidden field \"challenge\" is a required field" - end - - if !token - raise "Hidden field \"token\" is a required field" - end - - challenge = Base64.decode_string(challenge) - if challenge.split("-").size == 4 - expire, nonce, challenge_user_id, challenge_operation = challenge.split("-") - - expire = expire.to_i? - expire ||= 0 - else - raise "Invalid challenge" - end - - challenge = OpenSSL::HMAC.digest(:sha256, HMAC_KEY, challenge) - challenge = Base64.urlsafe_encode(challenge) - - if challenge != token - raise "Invalid token" - end - - if challenge_operation != operation - raise "Invalid token" - end - - if challenge_user_id != user_id - raise "Invalid token" - end - - if expire < Time.now.to_unix - raise "Token is expired, please try again" - end -end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 174f2993..f1339749 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -18,16 +18,28 @@ def elapsed_text(elapsed) "#{(millis * 1000).round(2)}µs" end -def make_client(url) +def make_client(url, proxies = {} of String => Array({ip: String, port: Int32}), region = nil) context = OpenSSL::SSL::Context::Client.new context.add_options( OpenSSL::SSL::Options::ALL | OpenSSL::SSL::Options::NO_SSL_V2 | OpenSSL::SSL::Options::NO_SSL_V3 ) - client = HTTP::Client.new(url, context) + client = HTTPClient.new(url, context) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds + + if region + proxies[region]?.try &.each do |proxy| + begin + proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) + client.set_proxy(proxy) + break + rescue ex + end + end + end + return client end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index b354306f..c8769090 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -70,7 +70,11 @@ class Preferences JSON.mapping({ video_loop: Bool, autoplay: Bool, - listen: { + continue: { + type: Bool, + default: false, + }, + listen: { type: Bool, default: false, }, @@ -195,3 +199,112 @@ def create_user(sid, email, password) return user end + +def create_response(user_id, operation, key, db, expire = 6.hours) + expire = Time.now + expire + nonce = Random::Secure.hex(16) + db.exec("INSERT INTO nonces VALUES ($1) ON CONFLICT DO NOTHING", nonce) + + challenge = "#{expire.to_unix}-#{nonce}-#{user_id}-#{operation}" + token = OpenSSL::HMAC.digest(:sha256, key, challenge) + + challenge = Base64.urlsafe_encode(challenge) + token = Base64.urlsafe_encode(token) + + return challenge, token +end + +def validate_response(challenge, token, user_id, operation, key, db) + if !challenge + raise "Hidden field \"challenge\" is a required field" + end + + if !token + raise "Hidden field \"token\" is a required field" + end + + challenge = Base64.decode_string(challenge) + if challenge.split("-").size == 4 + expire, nonce, challenge_user_id, challenge_operation = challenge.split("-") + + expire = expire.to_i? + expire ||= 0 + else + raise "Invalid challenge" + end + + challenge = OpenSSL::HMAC.digest(:sha256, HMAC_KEY, challenge) + challenge = Base64.urlsafe_encode(challenge) + + if db.query_one?("SELECT EXISTS (SELECT true FROM nonces WHERE nonce = $1)", nonce, as: Bool) + db.exec("DELETE FROM nonces * WHERE nonce = $1", nonce) + else + raise "Invalid token" + end + + if challenge != token + raise "Invalid token" + end + + if challenge_operation != operation + raise "Invalid token" + end + + if challenge_user_id != user_id + raise "Invalid user" + end + + if expire < Time.now.to_unix + raise "Token is expired, please try again" + end +end + +def generate_captcha(key, db) + minute = Random::Secure.rand(12) + minute_angle = minute * 30 + minute = minute * 5 + + hour = Random::Secure.rand(12) + hour_angle = hour * 30 + minute_angle.to_f / 12 + if hour == 0 + hour = 12 + end + + clock_svg = <<-END_SVG + + + + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + + + + + + END_SVG + + image = "" + convert = Process.run(%(convert -density 1200 -resize 400x400 -background none svg:- png:-), shell: true, + input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe) do |proc| + image = proc.output.gets_to_end + image = Base64.strict_encode(image) + image = "data:image/png;base64,#{image}" + end + + answer = "#{hour}:#{minute.to_s.rjust(2, '0')}" + answer = OpenSSL::HMAC.hexdigest(:sha256, key, answer) + + challenge, token = create_response(answer, "sign_in", key, db) + + return {image: image, challenge: challenge, token: token} +end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index a0fb7f22..89d4cfb1 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -507,14 +507,14 @@ end class VideoRedirect < Exception end -def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}), refresh = true) +def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}), refresh = true, region = nil) if db.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool) video = db.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video) # If record was last updated over 10 minutes ago, refresh (expire param in response lasts for 6 hours) if refresh && Time.now - video.updated > 10.minutes begin - video = fetch_video(id, proxies) + video = fetch_video(id, proxies, region) video_array = video.to_a args = arg_array(video_array[1..-1], 2) @@ -529,7 +529,7 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}) end end else - video = fetch_video(id, proxies) + video = fetch_video(id, proxies, region) video_array = video.to_a args = arg_array(video_array) @@ -540,12 +540,12 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}) return video end -def fetch_video(id, proxies) +def fetch_video(id, proxies, region) html_channel = Channel(XML::Node | String).new info_channel = Channel(HTTP::Params).new spawn do - client = make_client(YT_URL) + client = make_client(YT_URL, proxies, region) html = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") if md = html.headers["location"]?.try &.match(/v=(?[a-zA-Z0-9_-]{11})/) @@ -557,7 +557,7 @@ def fetch_video(id, proxies) end spawn do - client = make_client(YT_URL) + client = make_client(YT_URL, proxies, region) info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1") info = HTTP::Params.parse(info.body) @@ -628,9 +628,9 @@ def fetch_video(id, proxies) end proxy = {ip: proxy.proxy_host, port: proxy.proxy_port} - region = proxies.select { |region, list| list.includes? proxy } - if !region.empty? - info["region"] = region.keys[0] + region_proxies = proxies.select { |region, list| list.includes? proxy } + if !region_proxies.empty? + info["region"] = region_proxies.keys[0] end break @@ -677,7 +677,8 @@ def fetch_video(id, proxies) wilson_score = ci_lower_bound(likes, likes + dislikes) - published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).not_nil!["content"] + published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).try &.["content"] + published ||= Time.now.to_s("%Y-%m-%d") published = Time.parse(published, "%Y-%m-%d", Time::Location.local) allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).try &.["content"].split(",") @@ -685,15 +686,16 @@ def fetch_video(id, proxies) is_family_friendly = html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).try &.["content"] == "True" is_family_friendly ||= true - genre = html.xpath_node(%q(//meta[@itemprop="genre"])).not_nil!["content"] + genre = html.xpath_node(%q(//meta[@itemprop="genre"])).try &.["content"] + genre ||= "" + genre_url = html.xpath_node(%(//a[text()="#{genre}"])).try &.["href"] case genre when "Movies" genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g" when "Education" # Education channel is linked but does not exist - # genre_url = "/channel/UC3yA8nDwraeOfnYfBWun83g" - genre_url = "" + genre_url = "/channel/UC3yA8nDwraeOfnYfBWun83g" end genre_url ||= "" @@ -730,15 +732,19 @@ end def process_video_params(query, preferences) autoplay = query["autoplay"]?.try &.to_i? + continue = query["continue"]?.try &.to_i? listen = query["listen"]? && (query["listen"] == "true" || query["listen"] == "1").to_unsafe preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase } quality = query["quality"]? + region = query["region"]? speed = query["speed"]?.try &.to_f? video_loop = query["loop"]?.try &.to_i? volume = query["volume"]?.try &.to_i? if preferences + # region ||= preferences.region autoplay ||= preferences.autoplay.to_unsafe + continue ||= preferences.continue.to_unsafe listen ||= preferences.listen.to_unsafe preferred_captions ||= preferences.captions quality ||= preferences.quality @@ -748,6 +754,7 @@ def process_video_params(query, preferences) end autoplay ||= 0 + continue ||= 0 listen ||= 0 preferred_captions ||= [] of String quality ||= "hd720" @@ -756,6 +763,7 @@ def process_video_params(query, preferences) volume ||= 100 autoplay = autoplay == 1 + continue = continue == 1 listen = listen == 1 video_loop = video_loop == 1 @@ -786,11 +794,13 @@ def process_video_params(query, preferences) params = { autoplay: autoplay, + continue: continue, controls: controls, listen: listen, preferred_captions: preferred_captions, quality: quality, raw: raw, + region: region, speed: speed, video_end: video_end, video_loop: video_loop,