Merge branch 'master' into api-only

README.md

@@ -20,6 +20,7 @@
 - Support for Reddit comments in place of YT comments
 - Import/Export subscriptions, watch history, preferences
 - Does not use any of the official YouTube APIs
+- Developer [API](https://github.com/omarroth/invidious/wiki/API)
 
 Liberapay: https://liberapay.com/omarroth  
 Patreon: https://patreon.com/omarroth  

src/invidious.cr

@@ -100,10 +100,11 @@ get "/api/v1/captions/:id" do |env|
   env.response.content_type = "application/json"
 
   id = env.params.url["id"]
+  region = env.params.query["region"]?
 
   client = make_client(YT_URL)
   begin
-    video = fetch_video(id, proxies)
+    video = fetch_video(id, proxies, region)
   rescue ex : VideoRedirect
     next env.redirect "/api/v1/captions/#{ex.message}"
   rescue ex
@@ -332,9 +333,10 @@ get "/api/v1/videos/:id" do |env|
   env.response.content_type = "application/json"
 
   id = env.params.url["id"]
+  region = env.params.query["region"]?
 
   begin
-    video = fetch_video(id, proxies)
+    video = fetch_video(id, proxies, region)
   rescue ex : VideoRedirect
     next env.redirect "/api/v1/videos/#{ex.message}"
   rescue ex
@@ -570,6 +572,8 @@ get "/api/v1/channels/:ucid" do |env|
   env.response.content_type = "application/json"
 
   ucid = env.params.url["ucid"]
+  sort_by = env.params.query["sort_by"]?.try &.downcase
+  sort_by ||= "newest"
 
   begin
     author, ucid, auto_generated = get_about_info(ucid)
@@ -580,7 +584,7 @@ get "/api/v1/channels/:ucid" do |env|
 
   page = 1
   begin
-    videos, count = get_60_videos(ucid, page, auto_generated)
+    videos, count = get_60_videos(ucid, page, auto_generated, sort_by)
   rescue ex
     error_message = {"error" => ex.message}.to_json
     halt env, status_code: 500, response: error_message
@@ -717,6 +721,8 @@ end
     ucid = env.params.url["ucid"]
     page = env.params.query["page"]?.try &.to_i?
     page ||= 1
+    sort_by = env.params.query["sort_by"]?.try &.downcase
+    sort_by ||= "newest"
 
     begin
       author, ucid, auto_generated = get_about_info(ucid)
@@ -726,7 +732,7 @@ end
     end
 
     begin
-      videos, count = get_60_videos(ucid, page, auto_generated)
+      videos, count = get_60_videos(ucid, page, auto_generated, sort_by)
     rescue ex
       error_message = {"error" => ex.message}.to_json
       halt env, status_code: 500, response: error_message
@@ -1178,10 +1184,11 @@ get "/api/manifest/dash/id/:id" do |env|
 
   local = env.params.query["local"]?.try &.== "true"
   id = env.params.url["id"]
+  region = env.params.query["region"]?
 
   client = make_client(YT_URL)
   begin
-    video = fetch_video(id, proxies)
+    video = fetch_video(id, proxies, region)
   rescue ex : VideoRedirect
     next env.redirect "/api/manifest/dash/id/#{ex.message}"
   rescue ex

src/invidious/channels.cr

@@ -163,7 +163,7 @@ def fetch_channel(ucid, client, db, pull_all_videos = true)
   return channel
 end
 
-def produce_channel_videos_url(ucid, page = 1, auto_generated = nil)
+def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest")
   if auto_generated
     seed = Time.unix(1525757349)
 
@@ -190,6 +190,16 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil)
   meta += page.size.to_u8.unsafe_chr
   meta += page
 
+  case sort_by
+  when "newest"
+    # Empty tags can be omitted
+    # meta += "\x18\x00"
+  when "popular"
+    meta += "\x18\x01"
+  when "oldest"
+    meta += "\x18\x02"
+  end
+
   meta = Base64.urlsafe_encode(meta)
   meta = URI.escape(meta)
 
@@ -254,14 +264,14 @@ def get_about_info(ucid)
   return {author, ucid, auto_generated, sub_count}
 end
 
-def get_60_videos(ucid, page, auto_generated)
+def get_60_videos(ucid, page, auto_generated, sort_by = "newest")
   count = 0
   videos = [] of SearchVideo
 
   client = make_client(YT_URL)
 
   2.times do |i|
-    url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated)
+    url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
     response = client.get(url)
     json = JSON.parse(response.body)
 

src/invidious/helpers/helpers.cr

@@ -16,6 +16,7 @@ class Config
     hmac_key:     String?,
     full_refresh: Bool,
     geo_bypass:   Bool,
+    domain:       String?,
   })
 end
 
@@ -36,55 +37,6 @@ def login_req(login_form, f_req)
   return HTTP::Params.encode(data)
 end
 
-def generate_captcha(key)
-  minute = Random::Secure.rand(12)
-  minute_angle = minute * 30
-  minute = minute * 5
-
-  hour = Random::Secure.rand(12)
-  hour_angle = hour * 30 + minute_angle.to_f / 12
-  if hour == 0
-    hour = 12
-  end
-
-  clock_svg = <<-END_SVG
-  <svg viewBox="0 0 100 100" width="200px">
-  <circle cx="50" cy="50" r="45" fill="#eee" stroke="black" stroke-width="2"></circle>
-  
-  <text x="69"     y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 1</text>
-  <text x="82.909" y="34"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 2</text>
-  <text x="88"     y="53"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 3</text>
-  <text x="82.909" y="72"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 4</text>
-  <text x="69"     y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 5</text>
-  <text x="50"     y="91"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 6</text>
-  <text x="31"     y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 7</text>
-  <text x="17.091" y="72"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 8</text>
-  <text x="12"     y="53"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 9</text>
-  <text x="17.091" y="34"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px">10</text>
-  <text x="31"     y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">11</text>
-  <text x="50"     y="15"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px">12</text>
-
-  <circle cx="50" cy="50" r="3" fill="black"></circle>
-  <line id="minute" transform="rotate(#{minute_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="16" fill="black" stroke="black" stroke-width="2"></line>
-  <line id="hour"   transform="rotate(#{hour_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="24" fill="black" stroke="black" stroke-width="2"></line>
-  </svg>
-  END_SVG
-
-  challenge = ""
-  convert = Process.run(%(convert -density 1200 -resize 400x400 -background none svg:- png:-), shell: true,
-    input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe) do |proc|
-    challenge = proc.output.gets_to_end
-    challenge = Base64.strict_encode(challenge)
-    challenge = "data:image/png;base64,#{challenge}"
-  end
-
-  answer = "#{hour}:#{minute.to_s.rjust(2, '0')}"
-  token = OpenSSL::HMAC.digest(:sha256, key, answer)
-  token = Base64.urlsafe_encode(token)
-
-  return {challenge: challenge, token: token}
-end
-
 def html_to_content(description_html)
   if !description_html
     description = ""
@@ -296,55 +248,3 @@ def extract_items(nodeset, ucid = nil)
 
   return items
 end
-
-def create_response(user_id, operation, key, expire = 6.hours)
-  expire = Time.now + expire
-  nonce = Random::Secure.hex(4)
-
-  challenge = "#{expire.to_unix}-#{nonce}-#{user_id}-#{operation}"
-  token = OpenSSL::HMAC.digest(:sha256, key, challenge)
-
-  challenge = Base64.urlsafe_encode(challenge)
-  token = Base64.urlsafe_encode(token)
-
-  return challenge, token
-end
-
-def validate_response(challenge, token, user_id, operation, key)
-  if !challenge
-    raise "Hidden field \"challenge\" is a required field"
-  end
-
-  if !token
-    raise "Hidden field \"token\" is a required field"
-  end
-
-  challenge = Base64.decode_string(challenge)
-  if challenge.split("-").size == 4
-    expire, nonce, challenge_user_id, challenge_operation = challenge.split("-")
-
-    expire = expire.to_i?
-    expire ||= 0
-  else
-    raise "Invalid challenge"
-  end
-
-  challenge = OpenSSL::HMAC.digest(:sha256, HMAC_KEY, challenge)
-  challenge = Base64.urlsafe_encode(challenge)
-
-  if challenge != token
-    raise "Invalid token"
-  end
-
-  if challenge_operation != operation
-    raise "Invalid token"
-  end
-
-  if challenge_user_id != user_id
-    raise "Invalid token"
-  end
-
-  if expire < Time.now.to_unix
-    raise "Token is expired, please try again"
-  end
-end

src/invidious/helpers/utils.cr

@@ -18,16 +18,28 @@ def elapsed_text(elapsed)
   "#{(millis * 1000).round(2)}µs"
 end
 
-def make_client(url)
+def make_client(url, proxies = {} of String => Array({ip: String, port: Int32}), region = nil)
   context = OpenSSL::SSL::Context::Client.new
   context.add_options(
     OpenSSL::SSL::Options::ALL |
     OpenSSL::SSL::Options::NO_SSL_V2 |
     OpenSSL::SSL::Options::NO_SSL_V3
   )
-  client = HTTP::Client.new(url, context)
+  client = HTTPClient.new(url, context)
   client.read_timeout = 10.seconds
   client.connect_timeout = 10.seconds
+
+  if region
+    proxies[region]?.try &.each do |proxy|
+      begin
+        proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
+        client.set_proxy(proxy)
+        break
+      rescue ex
+      end
+    end
+  end
+
   return client
 end
 

src/invidious/users.cr

@@ -70,7 +70,11 @@ class Preferences
   JSON.mapping({
     video_loop: Bool,
     autoplay:   Bool,
-    listen:     {
+    continue:   {
+      type:    Bool,
+      default: false,
+    },
+    listen: {
       type:    Bool,
       default: false,
     },
@@ -195,3 +199,112 @@ def create_user(sid, email, password)
 
   return user
 end
+
+def create_response(user_id, operation, key, db, expire = 6.hours)
+  expire = Time.now + expire
+  nonce = Random::Secure.hex(16)
+  db.exec("INSERT INTO nonces VALUES ($1) ON CONFLICT DO NOTHING", nonce)
+
+  challenge = "#{expire.to_unix}-#{nonce}-#{user_id}-#{operation}"
+  token = OpenSSL::HMAC.digest(:sha256, key, challenge)
+
+  challenge = Base64.urlsafe_encode(challenge)
+  token = Base64.urlsafe_encode(token)
+
+  return challenge, token
+end
+
+def validate_response(challenge, token, user_id, operation, key, db)
+  if !challenge
+    raise "Hidden field \"challenge\" is a required field"
+  end
+
+  if !token
+    raise "Hidden field \"token\" is a required field"
+  end
+
+  challenge = Base64.decode_string(challenge)
+  if challenge.split("-").size == 4
+    expire, nonce, challenge_user_id, challenge_operation = challenge.split("-")
+
+    expire = expire.to_i?
+    expire ||= 0
+  else
+    raise "Invalid challenge"
+  end
+
+  challenge = OpenSSL::HMAC.digest(:sha256, HMAC_KEY, challenge)
+  challenge = Base64.urlsafe_encode(challenge)
+
+  if db.query_one?("SELECT EXISTS (SELECT true FROM nonces WHERE nonce = $1)", nonce, as: Bool)
+    db.exec("DELETE FROM nonces * WHERE nonce = $1", nonce)
+  else
+    raise "Invalid token"
+  end
+
+  if challenge != token
+    raise "Invalid token"
+  end
+
+  if challenge_operation != operation
+    raise "Invalid token"
+  end
+
+  if challenge_user_id != user_id
+    raise "Invalid user"
+  end
+
+  if expire < Time.now.to_unix
+    raise "Token is expired, please try again"
+  end
+end
+
+def generate_captcha(key, db)
+  minute = Random::Secure.rand(12)
+  minute_angle = minute * 30
+  minute = minute * 5
+
+  hour = Random::Secure.rand(12)
+  hour_angle = hour * 30 + minute_angle.to_f / 12
+  if hour == 0
+    hour = 12
+  end
+
+  clock_svg = <<-END_SVG
+  <svg viewBox="0 0 100 100" width="200px">
+  <circle cx="50" cy="50" r="45" fill="#eee" stroke="black" stroke-width="2"></circle>
+  
+  <text x="69"     y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 1</text>
+  <text x="82.909" y="34"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 2</text>
+  <text x="88"     y="53"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 3</text>
+  <text x="82.909" y="72"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 4</text>
+  <text x="69"     y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 5</text>
+  <text x="50"     y="91"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 6</text>
+  <text x="31"     y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 7</text>
+  <text x="17.091" y="72"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 8</text>
+  <text x="12"     y="53"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 9</text>
+  <text x="17.091" y="34"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px">10</text>
+  <text x="31"     y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">11</text>
+  <text x="50"     y="15"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px">12</text>
+
+  <circle cx="50" cy="50" r="3" fill="black"></circle>
+  <line id="minute" transform="rotate(#{minute_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="16" fill="black" stroke="black" stroke-width="2"></line>
+  <line id="hour"   transform="rotate(#{hour_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="24" fill="black" stroke="black" stroke-width="2"></line>
+  </svg>
+  END_SVG
+
+  image = ""
+  convert = Process.run(%(convert -density 1200 -resize 400x400 -background none svg:- png:-), shell: true,
+    input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe) do |proc|
+    image = proc.output.gets_to_end
+    image = Base64.strict_encode(image)
+    image = "data:image/png;base64,#{image}"
+  end
+
+  answer = "#{hour}:#{minute.to_s.rjust(2, '0')}"
+  answer = OpenSSL::HMAC.hexdigest(:sha256, key, answer)
+
+  challenge, token = create_response(answer, "sign_in", key, db)
+
+  return {image: image, challenge: challenge, token: token}
+end

src/invidious/videos.cr

@@ -507,14 +507,14 @@ end
 class VideoRedirect < Exception
 end
 
-def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}), refresh = true)
+def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}), refresh = true, region = nil)
   if db.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool)
     video = db.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video)
 
     # If record was last updated over 10 minutes ago, refresh (expire param in response lasts for 6 hours)
     if refresh && Time.now - video.updated > 10.minutes
       begin
-        video = fetch_video(id, proxies)
+        video = fetch_video(id, proxies, region)
         video_array = video.to_a
 
         args = arg_array(video_array[1..-1], 2)
@@ -529,7 +529,7 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32})
       end
     end
   else
-    video = fetch_video(id, proxies)
+    video = fetch_video(id, proxies, region)
     video_array = video.to_a
 
     args = arg_array(video_array)
@@ -540,12 +540,12 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32})
   return video
 end
 
-def fetch_video(id, proxies)
+def fetch_video(id, proxies, region)
   html_channel = Channel(XML::Node | String).new
   info_channel = Channel(HTTP::Params).new
 
   spawn do
-    client = make_client(YT_URL)
+    client = make_client(YT_URL, proxies, region)
     html = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
 
     if md = html.headers["location"]?.try &.match(/v=(?<id>[a-zA-Z0-9_-]{11})/)
@@ -557,7 +557,7 @@ def fetch_video(id, proxies)
   end
 
   spawn do
-    client = make_client(YT_URL)
+    client = make_client(YT_URL, proxies, region)
     info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1")
     info = HTTP::Params.parse(info.body)
 
@@ -628,9 +628,9 @@ def fetch_video(id, proxies)
           end
 
           proxy = {ip: proxy.proxy_host, port: proxy.proxy_port}
-          region = proxies.select { |region, list| list.includes? proxy }
+          region_proxies = proxies.select { |region, list| list.includes? proxy }
-          if !region.empty?
+          if !region_proxies.empty?
-            info["region"] = region.keys[0]
+            info["region"] = region_proxies.keys[0]
           end
 
           break
@@ -677,7 +677,8 @@ def fetch_video(id, proxies)
 
   wilson_score = ci_lower_bound(likes, likes + dislikes)
 
-  published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).not_nil!["content"]
+  published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).try &.["content"]
+  published ||= Time.now.to_s("%Y-%m-%d")
   published = Time.parse(published, "%Y-%m-%d", Time::Location.local)
 
   allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).try &.["content"].split(",")
@@ -685,15 +686,16 @@ def fetch_video(id, proxies)
   is_family_friendly = html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).try &.["content"] == "True"
   is_family_friendly ||= true
 
-  genre = html.xpath_node(%q(//meta[@itemprop="genre"])).not_nil!["content"]
+  genre = html.xpath_node(%q(//meta[@itemprop="genre"])).try &.["content"]
+  genre ||= ""
+
   genre_url = html.xpath_node(%(//a[text()="#{genre}"])).try &.["href"]
   case genre
   when "Movies"
     genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g"
   when "Education"
     # Education channel is linked but does not exist
-    # genre_url = "/channel/UC3yA8nDwraeOfnYfBWun83g"
+    genre_url = "/channel/UC3yA8nDwraeOfnYfBWun83g"
-    genre_url = ""
   end
   genre_url ||= ""
 
@@ -730,15 +732,19 @@ end
 
 def process_video_params(query, preferences)
   autoplay = query["autoplay"]?.try &.to_i?
+  continue = query["continue"]?.try &.to_i?
   listen = query["listen"]? && (query["listen"] == "true" || query["listen"] == "1").to_unsafe
   preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase }
   quality = query["quality"]?
+  region = query["region"]?
   speed = query["speed"]?.try &.to_f?
   video_loop = query["loop"]?.try &.to_i?
   volume = query["volume"]?.try &.to_i?
 
   if preferences
+    # region ||= preferences.region
     autoplay ||= preferences.autoplay.to_unsafe
+    continue ||= preferences.continue.to_unsafe
     listen ||= preferences.listen.to_unsafe
     preferred_captions ||= preferences.captions
     quality ||= preferences.quality
@@ -748,6 +754,7 @@ def process_video_params(query, preferences)
   end
 
   autoplay ||= 0
+  continue ||= 0
   listen ||= 0
   preferred_captions ||= [] of String
   quality ||= "hd720"
@@ -756,6 +763,7 @@ def process_video_params(query, preferences)
   volume ||= 100
 
   autoplay = autoplay == 1
+  continue = continue == 1
   listen = listen == 1
   video_loop = video_loop == 1
 
@@ -786,11 +794,13 @@ def process_video_params(query, preferences)
 
   params = {
     autoplay:           autoplay,
+    continue:           continue,
     controls:           controls,
     listen:             listen,
     preferred_captions: preferred_captions,
     quality:            quality,
     raw:                raw,
+    region:             region,
     speed:              speed,
     video_end:          video_end,
     video_loop:         video_loop,