Merge branch 'iv-org:master' into verified-badge

src/invidious/routes/account.cr

@@ -0,0 +1,358 @@
+{% skip_file if flag?(:api_only) %}
+
+module Invidious::Routes::Account
+  extend self
+
+  # -------------------
+  #  Password update
+  # -------------------
+
+  # Show the password change interface (GET request)
+  def get_change_password(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+    sid = sid.as(String)
+    csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY)
+
+    templated "user/change_password"
+  end
+
+  # Handle the password change (POST request)
+  def post_change_password(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+    sid = sid.as(String)
+    token = env.params.body["csrf_token"]?
+
+    # We don't store passwords for Google accounts
+    if !user.password
+      return error_template(400, "Cannot change password for Google accounts")
+    end
+
+    begin
+      validate_request(token, sid, env.request, HMAC_KEY, locale)
+    rescue ex
+      return error_template(400, ex)
+    end
+
+    password = env.params.body["password"]?
+    if !password
+      return error_template(401, "Password is a required field")
+    end
+
+    new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v }
+
+    if new_passwords.size <= 1 || new_passwords.uniq.size != 1
+      return error_template(400, "New passwords must match")
+    end
+
+    new_password = new_passwords.uniq[0]
+    if new_password.empty?
+      return error_template(401, "Password cannot be empty")
+    end
+
+    if new_password.bytesize > 55
+      return error_template(400, "Password cannot be longer than 55 characters")
+    end
+
+    if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
+      return error_template(401, "Incorrect password")
+    end
+
+    new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10)
+    Invidious::Database::Users.update_password(user, new_password.to_s)
+
+    env.redirect referer
+  end
+
+  # -------------------
+  #  Account deletion
+  # -------------------
+
+  # Show the account deletion confirmation prompt (GET request)
+  def get_delete(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+    sid = sid.as(String)
+    csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY)
+
+    templated "user/delete_account"
+  end
+
+  # Handle the account deletion (POST request)
+  def post_delete(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+    sid = sid.as(String)
+    token = env.params.body["csrf_token"]?
+
+    begin
+      validate_request(token, sid, env.request, HMAC_KEY, locale)
+    rescue ex
+      return error_template(400, ex)
+    end
+
+    view_name = "subscriptions_#{sha256(user.email)}"
+    Invidious::Database::Users.delete(user)
+    Invidious::Database::SessionIDs.delete(email: user.email)
+    PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}")
+
+    env.request.cookies.each do |cookie|
+      cookie.expires = Time.utc(1990, 1, 1)
+      env.response.cookies << cookie
+    end
+
+    env.redirect referer
+  end
+
+  # -------------------
+  #  Clear history
+  # -------------------
+
+  # Show the watch history deletion confirmation prompt (GET request)
+  def get_clear_history(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+    sid = sid.as(String)
+    csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY)
+
+    templated "user/clear_watch_history"
+  end
+
+  # Handle the watch history clearing (POST request)
+  def post_clear_history(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+    sid = sid.as(String)
+    token = env.params.body["csrf_token"]?
+
+    begin
+      validate_request(token, sid, env.request, HMAC_KEY, locale)
+    rescue ex
+      return error_template(400, ex)
+    end
+
+    Invidious::Database::Users.clear_watch_history(user)
+    env.redirect referer
+  end
+
+  # -------------------
+  #  Authorize tokens
+  # -------------------
+
+  # Show the "authorize token?" confirmation prompt (GET request)
+  def get_authorize_token(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+    sid = sid.as(String)
+    csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY)
+
+    scopes = env.params.query["scopes"]?.try &.split(",")
+    scopes ||= [] of String
+
+    callback_url = env.params.query["callback_url"]?
+    if callback_url
+      callback_url = URI.parse(callback_url)
+    end
+
+    expire = env.params.query["expire"]?.try &.to_i?
+
+    templated "user/authorize_token"
+  end
+
+  # Handle token authorization (POST request)
+  def post_authorize_token(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = env.get("user").as(User)
+    sid = sid.as(String)
+    token = env.params.body["csrf_token"]?
+
+    begin
+      validate_request(token, sid, env.request, HMAC_KEY, locale)
+    rescue ex
+      return error_template(400, ex)
+    end
+
+    scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v }
+    callback_url = env.params.body["callbackUrl"]?
+    expire = env.params.body["expire"]?.try &.to_i?
+
+    access_token = generate_token(user.email, scopes, expire, HMAC_KEY)
+
+    if callback_url
+      access_token = URI.encode_www_form(access_token)
+      url = URI.parse(callback_url)
+
+      if url.query
+        query = HTTP::Params.parse(url.query.not_nil!)
+      else
+        query = HTTP::Params.new
+      end
+
+      query["token"] = access_token
+      url.query = query.to_s
+
+      env.redirect url.to_s
+    else
+      csrf_token = ""
+      env.set "access_token", access_token
+      templated "user/authorize_token"
+    end
+  end
+
+  # -------------------
+  #  Manage tokens
+  # -------------------
+
+  # Show the token manager page (GET request)
+  def token_manager(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env, "/subscription_manager")
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+    tokens = Invidious::Database::SessionIDs.select_all(user.email)
+
+    templated "user/token_manager"
+  end
+
+  # -------------------
+  #  AJAX for tokens
+  # -------------------
+
+  # Handle internal (non-API) token actions (POST request)
+  def token_ajax(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    redirect = env.params.query["redirect"]?
+    redirect ||= "true"
+    redirect = redirect == "true"
+
+    if !user
+      if redirect
+        return env.redirect referer
+      else
+        return error_json(403, "No such user")
+      end
+    end
+
+    user = user.as(User)
+    sid = sid.as(String)
+    token = env.params.body["csrf_token"]?
+
+    begin
+      validate_request(token, sid, env.request, HMAC_KEY, locale)
+    rescue ex
+      if redirect
+        return error_template(400, ex)
+      else
+        return error_json(400, ex)
+      end
+    end
+
+    if env.params.query["action_revoke_token"]?
+      action = "action_revoke_token"
+    else
+      return env.redirect referer
+    end
+
+    session = env.params.query["session"]?
+    session ||= ""
+
+    case action
+    when .starts_with? "action_revoke_token"
+      Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
+    else
+      return error_json(400, "Unsupported action #{action}")
+    end
+
+    if redirect
+      return env.redirect referer
+    else
+      env.response.content_type = "application/json"
+      return "{}"
+    end
+  end
+end

src/invidious/routes/api/v1/authenticated.cr

@@ -343,7 +343,7 @@ module Invidious::Routes::API::V1::Authenticated
       env.response.content_type = "text/html"
 
       csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true)
-      return templated "authorize_token"
+      return templated "user/authorize_token"
     else
       env.response.content_type = "application/json"
 
@@ -397,4 +397,14 @@ module Invidious::Routes::API::V1::Authenticated
 
     env.response.status_code = 204
   end
+
+  def self.notifications(env)
+    env.response.content_type = "text/event-stream"
+
+    raw_topics = env.params.body["topics"]? || env.params.query["topics"]?
+    topics = raw_topics.try &.split(",").uniq.first(1000)
+    topics ||= [] of String
+
+    create_notification_stream(env, topics, CONNECTION_CHANNEL)
+  end
 end

src/invidious/routes/api/v1/channels.cr

@@ -96,7 +96,14 @@ module Invidious::Routes::API::V1::Channels
 
         json.field "relatedChannels" do
           json.array do
-            fetch_related_channels(channel).each do |related_channel|
+            # Fetch related channels
+            begin
+              related_channels = fetch_related_channels(channel)
+            rescue ex
+              related_channels = [] of AboutRelatedChannel
+            end
+
+            related_channels.each do |related_channel|
               json.object do
                 json.field "author", related_channel.author
                 json.field "authorId", related_channel.ucid
@@ -118,7 +125,8 @@ module Invidious::Routes::API::V1::Channels
               end
             end
           end
-        end
+        end # relatedChannels
+
       end
     end
   end

src/invidious/routes/api/v1/misc.cr

@@ -4,10 +4,10 @@ module Invidious::Routes::API::V1::Misc
     env.response.content_type = "application/json"
 
     if !CONFIG.statistics_enabled
-      return error_json(400, "Statistics are not enabled.")
+      return {"software" => SOFTWARE}.to_json
+    else
+      return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
     end
-
-    Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
   end
 
   # APIv1 currently uses the same logic for both

src/invidious/routes/api/v1/videos.cr

@@ -23,7 +23,11 @@ module Invidious::Routes::API::V1::Videos
     env.response.content_type = "application/json"
 
     id = env.params.url["id"]
-    region = env.params.query["region"]?
+    region = env.params.query["region"]? || env.params.body["region"]?
+
+    if id.nil? || id.size != 11 || !id.matches?(/^[\w-]+$/)
+      return error_json(400, "Invalid video ID")
+    end
 
     # See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354
     # It is possible to use `/api/timedtext?type=list&v=#{id}` and
@@ -130,7 +134,13 @@ module Invidious::Routes::API::V1::Videos
         end
       end
     else
+      # Some captions have "align:[start/end]" and "position:[num]%"
+      # attributes. Those are causing issues with VideoJS, which is unable
+      # to properly align the captions on the video, so we remove them.
+      #
+      # See: https://github.com/iv-org/invidious/issues/2391
       webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
+        .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
     end
 
     if title = env.params.query["title"]?

src/invidious/routes/channels.cr

@@ -147,6 +147,39 @@ module Invidious::Routes::Channels
     end
   end
 
+  def self.live(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    # Appears to be a bug in routing, having several routes configured
+    # as `/a/:a`, `/b/:a`, `/c/:a` results in 404
+    value = env.request.resource.split("/")[2]
+    body = ""
+    {"channel", "user", "c"}.each do |type|
+      response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1")
+      if response.status_code == 200
+        body = response.body
+      end
+    end
+
+    video_id = body.match(/'VIDEO_ID': "(?<id>[a-zA-Z0-9_-]{11})"/).try &.["id"]?
+    if video_id
+      params = [] of String
+      env.params.query.each do |k, v|
+        params << "#{k}=#{v}"
+      end
+      params = params.join("&")
+
+      url = "/watch?v=#{video_id}"
+      if !params.empty?
+        url += "&#{params}"
+      end
+
+      env.redirect url
+    else
+      env.redirect "/channel/#{value}"
+    end
+  end
+
   private def self.fetch_basic_information(env)
     locale = env.get("preferences").as(Preferences).locale
 

src/invidious/routes/login.cr

@@ -27,7 +27,7 @@ module Invidious::Routes::Login
     tfa = env.params.query["tfa"]?
     prompt = nil
 
-    templated "login"
+    templated "user/login"
   end
 
   def self.login(env)
@@ -133,7 +133,7 @@ module Invidious::Routes::Login
           tfa = tfa_code
           captcha = {tokens: [token], question: ""}
 
-          return templated "login"
+          return templated "user/login"
         end
 
         if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED"
@@ -190,7 +190,7 @@ module Invidious::Routes::Login
 
             tfa = nil
             captcha = nil
-            return templated "login"
+            return templated "user/login"
           end
 
           tl = challenge_results[1][2]
@@ -282,18 +282,8 @@ module Invidious::Routes::Login
 
         host = URI.parse(env.request.headers["Host"]).host
 
-        if Kemal.config.ssl || CONFIG.https_only
-          secure = true
-        else
-          secure = false
-        end
-
         cookies.each do |cookie|
-          if Kemal.config.ssl || CONFIG.https_only
-            cookie.secure = secure
-          else
-            cookie.secure = secure
-          end
+          cookie.secure = Invidious::User::Cookies::SECURE
 
           if cookie.extension
             cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host)
@@ -338,19 +328,7 @@ module Invidious::Routes::Login
           sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
           Invidious::Database::SessionIDs.insert(sid, email)
 
-          if Kemal.config.ssl || CONFIG.https_only
-            secure = true
-          else
-            secure = false
-          end
-
-          if CONFIG.domain
-            env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years,
-              secure: secure, http_only: true)
-          else
-            env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years,
-              secure: secure, http_only: true)
-          end
+          env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
         else
           return error_template(401, "Wrong username or password")
         end
@@ -393,12 +371,12 @@ module Invidious::Routes::Login
             prompt = ""
 
             if captcha_type == "image"
-              captcha = generate_captcha(HMAC_KEY)
+              captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
             else
-              captcha = generate_text_captcha(HMAC_KEY)
+              captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
             end
 
-            return templated "login"
+            return templated "user/login"
           end
 
           tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
@@ -455,19 +433,7 @@ module Invidious::Routes::Login
         view_name = "subscriptions_#{sha256(user.email)}"
         PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
 
-        if Kemal.config.ssl || CONFIG.https_only
-          secure = true
-        else
-          secure = false
-        end
-
-        if CONFIG.domain
-          env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years,
-            secure: secure, http_only: true)
-        else
-          env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years,
-            secure: secure, http_only: true)
-        end
+        env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
 
         if env.request.cookies["PREFS"]?
           user.preferences = env.get("preferences").as(Preferences)
@@ -515,4 +481,11 @@ module Invidious::Routes::Login
 
     env.redirect referer
   end
+
+  def self.captcha(env)
+    headers = HTTP::Headers{":authority" => "accounts.google.com"}
+    response = YT_POOL.client &.get(env.request.resource, headers)
+    env.response.headers["Content-Type"] = response.headers["Content-Type"]
+    response.body
+  end
 end

src/invidious/routes/playlists.cr

@@ -443,4 +443,15 @@ module Invidious::Routes::Playlists
 
     templated "mix"
   end
+
+  # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
+  def self.watch_videos(env)
+    response = YT_POOL.client &.get(env.request.resource)
+    if url = response.headers["Location"]?
+      url = URI.parse(url).request_target
+      return env.redirect url
+    end
+
+    env.response.status_code = response.status_code
+  end
 end

src/invidious/routes/preferences.cr

@@ -8,7 +8,7 @@ module Invidious::Routes::PreferencesRoute
 
     preferences = env.get("preferences").as(Preferences)
 
-    templated "preferences"
+    templated "user/preferences"
   end
 
   def self.update(env)
@@ -47,6 +47,10 @@ module Invidious::Routes::PreferencesRoute
     local ||= "off"
     local = local == "on"
 
+    watch_history = env.params.body["watch_history"]?.try &.as(String)
+    watch_history ||= "off"
+    watch_history = watch_history == "on"
+
     speed = env.params.body["speed"]?.try &.as(String).to_f32?
     speed ||= CONFIG.default_user_preferences.speed
 
@@ -136,7 +140,7 @@ module Invidious::Routes::PreferencesRoute
     notifications_only ||= "off"
     notifications_only = notifications_only == "on"
 
-    # Convert to JSON and back again to take advantage of converters used for compatability
+    # Convert to JSON and back again to take advantage of converters used for compatibility
     preferences = Preferences.from_json({
       annotations:                 annotations,
       annotations_subscribed:      annotations_subscribed,
@@ -149,6 +153,7 @@ module Invidious::Routes::PreferencesRoute
       latest_only:                 latest_only,
       listen:                      listen,
       local:                       local,
+      watch_history:               watch_history,
       locale:                      locale,
       max_results:                 max_results,
       notifications_only:          notifications_only,
@@ -214,19 +219,7 @@ module Invidious::Routes::PreferencesRoute
         File.write("config/config.yml", CONFIG.to_yaml)
       end
     else
-      if Kemal.config.ssl || CONFIG.https_only
-        secure = true
-      else
-        secure = false
-      end
-
-      if CONFIG.domain
-        env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years,
-          secure: secure, http_only: true)
-      else
-        env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years,
-          secure: secure, http_only: true)
-      end
+      env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences)
     end
 
     env.redirect referer
@@ -261,21 +254,7 @@ module Invidious::Routes::PreferencesRoute
         preferences.dark_mode = "dark"
       end
 
-      preferences = preferences.to_json
-
-      if Kemal.config.ssl || CONFIG.https_only
-        secure = true
-      else
-        secure = false
-      end
-
-      if CONFIG.domain
-        env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years,
-          secure: secure, http_only: true)
-      else
-        env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years,
-          secure: secure, http_only: true)
-      end
+      env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences)
     end
 
     if redirect
@@ -298,7 +277,7 @@ module Invidious::Routes::PreferencesRoute
 
     user = user.as(User)
 
-    templated "data_control"
+    templated "user/data_control"
   end
 
   def self.update_data_control(env)
@@ -321,149 +300,27 @@ module Invidious::Routes::PreferencesRoute
         # TODO: Unify into single import based on content-type
         case part.name
         when "import_invidious"
-          body = JSON.parse(body)
-
-          if body["subscriptions"]?
-            user.subscriptions += body["subscriptions"].as_a.map(&.as_s)
-            user.subscriptions.uniq!
-
-            user.subscriptions = get_batch_channels(user.subscriptions)
-
-            Invidious::Database::Users.update_subscriptions(user)
-          end
-
-          if body["watch_history"]?
-            user.watched += body["watch_history"].as_a.map(&.as_s)
-            user.watched.uniq!
-            Invidious::Database::Users.update_watch_history(user)
-          end
-
-          if body["preferences"]?
-            user.preferences = Preferences.from_json(body["preferences"].to_json)
-            Invidious::Database::Users.update_preferences(user)
-          end
-
-          if playlists = body["playlists"]?.try &.as_a?
-            playlists.each do |item|
-              title = item["title"]?.try &.as_s?.try &.delete("<>")
-              description = item["description"]?.try &.as_s?.try &.delete("\r")
-              privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
-
-              next if !title
-              next if !description
-              next if !privacy
-
-              playlist = create_playlist(title, privacy, user)
-              Invidious::Database::Playlists.update_description(playlist.id, description)
-
-              videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
-                raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
-
-                video_id = video_id.try &.as_s?
-                next if !video_id
-
-                begin
-                  video = get_video(video_id)
-                rescue ex
-                  next
-                end
-
-                playlist_video = PlaylistVideo.new({
-                  title:          video.title,
-                  id:             video.id,
-                  author:         video.author,
-                  ucid:           video.ucid,
-                  length_seconds: video.length_seconds,
-                  published:      video.published,
-                  plid:           playlist.id,
-                  live_now:       video.live_now,
-                  index:          Random::Secure.rand(0_i64..Int64::MAX),
-                })
-
-                Invidious::Database::PlaylistVideos.insert(playlist_video)
-                Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
-              end
-            end
-          end
+          Invidious::User::Import.from_invidious(user, body)
         when "import_youtube"
           filename = part.filename || ""
-          extension = filename.split(".").last
+          success = Invidious::User::Import.from_youtube(user, body, filename, type)
 
-          if extension == "xml" || type == "application/xml" || type == "text/xml"
-            subscriptions = XML.parse(body)
-            user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
-              channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
-            end
-          elsif extension == "json" || type == "application/json"
-            subscriptions = JSON.parse(body)
-            user.subscriptions += subscriptions.as_a.compact_map do |entry|
-              entry["snippet"]["resourceId"]["channelId"].as_s
-            end
-          elsif extension == "csv" || type == "text/csv"
-            subscriptions = parse_subscription_export_csv(body)
-            user.subscriptions += subscriptions
-          else
+          if !success
             haltf(env, status_code: 415,
               response: error_template(415, "Invalid subscription file uploaded")
             )
           end
-
-          user.subscriptions.uniq!
-          user.subscriptions = get_batch_channels(user.subscriptions)
-
-          Invidious::Database::Users.update_subscriptions(user)
         when "import_freetube"
-          user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md|
-            md["channel_id"]
-          end
-          user.subscriptions.uniq!
-
-          user.subscriptions = get_batch_channels(user.subscriptions)
-
-          Invidious::Database::Users.update_subscriptions(user)
+          Invidious::User::Import.from_freetube(user, body)
         when "import_newpipe_subscriptions"
-          body = JSON.parse(body)
-          user.subscriptions += body["subscriptions"].as_a.compact_map do |channel|
-            if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
-              next match["channel"]
-            elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
-              response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
-              html = XML.parse_html(response.body)
-              ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
-              next ucid if ucid
-            end
-
-            nil
-          end
-          user.subscriptions.uniq!
-
-          user.subscriptions = get_batch_channels(user.subscriptions)
-
-          Invidious::Database::Users.update_subscriptions(user)
+          Invidious::User::Import.from_newpipe_subs(user, body)
         when "import_newpipe"
-          Compress::Zip::Reader.open(IO::Memory.new(body)) do |file|
-            file.each_entry do |entry|
-              if entry.filename == "newpipe.db"
-                tempfile = File.tempfile(".db")
-                File.write(tempfile.path, entry.io.gets_to_end)
-                db = DB.open("sqlite3://" + tempfile.path)
+          success = Invidious::User::Import.from_newpipe(user, body)
 
-                user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v="))
-                user.watched.uniq!
-
-                Invidious::Database::Users.update_watch_history(user)
-
-                user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/"))
-                user.subscriptions.uniq!
-
-                user.subscriptions = get_batch_channels(user.subscriptions)
-
-                Invidious::Database::Users.update_subscriptions(user)
-
-                db.close
-                tempfile.delete
-              end
-            end
+          if !success
+            haltf(env, status_code: 415,
+              response: error_template(415, "Uploaded file is too large")
+            )
           end
         else nil # Ignore
         end

src/invidious/routes/subscriptions.cr

@@ -163,6 +163,6 @@ module Invidious::Routes::Subscriptions
       end
     end
 
-    templated "subscription_manager"
+    templated "user/subscription_manager"
   end
 end

src/invidious/routes/video_playback.cr

@@ -14,12 +14,18 @@ module Invidious::Routes::VideoPlayback
     end
 
     if query_params["host"]? && !query_params["host"].empty?
-      host = "https://#{query_params["host"]}"
+      host = query_params["host"]
       query_params.delete("host")
     else
-      host = "https://r#{fvip}---#{mns.pop}.googlevideo.com"
+      host = "r#{fvip}---#{mns.pop}.googlevideo.com"
     end
 
+    # Sanity check, to avoid being used as an open proxy
+    if !host.matches?(/[\w-]+.googlevideo.com/)
+      return error_template(400, "Invalid \"host\" parameter.")
+    end
+
+    host = "https://#{host}"
     url = "/videoplayback?#{query_params}"
 
     headers = HTTP::Headers.new
@@ -158,7 +164,9 @@ module Invidious::Routes::VideoPlayback
 
               if title = query_params["title"]?
                 # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
-                env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}"
+                filename = URI.encode_www_form(title, space_to_plus: false)
+                header = "attachment; filename=\"#{filename}\"; filename*=UTF-8''#{filename}"
+                env.response.headers["Content-Disposition"] = header
               end
 
               if !resp.headers.includes_word?("Transfer-Encoding", "chunked")
@@ -236,31 +244,25 @@ module Invidious::Routes::VideoPlayback
   # YouTube /videoplayback links expire after 6 hours,
   # so we have a mechanism here to redirect to the latest version
   def self.latest_version(env)
-    if env.params.query["download_widget"]?
-      download_widget = JSON.parse(env.params.query["download_widget"])
+    id = env.params.query["id"]?
+    itag = env.params.query["itag"]?.try &.to_i?
 
-      id = download_widget["id"].as_s
-      title = URI.decode_www_form(download_widget["title"].as_s)
-
-      if label = download_widget["label"]?
-        return env.redirect "/api/v1/captions/#{id}?label=#{label}&title=#{title}"
-      else
-        itag = download_widget["itag"].as_s.to_i
-        local = "true"
-      end
+    # Sanity checks
+    if id.nil? || id.size != 11 || !id.matches?(/^[\w-]+$/)
+      return error_template(400, "Invalid video ID")
     end
 
-    id ||= env.params.query["id"]?
-    itag ||= env.params.query["itag"]?.try &.to_i
+    if itag.nil? || itag <= 0 || itag >= 1000
+      return error_template(400, "Invalid itag")
+    end
 
     region = env.params.query["region"]?
+    local = (env.params.query["local"]? == "true")
 
-    local ||= env.params.query["local"]?
-    local ||= "false"
-    local = local == "true"
+    title = env.params.query["title"]?
 
-    if !id || !itag
-      haltf env, status_code: 400, response: "TESTING"
+    if title && CONFIG.disabled?("downloads")
+      return error_template(403, "Administrator has disabled this endpoint.")
     end
 
     video = get_video(id, region: region)
@@ -272,8 +274,10 @@ module Invidious::Routes::VideoPlayback
       haltf env, status_code: 404
     end
 
-    url = URI.parse(url).request_target.not_nil! if local
-    url = "#{url}&title=#{title}" if title
+    if local
+      url = URI.parse(url).request_target.not_nil!
+      url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title
+    end
 
     return env.redirect url
   end

src/invidious/routes/watch.cr

@@ -75,7 +75,7 @@ module Invidious::Routes::Watch
     end
     env.params.query.delete_all("iv_load_policy")
 
-    if watched && !watched.includes? id
+    if watched && preferences.watch_history && !watched.includes? id
       Invidious::Database::Users.mark_watched(user.as(User), id)
     end
 
@@ -189,6 +189,14 @@ module Invidious::Routes::Watch
       return env.redirect url
     end
 
+    # Structure used for the download widget
+    video_assets = Invidious::Frontend::WatchPage::VideoAssets.new(
+      full_videos: fmt_stream,
+      video_streams: video_streams,
+      audio_streams: audio_streams,
+      captions: video.captions
+    )
+
     templated "watch"
   end
 
@@ -281,4 +289,49 @@ module Invidious::Routes::Watch
       return error_template(404, "The requested clip doesn't exist")
     end
   end
+
+  def self.download(env)
+    if CONFIG.disabled?("downloads")
+      return error_template(403, "Administrator has disabled this endpoint.")
+    end
+
+    title = env.params.body["title"]? || ""
+    video_id = env.params.body["id"]? || ""
+    selection = env.params.body["download_widget"]?
+
+    if title.empty? || video_id.empty? || selection.nil?
+      return error_template(400, "Missing form data")
+    end
+
+    download_widget = JSON.parse(selection)
+
+    extension = download_widget["ext"].as_s
+    filename = "#{video_id}-#{title}.#{extension}"
+
+    # Pass form parameters as URL parameters for the handlers of both
+    # /latest_version and /api/v1/captions. This avoids an un-necessary
+    # redirect and duplicated (and hazardous) sanity checks.
+    env.params.query["id"] = video_id
+    env.params.query["title"] = filename
+
+    # Delete the useless ones
+    env.params.body.delete("id")
+    env.params.body.delete("title")
+    env.params.body.delete("download_widget")
+
+    if label = download_widget["label"]?
+      # URL params specific to /api/v1/captions/:id
+      env.params.query["label"] = URI.encode_www_form(label.as_s, space_to_plus: false)
+
+      return Invidious::Routes::API::V1::Videos.captions(env)
+    elsif itag = download_widget["itag"]?.try &.as_i
+      # URL params specific to /latest_version
+      env.params.query["itag"] = itag.to_s
+      env.params.query["local"] = "true"
+
+      return Invidious::Routes::VideoPlayback.latest_version(env)
+    else
+      return error_template(400, "Invalid label or itag")
+    end
+  end
 end