From d47aa3dd6a21700b64929c586e3cfdc18302b1f3 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Sun, 30 Mar 2025 20:08:15 -0300 Subject: [PATCH] feat: do all the backend balancing on the invidious side This will make invidious easier to maintain and escalate without the need of an overcomplicated reverse proxy configuration and multiple invidious instances with each one with a different configuration (in this case, invidious companion) --- src/invidious/helpers/backend_info.cr | 38 +- src/invidious/routes/api/manifest.cr | 3 +- src/invidious/routes/backend_switcher.cr | 16 + src/invidious/routes/before_all.cr | 33 +- src/invidious/routes/video_playback.cr | 3 +- src/invidious/routes/watch.cr | 10 +- src/invidious/routing.cr | 1 + src/invidious/user/cookies.cr | 24 ++ src/invidious/videos.cr | 12 +- src/invidious/videos/parser.cr | 10 +- src/invidious/views/template.ecr | 440 ++++++++++---------- src/invidious/yt_backend/connection_pool.cr | 10 +- src/invidious/yt_backend/youtube_api.cr | 6 +- 13 files changed, 341 insertions(+), 265 deletions(-) create mode 100644 src/invidious/routes/backend_switcher.cr diff --git a/src/invidious/helpers/backend_info.cr b/src/invidious/helpers/backend_info.cr index d1ff20fe..dd496e34 100644 --- a/src/invidious/helpers/backend_info.cr +++ b/src/invidious/helpers/backend_info.cr @@ -1,45 +1,49 @@ module BackendInfo extend self - @@exvpp_url : String = "" - @@status : Int32 = 0 + @@exvpp_url : Array(String) = Array.new(CONFIG.invidious_companion.size, "") + @@status : Array(Int32) = Array.new(CONFIG.invidious_companion.size, 0) def check_backends check_companion() end def check_companion - begin - response = HTTP::Client.get "#{CONFIG.invidious_companion.sample.private_url}/healthz" - if response.status_code == 200 - check_videoplayback_proxy() - else - @@status = 0 + CONFIG.invidious_companion.each_with_index do |companion, index| + spawn do + begin + response = HTTP::Client.get "#{companion.private_url}/healthz" + if response.status_code == 200 + check_videoplayback_proxy(companion, index) + else + @@status[index] = 0 + end + rescue + @@status[index] = 0 + end end - rescue - @@status = 0 end end - def check_videoplayback_proxy + private def check_videoplayback_proxy(companion : Config::CompanionConfig, index : Int32) begin - info = HTTP::Client.get "#{CONFIG.invidious_companion.sample.private_url}/info" + info = HTTP::Client.get "#{companion.private_url}/info" exvpp_url = JSON.parse(info.body)["external_videoplayback_proxy"]?.try &.to_s exvpp_url = "" if exvpp_url.nil? - @@exvpp_url = exvpp_url + @@exvpp_url[index] = exvpp_url if exvpp_url.empty? - @@status = 2 + @@status[index] = 2 return else begin exvpp_health = HTTP::Client.get "#{exvpp_url}/health" if exvpp_health.status_code == 200 - @@status = 2 + @@status[index] = 2 return else - @@status = 1 + @@status[index] = 1 end rescue - @@status = 1 + @@status[index] = 1 end end rescue diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 13a0f95a..d374788a 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -9,7 +9,8 @@ module Invidious::Routes::API::Manifest region = env.params.query["region"]? if CONFIG.invidious_companion.present? - invidious_companion = CONFIG.invidious_companion.sample + current_companion = env.get("current_companion").as(Int32) + invidious_companion = CONFIG.invidious_companion[current_companion] return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}" end diff --git a/src/invidious/routes/backend_switcher.cr b/src/invidious/routes/backend_switcher.cr new file mode 100644 index 00000000..ad7d7e3d --- /dev/null +++ b/src/invidious/routes/backend_switcher.cr @@ -0,0 +1,16 @@ +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::BackendSwitcher + def self.switch(env) + referer = get_referer(env) + backend_id = env.params.query["backend_id"]?.try &.to_i + + if backend_id.nil? + return error_template(400, "Backend ID is required") + end + + env.response.cookies[CONFIG.server_id_cookie_name] = Invidious::User::Cookies.server_id(env.request.headers["Host"], backend_id) + + env.redirect referer + end +end diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index e49bb300..a8218ea4 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -24,12 +24,33 @@ module Invidious::Routes::BeforeAll extra_connect_csp = "" if CONFIG.invidious_companion.present? - extra_media_csp = " #{CONFIG.invidious_companion.sample.public_url}" - extra_connect_csp = " #{CONFIG.invidious_companion.sample.public_url}" - exvpp_url = BackendInfo.get_exvpp - if !exvpp_url.empty? - extra_media_csp += " #{exvpp_url}" - extra_connect_csp += " #{exvpp_url}" + if env.request.cookies[CONFIG.server_id_cookie_name]?.nil? + env.response.cookies[CONFIG.server_id_cookie_name] = Invidious::User::Cookies.server_id(env.request.headers["Host"]) + end + + begin + current_companion = env.request.cookies[CONFIG.server_id_cookie_name].value.try &.to_i + rescue + current_companion = rand(CONFIG.invidious_companion.size) + end + + if current_companion > CONFIG.invidious_companion.size + current_companion = current_companion % CONFIG.invidious_companion.size + env.response.cookies[CONFIG.server_id_cookie_name] = Invidious::User::Cookies.server_id(env.request.headers["Host"], current_companion) + end + + env.set "current_companion", current_companion + + CONFIG.invidious_companion.each do |companion| + extra_media_csp += " #{companion.public_url}" + extra_connect_csp += " #{companion.public_url}" + end + exvpp_urls = BackendInfo.get_exvpp + exvpp_urls.each do |exvpp_url| + if !exvpp_url.empty? + extra_media_csp += " #{exvpp_url}" + extra_connect_csp += " #{exvpp_url}" + end end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 3c1b85a6..44a3474d 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -267,7 +267,8 @@ module Invidious::Routes::VideoPlayback # so we have a mechanism here to redirect to the latest version def self.latest_version(env) if CONFIG.invidious_companion.present? - invidious_companion = CONFIG.invidious_companion.sample + current_companion = env.get("current_companion").as(Int32) + invidious_companion = CONFIG.invidious_companion[current_companion] return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}" end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 83944c37..bca8f6b5 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -52,7 +52,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - video = get_video(id, region: params.region) + video = get_video(id, region: params.region, env: env) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) @@ -214,7 +214,8 @@ module Invidious::Routes::Watch end if CONFIG.invidious_companion.present? - invidious_companion = CONFIG.invidious_companion.sample + current_companion = env.get("current_companion").as(Int32) + invidious_companion = CONFIG.invidious_companion[current_companion] env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] .gsub("media-src", "media-src #{invidious_companion.public_url}") @@ -350,8 +351,9 @@ module Invidious::Routes::Watch env.params.query["local"] = "true" if (CONFIG.invidious_companion.present?) - video = get_video(video_id) - invidious_companion = CONFIG.invidious_companion.sample + video = get_video(video_id, env: env) + current_companion = env.get("current_companion").as(Int32) + invidious_companion = CONFIG.invidious_companion[current_companion] return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}" else return Invidious::Routes::VideoPlayback.latest_version(env) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 38ddb839..c0401cac 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -21,6 +21,7 @@ module Invidious::Routing get "/privacy", Routes::Misc, :privacy get "/licenses", Routes::Misc, :licenses get "/redirect", Routes::Misc, :cross_instance_redirect + get "/switchbackend", Routes::BackendSwitcher, :switch self.register_channel_routes self.register_watch_routes diff --git a/src/invidious/user/cookies.cr b/src/invidious/user/cookies.cr index a9928d0a..515718c8 100644 --- a/src/invidious/user/cookies.cr +++ b/src/invidious/user/cookies.cr @@ -45,5 +45,29 @@ struct Invidious::User samesite: HTTP::Cookie::SameSite::Lax ) end + + # Backend (CONFIG.server_id_cookie_name) cookie + # Parameter "domain" comes from the global config + def server_id(domain : String?, server_id : Int32? = nil) : HTTP::Cookie + if server_id.nil? + server_id = rand(CONFIG.invidious_companion.size) + end + # Strip the port from the domain if it's being accessed from another port + domain = domain.split(":")[0] + # Not secure if it's being accessed from I2P + # Browsers expect the domain to include https. On I2P there is no HTTPS + if domain.not_nil!.split(".").last == "i2p" + @@secure = false + end + return HTTP::Cookie.new( + name: CONFIG.server_id_cookie_name, + domain: domain, + path: "/", + value: server_id.to_s, + secure: @@secure, + http_only: true, + samesite: HTTP::Cookie::SameSite::Lax + ) + end end end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 87a47bdb..e288b152 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -298,7 +298,7 @@ struct Video predicate_bool upcoming, isUpcoming end -def get_video(id, refresh = true, region = nil, force_refresh = false) +def get_video(id, refresh = true, region = nil, force_refresh = false, env : HTTP::Server::Context | Nil = nil) if (video = Invidious::Database::Videos.select(id)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) @@ -308,7 +308,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) force_refresh || video.schema_version != Video::SCHEMA_VERSION # cache control begin - video = fetch_video(id, region) + video = fetch_video(id, region, env) Invidious::Database::Videos.insert(video) rescue ex Invidious::Database::Videos.delete(id) @@ -316,7 +316,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) end end else - video = fetch_video(id, region) + video = fetch_video(id, region, env) Invidious::Database::Videos.insert(video) if !region end @@ -324,11 +324,11 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) rescue DB::Error # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends # Note: All DB errors inherit from `DB::Error` - return fetch_video(id, region) + return fetch_video(id, region, env) end -def fetch_video(id, region) - info = extract_video_info(video_id: id) +def fetch_video(id, region, env) + info = extract_video_info(video_id: id, env: env) if reason = info["reason"]? if reason == "Video unavailable" diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 204ef833..f7853ec4 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -58,12 +58,12 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? } end -def extract_video_info(video_id : String) +def extract_video_info(video_id : String, env : HTTP::Server::Context | Nil = nil) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new # Fetch data from the player endpoint - player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config) + player_response = YoutubeAPI.player(env: env, video_id: video_id, params: "2AMB", client_config: client_config) playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s @@ -119,7 +119,7 @@ def extract_video_info(video_id : String) # following issue for an explanation about decrypted URLs: # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite - new_player_response = try_fetch_streaming_data(video_id, client_config) + new_player_response = try_fetch_streaming_data(video_id, client_config, env) end # Replace player response and reset reason @@ -154,9 +154,9 @@ def extract_video_info(video_id : String) return params end -def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? +def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig, env : HTTP::Server::Context | Nil = nil) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config) + response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config, env: env) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index b9504b52..a28e2e90 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -1,9 +1,7 @@ <% locale = env.get("preferences").as(Preferences).locale dark_mode = env.get("preferences").as(Preferences).dark_mode - current_backend = env.request.cookies[CONFIG.server_id_cookie_name]?.try &.value || env.request.headers["Host"] current_external_videoplayback_proxy = Invidious::HttpServer::Utils.get_external_proxy() - status = BackendInfo.get_status %> @@ -37,15 +35,6 @@
Invidious - <% if status == 0 %> - • - <% end %> - <% if status == 1 %> - • - <% end %> - <% if status == 2 %> - • - <% end %>
@@ -118,31 +107,35 @@
- <% if !CONFIG.backends.empty? %> - <% if !CONFIG.backend_domains.includes?(env.request.headers["Host"]) %> + <% + if CONFIG.invidious_companion.present? + current_backend = env.get("current_companion").as(Int32) + status = BackendInfo.get_status + %>
Switch Backend: - <% CONFIG.backends.each do | backend | %> - <% backend = backend.split(CONFIG.backends_delimiter) %> - <% if current_backend == backend[0] %> - - Backend<%= HTML.escape(backend[0]) %> - <% if backend.size == 2 %> - <%= HTML.escape(backend[1]) %> - <% end %> + <% CONFIG.invidious_companion.each_with_index do | backend, index | %> + <% if current_backend == index %> + + Backend<%= HTML.escape((index+1).to_s) %> + • + <% else %> + + Backend<%= HTML.escape((index+1).to_s) %> + • + <% end %> | - <% else %> - - Backend<%= HTML.escape(backend[0]) %> - <% if backend.size == 2 %> - <%= HTML.escape(backend[1]) %> - <% end %> - | - <% end %> - <% end %> + <% end %>
<% end %> - <% end %> <% if CONFIG.banner %>
@@ -152,202 +145,207 @@ <%= content %> - <% if buffer_footer %> - - <% end %> + <% if buffer_footer %> + + <% end %> -
- - - - <% if env.get? "user" %> - - - <% if CONFIG.enable_user_notifications %> - - <% end %> - <% end %> + + + + + <% if env.get? "user" %> + + + <% if CONFIG.enable_user_notifications %> + + <% end %> + <% end %> - + <% CONFIG.footer_instance_section_custom_fields.each do | field | %> +
  • + + <%= HTML.escape(field[0]) %> + +
    • + <% end %> + + + <% end %> + +
    + <%= translate(locale, "footer_support_section_header")%> + +
    + +
    +
    + <% + if CONFIG.invidious_companion.present? + current_backend = env.get("current_companion").as(Int32) + %> +
    You are currently using Backend: <%= CONFIG.invidious_companion[current_backend].public_url %>
    + <% end %> + <% if !current_external_videoplayback_proxy.empty? %> +
    External Videoplayback Proxy: <%= current_external_videoplayback_proxy %>
    + <% end %> + + <% if CONFIG.modified_source_code_url %> + <%= translate(locale, "footer_current_version_modified") %> + <% else %> + <%= translate(locale, "Current version: ") %> + <% end %> + + <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %> + +
    + "><%= translate(locale, "footer_privacy_policy_link") %> + | + "><%= translate(locale, "footer_licences_link") %> +
    +
    + + +
    + diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0daed46c..5e4525c5 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -63,7 +63,7 @@ struct CompanionConnectionPool end end - def client(&) + def client(env : HTTP::Server::Context | Nil, &) conn = pool.checkout begin @@ -71,7 +71,13 @@ struct CompanionConnectionPool rescue ex conn.close - companion = CONFIG.invidious_companion.sample + if env.nil? + companion = CONFIG.invidious_companion.sample + else + current_companion = env.get("current_companion").as(Int32) + companion = CONFIG.invidious_companion[current_companion] + end + conn = make_client(companion.private_url, use_http_proxy: false) response = yield conn diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 590b0b88..f8de1b0f 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -456,6 +456,7 @@ module YoutubeAPI *, # Force the following parameters to be passed by name params : String, client_config : ClientConfig | Nil = nil, + env : HTTP::Server::Context | Nil = nil, ) # Playback context, separate because it can be different between clients playback_ctx = { @@ -492,7 +493,7 @@ module YoutubeAPI end if CONFIG.invidious_companion.present? - return self._post_invidious_companion("/youtubei/v1/player", data) + return self._post_invidious_companion("/youtubei/v1/player", data, env) else return self._post_json("/youtubei/v1/player", data, client_config) end @@ -673,6 +674,7 @@ module YoutubeAPI def _post_invidious_companion( endpoint : String, data : Hash, + env : HTTP::Server::Context | Nil, ) : Hash(String, JSON::Any) headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", @@ -686,7 +688,7 @@ module YoutubeAPI # Send the POST request begin - response = COMPANION_POOL.client &.post(endpoint, headers: headers, body: data.to_json) + response = COMPANION_POOL.client(env, &.post(endpoint, headers: headers, body: data.to_json)) body = response.body if (response.status_code != 200) raise Exception.new(