diff --git a/.ameba.yml b/.ameba.yml index 36d7c48f..e9e67b16 100644 --- a/.ameba.yml +++ b/.ameba.yml @@ -25,7 +25,7 @@ Lint/NotNil: Lint/SpecFilename: Excluded: - - spec/parsers_helper.cr + - spec/*_helper.cr # diff --git a/config/config.example.yml b/config/config.example.yml index 8d3e6212..5171a65a 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -206,7 +206,7 @@ https_only: false #disable_proxy: false ## -## Size of the HTTP pool used to connect to youtube. Each +## Max size of the HTTP pool used to connect to youtube. Each ## domain ('youtube.com', 'ytimg.com', ...) has its own pool. ## ## Accepted values: a positive integer @@ -214,6 +214,16 @@ https_only: false ## #pool_size: 100 +## +## Amount of seconds to wait for a client to be free from the pool +## before raising an error +## +## +## Accepted values: a positive integer +## Default: 5 +## +#pool_checkout_timeout: 5 + ## ## Additional cookies to be sent when requesting the youtube API. diff --git a/spec/helpers/networking/connection_pool_spec.cr b/spec/helpers/networking/connection_pool_spec.cr new file mode 100644 index 00000000..3209d4e3 --- /dev/null +++ b/spec/helpers/networking/connection_pool_spec.cr @@ -0,0 +1,112 @@ +# Due to the way that specs are handled this file cannot be run +# together with everything else without causing a compile time error +# +# TODO: Allow running different isolated spec through make +# +# For now run this with `crystal spec -p spec/helpers/networking/connection_pool_spec.cr -Drunning_by_self` +{% skip_file unless flag?(:running_by_self) %} + +# Based on https://github.com/jgaskins/http_client/blob/958cf56064c0d31264a117467022b90397eb65d7/spec/http_client_spec.cr +require "wait_group" +require "uri" +require "http" +require "http/server" +require "http_proxy" + +require "db" +require "pg" +require "spectator" + +require "../../load_config_helper" +require "../../../src/invidious/helpers/crystal_class_overrides" +require "../../../src/invidious/connection/*" + +TEST_SERVER_URL = URI.parse("http://localhost:12345") + +server = HTTP::Server.new do |context| + request = context.request + response = context.response + + case {request.method, request.path} + when {"GET", "/get"} + response << "get" + when {"POST", "/post"} + response.status = :created + response << "post" + when {"GET", "/sleep"} + duration = request.query_params["duration_sec"].to_i.seconds + sleep duration + end +end + +spawn server.listen 12345 + +Fiber.yield + +Spectator.describe Invidious::ConnectionPool do + describe "Pool" do + it "Can make a requests through standard HTTP methods" do + pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) } + + expect(pool.get("/get").body).to eq("get") + expect(pool.post("/post").body).to eq("post") + end + + it "Can make streaming requests" do + pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) } + + expect(pool.get("/get", &.body_io.gets_to_end)).to eq("get") + expect(pool.get("/post", &.body)).to eq("") + expect(pool.post("/post", &.body_io.gets_to_end)).to eq("post") + end + + it "Allows more than one clients to be checked out (if applicable)" do + pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) } + + pool.checkout do |_| + expect(pool.post("/post").body).to eq("post") + end + end + + it "Can make multiple requests with the same client" do + pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) } + + pool.checkout do |client| + expect(client.get("/get").body).to eq("get") + expect(client.post("/post").body).to eq("post") + expect(client.get("/get").body).to eq("get") + end + end + + it "Allows concurrent requests" do + pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) } + responses = [] of HTTP::Client::Response + + WaitGroup.wait do |wg| + 100.times do + wg.spawn { responses << pool.get("/get") } + end + end + + expect(responses.map(&.body)).to eq(["get"] * 100) + end + + it "Raises on checkout timeout" do + pool = Invidious::ConnectionPool::Pool.new(max_capacity: 2, timeout: 0.01) { next make_client(TEST_SERVER_URL) } + + # Long running requests + 2.times do + spawn { pool.get("/sleep?duration_sec=2") } + end + + Fiber.yield + + expect { pool.get("/get") }.to raise_error(Invidious::ConnectionPool::Error) + end + + it "Raises when an error is encountered" do + pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) } + expect { pool.get("/get") { raise IO::Error.new } }.to raise_error(Invidious::ConnectionPool::Error) + end + end +end diff --git a/spec/load_config_helper.cr b/spec/load_config_helper.cr new file mode 100644 index 00000000..07a63718 --- /dev/null +++ b/spec/load_config_helper.cr @@ -0,0 +1,15 @@ +require "yaml" +require "log" + +abstract class Kemal::BaseLogHandler +end + +require "../src/invidious/config" +require "../src/invidious/jobs/base_job" +require "../src/invidious/jobs.cr" +require "../src/invidious/user/preferences.cr" +require "../src/invidious/helpers/logger" +require "../src/invidious/helpers/utils" + +CONFIG = Config.from_yaml(File.open("config/config.example.yml")) +HMAC_KEY = CONFIG.hmac_key diff --git a/src/invidious.cr b/src/invidious.cr index 69f8a26c..20704fd3 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -33,6 +33,7 @@ require "protodec/utils" require "./invidious/database/*" require "./invidious/database/migrations/*" +require "./invidious/connection/*" require "./invidious/http_server/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" @@ -90,15 +91,31 @@ SOFTWARE = { "branch" => "#{CURRENT_BRANCH}", } -YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +YT_POOL = Invidious::ConnectionPool::Pool.new( + max_capacity: CONFIG.pool_size, + timeout: CONFIG.pool_checkout_timeout +) do + next make_client(YT_URL, force_resolve: true) +end # Image request pool -GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) +GGPHT_URL = URI.parse("https://yt3.ggpht.com") -COMPANION_POOL = CompanionConnectionPool.new( - capacity: CONFIG.pool_size -) +GGPHT_POOL = Invidious::ConnectionPool::Pool.new( + max_capacity: CONFIG.pool_size, + timeout: CONFIG.pool_checkout_timeout +) do + next make_client(GGPHT_URL, force_resolve: true) +end + +COMPANION_POOL = Invidious::ConnectionPool::Pool.new( + max_capacity: CONFIG.pool_size, + reinitialize_proxy: false +) do + companion = CONFIG.invidious_companion.sample + next make_client(companion.private_url, use_http_proxy: false) +end # CLI Kemal.config.extra_options do |parser| diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 65982325..f71c4293 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -166,7 +166,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) } LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") - rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body + rss = YT_POOL.get("/feeds/videos.xml?channel_id=#{ucid}").body LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") rss = XML.parse(rss) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 4d69854c..d3a96cff 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -157,8 +157,13 @@ class Config property host_binding : String = "0.0.0.0" # Path and permissions to make Invidious listen on a UNIX socket instead of a TCP port property socket_binding : SocketBindingConfig? = nil - # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + + # Max pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool) property pool_size : Int32 = 100 + + # Amount of seconds to wait for a client to be free from the pool before rasing an error + property pool_checkout_timeout : Float64 = 5 + # HTTP Proxy configuration property http_proxy : HTTPProxyConfig? = nil diff --git a/src/invidious/connection/client.cr b/src/invidious/connection/client.cr new file mode 100644 index 00000000..ab3f8c50 --- /dev/null +++ b/src/invidious/connection/client.cr @@ -0,0 +1,53 @@ +def add_yt_headers(request) + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" + + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" + if !CONFIG.cookies.empty? + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true) + client = HTTP::Client.new(url) + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy + + # Force the usage of a specific configured IP Family + if force_resolve + client.family = CONFIG.force_resolve + client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC + end + + client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers + client.read_timeout = 10.seconds + client.connect_timeout = 10.seconds + + return client +end + +def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_proxy : Bool = true, &) + client = make_client(url, region, force_resolve: force_resolve, use_http_proxy: use_http_proxy) + begin + yield client + ensure + client.close + end +end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end diff --git a/src/invidious/connection/pool.cr b/src/invidious/connection/pool.cr new file mode 100644 index 00000000..a97b9983 --- /dev/null +++ b/src/invidious/connection/pool.cr @@ -0,0 +1,116 @@ +module Invidious::ConnectionPool + # A connection pool to reuse `HTTP::Client` connections + struct Pool + getter pool : DB::Pool(HTTP::Client) + + # Creates a connection pool with the provided options, and client factory block. + def initialize( + *, + max_capacity : Int32 = 5, + timeout : Float64 = 5.0, + @reinitialize_proxy : Bool = true, # Whether or not http-proxy should be reinitialized on checkout + &client_factory : -> HTTP::Client + ) + pool_options = DB::Pool::Options.new( + initial_pool_size: 0, + max_pool_size: max_capacity, + max_idle_pool_size: max_capacity, + checkout_timeout: timeout + ) + + @pool = DB::Pool(HTTP::Client).new(pool_options, &client_factory) + end + + {% for method in %w[get post put patch delete head options] %} + # Streaming API for {{method.id.upcase}} request. + # The response will have its body as an `IO` accessed via `HTTP::Client::Response#body_io`. + def {{method.id}}(*args, **kwargs, &) + self.checkout do | client | + client.{{method.id}}(*args, **kwargs) do | response | + result = yield response + return result + ensure + response.body_io?.try &.skip_to_end + end + end + end + + # Executes a {{method.id.upcase}} request. + # The response will have its body as a `String`, accessed via `HTTP::Client::Response#body`. + def {{method.id}}(*args, **kwargs) + self.checkout do | client | + return client.{{method.id}}(*args, **kwargs) + end + end + {% end %} + + # Checks out a client in the pool + def checkout(&) + # If a client has been deleted from the pool + # we won't try to release it + client_exists_in_pool = true + + http_client = pool.checkout + + # When the HTTP::Client connection is closed, the automatic reconnection + # feature will create a new IO to connect to the server with + # + # This new TCP IO will be a direct connection to the server and will not go + # through the proxy. As such we'll need to reinitialize the proxy connection + + http_client.proxy = make_configured_http_proxy_client() if @reinitialize_proxy && CONFIG.http_proxy + + response = yield http_client + rescue ex : DB::PoolTimeout + # Failed to checkout a client + raise ConnectionPool::PoolCheckoutError.new(ex.message) + rescue ex + # An error occurred with the client itself. + # Delete the client from the pool and close the connection + if http_client + client_exists_in_pool = false + @pool.delete(http_client) + http_client.close + end + + # Raise exception for outer methods to handle + raise ConnectionPool::Error.new(ex.message, cause: ex) + ensure + pool.release(http_client) if http_client && client_exists_in_pool + end + end + + class Error < Exception + end + + # Raised when the pool failed to get a client in time + class PoolCheckoutError < Error + end + + # Mapping of subdomain => Invidious::ConnectionPool::Pool + # This is needed as we may need to access arbitrary subdomains of ytimg + private YTIMG_POOLS = {} of String => ConnectionPool::Pool + + # Fetches a HTTP pool for the specified subdomain of ytimg.com + # + # Creates a new one when the specified pool for the subdomain does not exist + def self.get_ytimg_pool(subdomain) + if pool = YTIMG_POOLS[subdomain]? + return pool + else + LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") + url = URI.parse("https://#{subdomain}.ytimg.com") + + pool = ConnectionPool::Pool.new( + max_capacity: CONFIG.pool_size, + timeout: CONFIG.pool_checkout_timeout + ) do + next make_client(url, force_resolve: true) + end + + YTIMG_POOLS[subdomain] = pool + + return pool + end + end +end diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 28ff0ff6..6728ff47 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -26,7 +26,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) end video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_" - response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers) + response = YT_POOL.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers) initial_data = extract_initial_data(response.body) if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]? diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index c27caad7..bc1258f5 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -26,7 +26,7 @@ module Invidious::Routes::API::Manifest end if dashmpd = video.dash_manifest_url - response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) + response = YT_POOL.get(URI.parse(dashmpd).request_target) if response.status_code != 200 haltf env, status_code: response.status_code @@ -167,7 +167,7 @@ module Invidious::Routes::API::Manifest # /api/manifest/hls_playlist/* def self.get_hls_playlist(env) - response = YT_POOL.client &.get(env.request.path) + response = YT_POOL.get(env.request.path) if response.status_code != 200 haltf env, status_code: response.status_code @@ -223,7 +223,7 @@ module Invidious::Routes::API::Manifest # /api/manifest/hls_variant/* def self.get_hls_variant(env) - response = YT_POOL.client &.get(env.request.path) + response = YT_POOL.get(env.request.path) if response.status_code != 200 haltf env, status_code: response.status_code diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 6a3eb8ae..9d10b0e1 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -106,7 +106,7 @@ module Invidious::Routes::API::V1::Videos # Auto-generated captions often have cues that aren't aligned properly with the video, # as well as some other markup that makes it cumbersome, so we try to fix that here if caption.name.includes? "auto-generated" - caption_xml = YT_POOL.client &.get(url).body + caption_xml = YT_POOL.get(url).body settings_field = { "Kind" => "captions", @@ -147,7 +147,7 @@ module Invidious::Routes::API::V1::Videos query_params = uri.query_params query_params["fmt"] = "vtt" uri.query_params = query_params - webvtt = YT_POOL.client &.get(uri.request_target).body + webvtt = YT_POOL.get(uri.request_target).body if webvtt.starts_with?("[a-zA-Z0-9_-]{11})"/).try &.["video_id"] env.params.query.delete_all("channel") diff --git a/src/invidious/routes/errors.cr b/src/invidious/routes/errors.cr index 1e9ab44e..2f35f050 100644 --- a/src/invidious/routes/errors.cr +++ b/src/invidious/routes/errors.cr @@ -9,10 +9,10 @@ module Invidious::Routes::ErrorRoutes item = md["id"] # Check if item is branding URL e.g. https://youtube.com/gaming - response = YT_POOL.client &.get("/#{item}") + response = YT_POOL.get("/#{item}") if response.status_code == 301 - response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target) + response = YT_POOL.get(URI.parse(response.headers["Location"]).request_target) end if response.body.empty? @@ -40,7 +40,7 @@ module Invidious::Routes::ErrorRoutes end # Check if item is video ID - if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404 + if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.head("/watch?v=#{item}").status_code != 404 env.response.headers["Location"] = url haltf env, status_code: 302 end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 070c96eb..c7df71e2 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -160,8 +160,9 @@ module Invidious::Routes::Feeds "default" => "http://www.w3.org/2005/Atom", } - response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}") + response = YT_POOL.get("/feeds/videos.xml?channel_id=#{ucid}") return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404 + rss = XML.parse(response.body) videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry| @@ -310,7 +311,7 @@ module Invidious::Routes::Feeds end end - response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}") + response = YT_POOL.get("/feeds/videos.xml?playlist_id=#{plid}") return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404 document = XML.parse(response.body) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 51d85dfe..7097ab79 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -12,7 +12,7 @@ module Invidious::Routes::Images end begin - GGPHT_POOL.client &.get(url, headers) do |resp| + GGPHT_POOL.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -42,7 +42,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool(authority).client &.get(url, headers) do |resp| + ConnectionPool.get_ytimg_pool(authority).get(url, headers) do |resp| env.response.headers["Connection"] = "close" return self.proxy_image(env, resp) end @@ -65,7 +65,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool("i9").client &.get(url, headers) do |resp| + ConnectionPool.get_ytimg_pool("i9").get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -81,7 +81,7 @@ module Invidious::Routes::Images end begin - YT_POOL.client &.get(env.request.resource, headers) do |response| + YT_POOL.get(env.request.resource, headers) do |response| env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -111,7 +111,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - if get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200 + if ConnectionPool.get_ytimg_pool("i").head(thumbnail_resource_path, headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -127,7 +127,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool("i").client &.get(url, headers) do |resp| + ConnectionPool.get_ytimg_pool("i").get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index f2213da4..cb24648f 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -464,7 +464,7 @@ module Invidious::Routes::Playlists # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos def self.watch_videos(env) - response = YT_POOL.client &.get(env.request.resource) + response = YT_POOL.get(env.request.resource) if url = response.headers["Location"]? url = URI.parse(url).request_target return env.redirect url diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index 25edb936..4c635ab2 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -16,11 +16,11 @@ module Invidious::Search # Search a youtube channel # TODO: clean code, and rely more on YoutubeAPI def channel(query : Query) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{query.channel}") + response = YT_POOL.get("/channel/#{query.channel}") if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{query.channel}") - response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404 + response = YT_POOL.get("/user/#{query.channel}") + response = YT_POOL.get("/c/#{query.channel}") if response.status_code == 404 initial_data = extract_initial_data(response.body) ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) raise ChannelSearchException.new(query.channel) if !ucid diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr deleted file mode 100644 index 0daed46c..00000000 --- a/src/invidious/yt_backend/connection_pool.cr +++ /dev/null @@ -1,153 +0,0 @@ -# Mapping of subdomain => YoutubeConnectionPool -# This is needed as we may need to access arbitrary subdomains of ytimg -private YTIMG_POOLS = {} of String => YoutubeConnectionPool - -struct YoutubeConnectionPool - property! url : URI - property! capacity : Int32 - property! timeout : Float64 - property pool : DB::Pool(HTTP::Client) - - def initialize(url : URI, @capacity = 5, @timeout = 5.0) - @url = url - @pool = build_pool() - end - - def client(&) - conn = pool.checkout - # Proxy needs to be reinstated every time we get a client from the pool - conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy - - begin - response = yield conn - rescue ex - conn.close - conn = make_client(url, force_resolve: true) - - response = yield conn - ensure - pool.release(conn) - end - - response - end - - private def build_pool - options = DB::Pool::Options.new( - initial_pool_size: 0, - max_pool_size: capacity, - max_idle_pool_size: capacity, - checkout_timeout: timeout - ) - - DB::Pool(HTTP::Client).new(options) do - next make_client(url, force_resolve: true) - end - end -end - -struct CompanionConnectionPool - property pool : DB::Pool(HTTP::Client) - - def initialize(capacity = 5, timeout = 5.0) - options = DB::Pool::Options.new( - initial_pool_size: 0, - max_pool_size: capacity, - max_idle_pool_size: capacity, - checkout_timeout: timeout - ) - - @pool = DB::Pool(HTTP::Client).new(options) do - companion = CONFIG.invidious_companion.sample - next make_client(companion.private_url, use_http_proxy: false) - end - end - - def client(&) - conn = pool.checkout - - begin - response = yield conn - rescue ex - conn.close - - companion = CONFIG.invidious_companion.sample - conn = make_client(companion.private_url, use_http_proxy: false) - - response = yield conn - ensure - pool.release(conn) - end - - response - end -end - -def add_yt_headers(request) - request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" - - request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" - if !CONFIG.cookies.empty? - request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end - -def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true) - client = HTTP::Client.new(url) - client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy - - # Force the usage of a specific configured IP Family - if force_resolve - client.family = CONFIG.force_resolve - client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC - end - - client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - return client -end - -def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_proxy : Bool = true, &) - client = make_client(url, region, force_resolve: force_resolve, use_http_proxy: use_http_proxy) - begin - yield client - ensure - client.close - end -end - -def make_configured_http_proxy_client - # This method is only called when configuration for an HTTP proxy are set - config_proxy = CONFIG.http_proxy.not_nil! - - return HTTP::Proxy::Client.new( - config_proxy.host, - config_proxy.port, - - username: config_proxy.user, - password: config_proxy.password, - ) -end - -# Fetches a HTTP pool for the specified subdomain of ytimg.com -# -# Creates a new one when the specified pool for the subdomain does not exist -def get_ytimg_pool(subdomain) - if pool = YTIMG_POOLS[subdomain]? - return pool - else - LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") - pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size) - YTIMG_POOLS[subdomain] = pool - - return pool - end -end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index b40092a1..9a0c1cf8 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -639,15 +639,13 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: POST data: #{data}") # Send the POST request - body = YT_POOL.client() do |client| - client.post(url, headers: headers, body: data.to_json) do |response| - if response.status_code != 200 - raise InfoException.new("Error: non 200 status code. Youtube API returned \ - status code #{response.status_code}. See \ - https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") - end - self._decompress(response.body_io, response.headers["Content-Encoding"]?) + body = YT_POOL.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See \ + https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") end + self._decompress(response.body_io, response.headers["Content-Encoding"]?) end # Convert result to Hash @@ -695,7 +693,7 @@ module YoutubeAPI # Send the POST request begin - response = COMPANION_POOL.client &.post(endpoint, headers: headers, body: data.to_json) + response = COMPANION_POOL.post(endpoint, headers: headers, body: data.to_json) body = response.body if (response.status_code != 200) raise Exception.new(