diff --git a/.ameba.yml b/.ameba.yml
index 36d7c48f..e9e67b16 100644
--- a/.ameba.yml
+++ b/.ameba.yml
@@ -25,7 +25,7 @@ Lint/NotNil:
Lint/SpecFilename:
Excluded:
- - spec/parsers_helper.cr
+ - spec/*_helper.cr
#
diff --git a/config/config.example.yml b/config/config.example.yml
index 8d3e6212..5171a65a 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -206,7 +206,7 @@ https_only: false
#disable_proxy: false
##
-## Size of the HTTP pool used to connect to youtube. Each
+## Max size of the HTTP pool used to connect to youtube. Each
## domain ('youtube.com', 'ytimg.com', ...) has its own pool.
##
## Accepted values: a positive integer
@@ -214,6 +214,16 @@ https_only: false
##
#pool_size: 100
+##
+## Amount of seconds to wait for a client to be free from the pool
+## before raising an error
+##
+##
+## Accepted values: a positive integer
+## Default: 5
+##
+#pool_checkout_timeout: 5
+
##
## Additional cookies to be sent when requesting the youtube API.
diff --git a/spec/helpers/networking/connection_pool_spec.cr b/spec/helpers/networking/connection_pool_spec.cr
new file mode 100644
index 00000000..3209d4e3
--- /dev/null
+++ b/spec/helpers/networking/connection_pool_spec.cr
@@ -0,0 +1,112 @@
+# Due to the way that specs are handled this file cannot be run
+# together with everything else without causing a compile time error
+#
+# TODO: Allow running different isolated spec through make
+#
+# For now run this with `crystal spec -p spec/helpers/networking/connection_pool_spec.cr -Drunning_by_self`
+{% skip_file unless flag?(:running_by_self) %}
+
+# Based on https://github.com/jgaskins/http_client/blob/958cf56064c0d31264a117467022b90397eb65d7/spec/http_client_spec.cr
+require "wait_group"
+require "uri"
+require "http"
+require "http/server"
+require "http_proxy"
+
+require "db"
+require "pg"
+require "spectator"
+
+require "../../load_config_helper"
+require "../../../src/invidious/helpers/crystal_class_overrides"
+require "../../../src/invidious/connection/*"
+
+TEST_SERVER_URL = URI.parse("http://localhost:12345")
+
+server = HTTP::Server.new do |context|
+ request = context.request
+ response = context.response
+
+ case {request.method, request.path}
+ when {"GET", "/get"}
+ response << "get"
+ when {"POST", "/post"}
+ response.status = :created
+ response << "post"
+ when {"GET", "/sleep"}
+ duration = request.query_params["duration_sec"].to_i.seconds
+ sleep duration
+ end
+end
+
+spawn server.listen 12345
+
+Fiber.yield
+
+Spectator.describe Invidious::ConnectionPool do
+ describe "Pool" do
+ it "Can make a requests through standard HTTP methods" do
+ pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
+
+ expect(pool.get("/get").body).to eq("get")
+ expect(pool.post("/post").body).to eq("post")
+ end
+
+ it "Can make streaming requests" do
+ pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
+
+ expect(pool.get("/get", &.body_io.gets_to_end)).to eq("get")
+ expect(pool.get("/post", &.body)).to eq("")
+ expect(pool.post("/post", &.body_io.gets_to_end)).to eq("post")
+ end
+
+ it "Allows more than one clients to be checked out (if applicable)" do
+ pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
+
+ pool.checkout do |_|
+ expect(pool.post("/post").body).to eq("post")
+ end
+ end
+
+ it "Can make multiple requests with the same client" do
+ pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
+
+ pool.checkout do |client|
+ expect(client.get("/get").body).to eq("get")
+ expect(client.post("/post").body).to eq("post")
+ expect(client.get("/get").body).to eq("get")
+ end
+ end
+
+ it "Allows concurrent requests" do
+ pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
+ responses = [] of HTTP::Client::Response
+
+ WaitGroup.wait do |wg|
+ 100.times do
+ wg.spawn { responses << pool.get("/get") }
+ end
+ end
+
+ expect(responses.map(&.body)).to eq(["get"] * 100)
+ end
+
+ it "Raises on checkout timeout" do
+ pool = Invidious::ConnectionPool::Pool.new(max_capacity: 2, timeout: 0.01) { next make_client(TEST_SERVER_URL) }
+
+ # Long running requests
+ 2.times do
+ spawn { pool.get("/sleep?duration_sec=2") }
+ end
+
+ Fiber.yield
+
+ expect { pool.get("/get") }.to raise_error(Invidious::ConnectionPool::Error)
+ end
+
+ it "Raises when an error is encountered" do
+ pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
+ expect { pool.get("/get") { raise IO::Error.new } }.to raise_error(Invidious::ConnectionPool::Error)
+ end
+ end
+end
diff --git a/spec/load_config_helper.cr b/spec/load_config_helper.cr
new file mode 100644
index 00000000..07a63718
--- /dev/null
+++ b/spec/load_config_helper.cr
@@ -0,0 +1,15 @@
+require "yaml"
+require "log"
+
+abstract class Kemal::BaseLogHandler
+end
+
+require "../src/invidious/config"
+require "../src/invidious/jobs/base_job"
+require "../src/invidious/jobs.cr"
+require "../src/invidious/user/preferences.cr"
+require "../src/invidious/helpers/logger"
+require "../src/invidious/helpers/utils"
+
+CONFIG = Config.from_yaml(File.open("config/config.example.yml"))
+HMAC_KEY = CONFIG.hmac_key
diff --git a/src/invidious.cr b/src/invidious.cr
index 69f8a26c..20704fd3 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -33,6 +33,7 @@ require "protodec/utils"
require "./invidious/database/*"
require "./invidious/database/migrations/*"
+require "./invidious/connection/*"
require "./invidious/http_server/*"
require "./invidious/helpers/*"
require "./invidious/yt_backend/*"
@@ -90,15 +91,31 @@ SOFTWARE = {
"branch" => "#{CURRENT_BRANCH}",
}
-YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
+YT_POOL = Invidious::ConnectionPool::Pool.new(
+ max_capacity: CONFIG.pool_size,
+ timeout: CONFIG.pool_checkout_timeout
+) do
+ next make_client(YT_URL, force_resolve: true)
+end
# Image request pool
-GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size)
+GGPHT_URL = URI.parse("https://yt3.ggpht.com")
-COMPANION_POOL = CompanionConnectionPool.new(
- capacity: CONFIG.pool_size
-)
+GGPHT_POOL = Invidious::ConnectionPool::Pool.new(
+ max_capacity: CONFIG.pool_size,
+ timeout: CONFIG.pool_checkout_timeout
+) do
+ next make_client(GGPHT_URL, force_resolve: true)
+end
+
+COMPANION_POOL = Invidious::ConnectionPool::Pool.new(
+ max_capacity: CONFIG.pool_size,
+ reinitialize_proxy: false
+) do
+ companion = CONFIG.invidious_companion.sample
+ next make_client(companion.private_url, use_http_proxy: false)
+end
# CLI
Kemal.config.extra_options do |parser|
diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr
index 65982325..f71c4293 100644
--- a/src/invidious/channels/channels.cr
+++ b/src/invidious/channels/channels.cr
@@ -166,7 +166,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
}
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
- rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
+ rss = YT_POOL.get("/feeds/videos.xml?channel_id=#{ucid}").body
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
rss = XML.parse(rss)
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index 4d69854c..d3a96cff 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -157,8 +157,13 @@ class Config
property host_binding : String = "0.0.0.0"
# Path and permissions to make Invidious listen on a UNIX socket instead of a TCP port
property socket_binding : SocketBindingConfig? = nil
- # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
+
+ # Max pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool)
property pool_size : Int32 = 100
+
+ # Amount of seconds to wait for a client to be free from the pool before rasing an error
+ property pool_checkout_timeout : Float64 = 5
+
# HTTP Proxy configuration
property http_proxy : HTTPProxyConfig? = nil
diff --git a/src/invidious/connection/client.cr b/src/invidious/connection/client.cr
new file mode 100644
index 00000000..ab3f8c50
--- /dev/null
+++ b/src/invidious/connection/client.cr
@@ -0,0 +1,53 @@
+def add_yt_headers(request)
+ request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
+ request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+
+ request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
+ request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
+
+ # Preserve original cookies and add new YT consent cookie for EU servers
+ request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
+ if !CONFIG.cookies.empty?
+ request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
+ end
+end
+
+def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true)
+ client = HTTP::Client.new(url)
+ client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy
+
+ # Force the usage of a specific configured IP Family
+ if force_resolve
+ client.family = CONFIG.force_resolve
+ client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC
+ end
+
+ client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers
+ client.read_timeout = 10.seconds
+ client.connect_timeout = 10.seconds
+
+ return client
+end
+
+def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_proxy : Bool = true, &)
+ client = make_client(url, region, force_resolve: force_resolve, use_http_proxy: use_http_proxy)
+ begin
+ yield client
+ ensure
+ client.close
+ end
+end
+
+def make_configured_http_proxy_client
+ # This method is only called when configuration for an HTTP proxy are set
+ config_proxy = CONFIG.http_proxy.not_nil!
+
+ return HTTP::Proxy::Client.new(
+ config_proxy.host,
+ config_proxy.port,
+
+ username: config_proxy.user,
+ password: config_proxy.password,
+ )
+end
diff --git a/src/invidious/connection/pool.cr b/src/invidious/connection/pool.cr
new file mode 100644
index 00000000..a97b9983
--- /dev/null
+++ b/src/invidious/connection/pool.cr
@@ -0,0 +1,116 @@
+module Invidious::ConnectionPool
+ # A connection pool to reuse `HTTP::Client` connections
+ struct Pool
+ getter pool : DB::Pool(HTTP::Client)
+
+ # Creates a connection pool with the provided options, and client factory block.
+ def initialize(
+ *,
+ max_capacity : Int32 = 5,
+ timeout : Float64 = 5.0,
+ @reinitialize_proxy : Bool = true, # Whether or not http-proxy should be reinitialized on checkout
+ &client_factory : -> HTTP::Client
+ )
+ pool_options = DB::Pool::Options.new(
+ initial_pool_size: 0,
+ max_pool_size: max_capacity,
+ max_idle_pool_size: max_capacity,
+ checkout_timeout: timeout
+ )
+
+ @pool = DB::Pool(HTTP::Client).new(pool_options, &client_factory)
+ end
+
+ {% for method in %w[get post put patch delete head options] %}
+ # Streaming API for {{method.id.upcase}} request.
+ # The response will have its body as an `IO` accessed via `HTTP::Client::Response#body_io`.
+ def {{method.id}}(*args, **kwargs, &)
+ self.checkout do | client |
+ client.{{method.id}}(*args, **kwargs) do | response |
+ result = yield response
+ return result
+ ensure
+ response.body_io?.try &.skip_to_end
+ end
+ end
+ end
+
+ # Executes a {{method.id.upcase}} request.
+ # The response will have its body as a `String`, accessed via `HTTP::Client::Response#body`.
+ def {{method.id}}(*args, **kwargs)
+ self.checkout do | client |
+ return client.{{method.id}}(*args, **kwargs)
+ end
+ end
+ {% end %}
+
+ # Checks out a client in the pool
+ def checkout(&)
+ # If a client has been deleted from the pool
+ # we won't try to release it
+ client_exists_in_pool = true
+
+ http_client = pool.checkout
+
+ # When the HTTP::Client connection is closed, the automatic reconnection
+ # feature will create a new IO to connect to the server with
+ #
+ # This new TCP IO will be a direct connection to the server and will not go
+ # through the proxy. As such we'll need to reinitialize the proxy connection
+
+ http_client.proxy = make_configured_http_proxy_client() if @reinitialize_proxy && CONFIG.http_proxy
+
+ response = yield http_client
+ rescue ex : DB::PoolTimeout
+ # Failed to checkout a client
+ raise ConnectionPool::PoolCheckoutError.new(ex.message)
+ rescue ex
+ # An error occurred with the client itself.
+ # Delete the client from the pool and close the connection
+ if http_client
+ client_exists_in_pool = false
+ @pool.delete(http_client)
+ http_client.close
+ end
+
+ # Raise exception for outer methods to handle
+ raise ConnectionPool::Error.new(ex.message, cause: ex)
+ ensure
+ pool.release(http_client) if http_client && client_exists_in_pool
+ end
+ end
+
+ class Error < Exception
+ end
+
+ # Raised when the pool failed to get a client in time
+ class PoolCheckoutError < Error
+ end
+
+ # Mapping of subdomain => Invidious::ConnectionPool::Pool
+ # This is needed as we may need to access arbitrary subdomains of ytimg
+ private YTIMG_POOLS = {} of String => ConnectionPool::Pool
+
+ # Fetches a HTTP pool for the specified subdomain of ytimg.com
+ #
+ # Creates a new one when the specified pool for the subdomain does not exist
+ def self.get_ytimg_pool(subdomain)
+ if pool = YTIMG_POOLS[subdomain]?
+ return pool
+ else
+ LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"")
+ url = URI.parse("https://#{subdomain}.ytimg.com")
+
+ pool = ConnectionPool::Pool.new(
+ max_capacity: CONFIG.pool_size,
+ timeout: CONFIG.pool_checkout_timeout
+ ) do
+ next make_client(url, force_resolve: true)
+ end
+
+ YTIMG_POOLS[subdomain] = pool
+
+ return pool
+ end
+ end
+end
diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr
index 28ff0ff6..6728ff47 100644
--- a/src/invidious/mixes.cr
+++ b/src/invidious/mixes.cr
@@ -26,7 +26,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
end
video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
- response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
+ response = YT_POOL.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
initial_data = extract_initial_data(response.body)
if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?
diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr
index c27caad7..bc1258f5 100644
--- a/src/invidious/routes/api/manifest.cr
+++ b/src/invidious/routes/api/manifest.cr
@@ -26,7 +26,7 @@ module Invidious::Routes::API::Manifest
end
if dashmpd = video.dash_manifest_url
- response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
+ response = YT_POOL.get(URI.parse(dashmpd).request_target)
if response.status_code != 200
haltf env, status_code: response.status_code
@@ -167,7 +167,7 @@ module Invidious::Routes::API::Manifest
# /api/manifest/hls_playlist/*
def self.get_hls_playlist(env)
- response = YT_POOL.client &.get(env.request.path)
+ response = YT_POOL.get(env.request.path)
if response.status_code != 200
haltf env, status_code: response.status_code
@@ -223,7 +223,7 @@ module Invidious::Routes::API::Manifest
# /api/manifest/hls_variant/*
def self.get_hls_variant(env)
- response = YT_POOL.client &.get(env.request.path)
+ response = YT_POOL.get(env.request.path)
if response.status_code != 200
haltf env, status_code: response.status_code
diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index 6a3eb8ae..9d10b0e1 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -106,7 +106,7 @@ module Invidious::Routes::API::V1::Videos
# Auto-generated captions often have cues that aren't aligned properly with the video,
# as well as some other markup that makes it cumbersome, so we try to fix that here
if caption.name.includes? "auto-generated"
- caption_xml = YT_POOL.client &.get(url).body
+ caption_xml = YT_POOL.get(url).body
settings_field = {
"Kind" => "captions",
@@ -147,7 +147,7 @@ module Invidious::Routes::API::V1::Videos
query_params = uri.query_params
query_params["fmt"] = "vtt"
uri.query_params = query_params
- webvtt = YT_POOL.client &.get(uri.request_target).body
+ webvtt = YT_POOL.get(uri.request_target).body
if webvtt.starts_with?("[a-zA-Z0-9_-]{11})"/).try &.["video_id"]
env.params.query.delete_all("channel")
diff --git a/src/invidious/routes/errors.cr b/src/invidious/routes/errors.cr
index 1e9ab44e..2f35f050 100644
--- a/src/invidious/routes/errors.cr
+++ b/src/invidious/routes/errors.cr
@@ -9,10 +9,10 @@ module Invidious::Routes::ErrorRoutes
item = md["id"]
# Check if item is branding URL e.g. https://youtube.com/gaming
- response = YT_POOL.client &.get("/#{item}")
+ response = YT_POOL.get("/#{item}")
if response.status_code == 301
- response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target)
+ response = YT_POOL.get(URI.parse(response.headers["Location"]).request_target)
end
if response.body.empty?
@@ -40,7 +40,7 @@ module Invidious::Routes::ErrorRoutes
end
# Check if item is video ID
- if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
+ if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.head("/watch?v=#{item}").status_code != 404
env.response.headers["Location"] = url
haltf env, status_code: 302
end
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index 070c96eb..c7df71e2 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -160,8 +160,9 @@ module Invidious::Routes::Feeds
"default" => "http://www.w3.org/2005/Atom",
}
- response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
+ response = YT_POOL.get("/feeds/videos.xml?channel_id=#{ucid}")
return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
+
rss = XML.parse(response.body)
videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
@@ -310,7 +311,7 @@ module Invidious::Routes::Feeds
end
end
- response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
+ response = YT_POOL.get("/feeds/videos.xml?playlist_id=#{plid}")
return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
document = XML.parse(response.body)
diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 51d85dfe..7097ab79 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -12,7 +12,7 @@ module Invidious::Routes::Images
end
begin
- GGPHT_POOL.client &.get(url, headers) do |resp|
+ GGPHT_POOL.get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
@@ -42,7 +42,7 @@ module Invidious::Routes::Images
end
begin
- get_ytimg_pool(authority).client &.get(url, headers) do |resp|
+ ConnectionPool.get_ytimg_pool(authority).get(url, headers) do |resp|
env.response.headers["Connection"] = "close"
return self.proxy_image(env, resp)
end
@@ -65,7 +65,7 @@ module Invidious::Routes::Images
end
begin
- get_ytimg_pool("i9").client &.get(url, headers) do |resp|
+ ConnectionPool.get_ytimg_pool("i9").get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
@@ -81,7 +81,7 @@ module Invidious::Routes::Images
end
begin
- YT_POOL.client &.get(env.request.resource, headers) do |response|
+ YT_POOL.get(env.request.resource, headers) do |response|
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@@ -111,7 +111,7 @@ module Invidious::Routes::Images
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
- if get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200
+ if ConnectionPool.get_ytimg_pool("i").head(thumbnail_resource_path, headers).status_code == 200
name = thumb[:url] + ".jpg"
break
end
@@ -127,7 +127,7 @@ module Invidious::Routes::Images
end
begin
- get_ytimg_pool("i").client &.get(url, headers) do |resp|
+ ConnectionPool.get_ytimg_pool("i").get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr
index f2213da4..cb24648f 100644
--- a/src/invidious/routes/playlists.cr
+++ b/src/invidious/routes/playlists.cr
@@ -464,7 +464,7 @@ module Invidious::Routes::Playlists
# Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
def self.watch_videos(env)
- response = YT_POOL.client &.get(env.request.resource)
+ response = YT_POOL.get(env.request.resource)
if url = response.headers["Location"]?
url = URI.parse(url).request_target
return env.redirect url
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index 25edb936..4c635ab2 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -16,11 +16,11 @@ module Invidious::Search
# Search a youtube channel
# TODO: clean code, and rely more on YoutubeAPI
def channel(query : Query) : Array(SearchItem)
- response = YT_POOL.client &.get("/channel/#{query.channel}")
+ response = YT_POOL.get("/channel/#{query.channel}")
if response.status_code == 404
- response = YT_POOL.client &.get("/user/#{query.channel}")
- response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
+ response = YT_POOL.get("/user/#{query.channel}")
+ response = YT_POOL.get("/c/#{query.channel}") if response.status_code == 404
initial_data = extract_initial_data(response.body)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(query.channel) if !ucid
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
deleted file mode 100644
index 0daed46c..00000000
--- a/src/invidious/yt_backend/connection_pool.cr
+++ /dev/null
@@ -1,153 +0,0 @@
-# Mapping of subdomain => YoutubeConnectionPool
-# This is needed as we may need to access arbitrary subdomains of ytimg
-private YTIMG_POOLS = {} of String => YoutubeConnectionPool
-
-struct YoutubeConnectionPool
- property! url : URI
- property! capacity : Int32
- property! timeout : Float64
- property pool : DB::Pool(HTTP::Client)
-
- def initialize(url : URI, @capacity = 5, @timeout = 5.0)
- @url = url
- @pool = build_pool()
- end
-
- def client(&)
- conn = pool.checkout
- # Proxy needs to be reinstated every time we get a client from the pool
- conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
-
- begin
- response = yield conn
- rescue ex
- conn.close
- conn = make_client(url, force_resolve: true)
-
- response = yield conn
- ensure
- pool.release(conn)
- end
-
- response
- end
-
- private def build_pool
- options = DB::Pool::Options.new(
- initial_pool_size: 0,
- max_pool_size: capacity,
- max_idle_pool_size: capacity,
- checkout_timeout: timeout
- )
-
- DB::Pool(HTTP::Client).new(options) do
- next make_client(url, force_resolve: true)
- end
- end
-end
-
-struct CompanionConnectionPool
- property pool : DB::Pool(HTTP::Client)
-
- def initialize(capacity = 5, timeout = 5.0)
- options = DB::Pool::Options.new(
- initial_pool_size: 0,
- max_pool_size: capacity,
- max_idle_pool_size: capacity,
- checkout_timeout: timeout
- )
-
- @pool = DB::Pool(HTTP::Client).new(options) do
- companion = CONFIG.invidious_companion.sample
- next make_client(companion.private_url, use_http_proxy: false)
- end
- end
-
- def client(&)
- conn = pool.checkout
-
- begin
- response = yield conn
- rescue ex
- conn.close
-
- companion = CONFIG.invidious_companion.sample
- conn = make_client(companion.private_url, use_http_proxy: false)
-
- response = yield conn
- ensure
- pool.release(conn)
- end
-
- response
- end
-end
-
-def add_yt_headers(request)
- request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
- request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
-
- request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
- request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
- request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
-
- # Preserve original cookies and add new YT consent cookie for EU servers
- request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
- if !CONFIG.cookies.empty?
- request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
- end
-end
-
-def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true)
- client = HTTP::Client.new(url)
- client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy
-
- # Force the usage of a specific configured IP Family
- if force_resolve
- client.family = CONFIG.force_resolve
- client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC
- end
-
- client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers
- client.read_timeout = 10.seconds
- client.connect_timeout = 10.seconds
-
- return client
-end
-
-def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_proxy : Bool = true, &)
- client = make_client(url, region, force_resolve: force_resolve, use_http_proxy: use_http_proxy)
- begin
- yield client
- ensure
- client.close
- end
-end
-
-def make_configured_http_proxy_client
- # This method is only called when configuration for an HTTP proxy are set
- config_proxy = CONFIG.http_proxy.not_nil!
-
- return HTTP::Proxy::Client.new(
- config_proxy.host,
- config_proxy.port,
-
- username: config_proxy.user,
- password: config_proxy.password,
- )
-end
-
-# Fetches a HTTP pool for the specified subdomain of ytimg.com
-#
-# Creates a new one when the specified pool for the subdomain does not exist
-def get_ytimg_pool(subdomain)
- if pool = YTIMG_POOLS[subdomain]?
- return pool
- else
- LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"")
- pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size)
- YTIMG_POOLS[subdomain] = pool
-
- return pool
- end
-end
diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr
index b40092a1..9a0c1cf8 100644
--- a/src/invidious/yt_backend/youtube_api.cr
+++ b/src/invidious/yt_backend/youtube_api.cr
@@ -639,15 +639,13 @@ module YoutubeAPI
LOGGER.trace("YoutubeAPI: POST data: #{data}")
# Send the POST request
- body = YT_POOL.client() do |client|
- client.post(url, headers: headers, body: data.to_json) do |response|
- if response.status_code != 200
- raise InfoException.new("Error: non 200 status code. Youtube API returned \
- status code #{response.status_code}. See \
- https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.")
- end
- self._decompress(response.body_io, response.headers["Content-Encoding"]?)
+ body = YT_POOL.post(url, headers: headers, body: data.to_json) do |response|
+ if response.status_code != 200
+ raise InfoException.new("Error: non 200 status code. Youtube API returned \
+ status code #{response.status_code}. See \
+ https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.")
end
+ self._decompress(response.body_io, response.headers["Content-Encoding"]?)
end
# Convert result to Hash
@@ -695,7 +693,7 @@ module YoutubeAPI
# Send the POST request
begin
- response = COMPANION_POOL.client &.post(endpoint, headers: headers, body: data.to_json)
+ response = COMPANION_POOL.post(endpoint, headers: headers, body: data.to_json)
body = response.body
if (response.status_code != 200)
raise Exception.new(