From 3b62efb34f20122d00e06e9110767a6c4021d440 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 1 Jun 2025 14:53:04 -0700 Subject: [PATCH] Expose database connection pool settings Unless an instance maintainer was using the `database_url` attribute, and knew about the connection pool query parameters, the database connection pool settings was relegated to the default settings... And the default settings are... not great for any large instance. With it essentially only allowing a single connection within the pool, a maximum checkout time of a 5 seconds, and basically no additional retries whatsoever its no wonder that PgBouncer has became a staple among Invidious instances. This PR changes that by exposing the ability to configure the database connection pool that is used within the library that Invidious uses to interact with Postgres. --- config/config.example.yml | 88 +++++++++++++++++++++++++++++++++++++++ src/invidious/config.cr | 67 +++++++++++++++++++++++++---- 2 files changed, 147 insertions(+), 8 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 8d3e6212..377b90da 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -15,6 +15,90 @@ db: port: 5432 dbname: invidious + + ## ----------------------------------- + ## Database connection pool settings + ## ----------------------------------- + + ## + ## The maximum allowed number of connections within the connection pool. + ## When this number is reached and no connections are free, Invidious will + ## wait for up to `checkout_timeout` before raising an exception. + ## + ## Accepted values: a positive integer + ## Default: 100 + ## + #max_size: 100 + + ## + ## How many idle connections should be allowed to exist within the connection pool. + ## + ## There is no concept of whether a connection is closed or open, or how long it has been + ## idly sitting by. Instead Idle connections are defined as any connections that are not + ## currently being used. When this number is exceeded but the amount of connections is still + ## under max_pool_size, new connections will be created on a checkout but immediately closed + ## and destroyed during an release operation. + ## + ## For the most part, this should just be set to the same number as the `max_pool_size`. + ## + ## Accepted values: a positive integer + ## Default: 100 + ## + #max_idle_pool_size: 100 + + ## + ## The amount of connections to establish on start-up. + ## + ## When Invidious starts up, the database connection pool is immediately populated + ## with this many connections. This could allow an instance to avoid potentially experiencing + ## any degraded service during start-up; since a good amount of connections will be available + ## from the start rather than needing to be created on the fly as requests come in. + ## + ## + ## Accepted values: a positive integer + ## Default: 1 + ## + #initial_pool_size : 1 + + ## + ## How long to wait (in seconds) before timing out during a checkout + ## + ## Accepted values: a positive float + ## Default: 5.0 + ## + #checkout_timeout: 5.0 + + ## + ## How many attempts to retry a connection + ## + ## This allows Invidious to gracefully handle network problems that can + ## cause a connection to fail or be unable to get established in the first + ## place. + ## + ## Note: The mechanisms of the underlying library will first try to reuse all the + ## currently available idle connections within the pool, as well as one additional attempt, + ## at no delay. Afterwards, it will began counting down the `retry_attempts` and wait + ## `retry_delay` seconds between each attempt. + ## + ## This heuristic ensures that unnecessary new connections are not being made for no reason, and + ## in the event of widespread network failures, will help to throw out dead connections from the + ## pool. + ## + ## Accepted values: a positive integer + ## Default: 5.0 + ## + #retry_attempts: 5.0 + + ## + ## How long to wait (in-seconds) between each retry attempt + ## + ## Note: See description for `retry_attempts` for more information + ## + ## Accepted values: a positive float + ## Default: 1.0 + ## + #retry_delay: 1.0 + ## ## Database configuration using a single URI. This is an ## alternative to the 'db' parameter above. If both forms @@ -26,6 +110,10 @@ db: ## and append the 'host' parameter. E.g: ## postgres://kemal:kemal@/invidious?host=/var/run/postgresql ## +## You can also configure the connection pool here by +## adding query parameters with the same name as the value you +## want to change. +## ## Accepted values: a postgres:// URI ## Default: postgres://kemal:kemal@localhost:5432/invidious ## diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 4d69854c..3bbadd70 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -6,6 +6,52 @@ struct DBConfig property host : String property port : Int32 property dbname : String + + # How many connections to construct on start-up, and keep it there. + property initial_pool_size : Int32 = 1 + # The maximum size of the connection pool + property max_pool_size : Int32 = 100 + # The maximum amount of idle connections within the pool + # idle connections are defined as created connections that are + # sitting around in the pool. Exceeding this number will cause new connections + # to be created on checkout and then simply dropped on release, till the maximum pool size + # from which there will be a checkout timeout. + property max_idle_pool_size : Int32 = 100 + # The maximum amount of seconds to wait for a connection to become + # available when all connections can be checked out, and the pool has + # reached its maximum size. + property checkout_timeout : Float32 = 5.0 + # The number of tries allowed to establish a connection, reconnect, or retry + # the command in case of any network errors. + property retry_attempts : Int32 = 5 + # The number of seconds between each retry + property retry_delay : Float32 = 1.0 + + def to_url + URI.new( + scheme: "postgres", + user: user, + password: password, + host: host, + port: port, + path: dbname, + query: get_connection_pool_query_string + ) + end + + # Creates the query parameters for configuring the connection pool + private def get_connection_pool_query_string + {% begin %} + {% pool_vars = @type.instance_vars.reject { |v| {"user", "password", "host", "port", "dbname"}.includes?(v.name.stringify) } %} + {% raise "Error unable to isolate database connection pool properties" if pool_vars.size > 6 %} + + URI::Params.build do | build | + {% for vars in pool_vars %} + build.add {{vars.name.stringify}}, {{vars.name}}.to_s + {% end %} + end + {% end %} + end end struct SocketBindingConfig @@ -287,18 +333,23 @@ class Config # Build database_url from db.* if it's not set directly if config.database_url.to_s.empty? if db = config.db - config.database_url = URI.new( - scheme: "postgres", - user: db.user, - password: db.password, - host: db.host, - port: db.port, - path: db.dbname, - ) + config.database_url = db.to_url else puts "Config: Either database_url or db.* is required" exit(1) end + else + # Add default connection pool settings as needed + db_url_query_params = config.database_url.query_params + + {% begin %} + {% pool_vars = DBConfig.instance_vars.reject { |v| {"user", "password", "host", "port", "dbname"}.includes?(v.name.stringify) } %} + {% for vars in pool_vars %} + db_url_query_params[{{vars.name.stringify}}] = db_url_query_params[{{vars.name.stringify}}]? || {{vars.default_value}}.to_s + {% end %} + {% end %} + + config.database_url.query_params = db_url_query_params end # Check if the socket configuration is valid