diff --git a/config/config.example.yml b/config/config.example.yml index 8d3e6212..377b90da 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -15,6 +15,90 @@ db: port: 5432 dbname: invidious + + ## ----------------------------------- + ## Database connection pool settings + ## ----------------------------------- + + ## + ## The maximum allowed number of connections within the connection pool. + ## When this number is reached and no connections are free, Invidious will + ## wait for up to `checkout_timeout` before raising an exception. + ## + ## Accepted values: a positive integer + ## Default: 100 + ## + #max_size: 100 + + ## + ## How many idle connections should be allowed to exist within the connection pool. + ## + ## There is no concept of whether a connection is closed or open, or how long it has been + ## idly sitting by. Instead Idle connections are defined as any connections that are not + ## currently being used. When this number is exceeded but the amount of connections is still + ## under max_pool_size, new connections will be created on a checkout but immediately closed + ## and destroyed during an release operation. + ## + ## For the most part, this should just be set to the same number as the `max_pool_size`. + ## + ## Accepted values: a positive integer + ## Default: 100 + ## + #max_idle_pool_size: 100 + + ## + ## The amount of connections to establish on start-up. + ## + ## When Invidious starts up, the database connection pool is immediately populated + ## with this many connections. This could allow an instance to avoid potentially experiencing + ## any degraded service during start-up; since a good amount of connections will be available + ## from the start rather than needing to be created on the fly as requests come in. + ## + ## + ## Accepted values: a positive integer + ## Default: 1 + ## + #initial_pool_size : 1 + + ## + ## How long to wait (in seconds) before timing out during a checkout + ## + ## Accepted values: a positive float + ## Default: 5.0 + ## + #checkout_timeout: 5.0 + + ## + ## How many attempts to retry a connection + ## + ## This allows Invidious to gracefully handle network problems that can + ## cause a connection to fail or be unable to get established in the first + ## place. + ## + ## Note: The mechanisms of the underlying library will first try to reuse all the + ## currently available idle connections within the pool, as well as one additional attempt, + ## at no delay. Afterwards, it will began counting down the `retry_attempts` and wait + ## `retry_delay` seconds between each attempt. + ## + ## This heuristic ensures that unnecessary new connections are not being made for no reason, and + ## in the event of widespread network failures, will help to throw out dead connections from the + ## pool. + ## + ## Accepted values: a positive integer + ## Default: 5.0 + ## + #retry_attempts: 5.0 + + ## + ## How long to wait (in-seconds) between each retry attempt + ## + ## Note: See description for `retry_attempts` for more information + ## + ## Accepted values: a positive float + ## Default: 1.0 + ## + #retry_delay: 1.0 + ## ## Database configuration using a single URI. This is an ## alternative to the 'db' parameter above. If both forms @@ -26,6 +110,10 @@ db: ## and append the 'host' parameter. E.g: ## postgres://kemal:kemal@/invidious?host=/var/run/postgresql ## +## You can also configure the connection pool here by +## adding query parameters with the same name as the value you +## want to change. +## ## Accepted values: a postgres:// URI ## Default: postgres://kemal:kemal@localhost:5432/invidious ## diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 4d69854c..3bbadd70 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -6,6 +6,52 @@ struct DBConfig property host : String property port : Int32 property dbname : String + + # How many connections to construct on start-up, and keep it there. + property initial_pool_size : Int32 = 1 + # The maximum size of the connection pool + property max_pool_size : Int32 = 100 + # The maximum amount of idle connections within the pool + # idle connections are defined as created connections that are + # sitting around in the pool. Exceeding this number will cause new connections + # to be created on checkout and then simply dropped on release, till the maximum pool size + # from which there will be a checkout timeout. + property max_idle_pool_size : Int32 = 100 + # The maximum amount of seconds to wait for a connection to become + # available when all connections can be checked out, and the pool has + # reached its maximum size. + property checkout_timeout : Float32 = 5.0 + # The number of tries allowed to establish a connection, reconnect, or retry + # the command in case of any network errors. + property retry_attempts : Int32 = 5 + # The number of seconds between each retry + property retry_delay : Float32 = 1.0 + + def to_url + URI.new( + scheme: "postgres", + user: user, + password: password, + host: host, + port: port, + path: dbname, + query: get_connection_pool_query_string + ) + end + + # Creates the query parameters for configuring the connection pool + private def get_connection_pool_query_string + {% begin %} + {% pool_vars = @type.instance_vars.reject { |v| {"user", "password", "host", "port", "dbname"}.includes?(v.name.stringify) } %} + {% raise "Error unable to isolate database connection pool properties" if pool_vars.size > 6 %} + + URI::Params.build do | build | + {% for vars in pool_vars %} + build.add {{vars.name.stringify}}, {{vars.name}}.to_s + {% end %} + end + {% end %} + end end struct SocketBindingConfig @@ -287,18 +333,23 @@ class Config # Build database_url from db.* if it's not set directly if config.database_url.to_s.empty? if db = config.db - config.database_url = URI.new( - scheme: "postgres", - user: db.user, - password: db.password, - host: db.host, - port: db.port, - path: db.dbname, - ) + config.database_url = db.to_url else puts "Config: Either database_url or db.* is required" exit(1) end + else + # Add default connection pool settings as needed + db_url_query_params = config.database_url.query_params + + {% begin %} + {% pool_vars = DBConfig.instance_vars.reject { |v| {"user", "password", "host", "port", "dbname"}.includes?(v.name.stringify) } %} + {% for vars in pool_vars %} + db_url_query_params[{{vars.name.stringify}}] = db_url_query_params[{{vars.name.stringify}}]? || {{vars.default_value}}.to_s + {% end %} + {% end %} + + config.database_url.query_params = db_url_query_params end # Check if the socket configuration is valid