From dd1d509bb1c47e4da5a9f870080e14c07b760502 Mon Sep 17 00:00:00 2001 From: asonix Date: Mon, 17 Jul 2023 13:44:31 -0500 Subject: [PATCH] Enable configuring download client timeout --- defaults.toml | 5 ++++- pict-rs.toml | 14 ++++++++++++-- src/config/commandline.rs | 31 +++++++++++++++++++++++++++++-- src/config/defaults.rs | 19 +++++++++++++++++-- src/config/file.rs | 10 ++++++++-- src/lib.rs | 8 ++------ 6 files changed, 72 insertions(+), 15 deletions(-) diff --git a/defaults.toml b/defaults.toml index 7179742..adbc95e 100644 --- a/defaults.toml +++ b/defaults.toml @@ -1,7 +1,10 @@ [server] address = "0.0.0.0:8080" worker_id = "pict-rs-1" -client_pool_size = 100 + +[client] +pool_size = 100 +timeout = 30 [tracing.logging] format = "normal" diff --git a/pict-rs.toml b/pict-rs.toml index e3257c2..e6940c7 100644 --- a/pict-rs.toml +++ b/pict-rs.toml @@ -20,8 +20,10 @@ worker_id = 'pict-rs-1' # Not specifying api_key disables internal endpoints api_key = 'API_KEY' +## Client configuration +[client] ## Optional: connection pool size for internal http client -# environment variable: PICTRS__SERVER__CLIENT_POOL_SIZE +# environment variable: PICTRS__CLIENT__POOL_SIZE # default: 100 # # This number is multiplied the number of cores available to pict-rs. Running on a 2 core machine @@ -31,7 +33,15 @@ api_key = 'API_KEY' # This number can be lowered to keep pict-rs within ulimit bounds if you encounter errors related to # "Too many open files". Alternatively, increasing the ulimit of your system can solve this problem # as well. -client_pool_size = 100 +pool_size = 100 + +## Optional: time (in seconds) the client will wait for a response before giving up +# environment variable: PICTRS__CLIENT__TIMEOUT +# default: 30 +# +# This is used for the `/image/download` endpoint when fetching media from another server. It is +# distinct from the object storage client timeout, which can be configured separately +timeout = 30 ## Logging configuration diff --git a/src/config/commandline.rs b/src/config/commandline.rs index 58a4442..ba6ba2f 100644 --- a/src/config/commandline.rs +++ b/src/config/commandline.rs @@ -47,6 +47,7 @@ impl Args { api_key, worker_id, client_pool_size, + client_timeout, media_preprocess_steps, media_max_file_size, media_image_max_width, @@ -76,7 +77,11 @@ impl Args { address, api_key, worker_id, - client_pool_size, + }; + + let client = Client { + pool_size: client_pool_size, + timeout: client_timeout, }; let image = Image { @@ -124,6 +129,7 @@ impl Args { Output { config_format: ConfigFormat { server, + client, old_db, tracing, media, @@ -140,6 +146,7 @@ impl Args { Output { config_format: ConfigFormat { server, + client, old_db, tracing, media, @@ -154,6 +161,7 @@ impl Args { None => Output { config_format: ConfigFormat { server, + client, old_db, tracing, media, @@ -171,6 +179,7 @@ impl Args { store, }) => { let server = Server::default(); + let client = Client::default(); let media = Media::default(); match store { @@ -178,6 +187,7 @@ impl Args { MigrateStoreTo::Filesystem(MigrateFilesystemInner { to, repo }) => Output { config_format: ConfigFormat { server, + client, old_db, tracing, media, @@ -196,6 +206,7 @@ impl Args { Output { config_format: ConfigFormat { server, + client, old_db, tracing, media, @@ -218,6 +229,7 @@ impl Args { Output { config_format: ConfigFormat { server, + client, old_db, tracing, media, @@ -239,6 +251,7 @@ impl Args { }) => Output { config_format: ConfigFormat { server, + client, old_db, tracing, media, @@ -283,6 +296,7 @@ pub(crate) enum Operation { #[serde(rename_all = "snake_case")] pub(super) struct ConfigFormat { server: Server, + client: Client, old_db: OldDb, tracing: Tracing, media: Media, @@ -301,8 +315,15 @@ struct Server { worker_id: Option, #[serde(skip_serializing_if = "Option::is_none")] api_key: Option, +} + +#[derive(Debug, Default, serde::Serialize)] +#[serde(rename_all = "snake_case")] +struct Client { #[serde(skip_serializing_if = "Option::is_none")] - client_pool_size: Option, + pool_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + timeout: Option, } #[derive(Debug, Default, serde::Serialize)] @@ -549,6 +570,12 @@ struct Run { #[arg(long)] client_pool_size: Option, + /// How long (in seconds) the internel HTTP client should wait for responses + /// + /// This number defaults to 30 + #[arg(long)] + client_timeout: Option, + /// Optional pre-processing steps for uploaded media. /// /// All still images will be put through these steps before saving diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 5eebda8..55ddabd 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -9,6 +9,7 @@ use std::{net::SocketAddr, path::PathBuf}; #[serde(rename_all = "snake_case")] pub(crate) struct Defaults { server: ServerDefaults, + client: ClientDefaults, tracing: TracingDefaults, old_db: OldDbDefaults, media: MediaDefaults, @@ -21,7 +22,13 @@ pub(crate) struct Defaults { struct ServerDefaults { address: SocketAddr, worker_id: String, - client_pool_size: usize, +} + +#[derive(Clone, Debug, serde::Serialize)] +#[serde(rename_all = "snake_case")] +struct ClientDefaults { + pool_size: usize, + timeout: u64, } #[derive(Clone, Debug, Default, serde::Serialize)] @@ -149,7 +156,15 @@ impl Default for ServerDefaults { ServerDefaults { address: "0.0.0.0:8080".parse().expect("Valid address string"), worker_id: String::from("pict-rs-1"), - client_pool_size: 100, + } + } +} + +impl Default for ClientDefaults { + fn default() -> Self { + ClientDefaults { + pool_size: 100, + timeout: 30, } } } diff --git a/src/config/file.rs b/src/config/file.rs index b09140b..acdec9c 100644 --- a/src/config/file.rs +++ b/src/config/file.rs @@ -12,6 +12,8 @@ use url::Url; pub(crate) struct ConfigFile { pub(crate) server: Server, + pub(crate) client: Client, + pub(crate) tracing: Tracing, pub(crate) old_db: OldDb, @@ -95,9 +97,13 @@ pub(crate) struct Server { #[serde(skip_serializing_if = "Option::is_none")] pub(crate) api_key: Option, +} - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) client_pool_size: Option, +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub(crate) struct Client { + pub(crate) pool_size: usize, + + pub(crate) timeout: u64, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] diff --git a/src/lib.rs b/src/lib.rs index 86ef82e..af4fa17 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1128,17 +1128,13 @@ fn transform_error(error: actix_form_data::Error) -> actix_web::Error { } fn build_client() -> awc::Client { - let connector = CONFIG - .server - .client_pool_size - .map(|size| Connector::new().limit(size)) - .unwrap_or_else(Connector::new); + let connector = Connector::new().limit(CONFIG.client.pool_size); Client::builder() .connector(connector) .wrap(Tracing) .add_default_header(("User-Agent", "pict-rs v0.4.1")) - .timeout(Duration::from_secs(30)) + .timeout(Duration::from_secs(CONFIG.client.timeout)) .finish() }