Enable configuring download client timeout

defaults.toml

@@ -1,7 +1,10 @@
 [server]
 address = "0.0.0.0:8080"
 worker_id = "pict-rs-1"
-client_pool_size = 100
+
+[client]
+pool_size = 100
+timeout = 30
 
 [tracing.logging]
 format = "normal"

pict-rs.toml

@@ -20,8 +20,10 @@ worker_id = 'pict-rs-1'
 # Not specifying api_key disables internal endpoints
 api_key = 'API_KEY'
 
+## Client configuration
+[client]
 ## Optional: connection pool size for internal http client
-# environment variable: PICTRS__SERVER__CLIENT_POOL_SIZE
+# environment variable: PICTRS__CLIENT__POOL_SIZE
 # default: 100
 #
 # This number is multiplied the number of cores available to pict-rs. Running on a 2 core machine
@@ -31,7 +33,15 @@ api_key = 'API_KEY'
 # This number can be lowered to keep pict-rs within ulimit bounds if you encounter errors related to
 # "Too many open files". Alternatively, increasing the ulimit of your system can solve this problem
 # as well.
-client_pool_size = 100
+pool_size = 100
+
+## Optional: time (in seconds) the client will wait for a response before giving up
+# environment variable: PICTRS__CLIENT__TIMEOUT
+# default: 30
+#
+# This is used for the `/image/download` endpoint when fetching media from another server. It is
+# distinct from the object storage client timeout, which can be configured separately
+timeout = 30
 
 
 ## Logging configuration

src/config/commandline.rs

@@ -47,6 +47,7 @@ impl Args {
                 api_key,
                 worker_id,
                 client_pool_size,
+                client_timeout,
                 media_preprocess_steps,
                 media_max_file_size,
                 media_image_max_width,
@@ -76,7 +77,11 @@ impl Args {
                     address,
                     api_key,
                     worker_id,
-                    client_pool_size,
+                };
+
+                let client = Client {
+                    pool_size: client_pool_size,
+                    timeout: client_timeout,
                 };
 
                 let image = Image {
@@ -124,6 +129,7 @@ impl Args {
                         Output {
                             config_format: ConfigFormat {
                                 server,
+                                client,
                                 old_db,
                                 tracing,
                                 media,
@@ -140,6 +146,7 @@ impl Args {
                         Output {
                             config_format: ConfigFormat {
                                 server,
+                                client,
                                 old_db,
                                 tracing,
                                 media,
@@ -154,6 +161,7 @@ impl Args {
                     None => Output {
                         config_format: ConfigFormat {
                             server,
+                            client,
                             old_db,
                             tracing,
                             media,
@@ -171,6 +179,7 @@ impl Args {
                 store,
             }) => {
                 let server = Server::default();
+                let client = Client::default();
                 let media = Media::default();
 
                 match store {
@@ -178,6 +187,7 @@ impl Args {
                         MigrateStoreTo::Filesystem(MigrateFilesystemInner { to, repo }) => Output {
                             config_format: ConfigFormat {
                                 server,
+                                client,
                                 old_db,
                                 tracing,
                                 media,
@@ -196,6 +206,7 @@ impl Args {
                             Output {
                                 config_format: ConfigFormat {
                                     server,
+                                    client,
                                     old_db,
                                     tracing,
                                     media,
@@ -218,6 +229,7 @@ impl Args {
                                 Output {
                                     config_format: ConfigFormat {
                                         server,
+                                        client,
                                         old_db,
                                         tracing,
                                         media,
@@ -239,6 +251,7 @@ impl Args {
                             }) => Output {
                                 config_format: ConfigFormat {
                                     server,
+                                    client,
                                     old_db,
                                     tracing,
                                     media,
@@ -283,6 +296,7 @@ pub(crate) enum Operation {
 #[serde(rename_all = "snake_case")]
 pub(super) struct ConfigFormat {
     server: Server,
+    client: Client,
     old_db: OldDb,
     tracing: Tracing,
     media: Media,
@@ -301,8 +315,15 @@ struct Server {
     worker_id: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     api_key: Option<String>,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct Client {
     #[serde(skip_serializing_if = "Option::is_none")]
-    client_pool_size: Option<usize>,
+    pool_size: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    timeout: Option<u64>,
 }
 
 #[derive(Debug, Default, serde::Serialize)]
@@ -549,6 +570,12 @@ struct Run {
     #[arg(long)]
     client_pool_size: Option<usize>,
 
+    /// How long (in seconds) the internel HTTP client should wait for responses
+    ///
+    /// This number defaults to 30
+    #[arg(long)]
+    client_timeout: Option<u64>,
+
     /// Optional pre-processing steps for uploaded media.
     ///
     /// All still images will be put through these steps before saving

src/config/defaults.rs

@@ -9,6 +9,7 @@ use std::{net::SocketAddr, path::PathBuf};
 #[serde(rename_all = "snake_case")]
 pub(crate) struct Defaults {
     server: ServerDefaults,
+    client: ClientDefaults,
     tracing: TracingDefaults,
     old_db: OldDbDefaults,
     media: MediaDefaults,
@@ -21,7 +22,13 @@ pub(crate) struct Defaults {
 struct ServerDefaults {
     address: SocketAddr,
     worker_id: String,
-    client_pool_size: usize,
+}
+
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct ClientDefaults {
+    pool_size: usize,
+    timeout: u64,
 }
 
 #[derive(Clone, Debug, Default, serde::Serialize)]
@@ -149,7 +156,15 @@ impl Default for ServerDefaults {
         ServerDefaults {
             address: "0.0.0.0:8080".parse().expect("Valid address string"),
             worker_id: String::from("pict-rs-1"),
-            client_pool_size: 100,
+        }
+    }
+}
+
+impl Default for ClientDefaults {
+    fn default() -> Self {
+        ClientDefaults {
+            pool_size: 100,
+            timeout: 30,
         }
     }
 }

src/config/file.rs

@@ -12,6 +12,8 @@ use url::Url;
 pub(crate) struct ConfigFile {
     pub(crate) server: Server,
 
+    pub(crate) client: Client,
+
     pub(crate) tracing: Tracing,
 
     pub(crate) old_db: OldDb,
@@ -95,9 +97,13 @@ pub(crate) struct Server {
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) api_key: Option<String>,
+}
 
-    #[serde(skip_serializing_if = "Option::is_none")]
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
-    pub(crate) client_pool_size: Option<usize>,
+pub(crate) struct Client {
+    pub(crate) pool_size: usize,
+
+    pub(crate) timeout: u64,
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]

src/lib.rs

@@ -1128,17 +1128,13 @@ fn transform_error(error: actix_form_data::Error) -> actix_web::Error {
 }
 
 fn build_client() -> awc::Client {
-    let connector = CONFIG
+    let connector = Connector::new().limit(CONFIG.client.pool_size);
-        .server
-        .client_pool_size
-        .map(|size| Connector::new().limit(size))
-        .unwrap_or_else(Connector::new);
 
     Client::builder()
         .connector(connector)
         .wrap(Tracing)
         .add_default_header(("User-Agent", "pict-rs v0.4.1"))
-        .timeout(Duration::from_secs(30))
+        .timeout(Duration::from_secs(CONFIG.client.timeout))
         .finish()
 }