From 0aab38b40aed3aab94c5ec9119e283bfb1e188b5 Mon Sep 17 00:00:00 2001 From: asonix Date: Mon, 18 Sep 2023 19:50:17 -0500 Subject: [PATCH] Add timeout configuration for external validation --- defaults.toml | 1 + pict-rs.toml | 5 +++++ src/config/commandline.rs | 8 ++++++++ src/config/defaults.rs | 2 ++ src/config/file.rs | 2 ++ src/ingest.rs | 4 ++++ src/lib.rs | 3 +++ src/queue/process.rs | 3 +++ 8 files changed, 28 insertions(+) diff --git a/defaults.toml b/defaults.toml index 17faa2f..39f0e17 100644 --- a/defaults.toml +++ b/defaults.toml @@ -18,6 +18,7 @@ targets = "info" path = "/mnt" [media] +external_validation_timeout = 30 max_width = 10000 max_height = 10000 max_area = 40000000 diff --git a/pict-rs.toml b/pict-rs.toml index 923d007..5015f0a 100644 --- a/pict-rs.toml +++ b/pict-rs.toml @@ -131,6 +131,11 @@ path = '/mnt' # passes validation. Any other status code is considered a validation failure. external_validation = 'http://localhost:8076' +## Optional: Timeout (in seconds) for external validation endpoint +# environment variable: PICTRS__MEDIA__EXTERNAL_VALIDATION_TIMEOUT +# default: 30 +external_validation_timeout = 30 + ## Optional: preprocessing steps for uploaded images # environment variable: PICTRS__MEDIA__PREPROCESS_STEPS # default: empty diff --git a/src/config/commandline.rs b/src/config/commandline.rs index e0846c5..bc48b4d 100644 --- a/src/config/commandline.rs +++ b/src/config/commandline.rs @@ -47,6 +47,7 @@ impl Args { worker_id, client_pool_size, media_external_validation, + media_external_validation_timeout, media_preprocess_steps, media_skip_validate_imports, media_max_width, @@ -86,6 +87,7 @@ impl Args { }; let media = Media { external_validation: media_external_validation, + external_validation_timeout: media_external_validation_timeout, preprocess_steps: media_preprocess_steps, skip_validate_imports: media_skip_validate_imports, max_width: media_max_width, @@ -341,6 +343,8 @@ struct Media { #[serde(skip_serializing_if = "Option::is_none")] external_validation: Option, #[serde(skip_serializing_if = "Option::is_none")] + external_validation_timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] preprocess_steps: Option, #[serde(skip_serializing_if = "Option::is_none")] max_width: Option, @@ -462,6 +466,10 @@ struct Run { #[arg(long)] media_external_validation: Option, + /// Optional timeout for external validation requests + #[arg(long)] + media_external_validation_timeout: Option, + /// Optional pre-processing steps for uploaded media. /// /// All still images will be put through these steps before saving diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 0f718b9..c9cd659 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -62,6 +62,7 @@ struct OldDbDefaults { #[derive(Clone, Debug, serde::Serialize)] #[serde(rename_all = "snake_case")] struct MediaDefaults { + external_validation_timeout: u64, max_width: usize, max_height: usize, max_area: usize, @@ -176,6 +177,7 @@ impl Default for OldDbDefaults { impl Default for MediaDefaults { fn default() -> Self { MediaDefaults { + external_validation_timeout: 30, max_width: 10_000, max_height: 10_000, max_area: 40_000_000, diff --git a/src/config/file.rs b/src/config/file.rs index a2cb3d9..b1df7a4 100644 --- a/src/config/file.rs +++ b/src/config/file.rs @@ -146,6 +146,8 @@ pub(crate) struct Media { #[serde(skip_serializing_if = "Option::is_none")] pub(crate) external_validation: Option, + pub(crate) external_validation_timeout: u64, + #[serde(skip_serializing_if = "Option::is_none")] pub(crate) preprocess_steps: Option, diff --git a/src/ingest.rs b/src/ingest.rs index d773981..9527d30 100644 --- a/src/ingest.rs +++ b/src/ingest.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use crate::{ bytes_stream::BytesStream, either::Either, @@ -53,6 +55,7 @@ pub(crate) async fn ingest( stream: impl Stream> + Unpin + 'static, declared_alias: Option, external_validation: Option<&Url>, + external_validation_timeout: u64, should_validate: bool, timeout: u64, ) -> Result, Error> @@ -131,6 +134,7 @@ where let result = client .post(external_validation.as_str()) .header("Content-Type", input_type.content_type().to_string()) + .timeout(Duration::from_secs(external_validation_timeout)) .body(Body::wrap_stream(RxStream(rx))) .send() .await?; diff --git a/src/lib.rs b/src/lib.rs index 317fc0b..2d6d57a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -183,6 +183,7 @@ impl FormData for Upload { stream, None, CONFIG.media.external_validation.as_ref(), + CONFIG.media.external_validation_timeout, true, CONFIG.media.process_timeout, ) @@ -246,6 +247,7 @@ impl FormData for Import { stream, Some(Alias::from_existing(&filename)), CONFIG.media.external_validation.as_ref(), + CONFIG.media.external_validation_timeout, !CONFIG.media.skip_validate_imports, CONFIG.media.process_timeout, ) @@ -511,6 +513,7 @@ async fn do_download_inline( stream, None, CONFIG.media.external_validation.as_ref(), + CONFIG.media.external_validation_timeout, true, CONFIG.media.process_timeout, ) diff --git a/src/queue/process.rs b/src/queue/process.rs index 80c59e9..66ec37b 100644 --- a/src/queue/process.rs +++ b/src/queue/process.rs @@ -39,6 +39,7 @@ where Serde::into_inner(upload_id), declared_alias.map(Serde::into_inner), crate::CONFIG.media.external_validation.as_ref(), + crate::CONFIG.media.external_validation_timeout, should_validate, crate::CONFIG.media.process_timeout, ) @@ -110,6 +111,7 @@ async fn process_ingest( upload_id: UploadId, declared_alias: Option, external_validation: Option<&Url>, + external_validation_timeout: u64, should_validate: bool, timeout: u64, ) -> Result<(), Error> @@ -141,6 +143,7 @@ where stream, declared_alias, external_validation.as_ref(), + external_validation_timeout, should_validate, timeout, )