2
0
Fork 0
mirror of https://git.asonix.dog/asonix/pict-rs synced 2024-12-24 18:33:39 +00:00

Add timeout configuration for external validation

This commit is contained in:
asonix 2023-09-18 19:50:17 -05:00
parent d120e768ff
commit 0aab38b40a
8 changed files with 28 additions and 0 deletions

View file

@ -18,6 +18,7 @@ targets = "info"
path = "/mnt"
[media]
external_validation_timeout = 30
max_width = 10000
max_height = 10000
max_area = 40000000

View file

@ -131,6 +131,11 @@ path = '/mnt'
# passes validation. Any other status code is considered a validation failure.
external_validation = 'http://localhost:8076'
## Optional: Timeout (in seconds) for external validation endpoint
# environment variable: PICTRS__MEDIA__EXTERNAL_VALIDATION_TIMEOUT
# default: 30
external_validation_timeout = 30
## Optional: preprocessing steps for uploaded images
# environment variable: PICTRS__MEDIA__PREPROCESS_STEPS
# default: empty

View file

@ -47,6 +47,7 @@ impl Args {
worker_id,
client_pool_size,
media_external_validation,
media_external_validation_timeout,
media_preprocess_steps,
media_skip_validate_imports,
media_max_width,
@ -86,6 +87,7 @@ impl Args {
};
let media = Media {
external_validation: media_external_validation,
external_validation_timeout: media_external_validation_timeout,
preprocess_steps: media_preprocess_steps,
skip_validate_imports: media_skip_validate_imports,
max_width: media_max_width,
@ -341,6 +343,8 @@ struct Media {
#[serde(skip_serializing_if = "Option::is_none")]
external_validation: Option<Url>,
#[serde(skip_serializing_if = "Option::is_none")]
external_validation_timeout: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
preprocess_steps: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
max_width: Option<usize>,
@ -462,6 +466,10 @@ struct Run {
#[arg(long)]
media_external_validation: Option<Url>,
/// Optional timeout for external validation requests
#[arg(long)]
media_external_validation_timeout: Option<u64>,
/// Optional pre-processing steps for uploaded media.
///
/// All still images will be put through these steps before saving

View file

@ -62,6 +62,7 @@ struct OldDbDefaults {
#[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct MediaDefaults {
external_validation_timeout: u64,
max_width: usize,
max_height: usize,
max_area: usize,
@ -176,6 +177,7 @@ impl Default for OldDbDefaults {
impl Default for MediaDefaults {
fn default() -> Self {
MediaDefaults {
external_validation_timeout: 30,
max_width: 10_000,
max_height: 10_000,
max_area: 40_000_000,

View file

@ -146,6 +146,8 @@ pub(crate) struct Media {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) external_validation: Option<Url>,
pub(crate) external_validation_timeout: u64,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) preprocess_steps: Option<String>,

View file

@ -1,3 +1,5 @@
use std::time::Duration;
use crate::{
bytes_stream::BytesStream,
either::Either,
@ -53,6 +55,7 @@ pub(crate) async fn ingest<R, S>(
stream: impl Stream<Item = Result<Bytes, Error>> + Unpin + 'static,
declared_alias: Option<Alias>,
external_validation: Option<&Url>,
external_validation_timeout: u64,
should_validate: bool,
timeout: u64,
) -> Result<Session<R, S>, Error>
@ -131,6 +134,7 @@ where
let result = client
.post(external_validation.as_str())
.header("Content-Type", input_type.content_type().to_string())
.timeout(Duration::from_secs(external_validation_timeout))
.body(Body::wrap_stream(RxStream(rx)))
.send()
.await?;

View file

@ -183,6 +183,7 @@ impl<R: FullRepo, S: Store + 'static> FormData for Upload<R, S> {
stream,
None,
CONFIG.media.external_validation.as_ref(),
CONFIG.media.external_validation_timeout,
true,
CONFIG.media.process_timeout,
)
@ -246,6 +247,7 @@ impl<R: FullRepo, S: Store + 'static> FormData for Import<R, S> {
stream,
Some(Alias::from_existing(&filename)),
CONFIG.media.external_validation.as_ref(),
CONFIG.media.external_validation_timeout,
!CONFIG.media.skip_validate_imports,
CONFIG.media.process_timeout,
)
@ -511,6 +513,7 @@ async fn do_download_inline<R: FullRepo + 'static, S: Store + 'static>(
stream,
None,
CONFIG.media.external_validation.as_ref(),
CONFIG.media.external_validation_timeout,
true,
CONFIG.media.process_timeout,
)

View file

@ -39,6 +39,7 @@ where
Serde::into_inner(upload_id),
declared_alias.map(Serde::into_inner),
crate::CONFIG.media.external_validation.as_ref(),
crate::CONFIG.media.external_validation_timeout,
should_validate,
crate::CONFIG.media.process_timeout,
)
@ -110,6 +111,7 @@ async fn process_ingest<R, S>(
upload_id: UploadId,
declared_alias: Option<Alias>,
external_validation: Option<&Url>,
external_validation_timeout: u64,
should_validate: bool,
timeout: u64,
) -> Result<(), Error>
@ -141,6 +143,7 @@ where
stream,
declared_alias,
external_validation.as_ref(),
external_validation_timeout,
should_validate,
timeout,
)