mirror of
https://git.asonix.dog/asonix/pict-rs
synced 2024-12-25 03:51:24 +00:00
Add timeout configuration for external validation
This commit is contained in:
parent
d120e768ff
commit
0aab38b40a
8 changed files with 28 additions and 0 deletions
|
@ -18,6 +18,7 @@ targets = "info"
|
||||||
path = "/mnt"
|
path = "/mnt"
|
||||||
|
|
||||||
[media]
|
[media]
|
||||||
|
external_validation_timeout = 30
|
||||||
max_width = 10000
|
max_width = 10000
|
||||||
max_height = 10000
|
max_height = 10000
|
||||||
max_area = 40000000
|
max_area = 40000000
|
||||||
|
|
|
@ -131,6 +131,11 @@ path = '/mnt'
|
||||||
# passes validation. Any other status code is considered a validation failure.
|
# passes validation. Any other status code is considered a validation failure.
|
||||||
external_validation = 'http://localhost:8076'
|
external_validation = 'http://localhost:8076'
|
||||||
|
|
||||||
|
## Optional: Timeout (in seconds) for external validation endpoint
|
||||||
|
# environment variable: PICTRS__MEDIA__EXTERNAL_VALIDATION_TIMEOUT
|
||||||
|
# default: 30
|
||||||
|
external_validation_timeout = 30
|
||||||
|
|
||||||
## Optional: preprocessing steps for uploaded images
|
## Optional: preprocessing steps for uploaded images
|
||||||
# environment variable: PICTRS__MEDIA__PREPROCESS_STEPS
|
# environment variable: PICTRS__MEDIA__PREPROCESS_STEPS
|
||||||
# default: empty
|
# default: empty
|
||||||
|
|
|
@ -47,6 +47,7 @@ impl Args {
|
||||||
worker_id,
|
worker_id,
|
||||||
client_pool_size,
|
client_pool_size,
|
||||||
media_external_validation,
|
media_external_validation,
|
||||||
|
media_external_validation_timeout,
|
||||||
media_preprocess_steps,
|
media_preprocess_steps,
|
||||||
media_skip_validate_imports,
|
media_skip_validate_imports,
|
||||||
media_max_width,
|
media_max_width,
|
||||||
|
@ -86,6 +87,7 @@ impl Args {
|
||||||
};
|
};
|
||||||
let media = Media {
|
let media = Media {
|
||||||
external_validation: media_external_validation,
|
external_validation: media_external_validation,
|
||||||
|
external_validation_timeout: media_external_validation_timeout,
|
||||||
preprocess_steps: media_preprocess_steps,
|
preprocess_steps: media_preprocess_steps,
|
||||||
skip_validate_imports: media_skip_validate_imports,
|
skip_validate_imports: media_skip_validate_imports,
|
||||||
max_width: media_max_width,
|
max_width: media_max_width,
|
||||||
|
@ -341,6 +343,8 @@ struct Media {
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
external_validation: Option<Url>,
|
external_validation: Option<Url>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
external_validation_timeout: Option<u64>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
preprocess_steps: Option<String>,
|
preprocess_steps: Option<String>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
max_width: Option<usize>,
|
max_width: Option<usize>,
|
||||||
|
@ -462,6 +466,10 @@ struct Run {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
media_external_validation: Option<Url>,
|
media_external_validation: Option<Url>,
|
||||||
|
|
||||||
|
/// Optional timeout for external validation requests
|
||||||
|
#[arg(long)]
|
||||||
|
media_external_validation_timeout: Option<u64>,
|
||||||
|
|
||||||
/// Optional pre-processing steps for uploaded media.
|
/// Optional pre-processing steps for uploaded media.
|
||||||
///
|
///
|
||||||
/// All still images will be put through these steps before saving
|
/// All still images will be put through these steps before saving
|
||||||
|
|
|
@ -62,6 +62,7 @@ struct OldDbDefaults {
|
||||||
#[derive(Clone, Debug, serde::Serialize)]
|
#[derive(Clone, Debug, serde::Serialize)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
struct MediaDefaults {
|
struct MediaDefaults {
|
||||||
|
external_validation_timeout: u64,
|
||||||
max_width: usize,
|
max_width: usize,
|
||||||
max_height: usize,
|
max_height: usize,
|
||||||
max_area: usize,
|
max_area: usize,
|
||||||
|
@ -176,6 +177,7 @@ impl Default for OldDbDefaults {
|
||||||
impl Default for MediaDefaults {
|
impl Default for MediaDefaults {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
MediaDefaults {
|
MediaDefaults {
|
||||||
|
external_validation_timeout: 30,
|
||||||
max_width: 10_000,
|
max_width: 10_000,
|
||||||
max_height: 10_000,
|
max_height: 10_000,
|
||||||
max_area: 40_000_000,
|
max_area: 40_000_000,
|
||||||
|
|
|
@ -146,6 +146,8 @@ pub(crate) struct Media {
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub(crate) external_validation: Option<Url>,
|
pub(crate) external_validation: Option<Url>,
|
||||||
|
|
||||||
|
pub(crate) external_validation_timeout: u64,
|
||||||
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub(crate) preprocess_steps: Option<String>,
|
pub(crate) preprocess_steps: Option<String>,
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
bytes_stream::BytesStream,
|
bytes_stream::BytesStream,
|
||||||
either::Either,
|
either::Either,
|
||||||
|
@ -53,6 +55,7 @@ pub(crate) async fn ingest<R, S>(
|
||||||
stream: impl Stream<Item = Result<Bytes, Error>> + Unpin + 'static,
|
stream: impl Stream<Item = Result<Bytes, Error>> + Unpin + 'static,
|
||||||
declared_alias: Option<Alias>,
|
declared_alias: Option<Alias>,
|
||||||
external_validation: Option<&Url>,
|
external_validation: Option<&Url>,
|
||||||
|
external_validation_timeout: u64,
|
||||||
should_validate: bool,
|
should_validate: bool,
|
||||||
timeout: u64,
|
timeout: u64,
|
||||||
) -> Result<Session<R, S>, Error>
|
) -> Result<Session<R, S>, Error>
|
||||||
|
@ -131,6 +134,7 @@ where
|
||||||
let result = client
|
let result = client
|
||||||
.post(external_validation.as_str())
|
.post(external_validation.as_str())
|
||||||
.header("Content-Type", input_type.content_type().to_string())
|
.header("Content-Type", input_type.content_type().to_string())
|
||||||
|
.timeout(Duration::from_secs(external_validation_timeout))
|
||||||
.body(Body::wrap_stream(RxStream(rx)))
|
.body(Body::wrap_stream(RxStream(rx)))
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
|
@ -183,6 +183,7 @@ impl<R: FullRepo, S: Store + 'static> FormData for Upload<R, S> {
|
||||||
stream,
|
stream,
|
||||||
None,
|
None,
|
||||||
CONFIG.media.external_validation.as_ref(),
|
CONFIG.media.external_validation.as_ref(),
|
||||||
|
CONFIG.media.external_validation_timeout,
|
||||||
true,
|
true,
|
||||||
CONFIG.media.process_timeout,
|
CONFIG.media.process_timeout,
|
||||||
)
|
)
|
||||||
|
@ -246,6 +247,7 @@ impl<R: FullRepo, S: Store + 'static> FormData for Import<R, S> {
|
||||||
stream,
|
stream,
|
||||||
Some(Alias::from_existing(&filename)),
|
Some(Alias::from_existing(&filename)),
|
||||||
CONFIG.media.external_validation.as_ref(),
|
CONFIG.media.external_validation.as_ref(),
|
||||||
|
CONFIG.media.external_validation_timeout,
|
||||||
!CONFIG.media.skip_validate_imports,
|
!CONFIG.media.skip_validate_imports,
|
||||||
CONFIG.media.process_timeout,
|
CONFIG.media.process_timeout,
|
||||||
)
|
)
|
||||||
|
@ -511,6 +513,7 @@ async fn do_download_inline<R: FullRepo + 'static, S: Store + 'static>(
|
||||||
stream,
|
stream,
|
||||||
None,
|
None,
|
||||||
CONFIG.media.external_validation.as_ref(),
|
CONFIG.media.external_validation.as_ref(),
|
||||||
|
CONFIG.media.external_validation_timeout,
|
||||||
true,
|
true,
|
||||||
CONFIG.media.process_timeout,
|
CONFIG.media.process_timeout,
|
||||||
)
|
)
|
||||||
|
|
|
@ -39,6 +39,7 @@ where
|
||||||
Serde::into_inner(upload_id),
|
Serde::into_inner(upload_id),
|
||||||
declared_alias.map(Serde::into_inner),
|
declared_alias.map(Serde::into_inner),
|
||||||
crate::CONFIG.media.external_validation.as_ref(),
|
crate::CONFIG.media.external_validation.as_ref(),
|
||||||
|
crate::CONFIG.media.external_validation_timeout,
|
||||||
should_validate,
|
should_validate,
|
||||||
crate::CONFIG.media.process_timeout,
|
crate::CONFIG.media.process_timeout,
|
||||||
)
|
)
|
||||||
|
@ -110,6 +111,7 @@ async fn process_ingest<R, S>(
|
||||||
upload_id: UploadId,
|
upload_id: UploadId,
|
||||||
declared_alias: Option<Alias>,
|
declared_alias: Option<Alias>,
|
||||||
external_validation: Option<&Url>,
|
external_validation: Option<&Url>,
|
||||||
|
external_validation_timeout: u64,
|
||||||
should_validate: bool,
|
should_validate: bool,
|
||||||
timeout: u64,
|
timeout: u64,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
|
@ -141,6 +143,7 @@ where
|
||||||
stream,
|
stream,
|
||||||
declared_alias,
|
declared_alias,
|
||||||
external_validation.as_ref(),
|
external_validation.as_ref(),
|
||||||
|
external_validation_timeout,
|
||||||
should_validate,
|
should_validate,
|
||||||
timeout,
|
timeout,
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in a new issue