2
0
Fork 0
mirror of https://git.asonix.dog/asonix/pict-rs synced 2024-12-22 19:31:35 +00:00

Enable configuring download client timeout

This commit is contained in:
asonix 2023-07-17 13:44:31 -05:00
parent 5e8ab7856d
commit dd1d509bb1
6 changed files with 72 additions and 15 deletions

View file

@ -1,7 +1,10 @@
[server] [server]
address = "0.0.0.0:8080" address = "0.0.0.0:8080"
worker_id = "pict-rs-1" worker_id = "pict-rs-1"
client_pool_size = 100
[client]
pool_size = 100
timeout = 30
[tracing.logging] [tracing.logging]
format = "normal" format = "normal"

View file

@ -20,8 +20,10 @@ worker_id = 'pict-rs-1'
# Not specifying api_key disables internal endpoints # Not specifying api_key disables internal endpoints
api_key = 'API_KEY' api_key = 'API_KEY'
## Client configuration
[client]
## Optional: connection pool size for internal http client ## Optional: connection pool size for internal http client
# environment variable: PICTRS__SERVER__CLIENT_POOL_SIZE # environment variable: PICTRS__CLIENT__POOL_SIZE
# default: 100 # default: 100
# #
# This number is multiplied the number of cores available to pict-rs. Running on a 2 core machine # This number is multiplied the number of cores available to pict-rs. Running on a 2 core machine
@ -31,7 +33,15 @@ api_key = 'API_KEY'
# This number can be lowered to keep pict-rs within ulimit bounds if you encounter errors related to # This number can be lowered to keep pict-rs within ulimit bounds if you encounter errors related to
# "Too many open files". Alternatively, increasing the ulimit of your system can solve this problem # "Too many open files". Alternatively, increasing the ulimit of your system can solve this problem
# as well. # as well.
client_pool_size = 100 pool_size = 100
## Optional: time (in seconds) the client will wait for a response before giving up
# environment variable: PICTRS__CLIENT__TIMEOUT
# default: 30
#
# This is used for the `/image/download` endpoint when fetching media from another server. It is
# distinct from the object storage client timeout, which can be configured separately
timeout = 30
## Logging configuration ## Logging configuration

View file

@ -47,6 +47,7 @@ impl Args {
api_key, api_key,
worker_id, worker_id,
client_pool_size, client_pool_size,
client_timeout,
media_preprocess_steps, media_preprocess_steps,
media_max_file_size, media_max_file_size,
media_image_max_width, media_image_max_width,
@ -76,7 +77,11 @@ impl Args {
address, address,
api_key, api_key,
worker_id, worker_id,
client_pool_size, };
let client = Client {
pool_size: client_pool_size,
timeout: client_timeout,
}; };
let image = Image { let image = Image {
@ -124,6 +129,7 @@ impl Args {
Output { Output {
config_format: ConfigFormat { config_format: ConfigFormat {
server, server,
client,
old_db, old_db,
tracing, tracing,
media, media,
@ -140,6 +146,7 @@ impl Args {
Output { Output {
config_format: ConfigFormat { config_format: ConfigFormat {
server, server,
client,
old_db, old_db,
tracing, tracing,
media, media,
@ -154,6 +161,7 @@ impl Args {
None => Output { None => Output {
config_format: ConfigFormat { config_format: ConfigFormat {
server, server,
client,
old_db, old_db,
tracing, tracing,
media, media,
@ -171,6 +179,7 @@ impl Args {
store, store,
}) => { }) => {
let server = Server::default(); let server = Server::default();
let client = Client::default();
let media = Media::default(); let media = Media::default();
match store { match store {
@ -178,6 +187,7 @@ impl Args {
MigrateStoreTo::Filesystem(MigrateFilesystemInner { to, repo }) => Output { MigrateStoreTo::Filesystem(MigrateFilesystemInner { to, repo }) => Output {
config_format: ConfigFormat { config_format: ConfigFormat {
server, server,
client,
old_db, old_db,
tracing, tracing,
media, media,
@ -196,6 +206,7 @@ impl Args {
Output { Output {
config_format: ConfigFormat { config_format: ConfigFormat {
server, server,
client,
old_db, old_db,
tracing, tracing,
media, media,
@ -218,6 +229,7 @@ impl Args {
Output { Output {
config_format: ConfigFormat { config_format: ConfigFormat {
server, server,
client,
old_db, old_db,
tracing, tracing,
media, media,
@ -239,6 +251,7 @@ impl Args {
}) => Output { }) => Output {
config_format: ConfigFormat { config_format: ConfigFormat {
server, server,
client,
old_db, old_db,
tracing, tracing,
media, media,
@ -283,6 +296,7 @@ pub(crate) enum Operation {
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub(super) struct ConfigFormat { pub(super) struct ConfigFormat {
server: Server, server: Server,
client: Client,
old_db: OldDb, old_db: OldDb,
tracing: Tracing, tracing: Tracing,
media: Media, media: Media,
@ -301,8 +315,15 @@ struct Server {
worker_id: Option<String>, worker_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
api_key: Option<String>, api_key: Option<String>,
}
#[derive(Debug, Default, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct Client {
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
client_pool_size: Option<usize>, pool_size: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
timeout: Option<u64>,
} }
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
@ -549,6 +570,12 @@ struct Run {
#[arg(long)] #[arg(long)]
client_pool_size: Option<usize>, client_pool_size: Option<usize>,
/// How long (in seconds) the internel HTTP client should wait for responses
///
/// This number defaults to 30
#[arg(long)]
client_timeout: Option<u64>,
/// Optional pre-processing steps for uploaded media. /// Optional pre-processing steps for uploaded media.
/// ///
/// All still images will be put through these steps before saving /// All still images will be put through these steps before saving

View file

@ -9,6 +9,7 @@ use std::{net::SocketAddr, path::PathBuf};
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub(crate) struct Defaults { pub(crate) struct Defaults {
server: ServerDefaults, server: ServerDefaults,
client: ClientDefaults,
tracing: TracingDefaults, tracing: TracingDefaults,
old_db: OldDbDefaults, old_db: OldDbDefaults,
media: MediaDefaults, media: MediaDefaults,
@ -21,7 +22,13 @@ pub(crate) struct Defaults {
struct ServerDefaults { struct ServerDefaults {
address: SocketAddr, address: SocketAddr,
worker_id: String, worker_id: String,
client_pool_size: usize, }
#[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct ClientDefaults {
pool_size: usize,
timeout: u64,
} }
#[derive(Clone, Debug, Default, serde::Serialize)] #[derive(Clone, Debug, Default, serde::Serialize)]
@ -149,7 +156,15 @@ impl Default for ServerDefaults {
ServerDefaults { ServerDefaults {
address: "0.0.0.0:8080".parse().expect("Valid address string"), address: "0.0.0.0:8080".parse().expect("Valid address string"),
worker_id: String::from("pict-rs-1"), worker_id: String::from("pict-rs-1"),
client_pool_size: 100, }
}
}
impl Default for ClientDefaults {
fn default() -> Self {
ClientDefaults {
pool_size: 100,
timeout: 30,
} }
} }
} }

View file

@ -12,6 +12,8 @@ use url::Url;
pub(crate) struct ConfigFile { pub(crate) struct ConfigFile {
pub(crate) server: Server, pub(crate) server: Server,
pub(crate) client: Client,
pub(crate) tracing: Tracing, pub(crate) tracing: Tracing,
pub(crate) old_db: OldDb, pub(crate) old_db: OldDb,
@ -95,9 +97,13 @@ pub(crate) struct Server {
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub(crate) api_key: Option<String>, pub(crate) api_key: Option<String>,
}
#[serde(skip_serializing_if = "Option::is_none")] #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub(crate) client_pool_size: Option<usize>, pub(crate) struct Client {
pub(crate) pool_size: usize,
pub(crate) timeout: u64,
} }
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]

View file

@ -1128,17 +1128,13 @@ fn transform_error(error: actix_form_data::Error) -> actix_web::Error {
} }
fn build_client() -> awc::Client { fn build_client() -> awc::Client {
let connector = CONFIG let connector = Connector::new().limit(CONFIG.client.pool_size);
.server
.client_pool_size
.map(|size| Connector::new().limit(size))
.unwrap_or_else(Connector::new);
Client::builder() Client::builder()
.connector(connector) .connector(connector)
.wrap(Tracing) .wrap(Tracing)
.add_default_header(("User-Agent", "pict-rs v0.4.1")) .add_default_header(("User-Agent", "pict-rs v0.4.1"))
.timeout(Duration::from_secs(30)) .timeout(Duration::from_secs(CONFIG.client.timeout))
.finish() .finish()
} }