2
0
Fork 0
mirror of https://git.asonix.dog/asonix/pict-rs synced 2024-12-29 14:01:24 +00:00

Simplify old_repo, remove client_pool_size

This commit is contained in:
asonix 2023-09-30 22:43:40 -05:00
parent 47751f3875
commit 81c6e73b5f
6 changed files with 61 additions and 52 deletions

View file

@ -52,7 +52,6 @@ impl Args {
Command::Run(Run {
address,
api_key,
client_pool_size,
client_timeout,
metrics_prometheus_address,
media_preprocess_steps,
@ -111,7 +110,6 @@ impl Args {
};
let client = Client {
pool_size: client_pool_size,
timeout: client_timeout,
};
@ -493,8 +491,6 @@ struct Server {
#[derive(Debug, Default, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct Client {
#[serde(skip_serializing_if = "Option::is_none")]
pool_size: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
timeout: Option<u64>,
}
@ -862,14 +858,6 @@ struct Run {
#[arg(long)]
api_key: Option<String>,
/// Number of connections the internel HTTP client should maintain in its pool
///
/// This number defaults to 100, and the total number is multiplied by the number of cores
/// available to the program. This means that running on a 2 core system will result in 200
/// pooled connections, and running on a 32 core system will result in 3200 pooled connections.
#[arg(long)]
client_pool_size: Option<usize>,
/// How long (in seconds) the internel HTTP client should wait for responses
///
/// This number defaults to 30

View file

@ -10,7 +10,6 @@ pub(crate) struct Defaults {
server: ServerDefaults,
client: ClientDefaults,
tracing: TracingDefaults,
old_repo: SledDefaults,
media: MediaDefaults,
repo: RepoDefaults,
store: StoreDefaults,
@ -27,7 +26,6 @@ struct ServerDefaults {
#[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct ClientDefaults {
pool_size: usize,
timeout: u64,
}
@ -138,7 +136,7 @@ enum RepoDefaults {
#[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct SledDefaults {
pub(super) struct SledDefaults {
path: PathBuf,
cache_capacity: u64,
export_path: PathBuf,
@ -183,10 +181,7 @@ impl Default for ServerDefaults {
impl Default for ClientDefaults {
fn default() -> Self {
ClientDefaults {
pool_size: 100,
timeout: 30,
}
ClientDefaults { timeout: 30 }
}
}
@ -351,6 +346,16 @@ impl From<crate::config::commandline::Filesystem> for crate::config::primitives:
}
}
impl From<SledDefaults> for crate::config::file::Sled {
fn from(defaults: SledDefaults) -> Self {
crate::config::file::Sled {
path: defaults.path,
cache_capacity: defaults.cache_capacity,
export_path: defaults.export_path,
}
}
}
impl From<crate::config::commandline::Sled> for crate::config::file::Sled {
fn from(value: crate::config::commandline::Sled) -> Self {
let defaults = SledDefaults::default();

View file

@ -18,7 +18,7 @@ pub(crate) struct ConfigFile {
#[serde(default)]
pub(crate) metrics: Metrics,
pub(crate) old_repo: Sled,
old_repo: OldRepo,
pub(crate) media: Media,
@ -27,6 +27,15 @@ pub(crate) struct ConfigFile {
pub(crate) store: Store,
}
impl ConfigFile {
pub(crate) fn old_repo_path(&self) -> Option<&PathBuf> {
self.old_repo.path.as_ref().or_else(|| match self.repo {
Repo::Sled(ref sled) => Some(&sled.path),
_ => None,
})
}
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "snake_case")]
#[serde(tag = "type")]
@ -106,8 +115,6 @@ pub(crate) struct Server {
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub(crate) struct Client {
pub(crate) pool_size: usize,
pub(crate) timeout: u64,
}
@ -417,6 +424,12 @@ impl Media {
}
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct OldRepo {
pub(crate) path: Option<PathBuf>,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct Sled {

View file

@ -1598,11 +1598,10 @@ fn transform_error(error: actix_form_data::Error) -> actix_web::Error {
error
}
fn build_client(config: &Configuration) -> Result<ClientWithMiddleware, Error> {
fn build_client() -> Result<ClientWithMiddleware, Error> {
let client = reqwest::Client::builder()
.user_agent("pict-rs v0.5.0-main")
.use_rustls_tls()
.pool_max_idle_per_host(config.client.pool_size)
.build()
.map_err(UploadError::BuildClient)?;
@ -1979,7 +1978,7 @@ impl PictRsConfiguration {
pub async fn run(self) -> color_eyre::Result<()> {
let PictRsConfiguration { config, operation } = self;
let client = build_client(&config)?;
let client = build_client()?;
match operation {
Operation::Run => (),
@ -2073,12 +2072,19 @@ impl PictRsConfiguration {
let store = FileStore::build(path, arc_repo.clone()).await?;
if arc_repo.get("migrate-0.4").await?.is_none() {
if let Some(old_repo) = repo_04::open(&config.old_repo)? {
repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone())
.await?;
arc_repo
.set("migrate-0.4", Arc::from(b"migrated".to_vec()))
if let Some(path) = config.old_repo_path() {
if let Some(old_repo) = repo_04::open(path)? {
repo::migrate_04(
old_repo,
arc_repo.clone(),
store.clone(),
config.clone(),
)
.await?;
arc_repo
.set("migrate-0.4", Arc::from(b"migrated".to_vec()))
.await?;
}
}
}
@ -2129,12 +2135,19 @@ impl PictRsConfiguration {
.build(client.clone());
if arc_repo.get("migrate-0.4").await?.is_none() {
if let Some(old_repo) = repo_04::open(&config.old_repo)? {
repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone())
.await?;
arc_repo
.set("migrate-0.4", Arc::from(b"migrated".to_vec()))
if let Some(path) = config.old_repo_path() {
if let Some(old_repo) = repo_04::open(path)? {
repo::migrate_04(
old_repo,
arc_repo.clone(),
store.clone(),
config.clone(),
)
.await?;
arc_repo
.set("migrate-0.4", Arc::from(b"migrated".to_vec()))
.await?;
}
}
}

View file

@ -1,24 +1,17 @@
use crate::{
config,
details::Details,
repo::{Alias, DeleteToken},
};
use futures_core::Stream;
use std::{fmt::Debug, sync::Arc};
use std::{fmt::Debug, path::PathBuf, sync::Arc};
pub(crate) use self::sled::SledRepo;
mod sled;
#[tracing::instrument]
pub(crate) fn open(config: &config::Sled) -> color_eyre::Result<Option<SledRepo>> {
let config::Sled {
path,
cache_capacity,
export_path: _,
} = config;
SledRepo::build(path.clone(), *cache_capacity)
pub(crate) fn open(path: &PathBuf) -> color_eyre::Result<Option<SledRepo>> {
SledRepo::build(path.clone())
}
#[derive(Debug, thiserror::Error)]

View file

@ -109,8 +109,8 @@ impl OldDetails {
impl SledRepo {
#[tracing::instrument]
pub(crate) fn build(path: PathBuf, cache_capacity: u64) -> color_eyre::Result<Option<Self>> {
let Some(db) = Self::open(path, cache_capacity)? else {
pub(crate) fn build(path: PathBuf) -> color_eyre::Result<Option<Self>> {
let Some(db) = Self::open(path)? else {
return Ok(None);
};
@ -129,7 +129,7 @@ impl SledRepo {
}))
}
fn open(mut path: PathBuf, cache_capacity: u64) -> Result<Option<Db>, SledError> {
fn open(mut path: PathBuf) -> Result<Option<Db>, SledError> {
path.push("v0.4.0-alpha.1");
if let Err(e) = std::fs::metadata(&path) {
@ -138,10 +138,7 @@ impl SledRepo {
}
}
let db = ::sled::Config::new()
.cache_capacity(cache_capacity)
.path(path)
.open()?;
let db = ::sled::Config::new().path(path).open()?;
Ok(Some(db))
}