2
0
Fork 0
mirror of https://git.asonix.dog/asonix/pict-rs synced 2024-12-22 19:31:35 +00:00

Create unique errors for Repo and Store, separate from UploadError

Add .is_not_found() for Store errors, wire up Object and File storage to properly set NotFound
Allow skipping files that are not found during store migration
This commit is contained in:
asonix 2023-06-20 15:59:08 -05:00
parent 1e6705684e
commit 554d852e68
13 changed files with 510 additions and 316 deletions

View file

@ -148,13 +148,33 @@ impl Args {
}, },
} }
} }
Command::MigrateStore(migrate_store) => { Command::MigrateStore(MigrateStore {
skip_missing_files,
store,
}) => {
let server = Server::default(); let server = Server::default();
let media = Media::default(); let media = Media::default();
match migrate_store { match store {
MigrateStore::Filesystem(MigrateFilesystem { from, to }) => match to { MigrateStoreFrom::Filesystem(MigrateFilesystem { from, to }) => match to {
MigrateStoreInner::Filesystem(MigrateFilesystemInner { to, repo }) => { MigrateStoreTo::Filesystem(MigrateFilesystemInner { to, repo }) => Output {
config_format: ConfigFormat {
server,
old_db,
tracing,
media,
store: None,
repo,
},
operation: Operation::MigrateStore {
skip_missing_files,
from: from.into(),
to: to.into(),
},
config_file,
save_to,
},
MigrateStoreTo::ObjectStorage(MigrateObjectStorageInner { to, repo }) => {
Output { Output {
config_format: ConfigFormat { config_format: ConfigFormat {
server, server,
@ -165,6 +185,7 @@ impl Args {
repo, repo,
}, },
operation: Operation::MigrateStore { operation: Operation::MigrateStore {
skip_missing_files,
from: from.into(), from: from.into(),
to: to.into(), to: to.into(),
}, },
@ -172,29 +193,32 @@ impl Args {
save_to, save_to,
} }
} }
MigrateStoreInner::ObjectStorage(MigrateObjectStorageInner {
to,
repo,
}) => Output {
config_format: ConfigFormat {
server,
old_db,
tracing,
media,
store: None,
repo,
},
operation: Operation::MigrateStore {
from: from.into(),
to: to.into(),
},
config_file,
save_to,
},
}, },
MigrateStore::ObjectStorage(MigrateObjectStorage { from, to }) => match to { MigrateStoreFrom::ObjectStorage(MigrateObjectStorage { from, to }) => {
MigrateStoreInner::Filesystem(MigrateFilesystemInner { to, repo }) => { match to {
Output { MigrateStoreTo::Filesystem(MigrateFilesystemInner { to, repo }) => {
Output {
config_format: ConfigFormat {
server,
old_db,
tracing,
media,
store: None,
repo,
},
operation: Operation::MigrateStore {
skip_missing_files,
from: from.into(),
to: to.into(),
},
config_file,
save_to,
}
}
MigrateStoreTo::ObjectStorage(MigrateObjectStorageInner {
to,
repo,
}) => Output {
config_format: ConfigFormat { config_format: ConfigFormat {
server, server,
old_db, old_db,
@ -204,33 +228,15 @@ impl Args {
repo, repo,
}, },
operation: Operation::MigrateStore { operation: Operation::MigrateStore {
skip_missing_files,
from: from.into(), from: from.into(),
to: to.into(), to: to.into(),
}, },
config_file, config_file,
save_to, save_to,
} },
} }
MigrateStoreInner::ObjectStorage(MigrateObjectStorageInner { }
to,
repo,
}) => Output {
config_format: ConfigFormat {
server,
old_db,
tracing,
media,
store: None,
repo,
},
operation: Operation::MigrateStore {
from: from.into(),
to: to.into(),
},
config_file,
save_to,
},
},
} }
} }
} }
@ -249,6 +255,7 @@ pub(super) struct Output {
pub(crate) enum Operation { pub(crate) enum Operation {
Run, Run,
MigrateStore { MigrateStore {
skip_missing_files: bool,
from: crate::config::primitives::Store, from: crate::config::primitives::Store,
to: crate::config::primitives::Store, to: crate::config::primitives::Store,
}, },
@ -418,7 +425,6 @@ enum Command {
Run(Run), Run(Run),
/// Migrates from one provided media store to another /// Migrates from one provided media store to another
#[command(flatten)]
MigrateStore(MigrateStore), MigrateStore(MigrateStore),
} }
@ -527,9 +533,20 @@ enum RunStore {
ObjectStorage(RunObjectStorage), ObjectStorage(RunObjectStorage),
} }
#[derive(Debug, Parser)]
struct MigrateStore {
/// Normally, pict-rs will keep retrying when errors occur during migration. This flag tells
/// pict-rs to ignore errors that are caused by files not existing.
#[arg(long)]
skip_missing_files: bool,
#[command(subcommand)]
store: MigrateStoreFrom,
}
/// Configure the pict-rs storage migration /// Configure the pict-rs storage migration
#[derive(Debug, Subcommand)] #[derive(Debug, Subcommand)]
enum MigrateStore { enum MigrateStoreFrom {
/// Migrate from the provided filesystem storage /// Migrate from the provided filesystem storage
Filesystem(MigrateFilesystem), Filesystem(MigrateFilesystem),
@ -539,7 +556,7 @@ enum MigrateStore {
/// Configure the destination storage for pict-rs storage migration /// Configure the destination storage for pict-rs storage migration
#[derive(Debug, Subcommand)] #[derive(Debug, Subcommand)]
enum MigrateStoreInner { enum MigrateStoreTo {
/// Migrate to the provided filesystem storage /// Migrate to the provided filesystem storage
Filesystem(MigrateFilesystemInner), Filesystem(MigrateFilesystemInner),
@ -554,7 +571,7 @@ struct MigrateFilesystem {
from: Filesystem, from: Filesystem,
#[command(subcommand)] #[command(subcommand)]
to: MigrateStoreInner, to: MigrateStoreTo,
} }
/// Migrate pict-rs' storage to the provided filesystem storage /// Migrate pict-rs' storage to the provided filesystem storage
@ -574,7 +591,7 @@ struct MigrateObjectStorage {
from: crate::config::primitives::ObjectStorage, from: crate::config::primitives::ObjectStorage,
#[command(subcommand)] #[command(subcommand)]
to: MigrateStoreInner, to: MigrateStoreTo,
} }
/// Migrate pict-rs' storage to the provided object storage /// Migrate pict-rs' storage to the provided object storage

View file

@ -46,7 +46,7 @@ pub(crate) enum UploadError {
Upload(#[from] actix_form_data::Error), Upload(#[from] actix_form_data::Error),
#[error("Error in DB")] #[error("Error in DB")]
Sled(#[from] crate::repo::sled::SledError), Repo(#[from] crate::repo::RepoError),
#[error("Error in old sled DB")] #[error("Error in old sled DB")]
OldSled(#[from] ::sled::Error), OldSled(#[from] ::sled::Error),
@ -63,11 +63,8 @@ pub(crate) enum UploadError {
#[error("Error stripping prefix")] #[error("Error stripping prefix")]
StripPrefix(#[from] std::path::StripPrefixError), StripPrefix(#[from] std::path::StripPrefixError),
#[error("Error storing file")] #[error("Error in store")]
FileStore(#[from] crate::store::file_store::FileError), Store(#[source] crate::store::StoreError),
#[error("Error storing object")]
ObjectStore(#[from] crate::store::object_store::ObjectError),
#[error("Provided process path is invalid")] #[error("Provided process path is invalid")]
ParsePath, ParsePath,
@ -81,9 +78,6 @@ pub(crate) enum UploadError {
#[error("No files present in upload")] #[error("No files present in upload")]
NoFiles, NoFiles,
#[error("Upload was already claimed")]
AlreadyClaimed,
#[error("Requested a file that doesn't exist")] #[error("Requested a file that doesn't exist")]
MissingAlias, MissingAlias,
@ -151,6 +145,15 @@ impl From<tokio::sync::AcquireError> for UploadError {
} }
} }
impl From<crate::store::StoreError> for UploadError {
fn from(value: crate::store::StoreError) -> Self {
match value {
crate::store::StoreError::Repo(repo_error) => Self::Repo(repo_error),
e => Self::Store(e),
}
}
}
impl ResponseError for Error { impl ResponseError for Error {
fn status_code(&self) -> StatusCode { fn status_code(&self) -> StatusCode {
match self.kind() { match self.kind() {
@ -160,11 +163,16 @@ impl ResponseError for Error {
| UploadError::NoFiles | UploadError::NoFiles
| UploadError::Upload(_) | UploadError::Upload(_)
| UploadError::UnsupportedFormat | UploadError::UnsupportedFormat
| UploadError::AlreadyClaimed | UploadError::Store(crate::store::StoreError::Repo(
crate::repo::RepoError::AlreadyClaimed,
))
| UploadError::Repo(crate::repo::RepoError::AlreadyClaimed)
| UploadError::SilentVideoDisabled, | UploadError::SilentVideoDisabled,
) => StatusCode::BAD_REQUEST, ) => StatusCode::BAD_REQUEST,
Some( Some(
UploadError::Sled(crate::repo::sled::SledError::Missing) UploadError::Repo(crate::repo::RepoError::SledError(
crate::repo::sled::SledError::Missing,
))
| UploadError::MissingAlias, | UploadError::MissingAlias,
) => StatusCode::NOT_FOUND, ) => StatusCode::NOT_FOUND,
Some(UploadError::InvalidToken) => StatusCode::FORBIDDEN, Some(UploadError::InvalidToken) => StatusCode::FORBIDDEN,

View file

@ -3,7 +3,7 @@ use crate::{
error::{Error, UploadError}, error::{Error, UploadError},
magick::{Details, ValidInputType}, magick::{Details, ValidInputType},
process::Process, process::Process,
store::Store, store::{Store, StoreError},
}; };
use actix_web::web::Bytes; use actix_web::web::Bytes;
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
@ -413,7 +413,9 @@ where
{ {
let input_file = crate::tmp_file::tmp_file(None); let input_file = crate::tmp_file::tmp_file(None);
let input_file_str = input_file.to_str().ok_or(UploadError::Path)?; let input_file_str = input_file.to_str().ok_or(UploadError::Path)?;
crate::store::file_store::safe_create_parent(&input_file).await?; crate::store::file_store::safe_create_parent(&input_file)
.await
.map_err(StoreError::from)?;
let tmp_one = crate::file::File::create(&input_file).await?; let tmp_one = crate::file::File::create(&input_file).await?;
let tmp_one = (f)(tmp_one).await?; let tmp_one = (f)(tmp_one).await?;
@ -523,11 +525,15 @@ pub(crate) async fn transcode_bytes(
) -> Result<impl AsyncRead + Unpin, Error> { ) -> Result<impl AsyncRead + Unpin, Error> {
let input_file = crate::tmp_file::tmp_file(Some(transcode_options.input_file_extension())); let input_file = crate::tmp_file::tmp_file(Some(transcode_options.input_file_extension()));
let input_file_str = input_file.to_str().ok_or(UploadError::Path)?; let input_file_str = input_file.to_str().ok_or(UploadError::Path)?;
crate::store::file_store::safe_create_parent(&input_file).await?; crate::store::file_store::safe_create_parent(&input_file)
.await
.map_err(StoreError::from)?;
let output_file = crate::tmp_file::tmp_file(Some(transcode_options.output_file_extension())); let output_file = crate::tmp_file::tmp_file(Some(transcode_options.output_file_extension()));
let output_file_str = output_file.to_str().ok_or(UploadError::Path)?; let output_file_str = output_file.to_str().ok_or(UploadError::Path)?;
crate::store::file_store::safe_create_parent(&output_file).await?; crate::store::file_store::safe_create_parent(&output_file)
.await
.map_err(StoreError::from)?;
let mut tmp_one = crate::file::File::create(&input_file).await?; let mut tmp_one = crate::file::File::create(&input_file).await?;
tmp_one.write_from_bytes(input).await?; tmp_one.write_from_bytes(input).await?;
@ -557,7 +563,10 @@ pub(crate) async fn transcode_bytes(
tokio::fs::remove_file(input_file).await?; tokio::fs::remove_file(input_file).await?;
let tmp_two = crate::file::File::open(&output_file).await?; let tmp_two = crate::file::File::open(&output_file).await?;
let stream = tmp_two.read_to_stream(None, None).await?; let stream = tmp_two
.read_to_stream(None, None)
.await
.map_err(StoreError::from)?;
let reader = tokio_util::io::StreamReader::new(stream); let reader = tokio_util::io::StreamReader::new(stream);
let clean_reader = crate::tmp_file::cleanup_tmpfile(reader, output_file); let clean_reader = crate::tmp_file::cleanup_tmpfile(reader, output_file);
@ -573,11 +582,15 @@ pub(crate) async fn thumbnail<S: Store>(
) -> Result<impl AsyncRead + Unpin, Error> { ) -> Result<impl AsyncRead + Unpin, Error> {
let input_file = crate::tmp_file::tmp_file(Some(input_format.to_file_extension())); let input_file = crate::tmp_file::tmp_file(Some(input_format.to_file_extension()));
let input_file_str = input_file.to_str().ok_or(UploadError::Path)?; let input_file_str = input_file.to_str().ok_or(UploadError::Path)?;
crate::store::file_store::safe_create_parent(&input_file).await?; crate::store::file_store::safe_create_parent(&input_file)
.await
.map_err(StoreError::from)?;
let output_file = crate::tmp_file::tmp_file(Some(format.to_file_extension())); let output_file = crate::tmp_file::tmp_file(Some(format.to_file_extension()));
let output_file_str = output_file.to_str().ok_or(UploadError::Path)?; let output_file_str = output_file.to_str().ok_or(UploadError::Path)?;
crate::store::file_store::safe_create_parent(&output_file).await?; crate::store::file_store::safe_create_parent(&output_file)
.await
.map_err(StoreError::from)?;
let mut tmp_one = crate::file::File::create(&input_file).await?; let mut tmp_one = crate::file::File::create(&input_file).await?;
tmp_one tmp_one
@ -607,7 +620,10 @@ pub(crate) async fn thumbnail<S: Store>(
tokio::fs::remove_file(input_file).await?; tokio::fs::remove_file(input_file).await?;
let tmp_two = crate::file::File::open(&output_file).await?; let tmp_two = crate::file::File::open(&output_file).await?;
let stream = tmp_two.read_to_stream(None, None).await?; let stream = tmp_two
.read_to_stream(None, None)
.await
.map_err(StoreError::from)?;
let reader = tokio_util::io::StreamReader::new(stream); let reader = tokio_util::io::StreamReader::new(stream);
let clean_reader = crate::tmp_file::cleanup_tmpfile(reader, output_file); let clean_reader = crate::tmp_file::cleanup_tmpfile(reader, output_file);

View file

@ -1108,7 +1108,12 @@ async fn launch<R: FullRepo + 'static, SC: StoreConfig + 'static>(
Ok(()) Ok(())
} }
async fn migrate_inner<S1>(repo: &Repo, from: S1, to: config::Store) -> color_eyre::Result<()> async fn migrate_inner<S1>(
repo: &Repo,
from: S1,
to: config::Store,
skip_missing_files: bool,
) -> color_eyre::Result<()>
where where
S1: Store, S1: Store,
{ {
@ -1117,7 +1122,7 @@ where
let to = FileStore::build(path.clone(), repo.clone()).await?.build(); let to = FileStore::build(path.clone(), repo.clone()).await?.build();
match repo { match repo {
Repo::Sled(repo) => migrate_store(repo, from, to).await?, Repo::Sled(repo) => migrate_store(repo, from, to, skip_missing_files).await?,
} }
} }
config::Store::ObjectStorage(config::ObjectStorage { config::Store::ObjectStorage(config::ObjectStorage {
@ -1147,7 +1152,7 @@ where
.build(); .build();
match repo { match repo {
Repo::Sled(repo) => migrate_store(repo, from, to).await?, Repo::Sled(repo) => migrate_store(repo, from, to, skip_missing_files).await?,
} }
} }
} }
@ -1219,11 +1224,15 @@ pub async fn run() -> color_eyre::Result<()> {
match (*OPERATION).clone() { match (*OPERATION).clone() {
Operation::Run => (), Operation::Run => (),
Operation::MigrateStore { from, to } => { Operation::MigrateStore {
skip_missing_files,
from,
to,
} => {
match from { match from {
config::Store::Filesystem(config::Filesystem { path }) => { config::Store::Filesystem(config::Filesystem { path }) => {
let from = FileStore::build(path.clone(), repo.clone()).await?.build(); let from = FileStore::build(path.clone(), repo.clone()).await?.build();
migrate_inner(&repo, from, to).await?; migrate_inner(&repo, from, to, skip_missing_files).await?;
} }
config::Store::ObjectStorage(config::ObjectStorage { config::Store::ObjectStorage(config::ObjectStorage {
endpoint, endpoint,
@ -1251,7 +1260,7 @@ pub async fn run() -> color_eyre::Result<()> {
.await? .await?
.build(); .build();
migrate_inner(&repo, from, to).await?; migrate_inner(&repo, from, to, skip_missing_files).await?;
} }
} }
@ -1304,15 +1313,22 @@ const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress";
const STORE_MIGRATION_MOTION: &str = "store-migration-motion"; const STORE_MIGRATION_MOTION: &str = "store-migration-motion";
const STORE_MIGRATION_VARIANT: &str = "store-migration-variant"; const STORE_MIGRATION_VARIANT: &str = "store-migration-variant";
async fn migrate_store<R, S1, S2>(repo: &R, from: S1, to: S2) -> Result<(), Error> async fn migrate_store<R, S1, S2>(
repo: &R,
from: S1,
to: S2,
skip_missing_files: bool,
) -> Result<(), Error>
where where
S1: Store + Clone, S1: Store + Clone,
S2: Store + Clone, S2: Store + Clone,
R: IdentifierRepo + HashRepo + SettingsRepo, R: IdentifierRepo + HashRepo + SettingsRepo,
{ {
tracing::info!("Migrating store");
let mut failure_count = 0; let mut failure_count = 0;
while let Err(e) = do_migrate_store(repo, from.clone(), to.clone()).await { while let Err(e) = do_migrate_store(repo, from.clone(), to.clone(), skip_missing_files).await {
tracing::error!("Failed with {}", e.to_string()); tracing::error!("Failed with {}", e.to_string());
failure_count += 1; failure_count += 1;
@ -1326,7 +1342,12 @@ where
Ok(()) Ok(())
} }
async fn do_migrate_store<R, S1, S2>(repo: &R, from: S1, to: S2) -> Result<(), Error> async fn do_migrate_store<R, S1, S2>(
repo: &R,
from: S1,
to: S2,
skip_missing_files: bool,
) -> Result<(), Error>
where where
S1: Store, S1: Store,
S2: Store, S2: Store,
@ -1352,12 +1373,22 @@ where
.await? .await?
{ {
if repo.get(STORE_MIGRATION_MOTION).await?.is_none() { if repo.get(STORE_MIGRATION_MOTION).await?.is_none() {
let new_identifier = migrate_file(&from, &to, &identifier).await?; match migrate_file(&from, &to, &identifier, skip_missing_files).await {
migrate_details(repo, identifier, &new_identifier).await?; Ok(new_identifier) => {
repo.relate_motion_identifier(hash.as_ref().to_vec().into(), &new_identifier) migrate_details(repo, identifier, &new_identifier).await?;
.await?; repo.relate_motion_identifier(
repo.set(STORE_MIGRATION_MOTION, b"1".to_vec().into()) hash.as_ref().to_vec().into(),
.await?; &new_identifier,
)
.await?;
repo.set(STORE_MIGRATION_MOTION, b"1".to_vec().into())
.await?;
}
Err(e) if e.is_not_found() && skip_missing_files => {
tracing::warn!("Skipping motion file for hash {}", hex::encode(&hash));
}
Err(e) => return Err(e.into()),
}
} }
} }
@ -1371,22 +1402,45 @@ where
continue; continue;
} }
let new_identifier = migrate_file(&from, &to, &identifier).await?; match migrate_file(&from, &to, &identifier, skip_missing_files).await {
migrate_details(repo, identifier, &new_identifier).await?; Ok(new_identifier) => {
repo.remove_variant(hash.as_ref().to_vec().into(), variant.clone()) migrate_details(repo, identifier, &new_identifier).await?;
.await?; repo.remove_variant(hash.as_ref().to_vec().into(), variant.clone())
repo.relate_variant_identifier(hash.as_ref().to_vec().into(), variant, &new_identifier) .await?;
.await?; repo.relate_variant_identifier(
hash.as_ref().to_vec().into(),
variant,
&new_identifier,
)
.await?;
repo.set(STORE_MIGRATION_VARIANT, new_identifier.to_bytes()?.into()) repo.set(STORE_MIGRATION_VARIANT, new_identifier.to_bytes()?.into())
.await?; .await?;
}
Err(e) if e.is_not_found() && skip_missing_files => {
tracing::warn!(
"Skipping variant {} for hash {}",
variant,
hex::encode(&hash)
);
}
Err(e) => return Err(e.into()),
}
} }
let identifier = repo.identifier(hash.as_ref().to_vec().into()).await?; let identifier = repo.identifier(hash.as_ref().to_vec().into()).await?;
let new_identifier = migrate_file(&from, &to, &identifier).await?;
migrate_details(repo, identifier, &new_identifier).await?; match migrate_file(&from, &to, &identifier, skip_missing_files).await {
repo.relate_identifier(hash.as_ref().to_vec().into(), &new_identifier) Ok(new_identifier) => {
.await?; migrate_details(repo, identifier, &new_identifier).await?;
repo.relate_identifier(hash.as_ref().to_vec().into(), &new_identifier)
.await?;
}
Err(e) if e.is_not_found() && skip_missing_files => {
tracing::warn!("Skipping original file for hash {}", hex::encode(&hash));
}
Err(e) => return Err(e.into()),
}
repo.set(STORE_MIGRATION_PROGRESS, hash.as_ref().to_vec().into()) repo.set(STORE_MIGRATION_PROGRESS, hash.as_ref().to_vec().into())
.await?; .await?;
@ -1404,7 +1458,8 @@ async fn migrate_file<S1, S2>(
from: &S1, from: &S1,
to: &S2, to: &S2,
identifier: &S1::Identifier, identifier: &S1::Identifier,
) -> Result<S2::Identifier, Error> skip_missing_files: bool,
) -> Result<S2::Identifier, crate::store::StoreError>
where where
S1: Store, S1: Store,
S2: Store, S2: Store,
@ -1414,6 +1469,7 @@ where
loop { loop {
match do_migrate_file(from, to, identifier).await { match do_migrate_file(from, to, identifier).await {
Ok(identifier) => return Ok(identifier), Ok(identifier) => return Ok(identifier),
Err(e) if e.is_not_found() && skip_missing_files => return Err(e),
Err(e) => { Err(e) => {
failure_count += 1; failure_count += 1;
@ -1432,7 +1488,7 @@ async fn do_migrate_file<S1, S2>(
from: &S1, from: &S1,
to: &S2, to: &S2,
identifier: &S1::Identifier, identifier: &S1::Identifier,
) -> Result<S2::Identifier, Error> ) -> Result<S2::Identifier, crate::store::StoreError>
where where
S1: Store, S1: Store,
S2: Store, S2: Store,

View file

@ -3,7 +3,7 @@ use crate::{
error::{Error, UploadError}, error::{Error, UploadError},
process::Process, process::Process,
repo::Alias, repo::Alias,
store::Store, store::{Store, StoreError},
}; };
use actix_web::web::Bytes; use actix_web::web::Bytes;
use tokio::{ use tokio::{
@ -140,7 +140,9 @@ pub(crate) async fn details_bytes(
if let Some(hint) = hint.and_then(|hint| hint.video_hint()) { if let Some(hint) = hint.and_then(|hint| hint.video_hint()) {
let input_file = crate::tmp_file::tmp_file(Some(hint)); let input_file = crate::tmp_file::tmp_file(Some(hint));
let input_file_str = input_file.to_str().ok_or(UploadError::Path)?; let input_file_str = input_file.to_str().ok_or(UploadError::Path)?;
crate::store::file_store::safe_create_parent(&input_file).await?; crate::store::file_store::safe_create_parent(&input_file)
.await
.map_err(StoreError::from)?;
let mut tmp_one = crate::file::File::create(&input_file).await?; let mut tmp_one = crate::file::File::create(&input_file).await?;
tmp_one.write_from_bytes(input).await?; tmp_one.write_from_bytes(input).await?;
@ -178,7 +180,9 @@ pub(crate) async fn details_store<S: Store + 'static>(
if let Some(hint) = hint.and_then(|hint| hint.video_hint()) { if let Some(hint) = hint.and_then(|hint| hint.video_hint()) {
let input_file = crate::tmp_file::tmp_file(Some(hint)); let input_file = crate::tmp_file::tmp_file(Some(hint));
let input_file_str = input_file.to_str().ok_or(UploadError::Path)?; let input_file_str = input_file.to_str().ok_or(UploadError::Path)?;
crate::store::file_store::safe_create_parent(&input_file).await?; crate::store::file_store::safe_create_parent(&input_file)
.await
.map_err(StoreError::from)?;
let mut tmp_one = crate::file::File::create(&input_file).await?; let mut tmp_one = crate::file::File::create(&input_file).await?;
tmp_one tmp_one

View file

@ -34,7 +34,8 @@ pub(crate) async fn chop_store<S: Store>(
let end = end + 1; let end = end + 1;
return store return store
.to_stream(identifier, Some(start), Some(end.saturating_sub(start))) .to_stream(identifier, Some(start), Some(end.saturating_sub(start)))
.await; .await
.map_err(Error::from);
} }
Err(UploadError::Range.into()) Err(UploadError::Range.into())

View file

@ -1,8 +1,7 @@
use crate::{ use crate::{
config, config,
details::Details, details::Details,
error::Error, store::{file_store::FileId, Identifier, StoreError},
store::{file_store::FileId, Identifier},
}; };
use base64::{prelude::BASE64_STANDARD, Engine}; use base64::{prelude::BASE64_STANDARD, Engine};
use futures_util::Stream; use futures_util::Stream;
@ -47,6 +46,18 @@ pub(crate) enum UploadResult {
Failure { message: String }, Failure { message: String },
} }
#[derive(Debug, thiserror::Error)]
pub(crate) enum RepoError {
#[error("Error in sled")]
SledError(#[from] crate::repo::sled::SledError),
#[error("Upload was already claimed")]
AlreadyClaimed,
#[error("Panic in blocking operation")]
Canceled,
}
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
pub(crate) trait FullRepo: pub(crate) trait FullRepo:
UploadRepo UploadRepo
@ -60,19 +71,19 @@ pub(crate) trait FullRepo:
+ Clone + Clone
+ Debug + Debug
{ {
async fn health_check(&self) -> Result<(), Error>; async fn health_check(&self) -> Result<(), RepoError>;
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn identifier_from_alias<I: Identifier + 'static>( async fn identifier_from_alias<I: Identifier + 'static>(
&self, &self,
alias: &Alias, alias: &Alias,
) -> Result<I, Error> { ) -> Result<I, StoreError> {
let hash = self.hash(alias).await?; let hash = self.hash(alias).await?;
self.identifier(hash).await self.identifier(hash).await
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn aliases_from_alias(&self, alias: &Alias) -> Result<Vec<Alias>, Error> { async fn aliases_from_alias(&self, alias: &Alias) -> Result<Vec<Alias>, RepoError> {
let hash = self.hash(alias).await?; let hash = self.hash(alias).await?;
self.aliases(hash).await self.aliases(hash).await
} }
@ -81,7 +92,7 @@ pub(crate) trait FullRepo:
async fn still_identifier_from_alias<I: Identifier + 'static>( async fn still_identifier_from_alias<I: Identifier + 'static>(
&self, &self,
alias: &Alias, alias: &Alias,
) -> Result<Option<I>, Error> { ) -> Result<Option<I>, StoreError> {
let hash = self.hash(alias).await?; let hash = self.hash(alias).await?;
let identifier = self.identifier::<I>(hash.clone()).await?; let identifier = self.identifier::<I>(hash.clone()).await?;
@ -98,7 +109,7 @@ impl<T> FullRepo for actix_web::web::Data<T>
where where
T: FullRepo, T: FullRepo,
{ {
async fn health_check(&self) -> Result<(), Error> { async fn health_check(&self) -> Result<(), RepoError> {
T::health_check(self).await T::health_check(self).await
} }
} }
@ -116,13 +127,13 @@ where
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
pub(crate) trait UploadRepo: BaseRepo { pub(crate) trait UploadRepo: BaseRepo {
async fn create(&self, upload_id: UploadId) -> Result<(), Error>; async fn create(&self, upload_id: UploadId) -> Result<(), RepoError>;
async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, Error>; async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, RepoError>;
async fn claim(&self, upload_id: UploadId) -> Result<(), Error>; async fn claim(&self, upload_id: UploadId) -> Result<(), RepoError>;
async fn complete(&self, upload_id: UploadId, result: UploadResult) -> Result<(), Error>; async fn complete(&self, upload_id: UploadId, result: UploadResult) -> Result<(), RepoError>;
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
@ -130,30 +141,30 @@ impl<T> UploadRepo for actix_web::web::Data<T>
where where
T: UploadRepo, T: UploadRepo,
{ {
async fn create(&self, upload_id: UploadId) -> Result<(), Error> { async fn create(&self, upload_id: UploadId) -> Result<(), RepoError> {
T::create(self, upload_id).await T::create(self, upload_id).await
} }
async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, Error> { async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, RepoError> {
T::wait(self, upload_id).await T::wait(self, upload_id).await
} }
async fn claim(&self, upload_id: UploadId) -> Result<(), Error> { async fn claim(&self, upload_id: UploadId) -> Result<(), RepoError> {
T::claim(self, upload_id).await T::claim(self, upload_id).await
} }
async fn complete(&self, upload_id: UploadId, result: UploadResult) -> Result<(), Error> { async fn complete(&self, upload_id: UploadId, result: UploadResult) -> Result<(), RepoError> {
T::complete(self, upload_id, result).await T::complete(self, upload_id, result).await
} }
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
pub(crate) trait QueueRepo: BaseRepo { pub(crate) trait QueueRepo: BaseRepo {
async fn requeue_in_progress(&self, worker_prefix: Vec<u8>) -> Result<(), Error>; async fn requeue_in_progress(&self, worker_prefix: Vec<u8>) -> Result<(), RepoError>;
async fn push(&self, queue: &'static str, job: Self::Bytes) -> Result<(), Error>; async fn push(&self, queue: &'static str, job: Self::Bytes) -> Result<(), RepoError>;
async fn pop(&self, queue: &'static str, worker_id: Vec<u8>) -> Result<Self::Bytes, Error>; async fn pop(&self, queue: &'static str, worker_id: Vec<u8>) -> Result<Self::Bytes, RepoError>;
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
@ -161,24 +172,24 @@ impl<T> QueueRepo for actix_web::web::Data<T>
where where
T: QueueRepo, T: QueueRepo,
{ {
async fn requeue_in_progress(&self, worker_prefix: Vec<u8>) -> Result<(), Error> { async fn requeue_in_progress(&self, worker_prefix: Vec<u8>) -> Result<(), RepoError> {
T::requeue_in_progress(self, worker_prefix).await T::requeue_in_progress(self, worker_prefix).await
} }
async fn push(&self, queue: &'static str, job: Self::Bytes) -> Result<(), Error> { async fn push(&self, queue: &'static str, job: Self::Bytes) -> Result<(), RepoError> {
T::push(self, queue, job).await T::push(self, queue, job).await
} }
async fn pop(&self, queue: &'static str, worker_id: Vec<u8>) -> Result<Self::Bytes, Error> { async fn pop(&self, queue: &'static str, worker_id: Vec<u8>) -> Result<Self::Bytes, RepoError> {
T::pop(self, queue, worker_id).await T::pop(self, queue, worker_id).await
} }
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
pub(crate) trait SettingsRepo: BaseRepo { pub(crate) trait SettingsRepo: BaseRepo {
async fn set(&self, key: &'static str, value: Self::Bytes) -> Result<(), Error>; async fn set(&self, key: &'static str, value: Self::Bytes) -> Result<(), RepoError>;
async fn get(&self, key: &'static str) -> Result<Option<Self::Bytes>, Error>; async fn get(&self, key: &'static str) -> Result<Option<Self::Bytes>, RepoError>;
async fn remove(&self, key: &'static str) -> Result<(), Error>; async fn remove(&self, key: &'static str) -> Result<(), RepoError>;
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
@ -186,15 +197,15 @@ impl<T> SettingsRepo for actix_web::web::Data<T>
where where
T: SettingsRepo, T: SettingsRepo,
{ {
async fn set(&self, key: &'static str, value: Self::Bytes) -> Result<(), Error> { async fn set(&self, key: &'static str, value: Self::Bytes) -> Result<(), RepoError> {
T::set(self, key, value).await T::set(self, key, value).await
} }
async fn get(&self, key: &'static str) -> Result<Option<Self::Bytes>, Error> { async fn get(&self, key: &'static str) -> Result<Option<Self::Bytes>, RepoError> {
T::get(self, key).await T::get(self, key).await
} }
async fn remove(&self, key: &'static str) -> Result<(), Error> { async fn remove(&self, key: &'static str) -> Result<(), RepoError> {
T::remove(self, key).await T::remove(self, key).await
} }
} }
@ -205,10 +216,10 @@ pub(crate) trait IdentifierRepo: BaseRepo {
&self, &self,
identifier: &I, identifier: &I,
details: &Details, details: &Details,
) -> Result<(), Error>; ) -> Result<(), StoreError>;
async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Error>; async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, StoreError>;
async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Error>; async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), StoreError>;
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
@ -220,66 +231,67 @@ where
&self, &self,
identifier: &I, identifier: &I,
details: &Details, details: &Details,
) -> Result<(), Error> { ) -> Result<(), StoreError> {
T::relate_details(self, identifier, details).await T::relate_details(self, identifier, details).await
} }
async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Error> { async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, StoreError> {
T::details(self, identifier).await T::details(self, identifier).await
} }
async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Error> { async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), StoreError> {
T::cleanup(self, identifier).await T::cleanup(self, identifier).await
} }
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
pub(crate) trait HashRepo: BaseRepo { pub(crate) trait HashRepo: BaseRepo {
type Stream: Stream<Item = Result<Self::Bytes, Error>>; type Stream: Stream<Item = Result<Self::Bytes, RepoError>>;
async fn hashes(&self) -> Self::Stream; async fn hashes(&self) -> Self::Stream;
async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Error>; async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, RepoError>;
async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error>; async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), RepoError>;
async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error>; async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), RepoError>;
async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Error>; async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, RepoError>;
async fn relate_identifier<I: Identifier>( async fn relate_identifier<I: Identifier>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
identifier: &I, identifier: &I,
) -> Result<(), Error>; ) -> Result<(), StoreError>;
async fn identifier<I: Identifier + 'static>(&self, hash: Self::Bytes) -> Result<I, Error>; async fn identifier<I: Identifier + 'static>(&self, hash: Self::Bytes)
-> Result<I, StoreError>;
async fn relate_variant_identifier<I: Identifier>( async fn relate_variant_identifier<I: Identifier>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
variant: String, variant: String,
identifier: &I, identifier: &I,
) -> Result<(), Error>; ) -> Result<(), StoreError>;
async fn variant_identifier<I: Identifier + 'static>( async fn variant_identifier<I: Identifier + 'static>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
variant: String, variant: String,
) -> Result<Option<I>, Error>; ) -> Result<Option<I>, StoreError>;
async fn variants<I: Identifier + 'static>( async fn variants<I: Identifier + 'static>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
) -> Result<Vec<(String, I)>, Error>; ) -> Result<Vec<(String, I)>, StoreError>;
async fn remove_variant(&self, hash: Self::Bytes, variant: String) -> Result<(), Error>; async fn remove_variant(&self, hash: Self::Bytes, variant: String) -> Result<(), RepoError>;
async fn relate_motion_identifier<I: Identifier>( async fn relate_motion_identifier<I: Identifier>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
identifier: &I, identifier: &I,
) -> Result<(), Error>; ) -> Result<(), StoreError>;
async fn motion_identifier<I: Identifier + 'static>( async fn motion_identifier<I: Identifier + 'static>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
) -> Result<Option<I>, Error>; ) -> Result<Option<I>, StoreError>;
async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Error>; async fn cleanup(&self, hash: Self::Bytes) -> Result<(), RepoError>;
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
@ -293,19 +305,19 @@ where
T::hashes(self).await T::hashes(self).await
} }
async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Error> { async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, RepoError> {
T::create(self, hash).await T::create(self, hash).await
} }
async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error> { async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), RepoError> {
T::relate_alias(self, hash, alias).await T::relate_alias(self, hash, alias).await
} }
async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error> { async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), RepoError> {
T::remove_alias(self, hash, alias).await T::remove_alias(self, hash, alias).await
} }
async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Error> { async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, RepoError> {
T::aliases(self, hash).await T::aliases(self, hash).await
} }
@ -313,11 +325,14 @@ where
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
identifier: &I, identifier: &I,
) -> Result<(), Error> { ) -> Result<(), StoreError> {
T::relate_identifier(self, hash, identifier).await T::relate_identifier(self, hash, identifier).await
} }
async fn identifier<I: Identifier + 'static>(&self, hash: Self::Bytes) -> Result<I, Error> { async fn identifier<I: Identifier + 'static>(
&self,
hash: Self::Bytes,
) -> Result<I, StoreError> {
T::identifier(self, hash).await T::identifier(self, hash).await
} }
@ -326,7 +341,7 @@ where
hash: Self::Bytes, hash: Self::Bytes,
variant: String, variant: String,
identifier: &I, identifier: &I,
) -> Result<(), Error> { ) -> Result<(), StoreError> {
T::relate_variant_identifier(self, hash, variant, identifier).await T::relate_variant_identifier(self, hash, variant, identifier).await
} }
@ -334,18 +349,18 @@ where
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
variant: String, variant: String,
) -> Result<Option<I>, Error> { ) -> Result<Option<I>, StoreError> {
T::variant_identifier(self, hash, variant).await T::variant_identifier(self, hash, variant).await
} }
async fn variants<I: Identifier + 'static>( async fn variants<I: Identifier + 'static>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
) -> Result<Vec<(String, I)>, Error> { ) -> Result<Vec<(String, I)>, StoreError> {
T::variants(self, hash).await T::variants(self, hash).await
} }
async fn remove_variant(&self, hash: Self::Bytes, variant: String) -> Result<(), Error> { async fn remove_variant(&self, hash: Self::Bytes, variant: String) -> Result<(), RepoError> {
T::remove_variant(self, hash, variant).await T::remove_variant(self, hash, variant).await
} }
@ -353,37 +368,37 @@ where
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
identifier: &I, identifier: &I,
) -> Result<(), Error> { ) -> Result<(), StoreError> {
T::relate_motion_identifier(self, hash, identifier).await T::relate_motion_identifier(self, hash, identifier).await
} }
async fn motion_identifier<I: Identifier + 'static>( async fn motion_identifier<I: Identifier + 'static>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
) -> Result<Option<I>, Error> { ) -> Result<Option<I>, StoreError> {
T::motion_identifier(self, hash).await T::motion_identifier(self, hash).await
} }
async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Error> { async fn cleanup(&self, hash: Self::Bytes) -> Result<(), RepoError> {
T::cleanup(self, hash).await T::cleanup(self, hash).await
} }
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
pub(crate) trait AliasRepo: BaseRepo { pub(crate) trait AliasRepo: BaseRepo {
async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Error>; async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, RepoError>;
async fn relate_delete_token( async fn relate_delete_token(
&self, &self,
alias: &Alias, alias: &Alias,
delete_token: &DeleteToken, delete_token: &DeleteToken,
) -> Result<Result<(), AlreadyExists>, Error>; ) -> Result<Result<(), AlreadyExists>, RepoError>;
async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Error>; async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, RepoError>;
async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Error>; async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), RepoError>;
async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Error>; async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, RepoError>;
async fn cleanup(&self, alias: &Alias) -> Result<(), Error>; async fn cleanup(&self, alias: &Alias) -> Result<(), RepoError>;
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
@ -391,7 +406,7 @@ impl<T> AliasRepo for actix_web::web::Data<T>
where where
T: AliasRepo, T: AliasRepo,
{ {
async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Error> { async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, RepoError> {
T::create(self, alias).await T::create(self, alias).await
} }
@ -399,23 +414,23 @@ where
&self, &self,
alias: &Alias, alias: &Alias,
delete_token: &DeleteToken, delete_token: &DeleteToken,
) -> Result<Result<(), AlreadyExists>, Error> { ) -> Result<Result<(), AlreadyExists>, RepoError> {
T::relate_delete_token(self, alias, delete_token).await T::relate_delete_token(self, alias, delete_token).await
} }
async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Error> { async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, RepoError> {
T::delete_token(self, alias).await T::delete_token(self, alias).await
} }
async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Error> { async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), RepoError> {
T::relate_hash(self, alias, hash).await T::relate_hash(self, alias, hash).await
} }
async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Error> { async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, RepoError> {
T::hash(self, alias).await T::hash(self, alias).await
} }
async fn cleanup(&self, alias: &Alias) -> Result<(), Error> { async fn cleanup(&self, alias: &Alias) -> Result<(), RepoError> {
T::cleanup(self, alias).await T::cleanup(self, alias).await
} }
} }
@ -836,14 +851,14 @@ impl std::fmt::Display for Alias {
} }
impl Identifier for Vec<u8> { impl Identifier for Vec<u8> {
fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> fn from_bytes(bytes: Vec<u8>) -> Result<Self, StoreError>
where where
Self: Sized, Self: Sized,
{ {
Ok(bytes) Ok(bytes)
} }
fn to_bytes(&self) -> Result<Vec<u8>, Error> { fn to_bytes(&self) -> Result<Vec<u8>, StoreError> {
Ok(self.clone()) Ok(self.clone())
} }

View file

@ -1,10 +1,10 @@
use crate::{ use crate::{
error::{Error, UploadError},
repo::{ repo::{
Alias, AliasRepo, AlreadyExists, BaseRepo, DeleteToken, Details, FullRepo, HashRepo, Alias, AliasRepo, AlreadyExists, BaseRepo, DeleteToken, Details, FullRepo, HashRepo,
Identifier, IdentifierRepo, QueueRepo, SettingsRepo, UploadId, UploadRepo, UploadResult, Identifier, IdentifierRepo, QueueRepo, SettingsRepo, UploadId, UploadRepo, UploadResult,
}, },
serde_str::Serde, serde_str::Serde,
store::StoreError,
stream::from_iterator, stream::from_iterator,
}; };
use futures_util::Stream; use futures_util::Stream;
@ -19,6 +19,8 @@ use std::{
}; };
use tokio::sync::Notify; use tokio::sync::Notify;
use super::RepoError;
macro_rules! b { macro_rules! b {
($self:ident.$ident:ident, $expr:expr) => {{ ($self:ident.$ident:ident, $expr:expr) => {{
let $ident = $self.$ident.clone(); let $ident = $self.$ident.clone();
@ -27,7 +29,10 @@ macro_rules! b {
actix_rt::task::spawn_blocking(move || span.in_scope(|| $expr)) actix_rt::task::spawn_blocking(move || span.in_scope(|| $expr))
.await .await
.map_err(SledError::from)?? .map_err(SledError::from)
.map_err(RepoError::from)?
.map_err(SledError::from)
.map_err(RepoError::from)?
}}; }};
} }
@ -97,12 +102,12 @@ impl BaseRepo for SledRepo {
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
impl FullRepo for SledRepo { impl FullRepo for SledRepo {
async fn health_check(&self) -> Result<(), Error> { async fn health_check(&self) -> Result<(), RepoError> {
let next = self.healthz_count.fetch_add(1, Ordering::Relaxed); let next = self.healthz_count.fetch_add(1, Ordering::Relaxed);
b!(self.healthz, { b!(self.healthz, {
healthz.insert("healthz", &next.to_be_bytes()[..]) healthz.insert("healthz", &next.to_be_bytes()[..])
}); });
self.healthz.flush_async().await?; self.healthz.flush_async().await.map_err(SledError::from)?;
b!(self.healthz, healthz.get("healthz")); b!(self.healthz, healthz.get("healthz"));
Ok(()) Ok(())
} }
@ -146,13 +151,13 @@ impl From<InnerUploadResult> for UploadResult {
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
impl UploadRepo for SledRepo { impl UploadRepo for SledRepo {
#[tracing::instrument(level = "trace", skip(self))] #[tracing::instrument(level = "trace", skip(self))]
async fn create(&self, upload_id: UploadId) -> Result<(), Error> { async fn create(&self, upload_id: UploadId) -> Result<(), RepoError> {
b!(self.uploads, uploads.insert(upload_id.as_bytes(), b"1")); b!(self.uploads, uploads.insert(upload_id.as_bytes(), b"1"));
Ok(()) Ok(())
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, Error> { async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, RepoError> {
let mut subscriber = self.uploads.watch_prefix(upload_id.as_bytes()); let mut subscriber = self.uploads.watch_prefix(upload_id.as_bytes());
let bytes = upload_id.as_bytes().to_vec(); let bytes = upload_id.as_bytes().to_vec();
@ -160,40 +165,42 @@ impl UploadRepo for SledRepo {
if let Some(bytes) = opt { if let Some(bytes) = opt {
if bytes != b"1" { if bytes != b"1" {
let result: InnerUploadResult = serde_json::from_slice(&bytes)?; let result: InnerUploadResult =
serde_json::from_slice(&bytes).map_err(SledError::from)?;
return Ok(result.into()); return Ok(result.into());
} }
} else { } else {
return Err(UploadError::AlreadyClaimed.into()); return Err(RepoError::AlreadyClaimed);
} }
while let Some(event) = (&mut subscriber).await { while let Some(event) = (&mut subscriber).await {
match event { match event {
sled::Event::Remove { .. } => { sled::Event::Remove { .. } => {
return Err(UploadError::AlreadyClaimed.into()); return Err(RepoError::AlreadyClaimed);
} }
sled::Event::Insert { value, .. } => { sled::Event::Insert { value, .. } => {
if value != b"1" { if value != b"1" {
let result: InnerUploadResult = serde_json::from_slice(&value)?; let result: InnerUploadResult =
serde_json::from_slice(&value).map_err(SledError::from)?;
return Ok(result.into()); return Ok(result.into());
} }
} }
} }
} }
Err(UploadError::Canceled.into()) Err(RepoError::Canceled)
} }
#[tracing::instrument(level = "trace", skip(self))] #[tracing::instrument(level = "trace", skip(self))]
async fn claim(&self, upload_id: UploadId) -> Result<(), Error> { async fn claim(&self, upload_id: UploadId) -> Result<(), RepoError> {
b!(self.uploads, uploads.remove(upload_id.as_bytes())); b!(self.uploads, uploads.remove(upload_id.as_bytes()));
Ok(()) Ok(())
} }
#[tracing::instrument(level = "trace", skip(self, result))] #[tracing::instrument(level = "trace", skip(self, result))]
async fn complete(&self, upload_id: UploadId, result: UploadResult) -> Result<(), Error> { async fn complete(&self, upload_id: UploadId, result: UploadResult) -> Result<(), RepoError> {
let result: InnerUploadResult = result.into(); let result: InnerUploadResult = result.into();
let result = serde_json::to_vec(&result)?; let result = serde_json::to_vec(&result).map_err(SledError::from)?;
b!(self.uploads, uploads.insert(upload_id.as_bytes(), result)); b!(self.uploads, uploads.insert(upload_id.as_bytes(), result));
@ -204,7 +211,7 @@ impl UploadRepo for SledRepo {
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
impl QueueRepo for SledRepo { impl QueueRepo for SledRepo {
#[tracing::instrument(skip_all, fields(worker_id = %String::from_utf8_lossy(&worker_prefix)))] #[tracing::instrument(skip_all, fields(worker_id = %String::from_utf8_lossy(&worker_prefix)))]
async fn requeue_in_progress(&self, worker_prefix: Vec<u8>) -> Result<(), Error> { async fn requeue_in_progress(&self, worker_prefix: Vec<u8>) -> Result<(), RepoError> {
let vec: Vec<(String, IVec)> = b!(self.in_progress_queue, { let vec: Vec<(String, IVec)> = b!(self.in_progress_queue, {
let vec = in_progress_queue let vec = in_progress_queue
.scan_prefix(worker_prefix) .scan_prefix(worker_prefix)
@ -229,7 +236,7 @@ impl QueueRepo for SledRepo {
}) })
.collect::<Vec<(String, IVec)>>(); .collect::<Vec<(String, IVec)>>();
Ok(vec) as Result<_, Error> Ok(vec) as Result<_, SledError>
}); });
let db = self.db.clone(); let db = self.db.clone();
@ -242,15 +249,15 @@ impl QueueRepo for SledRepo {
queue.insert(key, job)?; queue.insert(key, job)?;
} }
Ok(()) as Result<(), Error> Ok(()) as Result<(), SledError>
}); });
Ok(()) Ok(())
} }
#[tracing::instrument(skip(self, job), fields(job = %String::from_utf8_lossy(&job)))] #[tracing::instrument(skip(self, job), fields(job = %String::from_utf8_lossy(&job)))]
async fn push(&self, queue_name: &'static str, job: Self::Bytes) -> Result<(), Error> { async fn push(&self, queue_name: &'static str, job: Self::Bytes) -> Result<(), RepoError> {
let id = self.db.generate_id()?; let id = self.db.generate_id().map_err(SledError::from)?;
let mut key = queue_name.as_bytes().to_vec(); let mut key = queue_name.as_bytes().to_vec();
key.extend(id.to_be_bytes()); key.extend(id.to_be_bytes());
@ -276,7 +283,7 @@ impl QueueRepo for SledRepo {
&self, &self,
queue_name: &'static str, queue_name: &'static str,
worker_id: Vec<u8>, worker_id: Vec<u8>,
) -> Result<Self::Bytes, Error> { ) -> Result<Self::Bytes, RepoError> {
loop { loop {
let in_progress_queue = self.in_progress_queue.clone(); let in_progress_queue = self.in_progress_queue.clone();
@ -333,21 +340,21 @@ impl QueueRepo for SledRepo {
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
impl SettingsRepo for SledRepo { impl SettingsRepo for SledRepo {
#[tracing::instrument(level = "trace", skip(value))] #[tracing::instrument(level = "trace", skip(value))]
async fn set(&self, key: &'static str, value: Self::Bytes) -> Result<(), Error> { async fn set(&self, key: &'static str, value: Self::Bytes) -> Result<(), RepoError> {
b!(self.settings, settings.insert(key, value)); b!(self.settings, settings.insert(key, value));
Ok(()) Ok(())
} }
#[tracing::instrument(level = "trace", skip(self))] #[tracing::instrument(level = "trace", skip(self))]
async fn get(&self, key: &'static str) -> Result<Option<Self::Bytes>, Error> { async fn get(&self, key: &'static str) -> Result<Option<Self::Bytes>, RepoError> {
let opt = b!(self.settings, settings.get(key)); let opt = b!(self.settings, settings.get(key));
Ok(opt) Ok(opt)
} }
#[tracing::instrument(level = "trace", skip(self))] #[tracing::instrument(level = "trace", skip(self))]
async fn remove(&self, key: &'static str) -> Result<(), Error> { async fn remove(&self, key: &'static str) -> Result<(), RepoError> {
b!(self.settings, settings.remove(key)); b!(self.settings, settings.remove(key));
Ok(()) Ok(())
@ -374,9 +381,11 @@ impl IdentifierRepo for SledRepo {
&self, &self,
identifier: &I, identifier: &I,
details: &Details, details: &Details,
) -> Result<(), Error> { ) -> Result<(), StoreError> {
let key = identifier.to_bytes()?; let key = identifier.to_bytes()?;
let details = serde_json::to_vec(&details)?; let details = serde_json::to_vec(&details)
.map_err(SledError::from)
.map_err(RepoError::from)?;
b!( b!(
self.identifier_details, self.identifier_details,
@ -387,18 +396,20 @@ impl IdentifierRepo for SledRepo {
} }
#[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))] #[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))]
async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Error> { async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, StoreError> {
let key = identifier.to_bytes()?; let key = identifier.to_bytes()?;
let opt = b!(self.identifier_details, identifier_details.get(key)); let opt = b!(self.identifier_details, identifier_details.get(key));
opt.map(|ivec| serde_json::from_slice(&ivec)) opt.map(|ivec| serde_json::from_slice(&ivec))
.transpose() .transpose()
.map_err(Error::from) .map_err(SledError::from)
.map_err(RepoError::from)
.map_err(StoreError::from)
} }
#[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))] #[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))]
async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Error> { async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), StoreError> {
let key = identifier.to_bytes()?; let key = identifier.to_bytes()?;
b!(self.identifier_details, identifier_details.remove(key)); b!(self.identifier_details, identifier_details.remove(key));
@ -407,7 +418,7 @@ impl IdentifierRepo for SledRepo {
} }
} }
type StreamItem = Result<IVec, Error>; type StreamItem = Result<IVec, RepoError>;
type LocalBoxStream<'a, T> = Pin<Box<dyn Stream<Item = T> + 'a>>; type LocalBoxStream<'a, T> = Pin<Box<dyn Stream<Item = T> + 'a>>;
fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> { fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> {
@ -425,13 +436,13 @@ impl HashRepo for SledRepo {
.hashes .hashes
.iter() .iter()
.keys() .keys()
.map(|res| res.map_err(Error::from)); .map(|res| res.map_err(SledError::from).map_err(RepoError::from));
Box::pin(from_iterator(iter, 8)) Box::pin(from_iterator(iter, 8))
} }
#[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))] #[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Error> { async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, RepoError> {
let res = b!(self.hashes, { let res = b!(self.hashes, {
let hash2 = hash.clone(); let hash2 = hash.clone();
hashes.compare_and_swap(hash, None as Option<Self::Bytes>, Some(hash2)) hashes.compare_and_swap(hash, None as Option<Self::Bytes>, Some(hash2))
@ -441,7 +452,7 @@ impl HashRepo for SledRepo {
} }
#[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))] #[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error> { async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), RepoError> {
let key = hash_alias_key(&hash, alias); let key = hash_alias_key(&hash, alias);
let value = alias.to_bytes(); let value = alias.to_bytes();
@ -451,7 +462,7 @@ impl HashRepo for SledRepo {
} }
#[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))] #[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error> { async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), RepoError> {
let key = hash_alias_key(&hash, alias); let key = hash_alias_key(&hash, alias);
b!(self.hash_aliases, hash_aliases.remove(key)); b!(self.hash_aliases, hash_aliases.remove(key));
@ -460,7 +471,7 @@ impl HashRepo for SledRepo {
} }
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Error> { async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, RepoError> {
let v = b!(self.hash_aliases, { let v = b!(self.hash_aliases, {
Ok(hash_aliases Ok(hash_aliases
.scan_prefix(hash) .scan_prefix(hash)
@ -478,7 +489,7 @@ impl HashRepo for SledRepo {
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
identifier: &I, identifier: &I,
) -> Result<(), Error> { ) -> Result<(), StoreError> {
let bytes = identifier.to_bytes()?; let bytes = identifier.to_bytes()?;
b!(self.hash_identifiers, hash_identifiers.insert(hash, bytes)); b!(self.hash_identifiers, hash_identifiers.insert(hash, bytes));
@ -487,11 +498,15 @@ impl HashRepo for SledRepo {
} }
#[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))] #[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn identifier<I: Identifier + 'static>(&self, hash: Self::Bytes) -> Result<I, Error> { async fn identifier<I: Identifier + 'static>(
&self,
hash: Self::Bytes,
) -> Result<I, StoreError> {
let opt = b!(self.hash_identifiers, hash_identifiers.get(hash)); let opt = b!(self.hash_identifiers, hash_identifiers.get(hash));
opt.ok_or(SledError::Missing) opt.ok_or(SledError::Missing)
.map_err(Error::from) .map_err(RepoError::from)
.map_err(StoreError::from)
.and_then(|ivec| I::from_bytes(ivec.to_vec())) .and_then(|ivec| I::from_bytes(ivec.to_vec()))
} }
@ -501,7 +516,7 @@ impl HashRepo for SledRepo {
hash: Self::Bytes, hash: Self::Bytes,
variant: String, variant: String,
identifier: &I, identifier: &I,
) -> Result<(), Error> { ) -> Result<(), StoreError> {
let key = variant_key(&hash, &variant); let key = variant_key(&hash, &variant);
let value = identifier.to_bytes()?; let value = identifier.to_bytes()?;
@ -518,7 +533,7 @@ impl HashRepo for SledRepo {
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
variant: String, variant: String,
) -> Result<Option<I>, Error> { ) -> Result<Option<I>, StoreError> {
let key = variant_key(&hash, &variant); let key = variant_key(&hash, &variant);
let opt = b!( let opt = b!(
@ -526,16 +541,14 @@ impl HashRepo for SledRepo {
hash_variant_identifiers.get(key) hash_variant_identifiers.get(key)
); );
opt.map(|ivec| I::from_bytes(ivec.to_vec())) opt.map(|ivec| I::from_bytes(ivec.to_vec())).transpose()
.transpose()
.map_err(Error::from)
} }
#[tracing::instrument(skip(self, hash), fields(hash = hex::encode(&hash)))] #[tracing::instrument(skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn variants<I: Identifier + 'static>( async fn variants<I: Identifier + 'static>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
) -> Result<Vec<(String, I)>, Error> { ) -> Result<Vec<(String, I)>, StoreError> {
let vec = b!( let vec = b!(
self.hash_variant_identifiers, self.hash_variant_identifiers,
Ok(hash_variant_identifiers Ok(hash_variant_identifiers
@ -557,14 +570,14 @@ impl HashRepo for SledRepo {
Some((variant?, identifier?)) Some((variant?, identifier?))
}) })
.collect::<Vec<_>>()) as Result<Vec<_>, sled::Error> .collect::<Vec<_>>()) as Result<Vec<_>, SledError>
); );
Ok(vec) Ok(vec)
} }
#[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))] #[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn remove_variant(&self, hash: Self::Bytes, variant: String) -> Result<(), Error> { async fn remove_variant(&self, hash: Self::Bytes, variant: String) -> Result<(), RepoError> {
let key = variant_key(&hash, &variant); let key = variant_key(&hash, &variant);
b!( b!(
@ -580,7 +593,7 @@ impl HashRepo for SledRepo {
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
identifier: &I, identifier: &I,
) -> Result<(), Error> { ) -> Result<(), StoreError> {
let bytes = identifier.to_bytes()?; let bytes = identifier.to_bytes()?;
b!( b!(
@ -595,19 +608,17 @@ impl HashRepo for SledRepo {
async fn motion_identifier<I: Identifier + 'static>( async fn motion_identifier<I: Identifier + 'static>(
&self, &self,
hash: Self::Bytes, hash: Self::Bytes,
) -> Result<Option<I>, Error> { ) -> Result<Option<I>, StoreError> {
let opt = b!( let opt = b!(
self.hash_motion_identifiers, self.hash_motion_identifiers,
hash_motion_identifiers.get(hash) hash_motion_identifiers.get(hash)
); );
opt.map(|ivec| I::from_bytes(ivec.to_vec())) opt.map(|ivec| I::from_bytes(ivec.to_vec())).transpose()
.transpose()
.map_err(Error::from)
} }
#[tracing::instrument(skip(self, hash), fields(hash = hex::encode(&hash)))] #[tracing::instrument(skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Error> { async fn cleanup(&self, hash: Self::Bytes) -> Result<(), RepoError> {
let hash2 = hash.clone(); let hash2 = hash.clone();
b!(self.hashes, hashes.remove(hash2)); b!(self.hashes, hashes.remove(hash2));
@ -628,7 +639,7 @@ impl HashRepo for SledRepo {
let _ = hash_aliases.remove(key); let _ = hash_aliases.remove(key);
} }
Ok(()) as Result<(), sled::Error> Ok(()) as Result<(), SledError>
}); });
let variant_keys = b!(self.hash_variant_identifiers, { let variant_keys = b!(self.hash_variant_identifiers, {
@ -638,13 +649,13 @@ impl HashRepo for SledRepo {
.filter_map(Result::ok) .filter_map(Result::ok)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
Ok(v) as Result<Vec<_>, sled::Error> Ok(v) as Result<Vec<_>, SledError>
}); });
b!(self.hash_variant_identifiers, { b!(self.hash_variant_identifiers, {
for key in variant_keys { for key in variant_keys {
let _ = hash_variant_identifiers.remove(key); let _ = hash_variant_identifiers.remove(key);
} }
Ok(()) as Result<(), sled::Error> Ok(()) as Result<(), SledError>
}); });
Ok(()) Ok(())
@ -654,7 +665,7 @@ impl HashRepo for SledRepo {
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
impl AliasRepo for SledRepo { impl AliasRepo for SledRepo {
#[tracing::instrument(level = "trace", skip(self))] #[tracing::instrument(level = "trace", skip(self))]
async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Error> { async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, RepoError> {
let bytes = alias.to_bytes(); let bytes = alias.to_bytes();
let bytes2 = bytes.clone(); let bytes2 = bytes.clone();
@ -671,7 +682,7 @@ impl AliasRepo for SledRepo {
&self, &self,
alias: &Alias, alias: &Alias,
delete_token: &DeleteToken, delete_token: &DeleteToken,
) -> Result<Result<(), AlreadyExists>, Error> { ) -> Result<Result<(), AlreadyExists>, RepoError> {
let key = alias.to_bytes(); let key = alias.to_bytes();
let token = delete_token.to_bytes(); let token = delete_token.to_bytes();
@ -684,18 +695,18 @@ impl AliasRepo for SledRepo {
} }
#[tracing::instrument(level = "trace", skip(self))] #[tracing::instrument(level = "trace", skip(self))]
async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Error> { async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, RepoError> {
let key = alias.to_bytes(); let key = alias.to_bytes();
let opt = b!(self.alias_delete_tokens, alias_delete_tokens.get(key)); let opt = b!(self.alias_delete_tokens, alias_delete_tokens.get(key));
opt.and_then(|ivec| DeleteToken::from_slice(&ivec)) opt.and_then(|ivec| DeleteToken::from_slice(&ivec))
.ok_or(SledError::Missing) .ok_or(SledError::Missing)
.map_err(Error::from) .map_err(RepoError::from)
} }
#[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))] #[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Error> { async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), RepoError> {
let key = alias.to_bytes(); let key = alias.to_bytes();
b!(self.alias_hashes, alias_hashes.insert(key, hash)); b!(self.alias_hashes, alias_hashes.insert(key, hash));
@ -704,16 +715,16 @@ impl AliasRepo for SledRepo {
} }
#[tracing::instrument(level = "trace", skip(self))] #[tracing::instrument(level = "trace", skip(self))]
async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Error> { async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, RepoError> {
let key = alias.to_bytes(); let key = alias.to_bytes();
let opt = b!(self.alias_hashes, alias_hashes.get(key)); let opt = b!(self.alias_hashes, alias_hashes.get(key));
opt.ok_or(SledError::Missing).map_err(Error::from) opt.ok_or(SledError::Missing).map_err(RepoError::from)
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn cleanup(&self, alias: &Alias) -> Result<(), Error> { async fn cleanup(&self, alias: &Alias) -> Result<(), RepoError> {
let key = alias.to_bytes(); let key = alias.to_bytes();
let key2 = key.clone(); let key2 = key.clone();

View file

@ -1,4 +1,3 @@
use crate::error::Error;
use actix_web::web::Bytes; use actix_web::web::Bytes;
use futures_util::stream::Stream; use futures_util::stream::Stream;
use std::fmt::Debug; use std::fmt::Debug;
@ -7,10 +6,56 @@ use tokio::io::{AsyncRead, AsyncWrite};
pub(crate) mod file_store; pub(crate) mod file_store;
pub(crate) mod object_store; pub(crate) mod object_store;
pub(crate) trait Identifier: Send + Sync + Clone + Debug { #[derive(Debug, thiserror::Error)]
fn to_bytes(&self) -> Result<Vec<u8>, Error>; pub(crate) enum StoreError {
#[error("Error in file store")]
FileStore(#[source] crate::store::file_store::FileError),
fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> #[error("Error in object store")]
ObjectStore(#[source] crate::store::object_store::ObjectError),
#[error("Error in DB")]
Repo(#[from] crate::repo::RepoError),
#[error("Requested file is not found")]
NotFound,
}
impl StoreError {
pub(crate) const fn is_not_found(&self) -> bool {
matches!(self, Self::NotFound)
}
}
impl From<crate::store::file_store::FileError> for StoreError {
fn from(value: crate::store::file_store::FileError) -> Self {
match value {
crate::store::file_store::FileError::Io(e)
if e.kind() == std::io::ErrorKind::NotFound =>
{
Self::NotFound
}
e => Self::FileStore(e),
}
}
}
impl From<crate::store::object_store::ObjectError> for StoreError {
fn from(value: crate::store::object_store::ObjectError) -> Self {
match value {
crate::store::object_store::ObjectError::Status(
actix_web::http::StatusCode::NOT_FOUND,
_,
) => Self::NotFound,
e => Self::ObjectStore(e),
}
}
}
pub(crate) trait Identifier: Send + Sync + Clone + Debug {
fn to_bytes(&self) -> Result<Vec<u8>, StoreError>;
fn from_bytes(bytes: Vec<u8>) -> Result<Self, StoreError>
where where
Self: Sized; Self: Sized;
@ -28,22 +73,22 @@ pub(crate) trait Store: Clone + Debug {
type Identifier: Identifier + 'static; type Identifier: Identifier + 'static;
type Stream: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static; type Stream: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static;
async fn save_async_read<Reader>(&self, reader: Reader) -> Result<Self::Identifier, Error> async fn save_async_read<Reader>(&self, reader: Reader) -> Result<Self::Identifier, StoreError>
where where
Reader: AsyncRead + Unpin + 'static; Reader: AsyncRead + Unpin + 'static;
async fn save_stream<S>(&self, stream: S) -> Result<Self::Identifier, Error> async fn save_stream<S>(&self, stream: S) -> Result<Self::Identifier, StoreError>
where where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static; S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static;
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Error>; async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, StoreError>;
async fn to_stream( async fn to_stream(
&self, &self,
identifier: &Self::Identifier, identifier: &Self::Identifier,
from_start: Option<u64>, from_start: Option<u64>,
len: Option<u64>, len: Option<u64>,
) -> Result<Self::Stream, Error>; ) -> Result<Self::Stream, StoreError>;
async fn read_into<Writer>( async fn read_into<Writer>(
&self, &self,
@ -53,9 +98,9 @@ pub(crate) trait Store: Clone + Debug {
where where
Writer: AsyncWrite + Unpin; Writer: AsyncWrite + Unpin;
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Error>; async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError>;
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error>; async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError>;
} }
#[async_trait::async_trait(?Send)] #[async_trait::async_trait(?Send)]
@ -66,21 +111,21 @@ where
type Identifier = T::Identifier; type Identifier = T::Identifier;
type Stream = T::Stream; type Stream = T::Stream;
async fn save_async_read<Reader>(&self, reader: Reader) -> Result<Self::Identifier, Error> async fn save_async_read<Reader>(&self, reader: Reader) -> Result<Self::Identifier, StoreError>
where where
Reader: AsyncRead + Unpin + 'static, Reader: AsyncRead + Unpin + 'static,
{ {
T::save_async_read(self, reader).await T::save_async_read(self, reader).await
} }
async fn save_stream<S>(&self, stream: S) -> Result<Self::Identifier, Error> async fn save_stream<S>(&self, stream: S) -> Result<Self::Identifier, StoreError>
where where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static, S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{ {
T::save_stream(self, stream).await T::save_stream(self, stream).await
} }
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Error> { async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, StoreError> {
T::save_bytes(self, bytes).await T::save_bytes(self, bytes).await
} }
@ -89,7 +134,7 @@ where
identifier: &Self::Identifier, identifier: &Self::Identifier,
from_start: Option<u64>, from_start: Option<u64>,
len: Option<u64>, len: Option<u64>,
) -> Result<Self::Stream, Error> { ) -> Result<Self::Stream, StoreError> {
T::to_stream(self, identifier, from_start, len).await T::to_stream(self, identifier, from_start, len).await
} }
@ -104,11 +149,11 @@ where
T::read_into(self, identifier, writer).await T::read_into(self, identifier, writer).await
} }
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Error> { async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError> {
T::len(self, identifier).await T::len(self, identifier).await
} }
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error> { async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError> {
T::remove(self, identifier).await T::remove(self, identifier).await
} }
} }
@ -121,21 +166,21 @@ where
type Identifier = T::Identifier; type Identifier = T::Identifier;
type Stream = T::Stream; type Stream = T::Stream;
async fn save_async_read<Reader>(&self, reader: Reader) -> Result<Self::Identifier, Error> async fn save_async_read<Reader>(&self, reader: Reader) -> Result<Self::Identifier, StoreError>
where where
Reader: AsyncRead + Unpin + 'static, Reader: AsyncRead + Unpin + 'static,
{ {
T::save_async_read(self, reader).await T::save_async_read(self, reader).await
} }
async fn save_stream<S>(&self, stream: S) -> Result<Self::Identifier, Error> async fn save_stream<S>(&self, stream: S) -> Result<Self::Identifier, StoreError>
where where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static, S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{ {
T::save_stream(self, stream).await T::save_stream(self, stream).await
} }
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Error> { async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, StoreError> {
T::save_bytes(self, bytes).await T::save_bytes(self, bytes).await
} }
@ -144,7 +189,7 @@ where
identifier: &Self::Identifier, identifier: &Self::Identifier,
from_start: Option<u64>, from_start: Option<u64>,
len: Option<u64>, len: Option<u64>,
) -> Result<Self::Stream, Error> { ) -> Result<Self::Stream, StoreError> {
T::to_stream(self, identifier, from_start, len).await T::to_stream(self, identifier, from_start, len).await
} }
@ -159,11 +204,11 @@ where
T::read_into(self, identifier, writer).await T::read_into(self, identifier, writer).await
} }
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Error> { async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError> {
T::len(self, identifier).await T::len(self, identifier).await
} }
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error> { async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError> {
T::remove(self, identifier).await T::remove(self, identifier).await
} }
} }

View file

@ -1,5 +1,4 @@
use crate::{ use crate::{
error::Error,
file::File, file::File,
repo::{Repo, SettingsRepo}, repo::{Repo, SettingsRepo},
store::{Store, StoreConfig}, store::{Store, StoreConfig},
@ -18,6 +17,8 @@ use tracing::Instrument;
mod file_id; mod file_id;
pub(crate) use file_id::FileId; pub(crate) use file_id::FileId;
use super::StoreError;
// - Settings Tree // - Settings Tree
// - last-path -> last generated path // - last-path -> last generated path
@ -62,7 +63,10 @@ impl Store for FileStore {
type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>; type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
#[tracing::instrument(skip(reader))] #[tracing::instrument(skip(reader))]
async fn save_async_read<Reader>(&self, mut reader: Reader) -> Result<Self::Identifier, Error> async fn save_async_read<Reader>(
&self,
mut reader: Reader,
) -> Result<Self::Identifier, StoreError>
where where
Reader: AsyncRead + Unpin + 'static, Reader: AsyncRead + Unpin + 'static,
{ {
@ -76,7 +80,7 @@ impl Store for FileStore {
Ok(self.file_id_from_path(path)?) Ok(self.file_id_from_path(path)?)
} }
async fn save_stream<S>(&self, stream: S) -> Result<Self::Identifier, Error> async fn save_stream<S>(&self, stream: S) -> Result<Self::Identifier, StoreError>
where where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static, S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{ {
@ -84,7 +88,7 @@ impl Store for FileStore {
} }
#[tracing::instrument(skip(bytes))] #[tracing::instrument(skip(bytes))]
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Error> { async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, StoreError> {
let path = self.next_file().await?; let path = self.next_file().await?;
if let Err(e) = self.safe_save_bytes(&path, bytes).await { if let Err(e) = self.safe_save_bytes(&path, bytes).await {
@ -101,14 +105,15 @@ impl Store for FileStore {
identifier: &Self::Identifier, identifier: &Self::Identifier,
from_start: Option<u64>, from_start: Option<u64>,
len: Option<u64>, len: Option<u64>,
) -> Result<Self::Stream, Error> { ) -> Result<Self::Stream, StoreError> {
let path = self.path_from_file_id(identifier); let path = self.path_from_file_id(identifier);
let file_span = tracing::trace_span!(parent: None, "File Stream"); let file_span = tracing::trace_span!(parent: None, "File Stream");
let file = file_span let file = file_span
.in_scope(|| File::open(path)) .in_scope(|| File::open(path))
.instrument(file_span.clone()) .instrument(file_span.clone())
.await?; .await
.map_err(FileError::from)?;
let stream = file_span let stream = file_span
.in_scope(|| file.read_to_stream(from_start, len)) .in_scope(|| file.read_to_stream(from_start, len))
@ -135,16 +140,19 @@ impl Store for FileStore {
} }
#[tracing::instrument] #[tracing::instrument]
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Error> { async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError> {
let path = self.path_from_file_id(identifier); let path = self.path_from_file_id(identifier);
let len = tokio::fs::metadata(path).await?.len(); let len = tokio::fs::metadata(path)
.await
.map_err(FileError::from)?
.len();
Ok(len) Ok(len)
} }
#[tracing::instrument] #[tracing::instrument]
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error> { async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError> {
let path = self.path_from_file_id(identifier); let path = self.path_from_file_id(identifier);
self.safe_remove_file(path).await?; self.safe_remove_file(path).await?;
@ -154,7 +162,7 @@ impl Store for FileStore {
} }
impl FileStore { impl FileStore {
pub(crate) async fn build(root_dir: PathBuf, repo: Repo) -> Result<Self, Error> { pub(crate) async fn build(root_dir: PathBuf, repo: Repo) -> Result<Self, StoreError> {
let path_gen = init_generator(&repo).await?; let path_gen = init_generator(&repo).await?;
Ok(FileStore { Ok(FileStore {
@ -164,7 +172,7 @@ impl FileStore {
}) })
} }
async fn next_directory(&self) -> Result<PathBuf, Error> { async fn next_directory(&self) -> Result<PathBuf, StoreError> {
let path = self.path_gen.next(); let path = self.path_gen.next();
match self.repo { match self.repo {
@ -183,7 +191,7 @@ impl FileStore {
Ok(target_path) Ok(target_path)
} }
async fn next_file(&self) -> Result<PathBuf, Error> { async fn next_file(&self) -> Result<PathBuf, StoreError> {
let target_path = self.next_directory().await?; let target_path = self.next_directory().await?;
let filename = uuid::Uuid::new_v4().to_string(); let filename = uuid::Uuid::new_v4().to_string();
@ -271,12 +279,13 @@ pub(crate) async fn safe_create_parent<P: AsRef<Path>>(path: P) -> Result<(), Fi
Ok(()) Ok(())
} }
async fn init_generator(repo: &Repo) -> Result<Generator, Error> { async fn init_generator(repo: &Repo) -> Result<Generator, StoreError> {
match repo { match repo {
Repo::Sled(sled_repo) => { Repo::Sled(sled_repo) => {
if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? { if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
Ok(Generator::from_existing( Ok(Generator::from_existing(
storage_path_generator::Path::from_be_bytes(ivec.to_vec())?, storage_path_generator::Path::from_be_bytes(ivec.to_vec())
.map_err(FileError::from)?,
)) ))
} else { } else {
Ok(Generator::new()) Ok(Generator::new())

View file

@ -1,9 +1,6 @@
use crate::{ use crate::store::{
error::Error, file_store::{FileError, FileStore},
store::{ Identifier, StoreError,
file_store::{FileError, FileStore},
Identifier,
},
}; };
use std::path::PathBuf; use std::path::PathBuf;
@ -11,7 +8,7 @@ use std::path::PathBuf;
pub(crate) struct FileId(PathBuf); pub(crate) struct FileId(PathBuf);
impl Identifier for FileId { impl Identifier for FileId {
fn to_bytes(&self) -> Result<Vec<u8>, Error> { fn to_bytes(&self) -> Result<Vec<u8>, StoreError> {
let vec = self let vec = self
.0 .0
.to_str() .to_str()
@ -22,7 +19,7 @@ impl Identifier for FileId {
Ok(vec) Ok(vec)
} }
fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> fn from_bytes(bytes: Vec<u8>) -> Result<Self, StoreError>
where where
Self: Sized, Self: Sized,
{ {

View file

@ -1,6 +1,5 @@
use crate::{ use crate::{
bytes_stream::BytesStream, bytes_stream::BytesStream,
error::Error,
repo::{Repo, SettingsRepo}, repo::{Repo, SettingsRepo},
store::{Store, StoreConfig}, store::{Store, StoreConfig},
}; };
@ -27,6 +26,8 @@ use url::Url;
mod object_id; mod object_id;
pub(crate) use object_id::ObjectId; pub(crate) use object_id::ObjectId;
use super::StoreError;
const CHUNK_SIZE: usize = 8_388_608; // 8 Mebibytes, min is 5 (5_242_880); const CHUNK_SIZE: usize = 8_388_608; // 8 Mebibytes, min is 5 (5_242_880);
// - Settings Tree // - Settings Tree
@ -42,6 +43,9 @@ pub(crate) enum ObjectError {
#[error("Failed to generate request")] #[error("Failed to generate request")]
S3(#[from] BucketError), S3(#[from] BucketError),
#[error("IO Error")]
IO(#[from] std::io::Error),
#[error("Error making request")] #[error("Error making request")]
SendRequest(String), SendRequest(String),
@ -62,6 +66,9 @@ pub(crate) enum ObjectError {
#[error("Invalid status: {0}\n{1}")] #[error("Invalid status: {0}\n{1}")]
Status(StatusCode, String), Status(StatusCode, String),
#[error("Unable to upload image")]
Upload(awc::error::PayloadError),
} }
impl From<SendRequestError> for ObjectError { impl From<SendRequestError> for ObjectError {
@ -131,7 +138,7 @@ fn payload_to_io_error(e: PayloadError) -> std::io::Error {
} }
#[tracing::instrument(skip(stream))] #[tracing::instrument(skip(stream))]
async fn read_chunk<S>(stream: &mut S) -> std::io::Result<BytesStream> async fn read_chunk<S>(stream: &mut S) -> Result<BytesStream, ObjectError>
where where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static, S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{ {
@ -148,9 +155,9 @@ where
Ok(buf) Ok(buf)
} }
async fn status_error(mut response: ClientResponse) -> Error { async fn status_error(mut response: ClientResponse) -> StoreError {
let body = match response.body().await { let body = match response.body().await {
Err(e) => return e.into(), Err(e) => return ObjectError::Upload(e).into(),
Ok(body) => body, Ok(body) => body,
}; };
@ -164,7 +171,7 @@ impl Store for ObjectStore {
type Identifier = ObjectId; type Identifier = ObjectId;
type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>; type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
async fn save_async_read<Reader>(&self, reader: Reader) -> Result<Self::Identifier, Error> async fn save_async_read<Reader>(&self, reader: Reader) -> Result<Self::Identifier, StoreError>
where where
Reader: AsyncRead + Unpin + 'static, Reader: AsyncRead + Unpin + 'static,
{ {
@ -172,7 +179,7 @@ impl Store for ObjectStore {
} }
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn save_stream<S>(&self, mut stream: S) -> Result<Self::Identifier, Error> async fn save_stream<S>(&self, mut stream: S) -> Result<Self::Identifier, StoreError>
where where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static, S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{ {
@ -181,7 +188,10 @@ impl Store for ObjectStore {
if first_chunk.len() < CHUNK_SIZE { if first_chunk.len() < CHUNK_SIZE {
drop(stream); drop(stream);
let (req, object_id) = self.put_object_request().await?; let (req, object_id) = self.put_object_request().await?;
let response = req.send_body(first_chunk).await?; let response = req
.send_body(first_chunk)
.await
.map_err(ObjectError::from)?;
if !response.status().is_success() { if !response.status().is_success() {
return Err(status_error(response).await); return Err(status_error(response).await);
@ -199,7 +209,7 @@ impl Store for ObjectStore {
return Err(status_error(response).await); return Err(status_error(response).await);
} }
let body = response.body().await?; let body = response.body().await.map_err(ObjectError::Upload)?;
let body: InitiateMultipartUploadResponse = let body: InitiateMultipartUploadResponse =
quick_xml::de::from_reader(&*body).map_err(ObjectError::from)?; quick_xml::de::from_reader(&*body).map_err(ObjectError::from)?;
let upload_id = &body.upload_id; let upload_id = &body.upload_id;
@ -236,7 +246,8 @@ impl Store for ObjectStore {
) )
.await? .await?
.send_body(buf) .send_body(buf)
.await?; .await
.map_err(ObjectError::from)?;
if !response.status().is_success() { if !response.status().is_success() {
return Err(status_error(response).await); return Err(status_error(response).await);
@ -253,7 +264,7 @@ impl Store for ObjectStore {
// early-drop response to close its tracing spans // early-drop response to close its tracing spans
drop(response); drop(response);
Ok(etag) as Result<String, Error> Ok(etag) as Result<String, StoreError>
} }
.instrument(tracing::Span::current()), .instrument(tracing::Span::current()),
); );
@ -276,20 +287,22 @@ impl Store for ObjectStore {
upload_id, upload_id,
etags.iter().map(|s| s.as_ref()), etags.iter().map(|s| s.as_ref()),
) )
.await?; .await
.map_err(ObjectError::from)?;
if !response.status().is_success() { if !response.status().is_success() {
return Err(status_error(response).await); return Err(status_error(response).await);
} }
Ok(()) as Result<(), Error> Ok(()) as Result<(), StoreError>
} }
.await; .await;
if let Err(e) = res { if let Err(e) = res {
self.create_abort_multipart_request(&object_id, upload_id) self.create_abort_multipart_request(&object_id, upload_id)
.send() .send()
.await?; .await
.map_err(ObjectError::from)?;
return Err(e); return Err(e);
} }
@ -297,7 +310,7 @@ impl Store for ObjectStore {
} }
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Error> { async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, StoreError> {
let (req, object_id) = self.put_object_request().await?; let (req, object_id) = self.put_object_request().await?;
let response = req.send_body(bytes).await.map_err(ObjectError::from)?; let response = req.send_body(bytes).await.map_err(ObjectError::from)?;
@ -315,7 +328,7 @@ impl Store for ObjectStore {
identifier: &Self::Identifier, identifier: &Self::Identifier,
from_start: Option<u64>, from_start: Option<u64>,
len: Option<u64>, len: Option<u64>,
) -> Result<Self::Stream, Error> { ) -> Result<Self::Stream, StoreError> {
let response = self let response = self
.get_object_request(identifier, from_start, len) .get_object_request(identifier, from_start, len)
.send() .send()
@ -361,7 +374,7 @@ impl Store for ObjectStore {
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Error> { async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError> {
let response = self let response = self
.head_object_request(identifier) .head_object_request(identifier)
.send() .send()
@ -385,8 +398,12 @@ impl Store for ObjectStore {
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error> { async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError> {
let response = self.delete_object_request(identifier).send().await?; let response = self
.delete_object_request(identifier)
.send()
.await
.map_err(ObjectError::from)?;
if !response.status().is_success() { if !response.status().is_success() {
return Err(status_error(response).await); return Err(status_error(response).await);
@ -407,7 +424,7 @@ impl ObjectStore {
secret_key: String, secret_key: String,
session_token: Option<String>, session_token: Option<String>,
repo: Repo, repo: Repo,
) -> Result<ObjectStoreConfig, Error> { ) -> Result<ObjectStoreConfig, StoreError> {
let path_gen = init_generator(&repo).await?; let path_gen = init_generator(&repo).await?;
Ok(ObjectStoreConfig { Ok(ObjectStoreConfig {
@ -423,7 +440,7 @@ impl ObjectStore {
}) })
} }
async fn put_object_request(&self) -> Result<(ClientRequest, ObjectId), Error> { async fn put_object_request(&self) -> Result<(ClientRequest, ObjectId), StoreError> {
let path = self.next_file().await?; let path = self.next_file().await?;
let mut action = self.bucket.put_object(Some(&self.credentials), &path); let mut action = self.bucket.put_object(Some(&self.credentials), &path);
@ -435,7 +452,7 @@ impl ObjectStore {
Ok((self.build_request(action), ObjectId::from_string(path))) Ok((self.build_request(action), ObjectId::from_string(path)))
} }
async fn create_multipart_request(&self) -> Result<(ClientRequest, ObjectId), Error> { async fn create_multipart_request(&self) -> Result<(ClientRequest, ObjectId), StoreError> {
let path = self.next_file().await?; let path = self.next_file().await?;
let mut action = self let mut action = self
@ -455,7 +472,7 @@ impl ObjectStore {
object_id: &ObjectId, object_id: &ObjectId,
part_number: u16, part_number: u16,
upload_id: &str, upload_id: &str,
) -> Result<ClientRequest, Error> { ) -> Result<ClientRequest, ObjectError> {
use md5::Digest; use md5::Digest;
let mut action = self.bucket.upload_part( let mut action = self.bucket.upload_part(
@ -593,7 +610,7 @@ impl ObjectStore {
self.build_request(action) self.build_request(action)
} }
async fn next_directory(&self) -> Result<Path, Error> { async fn next_directory(&self) -> Result<Path, StoreError> {
let path = self.path_gen.next(); let path = self.path_gen.next();
match self.repo { match self.repo {
@ -607,7 +624,7 @@ impl ObjectStore {
Ok(path) Ok(path)
} }
async fn next_file(&self) -> Result<String, Error> { async fn next_file(&self) -> Result<String, StoreError> {
let path = self.next_directory().await?.to_strings().join("/"); let path = self.next_directory().await?.to_strings().join("/");
let filename = uuid::Uuid::new_v4().to_string(); let filename = uuid::Uuid::new_v4().to_string();
@ -615,12 +632,13 @@ impl ObjectStore {
} }
} }
async fn init_generator(repo: &Repo) -> Result<Generator, Error> { async fn init_generator(repo: &Repo) -> Result<Generator, StoreError> {
match repo { match repo {
Repo::Sled(sled_repo) => { Repo::Sled(sled_repo) => {
if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? { if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
Ok(Generator::from_existing( Ok(Generator::from_existing(
storage_path_generator::Path::from_be_bytes(ivec.to_vec())?, storage_path_generator::Path::from_be_bytes(ivec.to_vec())
.map_err(ObjectError::from)?,
)) ))
} else { } else {
Ok(Generator::new()) Ok(Generator::new())

View file

@ -1,17 +1,14 @@
use crate::{ use crate::store::{object_store::ObjectError, Identifier, StoreError};
error::Error,
store::{object_store::ObjectError, Identifier},
};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub(crate) struct ObjectId(String); pub(crate) struct ObjectId(String);
impl Identifier for ObjectId { impl Identifier for ObjectId {
fn to_bytes(&self) -> Result<Vec<u8>, Error> { fn to_bytes(&self) -> Result<Vec<u8>, StoreError> {
Ok(self.0.as_bytes().to_vec()) Ok(self.0.as_bytes().to_vec())
} }
fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> { fn from_bytes(bytes: Vec<u8>) -> Result<Self, StoreError> {
Ok(ObjectId( Ok(ObjectId(
String::from_utf8(bytes).map_err(ObjectError::from)?, String::from_utf8(bytes).map_err(ObjectError::from)?,
)) ))