mirror of
https://git.asonix.dog/asonix/pict-rs
synced 2024-12-22 19:31:35 +00:00
Get it compiling again
This commit is contained in:
parent
323016f994
commit
15b52ba6ec
16 changed files with 1032 additions and 1263 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
@ -1610,7 +1610,7 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pict-rs"
|
name = "pict-rs"
|
||||||
version = "0.3.0-rc.7"
|
version = "0.4.0-alpha.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"actix-form-data",
|
"actix-form-data",
|
||||||
"actix-rt",
|
"actix-rt",
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
[package]
|
[package]
|
||||||
name = "pict-rs"
|
name = "pict-rs"
|
||||||
description = "A simple image hosting service"
|
description = "A simple image hosting service"
|
||||||
version = "0.3.0-rc.7"
|
version = "0.4.0-alpha.1"
|
||||||
authors = ["asonix <asonix@asonix.dog>"]
|
authors = ["asonix <asonix@asonix.dog>"]
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
|
|
39
src/error.rs
39
src/error.rs
|
@ -37,39 +37,33 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<sled::transaction::TransactionError<Error>> for Error {
|
|
||||||
fn from(e: sled::transaction::TransactionError<Error>) -> Self {
|
|
||||||
match e {
|
|
||||||
sled::transaction::TransactionError::Abort(t) => t,
|
|
||||||
sled::transaction::TransactionError::Storage(e) => e.into(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub(crate) enum UploadError {
|
pub(crate) enum UploadError {
|
||||||
#[error("Couln't upload file, {0}")]
|
#[error("Couln't upload file")]
|
||||||
Upload(#[from] actix_form_data::Error),
|
Upload(#[from] actix_form_data::Error),
|
||||||
|
|
||||||
#[error("Error in DB, {0}")]
|
#[error("Error in DB")]
|
||||||
Db(#[from] sled::Error),
|
Sled(#[from] crate::repo::sled::Error),
|
||||||
|
|
||||||
#[error("Error parsing string, {0}")]
|
#[error("Error in old sled DB")]
|
||||||
|
OldSled(#[from] ::sled::Error),
|
||||||
|
|
||||||
|
#[error("Error parsing string")]
|
||||||
ParseString(#[from] std::string::FromUtf8Error),
|
ParseString(#[from] std::string::FromUtf8Error),
|
||||||
|
|
||||||
#[error("Error interacting with filesystem, {0}")]
|
#[error("Error interacting with filesystem")]
|
||||||
Io(#[from] std::io::Error),
|
Io(#[from] std::io::Error),
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error("Error generating path")]
|
||||||
PathGenerator(#[from] storage_path_generator::PathError),
|
PathGenerator(#[from] storage_path_generator::PathError),
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error("Error stripping prefix")]
|
||||||
StripPrefix(#[from] std::path::StripPrefixError),
|
StripPrefix(#[from] std::path::StripPrefixError),
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error("Error storing file")]
|
||||||
FileStore(#[from] crate::store::file_store::FileError),
|
FileStore(#[from] crate::store::file_store::FileError),
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error("Error storing object")]
|
||||||
ObjectStore(#[from] crate::store::object_store::ObjectError),
|
ObjectStore(#[from] crate::store::object_store::ObjectError),
|
||||||
|
|
||||||
#[error("Provided process path is invalid")]
|
#[error("Provided process path is invalid")]
|
||||||
|
@ -87,9 +81,6 @@ pub(crate) enum UploadError {
|
||||||
#[error("Requested a file that doesn't exist")]
|
#[error("Requested a file that doesn't exist")]
|
||||||
MissingAlias,
|
MissingAlias,
|
||||||
|
|
||||||
#[error("Alias directed to missing file")]
|
|
||||||
MissingFile,
|
|
||||||
|
|
||||||
#[error("Provided token did not match expected token")]
|
#[error("Provided token did not match expected token")]
|
||||||
InvalidToken,
|
InvalidToken,
|
||||||
|
|
||||||
|
@ -102,7 +93,7 @@ pub(crate) enum UploadError {
|
||||||
#[error("Unable to download image, bad response {0}")]
|
#[error("Unable to download image, bad response {0}")]
|
||||||
Download(actix_web::http::StatusCode),
|
Download(actix_web::http::StatusCode),
|
||||||
|
|
||||||
#[error("Unable to download image, {0}")]
|
#[error("Unable to download image")]
|
||||||
Payload(#[from] awc::error::PayloadError),
|
Payload(#[from] awc::error::PayloadError),
|
||||||
|
|
||||||
#[error("Unable to send request, {0}")]
|
#[error("Unable to send request, {0}")]
|
||||||
|
@ -117,13 +108,13 @@ pub(crate) enum UploadError {
|
||||||
#[error("Tried to save an image with an already-taken name")]
|
#[error("Tried to save an image with an already-taken name")]
|
||||||
DuplicateAlias,
|
DuplicateAlias,
|
||||||
|
|
||||||
#[error("{0}")]
|
#[error("Error in json")]
|
||||||
Json(#[from] serde_json::Error),
|
Json(#[from] serde_json::Error),
|
||||||
|
|
||||||
#[error("Range header not satisfiable")]
|
#[error("Range header not satisfiable")]
|
||||||
Range,
|
Range,
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error("Hit limit")]
|
||||||
Limit(#[from] super::LimitError),
|
Limit(#[from] super::LimitError),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,7 @@ use crate::{
|
||||||
config::Format,
|
config::Format,
|
||||||
error::{Error, UploadError},
|
error::{Error, UploadError},
|
||||||
process::Process,
|
process::Process,
|
||||||
|
repo::Alias,
|
||||||
store::Store,
|
store::Store,
|
||||||
};
|
};
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
|
@ -11,8 +12,9 @@ use tokio::{
|
||||||
};
|
};
|
||||||
use tracing::instrument;
|
use tracing::instrument;
|
||||||
|
|
||||||
pub(crate) fn details_hint(filename: &str) -> Option<ValidInputType> {
|
pub(crate) fn details_hint(alias: &Alias) -> Option<ValidInputType> {
|
||||||
if filename.ends_with(".mp4") {
|
let ext = alias.extension()?;
|
||||||
|
if ext.ends_with(".mp4") {
|
||||||
Some(ValidInputType::Mp4)
|
Some(ValidInputType::Mp4)
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
|
|
207
src/main.rs
207
src/main.rs
|
@ -57,6 +57,7 @@ use self::{
|
||||||
magick::details_hint,
|
magick::details_hint,
|
||||||
middleware::{Deadline, Internal},
|
middleware::{Deadline, Internal},
|
||||||
migrate::LatestDb,
|
migrate::LatestDb,
|
||||||
|
repo::{Alias, DeleteToken, Repo},
|
||||||
store::{file_store::FileStore, object_store::ObjectStore, Store},
|
store::{file_store::FileStore, object_store::ObjectStore, Store},
|
||||||
upload_manager::{UploadManager, UploadManagerSession},
|
upload_manager::{UploadManager, UploadManagerSession},
|
||||||
};
|
};
|
||||||
|
@ -96,30 +97,26 @@ where
|
||||||
info!("Uploaded {} as {:?}", image.filename, alias);
|
info!("Uploaded {} as {:?}", image.filename, alias);
|
||||||
let delete_token = image.result.delete_token().await?;
|
let delete_token = image.result.delete_token().await?;
|
||||||
|
|
||||||
let name = manager.from_alias(alias.to_owned()).await?;
|
let identifier = manager.identifier_from_alias::<S>(alias).await?;
|
||||||
let identifier = manager.identifier_from_filename::<S>(name.clone()).await?;
|
let details = manager.details(&identifier).await?;
|
||||||
|
|
||||||
let details = manager.variant_details(&identifier, name.clone()).await?;
|
|
||||||
|
|
||||||
let details = if let Some(details) = details {
|
let details = if let Some(details) = details {
|
||||||
debug!("details exist");
|
debug!("details exist");
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
debug!("generating new details from {:?}", identifier);
|
debug!("generating new details from {:?}", identifier);
|
||||||
let hint = details_hint(&name);
|
let hint = details_hint(alias);
|
||||||
let new_details =
|
let new_details =
|
||||||
Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
||||||
debug!("storing details for {:?} {}", identifier, name);
|
debug!("storing details for {:?}", identifier);
|
||||||
manager
|
manager.store_details(&identifier, &new_details).await?;
|
||||||
.store_variant_details(&identifier, name, &new_details)
|
|
||||||
.await?;
|
|
||||||
debug!("stored");
|
debug!("stored");
|
||||||
new_details
|
new_details
|
||||||
};
|
};
|
||||||
|
|
||||||
files.push(serde_json::json!({
|
files.push(serde_json::json!({
|
||||||
"file": alias,
|
"file": alias.to_string(),
|
||||||
"delete_token": delete_token,
|
"delete_token": delete_token.to_string(),
|
||||||
"details": details,
|
"details": details,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
@ -222,19 +219,16 @@ where
|
||||||
drop(permit);
|
drop(permit);
|
||||||
let delete_token = session.delete_token().await?;
|
let delete_token = session.delete_token().await?;
|
||||||
|
|
||||||
let name = manager.from_alias(alias.to_owned()).await?;
|
let identifier = manager.identifier_from_alias::<S>(&alias).await?;
|
||||||
let identifier = manager.identifier_from_filename::<S>(name.clone()).await?;
|
|
||||||
|
|
||||||
let details = manager.variant_details(&identifier, name.clone()).await?;
|
let details = manager.details(&identifier).await?;
|
||||||
|
|
||||||
let details = if let Some(details) = details {
|
let details = if let Some(details) = details {
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
let hint = details_hint(&name);
|
let hint = details_hint(&alias);
|
||||||
let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
||||||
manager
|
manager.store_details(&identifier, &new_details).await?;
|
||||||
.store_variant_details(&identifier, name, &new_details)
|
|
||||||
.await?;
|
|
||||||
new_details
|
new_details
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -242,8 +236,8 @@ where
|
||||||
Ok(HttpResponse::Created().json(&serde_json::json!({
|
Ok(HttpResponse::Created().json(&serde_json::json!({
|
||||||
"msg": "ok",
|
"msg": "ok",
|
||||||
"files": [{
|
"files": [{
|
||||||
"file": alias,
|
"file": alias.to_string(),
|
||||||
"delete_token": delete_token,
|
"delete_token": delete_token.to_string(),
|
||||||
"details": details,
|
"details": details,
|
||||||
}]
|
}]
|
||||||
})))
|
})))
|
||||||
|
@ -261,19 +255,21 @@ where
|
||||||
{
|
{
|
||||||
let (alias, token) = path_entries.into_inner();
|
let (alias, token) = path_entries.into_inner();
|
||||||
|
|
||||||
manager.delete((**store).clone(), token, alias).await?;
|
let alias = Alias::from_existing(&alias);
|
||||||
|
let token = DeleteToken::from_existing(&token);
|
||||||
|
|
||||||
|
manager.delete((**store).clone(), alias, token).await?;
|
||||||
|
|
||||||
Ok(HttpResponse::NoContent().finish())
|
Ok(HttpResponse::NoContent().finish())
|
||||||
}
|
}
|
||||||
|
|
||||||
type ProcessQuery = Vec<(String, String)>;
|
type ProcessQuery = Vec<(String, String)>;
|
||||||
|
|
||||||
async fn prepare_process(
|
fn prepare_process(
|
||||||
query: web::Query<ProcessQuery>,
|
query: web::Query<ProcessQuery>,
|
||||||
ext: &str,
|
ext: &str,
|
||||||
manager: &UploadManager,
|
|
||||||
filters: &Option<HashSet<String>>,
|
filters: &Option<HashSet<String>>,
|
||||||
) -> Result<(Format, String, PathBuf, Vec<String>), Error> {
|
) -> Result<(Format, Alias, PathBuf, Vec<String>), Error> {
|
||||||
let (alias, operations) =
|
let (alias, operations) =
|
||||||
query
|
query
|
||||||
.into_inner()
|
.into_inner()
|
||||||
|
@ -291,7 +287,7 @@ async fn prepare_process(
|
||||||
return Err(UploadError::MissingFilename.into());
|
return Err(UploadError::MissingFilename.into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let name = manager.from_alias(alias).await?;
|
let alias = Alias::from_existing(&alias);
|
||||||
|
|
||||||
let operations = if let Some(filters) = filters.as_ref() {
|
let operations = if let Some(filters) = filters.as_ref() {
|
||||||
operations
|
operations
|
||||||
|
@ -305,12 +301,10 @@ async fn prepare_process(
|
||||||
let format = ext
|
let format = ext
|
||||||
.parse::<Format>()
|
.parse::<Format>()
|
||||||
.map_err(|_| UploadError::UnsupportedFormat)?;
|
.map_err(|_| UploadError::UnsupportedFormat)?;
|
||||||
let processed_name = format!("{}.{}", name, ext);
|
|
||||||
|
|
||||||
let (thumbnail_path, thumbnail_args) =
|
let (thumbnail_path, thumbnail_args) = self::processor::build_chain(&operations)?;
|
||||||
self::processor::build_chain(&operations, processed_name)?;
|
|
||||||
|
|
||||||
Ok((format, name, thumbnail_path, thumbnail_args))
|
Ok((format, alias, thumbnail_path, thumbnail_args))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Fetching derived details", skip(manager, filters))]
|
#[instrument(name = "Fetching derived details", skip(manager, filters))]
|
||||||
|
@ -324,15 +318,14 @@ async fn process_details<S: Store>(
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
let (_, name, thumbnail_path, _) =
|
let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
|
||||||
prepare_process(query, ext.as_str(), &manager, &filters).await?;
|
|
||||||
|
|
||||||
let identifier = manager
|
let identifier = manager
|
||||||
.variant_identifier::<S>(&thumbnail_path, &name)
|
.variant_identifier::<S>(&alias, &thumbnail_path)
|
||||||
.await?
|
.await?
|
||||||
.ok_or(UploadError::MissingAlias)?;
|
.ok_or(UploadError::MissingAlias)?;
|
||||||
|
|
||||||
let details = manager.variant_details(&identifier, name).await?;
|
let details = manager.details(&identifier).await?;
|
||||||
|
|
||||||
let details = details.ok_or(UploadError::NoFiles)?;
|
let details = details.ok_or(UploadError::NoFiles)?;
|
||||||
|
|
||||||
|
@ -352,24 +345,22 @@ async fn process<S: Store + 'static>(
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
let (format, name, thumbnail_path, thumbnail_args) =
|
let (format, alias, thumbnail_path, thumbnail_args) =
|
||||||
prepare_process(query, ext.as_str(), &manager, &filters).await?;
|
prepare_process(query, ext.as_str(), &filters)?;
|
||||||
|
|
||||||
let identifier_opt = manager
|
let identifier_opt = manager
|
||||||
.variant_identifier::<S>(&thumbnail_path, &name)
|
.variant_identifier::<S>(&alias, &thumbnail_path)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if let Some(identifier) = identifier_opt {
|
if let Some(identifier) = identifier_opt {
|
||||||
let details_opt = manager.variant_details(&identifier, name.clone()).await?;
|
let details_opt = manager.details(&identifier).await?;
|
||||||
|
|
||||||
let details = if let Some(details) = details_opt {
|
let details = if let Some(details) = details_opt {
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
let hint = details_hint(&name);
|
let hint = details_hint(&alias);
|
||||||
let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
||||||
manager
|
manager.store_details(&identifier, &details).await?;
|
||||||
.store_variant_details(&identifier, name, &details)
|
|
||||||
.await?;
|
|
||||||
details
|
details
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -377,7 +368,7 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
let identifier = manager
|
let identifier = manager
|
||||||
.still_identifier_from_filename((**store).clone(), name.clone())
|
.still_identifier_from_alias((**store).clone(), &alias)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let thumbnail_path2 = thumbnail_path.clone();
|
let thumbnail_path2 = thumbnail_path.clone();
|
||||||
|
@ -405,29 +396,27 @@ where
|
||||||
parent: None,
|
parent: None,
|
||||||
"Saving variant information",
|
"Saving variant information",
|
||||||
path = tracing::field::debug(&thumbnail_path),
|
path = tracing::field::debug(&thumbnail_path),
|
||||||
name = tracing::field::display(&name),
|
name = tracing::field::display(&alias),
|
||||||
);
|
);
|
||||||
save_span.follows_from(Span::current());
|
save_span.follows_from(Span::current());
|
||||||
let details2 = details.clone();
|
let details2 = details.clone();
|
||||||
let bytes2 = bytes.clone();
|
let bytes2 = bytes.clone();
|
||||||
|
let alias2 = alias.clone();
|
||||||
actix_rt::spawn(
|
actix_rt::spawn(
|
||||||
async move {
|
async move {
|
||||||
let identifier = match store.save_bytes(bytes2, &name).await {
|
let identifier = match store.save_bytes(bytes2).await {
|
||||||
Ok(identifier) => identifier,
|
Ok(identifier) => identifier,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("Failed to generate directory path: {}", e);
|
tracing::warn!("Failed to generate directory path: {}", e);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if let Err(e) = manager
|
if let Err(e) = manager.store_details(&identifier, &details2).await {
|
||||||
.store_variant_details(&identifier, name.clone(), &details2)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
tracing::warn!("Error saving variant details: {}", e);
|
tracing::warn!("Error saving variant details: {}", e);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if let Err(e) = manager
|
if let Err(e) = manager
|
||||||
.store_variant(Some(&thumbnail_path), &identifier, &name)
|
.store_variant(&alias2, &thumbnail_path, &identifier)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
tracing::warn!("Error saving variant info: {}", e);
|
tracing::warn!("Error saving variant info: {}", e);
|
||||||
|
@ -483,19 +472,19 @@ async fn details<S: Store>(
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
let name = manager.from_alias(alias.into_inner()).await?;
|
let alias = alias.into_inner();
|
||||||
let identifier = manager.identifier_from_filename::<S>(name.clone()).await?;
|
let alias = Alias::from_existing(&alias);
|
||||||
|
|
||||||
let details = manager.variant_details(&identifier, name.clone()).await?;
|
let identifier = manager.identifier_from_alias::<S>(&alias).await?;
|
||||||
|
|
||||||
|
let details = manager.details(&identifier).await?;
|
||||||
|
|
||||||
let details = if let Some(details) = details {
|
let details = if let Some(details) = details {
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
let hint = details_hint(&name);
|
let hint = details_hint(&alias);
|
||||||
let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
||||||
manager
|
manager.store_details(&identifier, &new_details).await?;
|
||||||
.store_variant_details(&identifier, name, &new_details)
|
|
||||||
.await?;
|
|
||||||
new_details
|
new_details
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -513,19 +502,18 @@ async fn serve<S: Store>(
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
let name = manager.from_alias(alias.into_inner()).await?;
|
let alias = alias.into_inner();
|
||||||
let identifier = manager.identifier_from_filename::<S>(name.clone()).await?;
|
let alias = Alias::from_existing(&alias);
|
||||||
|
let identifier = manager.identifier_from_alias::<S>(&alias).await?;
|
||||||
|
|
||||||
let details = manager.variant_details(&identifier, name.clone()).await?;
|
let details = manager.details(&identifier).await?;
|
||||||
|
|
||||||
let details = if let Some(details) = details {
|
let details = if let Some(details) = details {
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
let hint = details_hint(&name);
|
let hint = details_hint(&alias);
|
||||||
let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
||||||
manager
|
manager.store_details(&identifier, &details).await?;
|
||||||
.store_variant_details(&identifier, name, &details)
|
|
||||||
.await?;
|
|
||||||
details
|
details
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -605,25 +593,21 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, serde::Deserialize)]
|
#[derive(Debug, serde::Deserialize)]
|
||||||
#[serde(untagged)]
|
struct AliasQuery {
|
||||||
enum FileOrAlias {
|
alias: String,
|
||||||
File { file: String },
|
|
||||||
Alias { alias: String },
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Purging file", skip(upload_manager))]
|
#[instrument(name = "Purging file", skip(upload_manager))]
|
||||||
async fn purge<S: Store>(
|
async fn purge<S: Store>(
|
||||||
query: web::Query<FileOrAlias>,
|
query: web::Query<AliasQuery>,
|
||||||
upload_manager: web::Data<UploadManager>,
|
upload_manager: web::Data<UploadManager>,
|
||||||
store: web::Data<S>,
|
store: web::Data<S>,
|
||||||
) -> Result<HttpResponse, Error>
|
) -> Result<HttpResponse, Error>
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
let aliases = match query.into_inner() {
|
let alias = Alias::from_existing(&query.alias);
|
||||||
FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
|
let aliases = upload_manager.aliases_by_alias(&alias).await?;
|
||||||
FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
|
|
||||||
};
|
|
||||||
|
|
||||||
for alias in aliases.iter() {
|
for alias in aliases.iter() {
|
||||||
upload_manager
|
upload_manager
|
||||||
|
@ -633,49 +617,25 @@ where
|
||||||
|
|
||||||
Ok(HttpResponse::Ok().json(&serde_json::json!({
|
Ok(HttpResponse::Ok().json(&serde_json::json!({
|
||||||
"msg": "ok",
|
"msg": "ok",
|
||||||
"aliases": aliases
|
"aliases": aliases.iter().map(|a| a.to_string()).collect::<Vec<_>>()
|
||||||
})))
|
})))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Fetching aliases", skip(upload_manager))]
|
#[instrument(name = "Fetching aliases", skip(upload_manager))]
|
||||||
async fn aliases<S: Store>(
|
async fn aliases<S: Store>(
|
||||||
query: web::Query<FileOrAlias>,
|
query: web::Query<AliasQuery>,
|
||||||
upload_manager: web::Data<UploadManager>,
|
upload_manager: web::Data<UploadManager>,
|
||||||
store: web::Data<S>,
|
store: web::Data<S>,
|
||||||
) -> Result<HttpResponse, Error>
|
) -> Result<HttpResponse, Error>
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
let aliases = match query.into_inner() {
|
let alias = Alias::from_existing(&query.alias);
|
||||||
FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
|
let aliases = upload_manager.aliases_by_alias(&alias).await?;
|
||||||
FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(HttpResponse::Ok().json(&serde_json::json!({
|
Ok(HttpResponse::Ok().json(&serde_json::json!({
|
||||||
"msg": "ok",
|
"msg": "ok",
|
||||||
"aliases": aliases,
|
"aliases": aliases.iter().map(|a| a.to_string()).collect::<Vec<_>>()
|
||||||
})))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, serde::Deserialize)]
|
|
||||||
struct ByAlias {
|
|
||||||
alias: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[instrument(name = "Fetching filename", skip(upload_manager))]
|
|
||||||
async fn filename_by_alias<S: Store>(
|
|
||||||
query: web::Query<ByAlias>,
|
|
||||||
upload_manager: web::Data<UploadManager>,
|
|
||||||
store: web::Data<S>,
|
|
||||||
) -> Result<HttpResponse, Error>
|
|
||||||
where
|
|
||||||
Error: From<S::Error>,
|
|
||||||
{
|
|
||||||
let filename = upload_manager.from_alias(query.into_inner().alias).await?;
|
|
||||||
|
|
||||||
Ok(HttpResponse::Ok().json(&serde_json::json!({
|
|
||||||
"msg": "ok",
|
|
||||||
"filename": filename,
|
|
||||||
})))
|
})))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -817,10 +777,7 @@ where
|
||||||
.route(web::post().to(upload::<S>)),
|
.route(web::post().to(upload::<S>)),
|
||||||
)
|
)
|
||||||
.service(web::resource("/purge").route(web::post().to(purge::<S>)))
|
.service(web::resource("/purge").route(web::post().to(purge::<S>)))
|
||||||
.service(web::resource("/aliases").route(web::get().to(aliases::<S>)))
|
.service(web::resource("/aliases").route(web::get().to(aliases::<S>))),
|
||||||
.service(
|
|
||||||
web::resource("/filename").route(web::get().to(filename_by_alias::<S>)),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.bind(CONFIG.bind_address())?
|
.bind(CONFIG.bind_address())?
|
||||||
|
@ -834,7 +791,7 @@ where
|
||||||
|
|
||||||
async fn migrate_inner<S1>(
|
async fn migrate_inner<S1>(
|
||||||
manager: &UploadManager,
|
manager: &UploadManager,
|
||||||
db: &sled::Db,
|
repo: &Repo,
|
||||||
from: S1,
|
from: S1,
|
||||||
to: &config::Storage,
|
to: &config::Storage,
|
||||||
) -> anyhow::Result<()>
|
) -> anyhow::Result<()>
|
||||||
|
@ -844,9 +801,7 @@ where
|
||||||
{
|
{
|
||||||
match to {
|
match to {
|
||||||
config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
|
config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
|
||||||
let to = FileStore::build(path.clone(), db)?;
|
let to = FileStore::build(path.clone(), repo.clone()).await?;
|
||||||
manager.restructure(&to).await?;
|
|
||||||
|
|
||||||
manager.migrate_store::<S1, FileStore>(from, to).await?;
|
manager.migrate_store::<S1, FileStore>(from, to).await?;
|
||||||
}
|
}
|
||||||
config::Storage::ObjectStorage(RequiredObjectStorage {
|
config::Storage::ObjectStorage(RequiredObjectStorage {
|
||||||
|
@ -864,9 +819,10 @@ where
|
||||||
secret_key.clone(),
|
secret_key.clone(),
|
||||||
security_token.clone(),
|
security_token.clone(),
|
||||||
session_token.clone(),
|
session_token.clone(),
|
||||||
db,
|
repo.clone(),
|
||||||
build_reqwest_client()?,
|
build_reqwest_client()?,
|
||||||
)?;
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
manager.migrate_store::<S1, ObjectStore>(from, to).await?;
|
manager.migrate_store::<S1, ObjectStore>(from, to).await?;
|
||||||
}
|
}
|
||||||
|
@ -883,13 +839,12 @@ async fn main() -> anyhow::Result<()> {
|
||||||
CONFIG.console_buffer_capacity(),
|
CONFIG.console_buffer_capacity(),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
let repo = Repo::open(CONFIG.repo())?;
|
||||||
|
|
||||||
let db = LatestDb::exists(CONFIG.data_dir()).migrate()?;
|
let db = LatestDb::exists(CONFIG.data_dir()).migrate()?;
|
||||||
|
|
||||||
let repo = self::repo::Repo::open(CONFIG.repo())?;
|
|
||||||
|
|
||||||
repo.from_db(db).await?;
|
repo.from_db(db).await?;
|
||||||
|
|
||||||
let manager = UploadManager::new(db.clone(), CONFIG.format()).await?;
|
let manager = UploadManager::new(repo.clone(), CONFIG.format()).await?;
|
||||||
|
|
||||||
match CONFIG.command()? {
|
match CONFIG.command()? {
|
||||||
CommandConfig::Run => (),
|
CommandConfig::Run => (),
|
||||||
|
@ -901,10 +856,8 @@ async fn main() -> anyhow::Result<()> {
|
||||||
|
|
||||||
match from {
|
match from {
|
||||||
config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
|
config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
|
||||||
let from = FileStore::build(path.clone(), &db)?;
|
let from = FileStore::build(path.clone(), repo.clone()).await?;
|
||||||
manager.restructure(&from).await?;
|
migrate_inner(&manager, &repo, from, &to).await?;
|
||||||
|
|
||||||
migrate_inner(&manager, &db, from, &to).await?;
|
|
||||||
}
|
}
|
||||||
config::Storage::ObjectStorage(RequiredObjectStorage {
|
config::Storage::ObjectStorage(RequiredObjectStorage {
|
||||||
bucket_name,
|
bucket_name,
|
||||||
|
@ -921,11 +874,12 @@ async fn main() -> anyhow::Result<()> {
|
||||||
secret_key,
|
secret_key,
|
||||||
security_token,
|
security_token,
|
||||||
session_token,
|
session_token,
|
||||||
&db,
|
repo.clone(),
|
||||||
build_reqwest_client()?,
|
build_reqwest_client()?,
|
||||||
)?;
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
migrate_inner(&manager, &db, from, &to).await?;
|
migrate_inner(&manager, &repo, from, &to).await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -935,9 +889,7 @@ async fn main() -> anyhow::Result<()> {
|
||||||
|
|
||||||
match CONFIG.store()? {
|
match CONFIG.store()? {
|
||||||
config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
|
config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
|
||||||
let store = FileStore::build(path, &db)?;
|
let store = FileStore::build(path, repo).await?;
|
||||||
manager.restructure(&store).await?;
|
|
||||||
|
|
||||||
launch(manager, store).await
|
launch(manager, store).await
|
||||||
}
|
}
|
||||||
config::Storage::ObjectStorage(RequiredObjectStorage {
|
config::Storage::ObjectStorage(RequiredObjectStorage {
|
||||||
|
@ -955,9 +907,10 @@ async fn main() -> anyhow::Result<()> {
|
||||||
secret_key,
|
secret_key,
|
||||||
security_token,
|
security_token,
|
||||||
session_token,
|
session_token,
|
||||||
&db,
|
repo,
|
||||||
build_reqwest_client()?,
|
build_reqwest_client()?,
|
||||||
)?;
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
launch(manager, store).await
|
launch(manager, store).await
|
||||||
}
|
}
|
||||||
|
|
|
@ -95,26 +95,3 @@ impl DbVersion {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn alias_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
|
|
||||||
let mut start = hash.to_vec();
|
|
||||||
start.extend(&[0]);
|
|
||||||
|
|
||||||
let mut end = hash.to_vec();
|
|
||||||
end.extend(&[1]);
|
|
||||||
|
|
||||||
(start, end)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn alias_id_key(alias: &str) -> String {
|
|
||||||
format!("{}/id", alias)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn alias_key(hash: &[u8], id: &str) -> Vec<u8> {
|
|
||||||
let mut key = hash.to_vec();
|
|
||||||
// add a separator to the key between the hash and the ID
|
|
||||||
key.extend(&[0]);
|
|
||||||
key.extend(id.as_bytes());
|
|
||||||
|
|
||||||
key
|
|
||||||
}
|
|
||||||
|
|
|
@ -20,10 +20,7 @@ pub(crate) struct Crop(usize, usize);
|
||||||
pub(crate) struct Blur(f64);
|
pub(crate) struct Blur(f64);
|
||||||
|
|
||||||
#[instrument]
|
#[instrument]
|
||||||
pub(crate) fn build_chain(
|
pub(crate) fn build_chain(args: &[(String, String)]) -> Result<(PathBuf, Vec<String>), Error> {
|
||||||
args: &[(String, String)],
|
|
||||||
filename: String,
|
|
||||||
) -> Result<(PathBuf, Vec<String>), Error> {
|
|
||||||
fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, UploadError> {
|
fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, UploadError> {
|
||||||
if key == P::NAME {
|
if key == P::NAME {
|
||||||
return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?));
|
return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?));
|
||||||
|
@ -56,7 +53,7 @@ pub(crate) fn build_chain(
|
||||||
}
|
}
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
Ok((path.join(filename), args))
|
Ok((path, args))
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Processor for Identity {
|
impl Processor for Identity {
|
||||||
|
|
488
src/repo.rs
488
src/repo.rs
|
@ -1,105 +1,131 @@
|
||||||
use crate::config::RequiredSledRepo;
|
use crate::config::RequiredSledRepo;
|
||||||
use crate::{config::Repository, details::Details, store::Identifier};
|
use crate::{config::Repository, details::Details, store::Identifier};
|
||||||
use futures_util::Stream;
|
use futures_util::Stream;
|
||||||
|
use tracing::debug;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
mod old;
|
mod old;
|
||||||
pub(crate) mod sled;
|
pub(crate) mod sled;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub(crate) enum Repo {
|
pub(crate) enum Repo {
|
||||||
Sled(self::sled::SledRepo),
|
Sled(self::sled::SledRepo),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||||
pub(crate) struct Alias {
|
enum MaybeUuid {
|
||||||
id: Uuid,
|
Uuid(Uuid),
|
||||||
extension: String,
|
Name(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||||
|
pub(crate) struct Alias {
|
||||||
|
id: MaybeUuid,
|
||||||
|
extension: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||||
pub(crate) struct DeleteToken {
|
pub(crate) struct DeleteToken {
|
||||||
id: Uuid,
|
id: MaybeUuid,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct AlreadyExists;
|
pub(crate) struct AlreadyExists;
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait(?Send)]
|
||||||
pub(crate) trait SettingsRepo {
|
pub(crate) trait SettingsRepo {
|
||||||
type Bytes: AsRef<[u8]> + From<Vec<u8>>;
|
type Bytes: AsRef<[u8]> + From<Vec<u8>>;
|
||||||
type Error: std::error::Error;
|
type Error: std::error::Error + Send + Sync + 'static;
|
||||||
|
|
||||||
async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Self::Error>;
|
async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Self::Error>;
|
||||||
async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Self::Error>;
|
async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Self::Error>;
|
||||||
async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error>;
|
async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error>;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait(?Send)]
|
||||||
pub(crate) trait IdentifierRepo<I: Identifier> {
|
pub(crate) trait IdentifierRepo {
|
||||||
type Error: std::error::Error;
|
type Error: std::error::Error + Send + Sync + 'static;
|
||||||
|
|
||||||
async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error>;
|
async fn relate_details<I: Identifier>(
|
||||||
async fn details(&self, identifier: I) -> Result<Option<Details>, Self::Error>;
|
&self,
|
||||||
|
identifier: &I,
|
||||||
|
details: &Details,
|
||||||
|
) -> Result<(), Self::Error>;
|
||||||
|
async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Self::Error>;
|
||||||
|
|
||||||
async fn cleanup(&self, identifier: I) -> Result<(), Self::Error>;
|
async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Self::Error>;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait(?Send)]
|
||||||
pub(crate) trait HashRepo<I: Identifier> {
|
pub(crate) trait HashRepo {
|
||||||
type Bytes: AsRef<[u8]> + From<Vec<u8>>;
|
type Bytes: AsRef<[u8]> + From<Vec<u8>>;
|
||||||
type Error: std::error::Error;
|
type Error: std::error::Error + Send + Sync + 'static;
|
||||||
type Stream: Stream<Item = Result<Self::Bytes, Self::Error>>;
|
type Stream: Stream<Item = Result<Self::Bytes, Self::Error>>;
|
||||||
|
|
||||||
async fn hashes(&self) -> Self::Stream;
|
async fn hashes(&self) -> Self::Stream;
|
||||||
|
|
||||||
async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Self::Error>;
|
async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Self::Error>;
|
||||||
|
|
||||||
async fn relate_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
|
async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error>;
|
||||||
async fn remove_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
|
async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error>;
|
||||||
async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Self::Error>;
|
async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Self::Error>;
|
||||||
|
|
||||||
async fn relate_identifier(&self, hash: Self::Bytes, identifier: I) -> Result<(), Self::Error>;
|
async fn relate_identifier<I: Identifier>(
|
||||||
async fn identifier(&self, hash: Self::Bytes) -> Result<I, Self::Error>;
|
&self,
|
||||||
|
hash: Self::Bytes,
|
||||||
|
identifier: &I,
|
||||||
|
) -> Result<(), Self::Error>;
|
||||||
|
async fn identifier<I: Identifier + 'static>(
|
||||||
|
&self,
|
||||||
|
hash: Self::Bytes,
|
||||||
|
) -> Result<I, Self::Error>;
|
||||||
|
|
||||||
async fn relate_variant_identifier(
|
async fn relate_variant_identifier<I: Identifier>(
|
||||||
&self,
|
&self,
|
||||||
hash: Self::Bytes,
|
hash: Self::Bytes,
|
||||||
variant: String,
|
variant: String,
|
||||||
identifier: I,
|
identifier: &I,
|
||||||
) -> Result<(), Self::Error>;
|
) -> Result<(), Self::Error>;
|
||||||
async fn variant_identifier(
|
async fn variant_identifier<I: Identifier + 'static>(
|
||||||
&self,
|
&self,
|
||||||
hash: Self::Bytes,
|
hash: Self::Bytes,
|
||||||
variant: String,
|
variant: String,
|
||||||
) -> Result<Option<I>, Self::Error>;
|
) -> Result<Option<I>, Self::Error>;
|
||||||
|
async fn variants<I: Identifier + 'static>(
|
||||||
async fn relate_motion_identifier(
|
|
||||||
&self,
|
&self,
|
||||||
hash: Self::Bytes,
|
hash: Self::Bytes,
|
||||||
identifier: I,
|
) -> Result<Vec<(String, I)>, Self::Error>;
|
||||||
|
|
||||||
|
async fn relate_motion_identifier<I: Identifier>(
|
||||||
|
&self,
|
||||||
|
hash: Self::Bytes,
|
||||||
|
identifier: &I,
|
||||||
) -> Result<(), Self::Error>;
|
) -> Result<(), Self::Error>;
|
||||||
async fn motion_identifier(&self, hash: Self::Bytes) -> Result<Option<I>, Self::Error>;
|
async fn motion_identifier<I: Identifier + 'static>(
|
||||||
|
&self,
|
||||||
|
hash: Self::Bytes,
|
||||||
|
) -> Result<Option<I>, Self::Error>;
|
||||||
|
|
||||||
async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error>;
|
async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error>;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait(?Send)]
|
||||||
pub(crate) trait AliasRepo {
|
pub(crate) trait AliasRepo {
|
||||||
type Bytes: AsRef<[u8]> + From<Vec<u8>>;
|
type Bytes: AsRef<[u8]> + From<Vec<u8>>;
|
||||||
type Error: std::error::Error;
|
type Error: std::error::Error + Send + Sync + 'static;
|
||||||
|
|
||||||
async fn create(&self, alias: Alias) -> Result<Result<(), AlreadyExists>, Self::Error>;
|
async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Self::Error>;
|
||||||
|
|
||||||
async fn relate_delete_token(
|
async fn relate_delete_token(
|
||||||
&self,
|
&self,
|
||||||
alias: Alias,
|
alias: &Alias,
|
||||||
delete_token: DeleteToken,
|
delete_token: &DeleteToken,
|
||||||
) -> Result<Result<(), AlreadyExists>, Self::Error>;
|
) -> Result<Result<(), AlreadyExists>, Self::Error>;
|
||||||
async fn delete_token(&self, alias: Alias) -> Result<DeleteToken, Self::Error>;
|
async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Self::Error>;
|
||||||
|
|
||||||
async fn relate_hash(&self, alias: Alias, hash: Self::Bytes) -> Result<(), Self::Error>;
|
async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Self::Error>;
|
||||||
async fn hash(&self, alias: Alias) -> Result<Self::Bytes, Self::Error>;
|
async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Self::Error>;
|
||||||
|
|
||||||
async fn cleanup(&self, alias: Alias) -> Result<(), Self::Error>;
|
async fn cleanup(&self, alias: &Alias) -> Result<(), Self::Error>;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Repo {
|
impl Repo {
|
||||||
|
@ -167,72 +193,134 @@ const GENERATOR_KEY: &[u8] = b"last-path";
|
||||||
|
|
||||||
async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> anyhow::Result<()>
|
async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> anyhow::Result<()>
|
||||||
where
|
where
|
||||||
T: IdentifierRepo<::sled::IVec>,
|
T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo,
|
||||||
<T as IdentifierRepo<::sled::IVec>>::Error: Send + Sync + 'static,
|
|
||||||
T: HashRepo<::sled::IVec>,
|
|
||||||
<T as HashRepo<::sled::IVec>>::Error: Send + Sync + 'static,
|
|
||||||
T: AliasRepo,
|
|
||||||
<T as AliasRepo>::Error: Send + Sync + 'static,
|
|
||||||
T: SettingsRepo,
|
|
||||||
<T as SettingsRepo>::Error: Send + Sync + 'static,
|
|
||||||
{
|
{
|
||||||
HashRepo::create(repo, hash.to_vec().into()).await?;
|
if HashRepo::create(repo, hash.to_vec().into()).await?.is_err() {
|
||||||
|
debug!("Duplicate hash detected");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
let main_ident = old.main_identifier(&hash)?;
|
let main_ident = old.main_identifier(&hash)?.to_vec();
|
||||||
|
|
||||||
HashRepo::relate_identifier(repo, hash.to_vec().into(), main_ident.clone()).await?;
|
repo.relate_identifier(hash.to_vec().into(), &main_ident)
|
||||||
|
.await?;
|
||||||
|
|
||||||
for alias in old.aliases(&hash) {
|
for alias in old.aliases(&hash) {
|
||||||
if let Ok(Ok(())) = AliasRepo::create(repo, alias.clone()).await {
|
if let Ok(Ok(())) = AliasRepo::create(repo, &alias).await {
|
||||||
let _ = HashRepo::relate_alias(repo, hash.to_vec().into(), alias.clone()).await;
|
let _ = repo.relate_alias(hash.to_vec().into(), &alias).await;
|
||||||
let _ = AliasRepo::relate_hash(repo, alias.clone(), hash.to_vec().into()).await;
|
let _ = repo.relate_hash(&alias, hash.to_vec().into()).await;
|
||||||
|
|
||||||
if let Ok(Some(delete_token)) = old.delete_token(&alias) {
|
if let Ok(Some(delete_token)) = old.delete_token(&alias) {
|
||||||
let _ = AliasRepo::relate_delete_token(repo, alias, delete_token).await;
|
let _ = repo.relate_delete_token(&alias, &delete_token).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Ok(Some(identifier)) = old.motion_identifier(&hash) {
|
if let Ok(Some(identifier)) = old.motion_identifier(&hash) {
|
||||||
HashRepo::relate_motion_identifier(repo, hash.to_vec().into(), identifier).await;
|
let _ = repo
|
||||||
|
.relate_motion_identifier(hash.to_vec().into(), &identifier.to_vec())
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (variant, identifier) in old.variants(&hash)? {
|
for (variant_path, identifier) in old.variants(&hash)? {
|
||||||
let _ =
|
let variant = variant_path.to_string_lossy().to_string();
|
||||||
HashRepo::relate_variant_identifier(repo, hash.to_vec().into(), variant, identifier)
|
|
||||||
.await;
|
let _ = repo
|
||||||
|
.relate_variant_identifier(hash.to_vec().into(), variant, &identifier.to_vec())
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (identifier, details) in old.details(&hash)? {
|
for (identifier, details) in old.details(&hash)? {
|
||||||
let _ = IdentifierRepo::relate_details(repo, identifier, details).await;
|
let _ = repo.relate_details(&identifier.to_vec(), &details).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Ok(Some(value)) = old.setting(STORE_MIGRATION_PROGRESS) {
|
if let Ok(Some(value)) = old.setting(STORE_MIGRATION_PROGRESS) {
|
||||||
SettingsRepo::set(repo, STORE_MIGRATION_PROGRESS, value.to_vec().into()).await?;
|
repo.set(STORE_MIGRATION_PROGRESS, value.to_vec().into())
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Ok(Some(value)) = old.setting(GENERATOR_KEY) {
|
if let Ok(Some(value)) = old.setting(GENERATOR_KEY) {
|
||||||
SettingsRepo::set(repo, GENERATOR_KEY, value.to_vec().into()).await?;
|
repo.set(GENERATOR_KEY, value.to_vec().into()).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl MaybeUuid {
|
||||||
|
fn from_str(s: &str) -> Self {
|
||||||
|
if let Ok(uuid) = Uuid::parse_str(s) {
|
||||||
|
MaybeUuid::Uuid(uuid)
|
||||||
|
} else {
|
||||||
|
MaybeUuid::Name(s.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn as_bytes(&self) -> &[u8] {
|
||||||
|
match self {
|
||||||
|
Self::Uuid(uuid) => &uuid.as_bytes()[..],
|
||||||
|
Self::Name(name) => name.as_bytes(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn split_at_dot(s: &str) -> Option<(&str, &str)> {
|
||||||
|
let index = s.find('.')?;
|
||||||
|
|
||||||
|
Some(s.split_at(index))
|
||||||
|
}
|
||||||
|
|
||||||
impl Alias {
|
impl Alias {
|
||||||
|
pub(crate) fn generate(extension: String) -> Self {
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Uuid(Uuid::new_v4()),
|
||||||
|
extension: Some(extension),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn from_existing(alias: &str) -> Self {
|
||||||
|
if let Some((start, end)) = split_at_dot(alias) {
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::from_str(start),
|
||||||
|
extension: Some(end.into()),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::from_str(alias),
|
||||||
|
extension: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn extension(&self) -> Option<&str> {
|
||||||
|
self.extension.as_deref()
|
||||||
|
}
|
||||||
|
|
||||||
fn to_bytes(&self) -> Vec<u8> {
|
fn to_bytes(&self) -> Vec<u8> {
|
||||||
let mut v = self.id.as_bytes().to_vec();
|
let mut v = self.id.as_bytes().to_vec();
|
||||||
|
|
||||||
v.extend_from_slice(self.extension.as_bytes());
|
if let Some(ext) = self.extension() {
|
||||||
|
v.extend_from_slice(ext.as_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
v
|
v
|
||||||
}
|
}
|
||||||
|
|
||||||
fn from_slice(bytes: &[u8]) -> Option<Self> {
|
fn from_slice(bytes: &[u8]) -> Option<Self> {
|
||||||
if bytes.len() > 16 {
|
if let Ok(s) = std::str::from_utf8(bytes) {
|
||||||
|
Some(Self::from_existing(s))
|
||||||
|
} else if bytes.len() >= 16 {
|
||||||
let id = Uuid::from_slice(&bytes[0..16]).expect("Already checked length");
|
let id = Uuid::from_slice(&bytes[0..16]).expect("Already checked length");
|
||||||
let extension = String::from_utf8_lossy(&bytes[16..]).to_string();
|
|
||||||
|
|
||||||
Some(Self { id, extension })
|
let extension = if bytes.len() > 16 {
|
||||||
|
Some(String::from_utf8_lossy(&bytes[16..]).to_string())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
Some(Self {
|
||||||
|
id: MaybeUuid::Uuid(id),
|
||||||
|
extension,
|
||||||
|
})
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
@ -240,20 +328,63 @@ impl Alias {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DeleteToken {
|
impl DeleteToken {
|
||||||
|
pub(crate) fn from_existing(existing: &str) -> Self {
|
||||||
|
if let Ok(uuid) = Uuid::parse_str(existing) {
|
||||||
|
DeleteToken {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
DeleteToken {
|
||||||
|
id: MaybeUuid::Name(existing.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn generate() -> Self {
|
||||||
|
Self {
|
||||||
|
id: MaybeUuid::Uuid(Uuid::new_v4()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn to_bytes(&self) -> Vec<u8> {
|
fn to_bytes(&self) -> Vec<u8> {
|
||||||
self.id.as_bytes().to_vec()
|
self.id.as_bytes().to_vec()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn from_slice(bytes: &[u8]) -> Option<Self> {
|
fn from_slice(bytes: &[u8]) -> Option<Self> {
|
||||||
Some(DeleteToken {
|
if let Ok(s) = std::str::from_utf8(bytes) {
|
||||||
id: Uuid::from_slice(bytes).ok()?,
|
Some(DeleteToken::from_existing(s))
|
||||||
})
|
} else if bytes.len() == 16 {
|
||||||
|
Some(DeleteToken {
|
||||||
|
id: MaybeUuid::Uuid(Uuid::from_slice(bytes).ok()?),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for MaybeUuid {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::Uuid(id) => write!(f, "{}", id),
|
||||||
|
Self::Name(name) => write!(f, "{}", name),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for DeleteToken {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(f, "{}", self.id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Display for Alias {
|
impl std::fmt::Display for Alias {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
write!(f, "{}{}", self.id, self.extension)
|
if let Some(ext) = self.extension() {
|
||||||
|
write!(f, "{}{}", self.id, ext)
|
||||||
|
} else {
|
||||||
|
write!(f, "{}", self.id)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -272,17 +403,220 @@ impl Identifier for Vec<u8> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Identifier for ::sled::IVec {
|
#[cfg(test)]
|
||||||
type Error = std::convert::Infallible;
|
mod tests {
|
||||||
|
use super::{Alias, DeleteToken, MaybeUuid, Uuid};
|
||||||
|
|
||||||
fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error>
|
#[test]
|
||||||
where
|
fn string_delete_token() {
|
||||||
Self: Sized,
|
let delete_token = DeleteToken::from_existing("blah");
|
||||||
{
|
|
||||||
Ok(bytes.into())
|
assert_eq!(
|
||||||
|
delete_token,
|
||||||
|
DeleteToken {
|
||||||
|
id: MaybeUuid::Name(String::from("blah"))
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> {
|
#[test]
|
||||||
Ok(self.to_vec())
|
fn uuid_string_delete_token() {
|
||||||
|
let uuid = Uuid::new_v4();
|
||||||
|
|
||||||
|
let delete_token = DeleteToken::from_existing(&uuid.to_string());
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
delete_token,
|
||||||
|
DeleteToken {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bytes_delete_token() {
|
||||||
|
let delete_token = DeleteToken::from_slice(b"blah").unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
delete_token,
|
||||||
|
DeleteToken {
|
||||||
|
id: MaybeUuid::Name(String::from("blah"))
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uuid_bytes_delete_token() {
|
||||||
|
let uuid = Uuid::new_v4();
|
||||||
|
|
||||||
|
let delete_token = DeleteToken::from_slice(&uuid.as_bytes()[..]).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
delete_token,
|
||||||
|
DeleteToken {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uuid_bytes_string_delete_token() {
|
||||||
|
let uuid = Uuid::new_v4();
|
||||||
|
|
||||||
|
let delete_token = DeleteToken::from_slice(uuid.to_string().as_bytes()).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
delete_token,
|
||||||
|
DeleteToken {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn string_alias() {
|
||||||
|
let alias = Alias::from_existing("blah");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Name(String::from("blah")),
|
||||||
|
extension: None
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn string_alias_ext() {
|
||||||
|
let alias = Alias::from_existing("blah.mp4");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Name(String::from("blah")),
|
||||||
|
extension: Some(String::from(".mp4")),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uuid_string_alias() {
|
||||||
|
let uuid = Uuid::new_v4();
|
||||||
|
|
||||||
|
let alias = Alias::from_existing(&uuid.to_string());
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
extension: None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uuid_string_alias_ext() {
|
||||||
|
let uuid = Uuid::new_v4();
|
||||||
|
|
||||||
|
let alias_str = format!("{}.mp4", uuid);
|
||||||
|
let alias = Alias::from_existing(&alias_str);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
extension: Some(String::from(".mp4")),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bytes_alias() {
|
||||||
|
let alias = Alias::from_slice(b"blah").unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Name(String::from("blah")),
|
||||||
|
extension: None
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bytes_alias_ext() {
|
||||||
|
let alias = Alias::from_slice(b"blah.mp4").unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Name(String::from("blah")),
|
||||||
|
extension: Some(String::from(".mp4")),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uuid_bytes_alias() {
|
||||||
|
let uuid = Uuid::new_v4();
|
||||||
|
|
||||||
|
let alias = Alias::from_slice(&uuid.as_bytes()[..]).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
extension: None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uuid_bytes_string_alias() {
|
||||||
|
let uuid = Uuid::new_v4();
|
||||||
|
|
||||||
|
let alias = Alias::from_slice(uuid.to_string().as_bytes()).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
extension: None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uuid_bytes_alias_ext() {
|
||||||
|
let uuid = Uuid::new_v4();
|
||||||
|
|
||||||
|
let mut alias_bytes = uuid.as_bytes().to_vec();
|
||||||
|
alias_bytes.extend_from_slice(b".mp4");
|
||||||
|
|
||||||
|
let alias = Alias::from_slice(&alias_bytes).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
extension: Some(String::from(".mp4")),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uuid_bytes_string_alias_ext() {
|
||||||
|
let uuid = Uuid::new_v4();
|
||||||
|
|
||||||
|
let alias_str = format!("{}.mp4", uuid);
|
||||||
|
let alias = Alias::from_slice(alias_str.as_bytes()).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
alias,
|
||||||
|
Alias {
|
||||||
|
id: MaybeUuid::Uuid(uuid),
|
||||||
|
extension: Some(String::from(".mp4")),
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,8 +17,9 @@
|
||||||
// - Settings Tree
|
// - Settings Tree
|
||||||
// - store-migration-progress -> Path Tree Key
|
// - store-migration-progress -> Path Tree Key
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
use super::{Alias, DeleteToken, Details};
|
use super::{Alias, DeleteToken, Details};
|
||||||
use uuid::Uuid;
|
|
||||||
|
|
||||||
pub(super) struct Old {
|
pub(super) struct Old {
|
||||||
alias_tree: ::sled::Tree,
|
alias_tree: ::sled::Tree,
|
||||||
|
@ -91,7 +92,7 @@ impl Old {
|
||||||
.ok_or_else(|| anyhow::anyhow!("Missing identifier"))
|
.ok_or_else(|| anyhow::anyhow!("Missing identifier"))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn variants(&self, hash: &sled::IVec) -> anyhow::Result<Vec<(String, sled::IVec)>> {
|
pub(super) fn variants(&self, hash: &sled::IVec) -> anyhow::Result<Vec<(PathBuf, sled::IVec)>> {
|
||||||
let filename = self
|
let filename = self
|
||||||
.main_tree
|
.main_tree
|
||||||
.get(hash)?
|
.get(hash)?
|
||||||
|
@ -106,13 +107,16 @@ impl Old {
|
||||||
.scan_prefix(&variant_prefix)
|
.scan_prefix(&variant_prefix)
|
||||||
.filter_map(|res| res.ok())
|
.filter_map(|res| res.ok())
|
||||||
.filter_map(|(key, value)| {
|
.filter_map(|(key, value)| {
|
||||||
let key_str = String::from_utf8_lossy(&key);
|
let variant_path_bytes = &key[variant_prefix.as_bytes().len()..];
|
||||||
let variant_path = key_str.trim_start_matches(&variant_prefix);
|
if variant_path_bytes == b"motion" {
|
||||||
if variant_path == "motion" {
|
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
Some((variant_path.to_string(), value))
|
let path = String::from_utf8(variant_path_bytes.to_vec()).ok()?;
|
||||||
|
let mut path = PathBuf::from(path);
|
||||||
|
path.pop();
|
||||||
|
|
||||||
|
Some((path, value))
|
||||||
})
|
})
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
@ -141,29 +145,15 @@ impl Old {
|
||||||
.scan_prefix(key)
|
.scan_prefix(key)
|
||||||
.values()
|
.values()
|
||||||
.filter_map(|res| res.ok())
|
.filter_map(|res| res.ok())
|
||||||
.filter_map(|alias| {
|
.filter_map(|alias| Alias::from_slice(&alias))
|
||||||
let alias_str = String::from_utf8_lossy(&alias);
|
|
||||||
|
|
||||||
let (uuid, ext) = alias_str.split_once('.')?;
|
|
||||||
|
|
||||||
let uuid = uuid.parse::<Uuid>().ok()?;
|
|
||||||
|
|
||||||
Some(Alias {
|
|
||||||
id: uuid,
|
|
||||||
extension: ext.to_string(),
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn delete_token(&self, alias: &Alias) -> anyhow::Result<Option<DeleteToken>> {
|
pub(super) fn delete_token(&self, alias: &Alias) -> anyhow::Result<Option<DeleteToken>> {
|
||||||
let key = format!("{}{}/delete", alias.id, alias.extension);
|
let key = format!("{}/delete", alias);
|
||||||
|
|
||||||
if let Some(ivec) = self.alias_tree.get(key)? {
|
if let Some(ivec) = self.alias_tree.get(key)? {
|
||||||
let token_str = String::from_utf8_lossy(&ivec);
|
return Ok(DeleteToken::from_slice(&ivec));
|
||||||
if let Ok(uuid) = token_str.parse::<Uuid>() {
|
|
||||||
return Ok(Some(DeleteToken { id: uuid }));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(None)
|
Ok(None)
|
||||||
|
|
149
src/repo/sled.rs
149
src/repo/sled.rs
|
@ -18,7 +18,7 @@ pub(crate) enum Error {
|
||||||
Sled(#[from] sled::Error),
|
Sled(#[from] sled::Error),
|
||||||
|
|
||||||
#[error("Invalid identifier")]
|
#[error("Invalid identifier")]
|
||||||
Identifier(#[source] Box<dyn std::error::Error + Send + Sync>),
|
Identifier(#[source] Box<dyn std::error::Error + Sync + Send>),
|
||||||
|
|
||||||
#[error("Invalid details json")]
|
#[error("Invalid details json")]
|
||||||
Details(#[from] serde_json::Error),
|
Details(#[from] serde_json::Error),
|
||||||
|
@ -30,6 +30,7 @@ pub(crate) enum Error {
|
||||||
Panic,
|
Panic,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
pub(crate) struct SledRepo {
|
pub(crate) struct SledRepo {
|
||||||
settings: Tree,
|
settings: Tree,
|
||||||
identifier_details: Tree,
|
identifier_details: Tree,
|
||||||
|
@ -62,7 +63,7 @@ impl SledRepo {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait(?Send)]
|
||||||
impl SettingsRepo for SledRepo {
|
impl SettingsRepo for SledRepo {
|
||||||
type Bytes = IVec;
|
type Bytes = IVec;
|
||||||
type Error = Error;
|
type Error = Error;
|
||||||
|
@ -89,31 +90,35 @@ impl SettingsRepo for SledRepo {
|
||||||
fn identifier_bytes<I>(identifier: &I) -> Result<Vec<u8>, Error>
|
fn identifier_bytes<I>(identifier: &I) -> Result<Vec<u8>, Error>
|
||||||
where
|
where
|
||||||
I: Identifier,
|
I: Identifier,
|
||||||
I::Error: Send + Sync + 'static,
|
|
||||||
{
|
{
|
||||||
identifier
|
identifier
|
||||||
.to_bytes()
|
.to_bytes()
|
||||||
.map_err(|e| Error::Identifier(Box::new(e)))
|
.map_err(|e| Error::Identifier(Box::new(e)))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn variant_key(hash: &[u8], variant: &str) -> Result<Vec<u8>, Error> {
|
fn variant_key(hash: &[u8], variant: &str) -> Vec<u8> {
|
||||||
let mut bytes = hash.to_vec();
|
let mut bytes = hash.to_vec();
|
||||||
bytes.push(b'/');
|
bytes.push(b'/');
|
||||||
bytes.extend_from_slice(variant.as_bytes());
|
bytes.extend_from_slice(variant.as_bytes());
|
||||||
Ok(bytes)
|
bytes
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
fn variant_from_key(hash: &[u8], key: &[u8]) -> Option<String> {
|
||||||
impl<I> IdentifierRepo<I> for SledRepo
|
let prefix_len = hash.len() + 1;
|
||||||
where
|
let variant_bytes = key.get(prefix_len..)?.to_vec();
|
||||||
I: Identifier + 'static,
|
String::from_utf8(variant_bytes).ok()
|
||||||
I::Error: Send + Sync + 'static,
|
}
|
||||||
{
|
|
||||||
|
#[async_trait::async_trait(?Send)]
|
||||||
|
impl IdentifierRepo for SledRepo {
|
||||||
type Error = Error;
|
type Error = Error;
|
||||||
|
|
||||||
async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error> {
|
async fn relate_details<I: Identifier>(
|
||||||
let key = identifier_bytes(&identifier)?;
|
&self,
|
||||||
|
identifier: &I,
|
||||||
|
details: &Details,
|
||||||
|
) -> Result<(), Self::Error> {
|
||||||
|
let key = identifier_bytes(identifier)?;
|
||||||
let details = serde_json::to_vec(&details)?;
|
let details = serde_json::to_vec(&details)?;
|
||||||
|
|
||||||
b!(
|
b!(
|
||||||
|
@ -124,8 +129,8 @@ where
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn details(&self, identifier: I) -> Result<Option<Details>, Self::Error> {
|
async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Self::Error> {
|
||||||
let key = identifier_bytes(&identifier)?;
|
let key = identifier_bytes(identifier)?;
|
||||||
|
|
||||||
let opt = b!(self.identifier_details, identifier_details.get(key));
|
let opt = b!(self.identifier_details, identifier_details.get(key));
|
||||||
|
|
||||||
|
@ -136,8 +141,8 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn cleanup(&self, identifier: I) -> Result<(), Self::Error> {
|
async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Self::Error> {
|
||||||
let key = identifier_bytes(&identifier)?;
|
let key = identifier_bytes(identifier)?;
|
||||||
|
|
||||||
b!(self.identifier_details, identifier_details.remove(key));
|
b!(self.identifier_details, identifier_details.remove(key));
|
||||||
|
|
||||||
|
@ -182,11 +187,12 @@ impl futures_util::Stream for HashStream {
|
||||||
} else if let Some(mut iter) = this.hashes.take() {
|
} else if let Some(mut iter) = this.hashes.take() {
|
||||||
let fut = Box::pin(async move {
|
let fut = Box::pin(async move {
|
||||||
actix_rt::task::spawn_blocking(move || {
|
actix_rt::task::spawn_blocking(move || {
|
||||||
let opt = iter.next().map(|res| res.map_err(Error::from));
|
let opt = iter.next();
|
||||||
|
|
||||||
(iter, opt)
|
(iter, opt)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
|
.map(|(iter, opt)| (iter, opt.map(|res| res.map_err(Error::from))))
|
||||||
.map_err(Error::from)
|
.map_err(Error::from)
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -204,12 +210,8 @@ fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> {
|
||||||
v
|
v
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait(?Send)]
|
||||||
impl<I> HashRepo<I> for SledRepo
|
impl HashRepo for SledRepo {
|
||||||
where
|
|
||||||
I: Identifier + 'static,
|
|
||||||
I::Error: Send + Sync + 'static,
|
|
||||||
{
|
|
||||||
type Bytes = IVec;
|
type Bytes = IVec;
|
||||||
type Error = Error;
|
type Error = Error;
|
||||||
type Stream = HashStream;
|
type Stream = HashStream;
|
||||||
|
@ -232,19 +234,17 @@ where
|
||||||
Ok(res.map_err(|_| AlreadyExists))
|
Ok(res.map_err(|_| AlreadyExists))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn relate_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error> {
|
async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error> {
|
||||||
let key = hash_alias_key(&hash, &alias);
|
let key = hash_alias_key(&hash, alias);
|
||||||
|
let value = alias.to_bytes();
|
||||||
|
|
||||||
b!(
|
b!(self.hash_aliases, hash_aliases.insert(key, value));
|
||||||
self.hash_aliases,
|
|
||||||
hash_aliases.insert(key, alias.to_bytes())
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn remove_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error> {
|
async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error> {
|
||||||
let key = hash_alias_key(&hash, &alias);
|
let key = hash_alias_key(&hash, alias);
|
||||||
|
|
||||||
b!(self.hash_aliases, hash_aliases.remove(key));
|
b!(self.hash_aliases, hash_aliases.remove(key));
|
||||||
|
|
||||||
|
@ -258,21 +258,28 @@ where
|
||||||
.values()
|
.values()
|
||||||
.filter_map(Result::ok)
|
.filter_map(Result::ok)
|
||||||
.filter_map(|ivec| Alias::from_slice(&ivec))
|
.filter_map(|ivec| Alias::from_slice(&ivec))
|
||||||
.collect::<Vec<_>>()) as Result<_, Error>
|
.collect::<Vec<_>>()) as Result<_, sled::Error>
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(v)
|
Ok(v)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn relate_identifier(&self, hash: Self::Bytes, identifier: I) -> Result<(), Self::Error> {
|
async fn relate_identifier<I: Identifier>(
|
||||||
let bytes = identifier_bytes(&identifier)?;
|
&self,
|
||||||
|
hash: Self::Bytes,
|
||||||
|
identifier: &I,
|
||||||
|
) -> Result<(), Self::Error> {
|
||||||
|
let bytes = identifier_bytes(identifier)?;
|
||||||
|
|
||||||
b!(self.hash_identifiers, hash_identifiers.insert(hash, bytes));
|
b!(self.hash_identifiers, hash_identifiers.insert(hash, bytes));
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn identifier(&self, hash: Self::Bytes) -> Result<I, Self::Error> {
|
async fn identifier<I: Identifier + 'static>(
|
||||||
|
&self,
|
||||||
|
hash: Self::Bytes,
|
||||||
|
) -> Result<I, Self::Error> {
|
||||||
let opt = b!(self.hash_identifiers, hash_identifiers.get(hash));
|
let opt = b!(self.hash_identifiers, hash_identifiers.get(hash));
|
||||||
|
|
||||||
opt.ok_or(Error::Missing).and_then(|ivec| {
|
opt.ok_or(Error::Missing).and_then(|ivec| {
|
||||||
|
@ -280,14 +287,14 @@ where
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn relate_variant_identifier(
|
async fn relate_variant_identifier<I: Identifier>(
|
||||||
&self,
|
&self,
|
||||||
hash: Self::Bytes,
|
hash: Self::Bytes,
|
||||||
variant: String,
|
variant: String,
|
||||||
identifier: I,
|
identifier: &I,
|
||||||
) -> Result<(), Self::Error> {
|
) -> Result<(), Self::Error> {
|
||||||
let key = variant_key(&hash, &variant)?;
|
let key = variant_key(&hash, &variant);
|
||||||
let value = identifier_bytes(&identifier)?;
|
let value = identifier_bytes(identifier)?;
|
||||||
|
|
||||||
b!(
|
b!(
|
||||||
self.hash_variant_identifiers,
|
self.hash_variant_identifiers,
|
||||||
|
@ -297,12 +304,12 @@ where
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn variant_identifier(
|
async fn variant_identifier<I: Identifier + 'static>(
|
||||||
&self,
|
&self,
|
||||||
hash: Self::Bytes,
|
hash: Self::Bytes,
|
||||||
variant: String,
|
variant: String,
|
||||||
) -> Result<Option<I>, Self::Error> {
|
) -> Result<Option<I>, Self::Error> {
|
||||||
let key = variant_key(&hash, &variant)?;
|
let key = variant_key(&hash, &variant);
|
||||||
|
|
||||||
let opt = b!(
|
let opt = b!(
|
||||||
self.hash_variant_identifiers,
|
self.hash_variant_identifiers,
|
||||||
|
@ -318,12 +325,33 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn relate_motion_identifier(
|
async fn variants<I: Identifier + 'static>(
|
||||||
&self,
|
&self,
|
||||||
hash: Self::Bytes,
|
hash: Self::Bytes,
|
||||||
identifier: I,
|
) -> Result<Vec<(String, I)>, Self::Error> {
|
||||||
|
let vec = b!(
|
||||||
|
self.hash_variant_identifiers,
|
||||||
|
Ok(hash_variant_identifiers
|
||||||
|
.scan_prefix(&hash)
|
||||||
|
.filter_map(|res| res.ok())
|
||||||
|
.filter_map(|(key, ivec)| {
|
||||||
|
let identifier = I::from_bytes(ivec.to_vec()).ok()?;
|
||||||
|
let variant = variant_from_key(&hash, &key)?;
|
||||||
|
|
||||||
|
Some((variant, identifier))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()) as Result<Vec<_>, sled::Error>
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(vec)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn relate_motion_identifier<I: Identifier>(
|
||||||
|
&self,
|
||||||
|
hash: Self::Bytes,
|
||||||
|
identifier: &I,
|
||||||
) -> Result<(), Self::Error> {
|
) -> Result<(), Self::Error> {
|
||||||
let bytes = identifier_bytes(&identifier)?;
|
let bytes = identifier_bytes(identifier)?;
|
||||||
|
|
||||||
b!(
|
b!(
|
||||||
self.hash_motion_identifiers,
|
self.hash_motion_identifiers,
|
||||||
|
@ -333,7 +361,10 @@ where
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn motion_identifier(&self, hash: Self::Bytes) -> Result<Option<I>, Self::Error> {
|
async fn motion_identifier<I: Identifier + 'static>(
|
||||||
|
&self,
|
||||||
|
hash: Self::Bytes,
|
||||||
|
) -> Result<Option<I>, Self::Error> {
|
||||||
let opt = b!(
|
let opt = b!(
|
||||||
self.hash_motion_identifiers,
|
self.hash_motion_identifiers,
|
||||||
hash_motion_identifiers.get(hash)
|
hash_motion_identifiers.get(hash)
|
||||||
|
@ -361,7 +392,7 @@ where
|
||||||
hash_motion_identifiers.remove(hash2)
|
hash_motion_identifiers.remove(hash2)
|
||||||
);
|
);
|
||||||
|
|
||||||
let aliases = HashRepo::<I>::aliases(self, hash.clone()).await?;
|
let aliases = self.aliases(hash.clone()).await?;
|
||||||
let hash2 = hash.clone();
|
let hash2 = hash.clone();
|
||||||
b!(self.hash_aliases, {
|
b!(self.hash_aliases, {
|
||||||
for alias in aliases {
|
for alias in aliases {
|
||||||
|
@ -369,7 +400,7 @@ where
|
||||||
|
|
||||||
let _ = hash_aliases.remove(key);
|
let _ = hash_aliases.remove(key);
|
||||||
}
|
}
|
||||||
Ok(()) as Result<(), Error>
|
Ok(()) as Result<(), sled::Error>
|
||||||
});
|
});
|
||||||
|
|
||||||
let variant_keys = b!(self.hash_variant_identifiers, {
|
let variant_keys = b!(self.hash_variant_identifiers, {
|
||||||
|
@ -379,25 +410,25 @@ where
|
||||||
.filter_map(Result::ok)
|
.filter_map(Result::ok)
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
Ok(v) as Result<Vec<_>, Error>
|
Ok(v) as Result<Vec<_>, sled::Error>
|
||||||
});
|
});
|
||||||
b!(self.hash_variant_identifiers, {
|
b!(self.hash_variant_identifiers, {
|
||||||
for key in variant_keys {
|
for key in variant_keys {
|
||||||
let _ = hash_variant_identifiers.remove(key);
|
let _ = hash_variant_identifiers.remove(key);
|
||||||
}
|
}
|
||||||
Ok(()) as Result<(), Error>
|
Ok(()) as Result<(), sled::Error>
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait(?Send)]
|
||||||
impl AliasRepo for SledRepo {
|
impl AliasRepo for SledRepo {
|
||||||
type Bytes = sled::IVec;
|
type Bytes = sled::IVec;
|
||||||
type Error = Error;
|
type Error = Error;
|
||||||
|
|
||||||
async fn create(&self, alias: Alias) -> Result<Result<(), AlreadyExists>, Self::Error> {
|
async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Self::Error> {
|
||||||
let bytes = alias.to_bytes();
|
let bytes = alias.to_bytes();
|
||||||
let bytes2 = bytes.clone();
|
let bytes2 = bytes.clone();
|
||||||
|
|
||||||
|
@ -411,8 +442,8 @@ impl AliasRepo for SledRepo {
|
||||||
|
|
||||||
async fn relate_delete_token(
|
async fn relate_delete_token(
|
||||||
&self,
|
&self,
|
||||||
alias: Alias,
|
alias: &Alias,
|
||||||
delete_token: DeleteToken,
|
delete_token: &DeleteToken,
|
||||||
) -> Result<Result<(), AlreadyExists>, Self::Error> {
|
) -> Result<Result<(), AlreadyExists>, Self::Error> {
|
||||||
let key = alias.to_bytes();
|
let key = alias.to_bytes();
|
||||||
let token = delete_token.to_bytes();
|
let token = delete_token.to_bytes();
|
||||||
|
@ -425,7 +456,7 @@ impl AliasRepo for SledRepo {
|
||||||
Ok(res.map_err(|_| AlreadyExists))
|
Ok(res.map_err(|_| AlreadyExists))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn delete_token(&self, alias: Alias) -> Result<DeleteToken, Self::Error> {
|
async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Self::Error> {
|
||||||
let key = alias.to_bytes();
|
let key = alias.to_bytes();
|
||||||
|
|
||||||
let opt = b!(self.alias_delete_tokens, alias_delete_tokens.get(key));
|
let opt = b!(self.alias_delete_tokens, alias_delete_tokens.get(key));
|
||||||
|
@ -434,7 +465,7 @@ impl AliasRepo for SledRepo {
|
||||||
.ok_or(Error::Missing)
|
.ok_or(Error::Missing)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn relate_hash(&self, alias: Alias, hash: Self::Bytes) -> Result<(), Self::Error> {
|
async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Self::Error> {
|
||||||
let key = alias.to_bytes();
|
let key = alias.to_bytes();
|
||||||
|
|
||||||
b!(self.alias_hashes, alias_hashes.insert(key, hash));
|
b!(self.alias_hashes, alias_hashes.insert(key, hash));
|
||||||
|
@ -442,7 +473,7 @@ impl AliasRepo for SledRepo {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn hash(&self, alias: Alias) -> Result<Self::Bytes, Self::Error> {
|
async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Self::Error> {
|
||||||
let key = alias.to_bytes();
|
let key = alias.to_bytes();
|
||||||
|
|
||||||
let opt = b!(self.alias_hashes, alias_hashes.get(key));
|
let opt = b!(self.alias_hashes, alias_hashes.get(key));
|
||||||
|
@ -450,7 +481,7 @@ impl AliasRepo for SledRepo {
|
||||||
opt.ok_or(Error::Missing)
|
opt.ok_or(Error::Missing)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn cleanup(&self, alias: Alias) -> Result<(), Self::Error> {
|
async fn cleanup(&self, alias: &Alias) -> Result<(), Self::Error> {
|
||||||
let key = alias.to_bytes();
|
let key = alias.to_bytes();
|
||||||
|
|
||||||
let key2 = key.clone();
|
let key2 = key.clone();
|
||||||
|
|
11
src/store.rs
11
src/store.rs
|
@ -8,7 +8,7 @@ pub(crate) mod file_store;
|
||||||
pub(crate) mod object_store;
|
pub(crate) mod object_store;
|
||||||
|
|
||||||
pub(crate) trait Identifier: Send + Sync + Clone + Debug {
|
pub(crate) trait Identifier: Send + Sync + Clone + Debug {
|
||||||
type Error: std::error::Error;
|
type Error: std::error::Error + Send + Sync + 'static;
|
||||||
|
|
||||||
fn to_bytes(&self) -> Result<Vec<u8>, Self::Error>;
|
fn to_bytes(&self) -> Result<Vec<u8>, Self::Error>;
|
||||||
|
|
||||||
|
@ -19,23 +19,18 @@ pub(crate) trait Identifier: Send + Sync + Clone + Debug {
|
||||||
|
|
||||||
#[async_trait::async_trait(?Send)]
|
#[async_trait::async_trait(?Send)]
|
||||||
pub(crate) trait Store: Send + Sync + Clone + Debug + 'static {
|
pub(crate) trait Store: Send + Sync + Clone + Debug + 'static {
|
||||||
type Error: std::error::Error;
|
type Error: std::error::Error + Send + Sync + 'static;
|
||||||
type Identifier: Identifier<Error = Self::Error>;
|
type Identifier: Identifier<Error = Self::Error>;
|
||||||
type Stream: Stream<Item = std::io::Result<Bytes>>;
|
type Stream: Stream<Item = std::io::Result<Bytes>>;
|
||||||
|
|
||||||
async fn save_async_read<Reader>(
|
async fn save_async_read<Reader>(
|
||||||
&self,
|
&self,
|
||||||
reader: &mut Reader,
|
reader: &mut Reader,
|
||||||
filename: &str,
|
|
||||||
) -> Result<Self::Identifier, Self::Error>
|
) -> Result<Self::Identifier, Self::Error>
|
||||||
where
|
where
|
||||||
Reader: AsyncRead + Unpin;
|
Reader: AsyncRead + Unpin;
|
||||||
|
|
||||||
async fn save_bytes(
|
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error>;
|
||||||
&self,
|
|
||||||
bytes: Bytes,
|
|
||||||
filename: &str,
|
|
||||||
) -> Result<Self::Identifier, Self::Error>;
|
|
||||||
|
|
||||||
async fn to_stream(
|
async fn to_stream(
|
||||||
&self,
|
&self,
|
||||||
|
|
|
@ -1,4 +1,8 @@
|
||||||
use crate::{file::File, store::Store};
|
use crate::{
|
||||||
|
file::File,
|
||||||
|
repo::{Repo, SettingsRepo},
|
||||||
|
store::Store,
|
||||||
|
};
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
use futures_util::stream::Stream;
|
use futures_util::stream::Stream;
|
||||||
use std::{
|
use std::{
|
||||||
|
@ -10,24 +14,22 @@ use tokio::io::{AsyncRead, AsyncWrite};
|
||||||
use tracing::{debug, error, instrument};
|
use tracing::{debug, error, instrument};
|
||||||
|
|
||||||
mod file_id;
|
mod file_id;
|
||||||
mod restructure;
|
|
||||||
pub(crate) use file_id::FileId;
|
pub(crate) use file_id::FileId;
|
||||||
|
|
||||||
// - Settings Tree
|
// - Settings Tree
|
||||||
// - last-path -> last generated path
|
// - last-path -> last generated path
|
||||||
// - fs-restructure-01-complete -> bool
|
|
||||||
|
|
||||||
const GENERATOR_KEY: &[u8] = b"last-path";
|
const GENERATOR_KEY: &[u8] = b"last-path";
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub(crate) enum FileError {
|
pub(crate) enum FileError {
|
||||||
#[error(transparent)]
|
#[error("Failed to interact with sled db")]
|
||||||
Sled(#[from] sled::Error),
|
Sled(#[from] crate::repo::sled::Error),
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error("Failed to read or write file")]
|
||||||
Io(#[from] std::io::Error),
|
Io(#[from] std::io::Error),
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error("Failed to generate path")]
|
||||||
PathGenerator(#[from] storage_path_generator::PathError),
|
PathGenerator(#[from] storage_path_generator::PathError),
|
||||||
|
|
||||||
#[error("Error formatting file store identifier")]
|
#[error("Error formatting file store identifier")]
|
||||||
|
@ -44,7 +46,7 @@ pub(crate) enum FileError {
|
||||||
pub(crate) struct FileStore {
|
pub(crate) struct FileStore {
|
||||||
path_gen: Generator,
|
path_gen: Generator,
|
||||||
root_dir: PathBuf,
|
root_dir: PathBuf,
|
||||||
settings_tree: sled::Tree,
|
repo: Repo,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait(?Send)]
|
#[async_trait::async_trait(?Send)]
|
||||||
|
@ -57,12 +59,11 @@ impl Store for FileStore {
|
||||||
async fn save_async_read<Reader>(
|
async fn save_async_read<Reader>(
|
||||||
&self,
|
&self,
|
||||||
reader: &mut Reader,
|
reader: &mut Reader,
|
||||||
filename: &str,
|
|
||||||
) -> Result<Self::Identifier, Self::Error>
|
) -> Result<Self::Identifier, Self::Error>
|
||||||
where
|
where
|
||||||
Reader: AsyncRead + Unpin,
|
Reader: AsyncRead + Unpin,
|
||||||
{
|
{
|
||||||
let path = self.next_file(filename)?;
|
let path = self.next_file().await?;
|
||||||
|
|
||||||
if let Err(e) = self.safe_save_reader(&path, reader).await {
|
if let Err(e) = self.safe_save_reader(&path, reader).await {
|
||||||
self.safe_remove_file(&path).await?;
|
self.safe_remove_file(&path).await?;
|
||||||
|
@ -73,12 +74,8 @@ impl Store for FileStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip(bytes))]
|
#[tracing::instrument(skip(bytes))]
|
||||||
async fn save_bytes(
|
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
|
||||||
&self,
|
let path = self.next_file().await?;
|
||||||
bytes: Bytes,
|
|
||||||
filename: &str,
|
|
||||||
) -> Result<Self::Identifier, Self::Error> {
|
|
||||||
let path = self.next_file(filename)?;
|
|
||||||
|
|
||||||
if let Err(e) = self.safe_save_bytes(&path, bytes).await {
|
if let Err(e) = self.safe_save_bytes(&path, bytes).await {
|
||||||
self.safe_remove_file(&path).await?;
|
self.safe_remove_file(&path).await?;
|
||||||
|
@ -141,23 +138,26 @@ impl Store for FileStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FileStore {
|
impl FileStore {
|
||||||
pub fn build(root_dir: PathBuf, db: &sled::Db) -> Result<Self, FileError> {
|
pub(crate) async fn build(root_dir: PathBuf, repo: Repo) -> Result<Self, FileError> {
|
||||||
let settings_tree = db.open_tree("settings")?;
|
let path_gen = init_generator(&repo).await?;
|
||||||
|
|
||||||
let path_gen = init_generator(&settings_tree)?;
|
|
||||||
|
|
||||||
Ok(FileStore {
|
Ok(FileStore {
|
||||||
root_dir,
|
root_dir,
|
||||||
path_gen,
|
path_gen,
|
||||||
settings_tree,
|
repo,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn next_directory(&self) -> Result<PathBuf, FileError> {
|
async fn next_directory(&self) -> Result<PathBuf, FileError> {
|
||||||
let path = self.path_gen.next();
|
let path = self.path_gen.next();
|
||||||
|
|
||||||
self.settings_tree
|
match self.repo {
|
||||||
.insert(GENERATOR_KEY, path.to_be_bytes())?;
|
Repo::Sled(ref sled_repo) => {
|
||||||
|
sled_repo
|
||||||
|
.set(GENERATOR_KEY, path.to_be_bytes().into())
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let mut target_path = self.root_dir.clone();
|
let mut target_path = self.root_dir.clone();
|
||||||
for dir in path.to_strings() {
|
for dir in path.to_strings() {
|
||||||
|
@ -167,8 +167,9 @@ impl FileStore {
|
||||||
Ok(target_path)
|
Ok(target_path)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn next_file(&self, filename: &str) -> Result<PathBuf, FileError> {
|
async fn next_file(&self) -> Result<PathBuf, FileError> {
|
||||||
let target_path = self.next_directory()?;
|
let target_path = self.next_directory().await?;
|
||||||
|
let filename = uuid::Uuid::new_v4().to_string();
|
||||||
|
|
||||||
Ok(target_path.join(filename))
|
Ok(target_path.join(filename))
|
||||||
}
|
}
|
||||||
|
@ -289,13 +290,17 @@ pub(crate) async fn safe_create_parent<P: AsRef<Path>>(path: P) -> Result<(), Fi
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn init_generator(settings: &sled::Tree) -> Result<Generator, FileError> {
|
async fn init_generator(repo: &Repo) -> Result<Generator, FileError> {
|
||||||
if let Some(ivec) = settings.get(GENERATOR_KEY)? {
|
match repo {
|
||||||
Ok(Generator::from_existing(
|
Repo::Sled(sled_repo) => {
|
||||||
storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
|
if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
|
||||||
))
|
Ok(Generator::from_existing(
|
||||||
} else {
|
storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
|
||||||
Ok(Generator::new())
|
))
|
||||||
|
} else {
|
||||||
|
Ok(Generator::new())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,118 +0,0 @@
|
||||||
use crate::{
|
|
||||||
error::{Error, UploadError},
|
|
||||||
store::file_store::FileStore,
|
|
||||||
upload_manager::UploadManager,
|
|
||||||
};
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
|
|
||||||
const RESTRUCTURE_COMPLETE: &[u8] = b"fs-restructure-01-complete";
|
|
||||||
const DETAILS: &[u8] = b"details";
|
|
||||||
|
|
||||||
impl UploadManager {
|
|
||||||
#[tracing::instrument(skip(self))]
|
|
||||||
pub(crate) async fn restructure(&self, store: &FileStore) -> Result<(), Error> {
|
|
||||||
if self.restructure_complete(store)? {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
for res in self.inner().filename_tree.iter() {
|
|
||||||
let (filename, hash) = res?;
|
|
||||||
let filename = String::from_utf8(filename.to_vec())?;
|
|
||||||
tracing::info!("Migrating {}", filename);
|
|
||||||
|
|
||||||
let file_path = store.root_dir.join("files").join(&filename);
|
|
||||||
|
|
||||||
if tokio::fs::metadata(&file_path).await.is_ok() {
|
|
||||||
let target_path = store.next_directory()?.join(&filename);
|
|
||||||
|
|
||||||
let target_path_bytes = self
|
|
||||||
.generalize_path(store, &target_path)?
|
|
||||||
.to_str()
|
|
||||||
.ok_or(UploadError::Path)?
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec();
|
|
||||||
|
|
||||||
self.inner()
|
|
||||||
.identifier_tree
|
|
||||||
.insert(filename.as_bytes(), target_path_bytes)?;
|
|
||||||
|
|
||||||
store.safe_move_file(file_path, target_path).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
let (start, end) = variant_key_bounds(&hash);
|
|
||||||
|
|
||||||
for res in self.inner().main_tree.range(start..end) {
|
|
||||||
let (hash_variant_key, variant_path_or_details) = res?;
|
|
||||||
|
|
||||||
if !hash_variant_key.ends_with(DETAILS) {
|
|
||||||
let variant_path =
|
|
||||||
PathBuf::from(String::from_utf8(variant_path_or_details.to_vec())?);
|
|
||||||
if tokio::fs::metadata(&variant_path).await.is_ok() {
|
|
||||||
let target_path = store.next_directory()?.join(&filename);
|
|
||||||
|
|
||||||
let relative_target_path_bytes = self
|
|
||||||
.generalize_path(store, &target_path)?
|
|
||||||
.to_str()
|
|
||||||
.ok_or(UploadError::Path)?
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec();
|
|
||||||
|
|
||||||
let variant_key =
|
|
||||||
self.migrate_variant_key(store, &variant_path, &filename)?;
|
|
||||||
|
|
||||||
self.inner()
|
|
||||||
.identifier_tree
|
|
||||||
.insert(variant_key, relative_target_path_bytes)?;
|
|
||||||
|
|
||||||
store
|
|
||||||
.safe_move_file(variant_path.clone(), target_path)
|
|
||||||
.await?;
|
|
||||||
store.try_remove_parents(&variant_path).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
self.inner().main_tree.remove(hash_variant_key)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
self.mark_restructure_complete(store)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn restructure_complete(&self, store: &FileStore) -> Result<bool, Error> {
|
|
||||||
Ok(store.settings_tree.get(RESTRUCTURE_COMPLETE)?.is_some())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn mark_restructure_complete(&self, store: &FileStore) -> Result<(), Error> {
|
|
||||||
store.settings_tree.insert(RESTRUCTURE_COMPLETE, b"true")?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn generalize_path<'a>(&self, store: &FileStore, path: &'a Path) -> Result<&'a Path, Error> {
|
|
||||||
Ok(path.strip_prefix(&store.root_dir)?)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn migrate_variant_key(
|
|
||||||
&self,
|
|
||||||
store: &FileStore,
|
|
||||||
variant_process_path: &Path,
|
|
||||||
filename: &str,
|
|
||||||
) -> Result<Vec<u8>, Error> {
|
|
||||||
let path = self
|
|
||||||
.generalize_path(store, variant_process_path)?
|
|
||||||
.strip_prefix("files")?;
|
|
||||||
|
|
||||||
self.variant_key(path, filename)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn variant_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
|
|
||||||
let mut start = hash.to_vec();
|
|
||||||
start.extend(&[2]);
|
|
||||||
|
|
||||||
let mut end = hash.to_vec();
|
|
||||||
end.extend(&[3]);
|
|
||||||
|
|
||||||
(start, end)
|
|
||||||
}
|
|
|
@ -1,4 +1,7 @@
|
||||||
use crate::store::Store;
|
use crate::{
|
||||||
|
repo::{Repo, SettingsRepo},
|
||||||
|
store::Store,
|
||||||
|
};
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
use futures_util::stream::Stream;
|
use futures_util::stream::Stream;
|
||||||
use s3::{
|
use s3::{
|
||||||
|
@ -22,26 +25,26 @@ const GENERATOR_KEY: &[u8] = b"last-path";
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub(crate) enum ObjectError {
|
pub(crate) enum ObjectError {
|
||||||
#[error(transparent)]
|
#[error("Failed to generate path")]
|
||||||
PathGenerator(#[from] storage_path_generator::PathError),
|
PathGenerator(#[from] storage_path_generator::PathError),
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error("Failed to interact with sled repo")]
|
||||||
Sled(#[from] sled::Error),
|
Sled(#[from] crate::repo::sled::Error),
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error("Failed to parse string")]
|
||||||
Utf8(#[from] FromUtf8Error),
|
Utf8(#[from] FromUtf8Error),
|
||||||
|
|
||||||
#[error("Invalid length")]
|
#[error("Invalid length")]
|
||||||
Length,
|
Length,
|
||||||
|
|
||||||
#[error("Storage error: {0}")]
|
#[error("Storage error")]
|
||||||
Anyhow(#[from] anyhow::Error),
|
Anyhow(#[from] anyhow::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub(crate) struct ObjectStore {
|
pub(crate) struct ObjectStore {
|
||||||
path_gen: Generator,
|
path_gen: Generator,
|
||||||
settings_tree: sled::Tree,
|
repo: Repo,
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
client: reqwest::Client,
|
client: reqwest::Client,
|
||||||
}
|
}
|
||||||
|
@ -63,12 +66,11 @@ impl Store for ObjectStore {
|
||||||
async fn save_async_read<Reader>(
|
async fn save_async_read<Reader>(
|
||||||
&self,
|
&self,
|
||||||
reader: &mut Reader,
|
reader: &mut Reader,
|
||||||
filename: &str,
|
|
||||||
) -> Result<Self::Identifier, Self::Error>
|
) -> Result<Self::Identifier, Self::Error>
|
||||||
where
|
where
|
||||||
Reader: AsyncRead + Unpin,
|
Reader: AsyncRead + Unpin,
|
||||||
{
|
{
|
||||||
let path = self.next_file(filename)?;
|
let path = self.next_file().await?;
|
||||||
|
|
||||||
self.bucket
|
self.bucket
|
||||||
.put_object_stream(&self.client, reader, &path)
|
.put_object_stream(&self.client, reader, &path)
|
||||||
|
@ -78,12 +80,8 @@ impl Store for ObjectStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip(bytes))]
|
#[tracing::instrument(skip(bytes))]
|
||||||
async fn save_bytes(
|
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
|
||||||
&self,
|
let path = self.next_file().await?;
|
||||||
bytes: Bytes,
|
|
||||||
filename: &str,
|
|
||||||
) -> Result<Self::Identifier, Self::Error> {
|
|
||||||
let path = self.next_file(filename)?;
|
|
||||||
|
|
||||||
self.bucket.put_object(&self.client, &path, &bytes).await?;
|
self.bucket.put_object(&self.client, &path, &bytes).await?;
|
||||||
|
|
||||||
|
@ -154,23 +152,21 @@ impl Store for ObjectStore {
|
||||||
|
|
||||||
impl ObjectStore {
|
impl ObjectStore {
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub(crate) fn build(
|
pub(crate) async fn build(
|
||||||
bucket_name: &str,
|
bucket_name: &str,
|
||||||
region: Region,
|
region: Region,
|
||||||
access_key: Option<String>,
|
access_key: Option<String>,
|
||||||
secret_key: Option<String>,
|
secret_key: Option<String>,
|
||||||
security_token: Option<String>,
|
security_token: Option<String>,
|
||||||
session_token: Option<String>,
|
session_token: Option<String>,
|
||||||
db: &sled::Db,
|
repo: Repo,
|
||||||
client: reqwest::Client,
|
client: reqwest::Client,
|
||||||
) -> Result<ObjectStore, ObjectError> {
|
) -> Result<ObjectStore, ObjectError> {
|
||||||
let settings_tree = db.open_tree("settings")?;
|
let path_gen = init_generator(&repo).await?;
|
||||||
|
|
||||||
let path_gen = init_generator(&settings_tree)?;
|
|
||||||
|
|
||||||
Ok(ObjectStore {
|
Ok(ObjectStore {
|
||||||
path_gen,
|
path_gen,
|
||||||
settings_tree,
|
repo,
|
||||||
bucket: Bucket::new_with_path_style(
|
bucket: Bucket::new_with_path_style(
|
||||||
bucket_name,
|
bucket_name,
|
||||||
match region {
|
match region {
|
||||||
|
@ -191,29 +187,39 @@ impl ObjectStore {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn next_directory(&self) -> Result<Path, ObjectError> {
|
async fn next_directory(&self) -> Result<Path, ObjectError> {
|
||||||
let path = self.path_gen.next();
|
let path = self.path_gen.next();
|
||||||
|
|
||||||
self.settings_tree
|
match self.repo {
|
||||||
.insert(GENERATOR_KEY, path.to_be_bytes())?;
|
Repo::Sled(ref sled_repo) => {
|
||||||
|
sled_repo
|
||||||
|
.set(GENERATOR_KEY, path.to_be_bytes().into())
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Ok(path)
|
Ok(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn next_file(&self, filename: &str) -> Result<String, ObjectError> {
|
async fn next_file(&self) -> Result<String, ObjectError> {
|
||||||
let path = self.next_directory()?.to_strings().join("/");
|
let path = self.next_directory().await?.to_strings().join("/");
|
||||||
|
let filename = uuid::Uuid::new_v4().to_string();
|
||||||
|
|
||||||
Ok(format!("{}/{}", path, filename))
|
Ok(format!("{}/{}", path, filename))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn init_generator(settings: &sled::Tree) -> Result<Generator, ObjectError> {
|
async fn init_generator(repo: &Repo) -> Result<Generator, ObjectError> {
|
||||||
if let Some(ivec) = settings.get(GENERATOR_KEY)? {
|
match repo {
|
||||||
Ok(Generator::from_existing(
|
Repo::Sled(sled_repo) => {
|
||||||
storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
|
if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
|
||||||
))
|
Ok(Generator::from_existing(
|
||||||
} else {
|
storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
|
||||||
Ok(Generator::new())
|
))
|
||||||
|
} else {
|
||||||
|
Ok(Generator::new())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,13 +4,15 @@ use crate::{
|
||||||
error::{Error, UploadError},
|
error::{Error, UploadError},
|
||||||
ffmpeg::{InputFormat, ThumbnailFormat},
|
ffmpeg::{InputFormat, ThumbnailFormat},
|
||||||
magick::details_hint,
|
magick::details_hint,
|
||||||
migrate::{alias_id_key, alias_key, alias_key_bounds},
|
repo::{
|
||||||
|
sled::SledRepo, Alias, AliasRepo, DeleteToken, HashRepo, IdentifierRepo, Repo, SettingsRepo,
|
||||||
|
},
|
||||||
store::{Identifier, Store},
|
store::{Identifier, Store},
|
||||||
};
|
};
|
||||||
use actix_web::web;
|
use futures_util::StreamExt;
|
||||||
use sha2::Digest;
|
use sha2::Digest;
|
||||||
use std::{string::FromUtf8Error, sync::Arc};
|
use std::sync::Arc;
|
||||||
use tracing::{debug, error, info, instrument, warn, Span};
|
use tracing::{debug, error, instrument, Span};
|
||||||
use tracing_futures::Instrument;
|
use tracing_futures::Instrument;
|
||||||
|
|
||||||
mod hasher;
|
mod hasher;
|
||||||
|
@ -18,28 +20,6 @@ mod session;
|
||||||
|
|
||||||
pub(super) use session::UploadManagerSession;
|
pub(super) use session::UploadManagerSession;
|
||||||
|
|
||||||
// TREE STRUCTURE
|
|
||||||
// - Alias Tree
|
|
||||||
// - alias -> hash
|
|
||||||
// - alias / id -> u64(id)
|
|
||||||
// - alias / delete -> delete token
|
|
||||||
// - Main Tree
|
|
||||||
// - hash -> filename
|
|
||||||
// - hash 0 u64(id) -> alias
|
|
||||||
// - DEPRECATED:
|
|
||||||
// - hash 2 variant path -> variant path
|
|
||||||
// - hash 2 vairant path details -> details
|
|
||||||
// - Filename Tree
|
|
||||||
// - filename -> hash
|
|
||||||
// - Details Tree
|
|
||||||
// - filename / S::Identifier -> details
|
|
||||||
// - Identifier Tree
|
|
||||||
// - filename -> S::Identifier
|
|
||||||
// - filename / variant path -> S::Identifier
|
|
||||||
// - filename / motion -> S::Identifier
|
|
||||||
// - Settings Tree
|
|
||||||
// - store-migration-progress -> Path Tree Key
|
|
||||||
|
|
||||||
const STORE_MIGRATION_PROGRESS: &[u8] = b"store-migration-progress";
|
const STORE_MIGRATION_PROGRESS: &[u8] = b"store-migration-progress";
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
|
@ -50,33 +30,17 @@ pub(crate) struct UploadManager {
|
||||||
pub(crate) struct UploadManagerInner {
|
pub(crate) struct UploadManagerInner {
|
||||||
format: Option<Format>,
|
format: Option<Format>,
|
||||||
hasher: sha2::Sha256,
|
hasher: sha2::Sha256,
|
||||||
pub(crate) alias_tree: sled::Tree,
|
repo: Repo,
|
||||||
pub(crate) filename_tree: sled::Tree,
|
|
||||||
pub(crate) main_tree: sled::Tree,
|
|
||||||
details_tree: sled::Tree,
|
|
||||||
settings_tree: sled::Tree,
|
|
||||||
pub(crate) identifier_tree: sled::Tree,
|
|
||||||
db: sled::Db,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct FilenameIVec {
|
|
||||||
inner: sled::IVec,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl UploadManager {
|
impl UploadManager {
|
||||||
/// Create a new UploadManager
|
/// Create a new UploadManager
|
||||||
pub(crate) async fn new(db: sled::Db, format: Option<Format>) -> Result<Self, Error> {
|
pub(crate) async fn new(repo: Repo, format: Option<Format>) -> Result<Self, Error> {
|
||||||
let manager = UploadManager {
|
let manager = UploadManager {
|
||||||
inner: Arc::new(UploadManagerInner {
|
inner: Arc::new(UploadManagerInner {
|
||||||
format,
|
format,
|
||||||
hasher: sha2::Sha256::new(),
|
hasher: sha2::Sha256::new(),
|
||||||
alias_tree: db.open_tree("alias")?,
|
repo,
|
||||||
filename_tree: db.open_tree("filename")?,
|
|
||||||
main_tree: db.open_tree("main")?,
|
|
||||||
details_tree: db.open_tree("details")?,
|
|
||||||
settings_tree: db.open_tree("settings")?,
|
|
||||||
identifier_tree: db.open_tree("path")?,
|
|
||||||
db,
|
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -89,92 +53,32 @@ impl UploadManager {
|
||||||
S2: Store,
|
S2: Store,
|
||||||
Error: From<S1::Error> + From<S2::Error>,
|
Error: From<S1::Error> + From<S2::Error>,
|
||||||
{
|
{
|
||||||
let iter =
|
match self.inner.repo {
|
||||||
if let Some(starting_line) = self.inner.settings_tree.get(STORE_MIGRATION_PROGRESS)? {
|
Repo::Sled(ref sled_repo) => do_migrate_store(sled_repo, from, to).await,
|
||||||
self.inner.identifier_tree.range(starting_line..)
|
|
||||||
} else {
|
|
||||||
self.inner.identifier_tree.iter()
|
|
||||||
};
|
|
||||||
|
|
||||||
for res in iter {
|
|
||||||
let (key, identifier) = res?;
|
|
||||||
|
|
||||||
let identifier = S1::Identifier::from_bytes(identifier.to_vec())?;
|
|
||||||
|
|
||||||
let filename =
|
|
||||||
if let Some((filename, _)) = String::from_utf8_lossy(&key).split_once('/') {
|
|
||||||
filename.to_string()
|
|
||||||
} else {
|
|
||||||
String::from_utf8_lossy(&key).to_string()
|
|
||||||
};
|
|
||||||
|
|
||||||
let stream = from.to_stream(&identifier, None, None).await?;
|
|
||||||
futures_util::pin_mut!(stream);
|
|
||||||
let mut reader = tokio_util::io::StreamReader::new(stream);
|
|
||||||
|
|
||||||
let new_identifier = to.save_async_read(&mut reader, &filename).await?;
|
|
||||||
|
|
||||||
let details_key = self.details_key(&identifier, &filename)?;
|
|
||||||
|
|
||||||
if let Some(details) = self.inner.details_tree.get(details_key.clone())? {
|
|
||||||
let new_details_key = self.details_key(&new_identifier, &filename)?;
|
|
||||||
|
|
||||||
self.inner.details_tree.insert(new_details_key, details)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
self.inner
|
|
||||||
.identifier_tree
|
|
||||||
.insert(key.clone(), new_identifier.to_bytes()?)?;
|
|
||||||
self.inner.details_tree.remove(details_key)?;
|
|
||||||
self.inner
|
|
||||||
.settings_tree
|
|
||||||
.insert(STORE_MIGRATION_PROGRESS, key)?;
|
|
||||||
|
|
||||||
let (ident, detail, settings) = futures_util::future::join3(
|
|
||||||
self.inner.identifier_tree.flush_async(),
|
|
||||||
self.inner.details_tree.flush_async(),
|
|
||||||
self.inner.settings_tree.flush_async(),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
ident?;
|
|
||||||
detail?;
|
|
||||||
settings?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// clean up the migration key to avoid interfering with future migrations
|
|
||||||
self.inner.settings_tree.remove(STORE_MIGRATION_PROGRESS)?;
|
|
||||||
self.inner.settings_tree.flush_async().await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn inner(&self) -> &UploadManagerInner {
|
pub(crate) async fn still_identifier_from_alias<S: Store + Clone>(
|
||||||
&self.inner
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn still_identifier_from_filename<S: Store + Clone>(
|
|
||||||
&self,
|
&self,
|
||||||
store: S,
|
store: S,
|
||||||
filename: String,
|
alias: &Alias,
|
||||||
) -> Result<S::Identifier, Error>
|
) -> Result<S::Identifier, Error>
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
let identifier = self.identifier_from_filename::<S>(filename.clone()).await?;
|
let identifier = self.identifier_from_alias::<S>(alias).await?;
|
||||||
let details =
|
let details = if let Some(details) = self.details(&identifier).await? {
|
||||||
if let Some(details) = self.variant_details(&identifier, filename.clone()).await? {
|
details
|
||||||
details
|
} else {
|
||||||
} else {
|
let hint = details_hint(alias);
|
||||||
let hint = details_hint(&filename);
|
Details::from_store(store.clone(), identifier.clone(), hint).await?
|
||||||
Details::from_store(store.clone(), identifier.clone(), hint).await?
|
};
|
||||||
};
|
|
||||||
|
|
||||||
if !details.is_motion() {
|
if !details.is_motion() {
|
||||||
return Ok(identifier);
|
return Ok(identifier);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(motion_identifier) = self.motion_identifier::<S>(&filename).await? {
|
if let Some(motion_identifier) = self.motion_identifier::<S>(alias).await? {
|
||||||
return Ok(motion_identifier);
|
return Ok(motion_identifier);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -186,101 +90,93 @@ impl UploadManager {
|
||||||
ThumbnailFormat::Jpeg,
|
ThumbnailFormat::Jpeg,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let motion_identifier = store.save_async_read(&mut reader, &filename).await?;
|
let motion_identifier = store.save_async_read(&mut reader).await?;
|
||||||
drop(permit);
|
drop(permit);
|
||||||
|
|
||||||
self.store_motion_path(&filename, &motion_identifier)
|
self.store_motion_identifier(alias, &motion_identifier)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(motion_identifier)
|
Ok(motion_identifier)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn motion_identifier<S: Store>(
|
async fn motion_identifier<S: Store>(
|
||||||
&self,
|
&self,
|
||||||
filename: &str,
|
alias: &Alias,
|
||||||
) -> Result<Option<S::Identifier>, Error>
|
) -> Result<Option<S::Identifier>, Error> {
|
||||||
where
|
match self.inner.repo {
|
||||||
Error: From<S::Error>,
|
Repo::Sled(ref sled_repo) => {
|
||||||
{
|
let hash = sled_repo.hash(alias).await?;
|
||||||
let identifier_tree = self.inner.identifier_tree.clone();
|
Ok(sled_repo.motion_identifier(hash).await?)
|
||||||
let motion_key = format!("{}/motion", filename);
|
}
|
||||||
|
|
||||||
let opt = web::block(move || identifier_tree.get(motion_key.as_bytes())).await??;
|
|
||||||
|
|
||||||
if let Some(ivec) = opt {
|
|
||||||
return Ok(Some(S::Identifier::from_bytes(ivec.to_vec())?));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(None)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn store_motion_path<I: Identifier>(
|
async fn store_motion_identifier<I: Identifier + 'static>(
|
||||||
&self,
|
&self,
|
||||||
filename: &str,
|
alias: &Alias,
|
||||||
identifier: &I,
|
identifier: &I,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error> {
|
||||||
where
|
match self.inner.repo {
|
||||||
Error: From<I::Error>,
|
Repo::Sled(ref sled_repo) => {
|
||||||
{
|
let hash = sled_repo.hash(alias).await?;
|
||||||
let identifier_bytes = identifier.to_bytes()?;
|
Ok(sled_repo.relate_motion_identifier(hash, identifier).await?)
|
||||||
let motion_key = format!("{}/motion", filename);
|
}
|
||||||
let identifier_tree = self.inner.identifier_tree.clone();
|
}
|
||||||
|
|
||||||
web::block(move || identifier_tree.insert(motion_key.as_bytes(), identifier_bytes))
|
|
||||||
.await??;
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn identifier_from_filename<S: Store>(
|
pub(crate) async fn identifier_from_alias<S: Store>(
|
||||||
&self,
|
&self,
|
||||||
filename: String,
|
alias: &Alias,
|
||||||
) -> Result<S::Identifier, Error>
|
) -> Result<S::Identifier, Error> {
|
||||||
where
|
match self.inner.repo {
|
||||||
Error: From<S::Error>,
|
Repo::Sled(ref sled_repo) => {
|
||||||
{
|
let hash = sled_repo.hash(alias).await?;
|
||||||
let identifier_tree = self.inner.identifier_tree.clone();
|
Ok(sled_repo.identifier(hash).await?)
|
||||||
let path_ivec = web::block(move || identifier_tree.get(filename.as_bytes()))
|
}
|
||||||
.await??
|
}
|
||||||
.ok_or(UploadError::MissingFile)?;
|
|
||||||
|
|
||||||
let identifier = S::Identifier::from_bytes(path_ivec.to_vec())?;
|
|
||||||
|
|
||||||
Ok(identifier)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
async fn store_identifier<I: Identifier>(
|
async fn store_identifier<I: Identifier>(
|
||||||
&self,
|
&self,
|
||||||
filename: String,
|
hash: Vec<u8>,
|
||||||
identifier: &I,
|
identifier: &I,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error> {
|
||||||
where
|
match self.inner.repo {
|
||||||
Error: From<I::Error>,
|
Repo::Sled(ref sled_repo) => {
|
||||||
{
|
Ok(sled_repo.relate_identifier(hash.into(), identifier).await?)
|
||||||
let identifier_bytes = identifier.to_bytes()?;
|
}
|
||||||
let identifier_tree = self.inner.identifier_tree.clone();
|
}
|
||||||
web::block(move || identifier_tree.insert(filename.as_bytes(), identifier_bytes)).await??;
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn variant_identifier<S: Store>(
|
pub(crate) async fn variant_identifier<S: Store>(
|
||||||
&self,
|
&self,
|
||||||
|
alias: &Alias,
|
||||||
process_path: &std::path::Path,
|
process_path: &std::path::Path,
|
||||||
filename: &str,
|
) -> Result<Option<S::Identifier>, Error> {
|
||||||
) -> Result<Option<S::Identifier>, Error>
|
let variant = process_path.to_string_lossy().to_string();
|
||||||
where
|
|
||||||
Error: From<S::Error>,
|
|
||||||
{
|
|
||||||
let key = self.variant_key(process_path, filename)?;
|
|
||||||
let identifier_tree = self.inner.identifier_tree.clone();
|
|
||||||
let path_opt = web::block(move || identifier_tree.get(key)).await??;
|
|
||||||
|
|
||||||
if let Some(ivec) = path_opt {
|
match self.inner.repo {
|
||||||
let identifier = S::Identifier::from_bytes(ivec.to_vec())?;
|
Repo::Sled(ref sled_repo) => {
|
||||||
Ok(Some(identifier))
|
let hash = sled_repo.hash(alias).await?;
|
||||||
} else {
|
Ok(sled_repo.variant_identifier(hash, variant).await?)
|
||||||
Ok(None)
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Store the path to a generated image variant so we can easily clean it up later
|
||||||
|
#[instrument(skip(self))]
|
||||||
|
pub(crate) async fn store_full_res<I: Identifier>(
|
||||||
|
&self,
|
||||||
|
alias: &Alias,
|
||||||
|
identifier: &I,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
match self.inner.repo {
|
||||||
|
Repo::Sled(ref sled_repo) => {
|
||||||
|
let hash = sled_repo.hash(alias).await?;
|
||||||
|
Ok(sled_repo.relate_identifier(hash, identifier).await?)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -288,139 +184,71 @@ impl UploadManager {
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn store_variant<I: Identifier>(
|
pub(crate) async fn store_variant<I: Identifier>(
|
||||||
&self,
|
&self,
|
||||||
variant_process_path: Option<&std::path::Path>,
|
alias: &Alias,
|
||||||
|
variant_process_path: &std::path::Path,
|
||||||
identifier: &I,
|
identifier: &I,
|
||||||
filename: &str,
|
) -> Result<(), Error> {
|
||||||
) -> Result<(), Error>
|
let variant = variant_process_path.to_string_lossy().to_string();
|
||||||
where
|
|
||||||
Error: From<I::Error>,
|
|
||||||
{
|
|
||||||
let key = if let Some(path) = variant_process_path {
|
|
||||||
self.variant_key(path, filename)?
|
|
||||||
} else {
|
|
||||||
let mut vec = filename.as_bytes().to_vec();
|
|
||||||
vec.extend(b"/");
|
|
||||||
vec.extend(&identifier.to_bytes()?);
|
|
||||||
vec
|
|
||||||
};
|
|
||||||
let identifier_tree = self.inner.identifier_tree.clone();
|
|
||||||
let identifier_bytes = identifier.to_bytes()?;
|
|
||||||
|
|
||||||
debug!("Storing variant");
|
match self.inner.repo {
|
||||||
web::block(move || identifier_tree.insert(key, identifier_bytes)).await??;
|
Repo::Sled(ref sled_repo) => {
|
||||||
debug!("Stored variant");
|
let hash = sled_repo.hash(alias).await?;
|
||||||
|
Ok(sled_repo
|
||||||
Ok(())
|
.relate_variant_identifier(hash, variant, identifier)
|
||||||
|
.await?)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the image details for a given variant
|
/// Get the image details for a given variant
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn variant_details<I: Identifier>(
|
pub(crate) async fn details<I: Identifier>(
|
||||||
&self,
|
&self,
|
||||||
identifier: &I,
|
identifier: &I,
|
||||||
filename: String,
|
|
||||||
) -> Result<Option<Details>, Error>
|
) -> Result<Option<Details>, Error>
|
||||||
where
|
where
|
||||||
Error: From<I::Error>,
|
Error: From<I::Error>,
|
||||||
{
|
{
|
||||||
let key = self.details_key(identifier, &filename)?;
|
match self.inner.repo {
|
||||||
let details_tree = self.inner.details_tree.clone();
|
Repo::Sled(ref sled_repo) => Ok(sled_repo.details(identifier).await?),
|
||||||
|
}
|
||||||
debug!("Getting details");
|
|
||||||
let opt = match web::block(move || details_tree.get(key)).await?? {
|
|
||||||
Some(ivec) => match serde_json::from_slice(&ivec) {
|
|
||||||
Ok(details) => Some(details),
|
|
||||||
Err(_) => None,
|
|
||||||
},
|
|
||||||
None => None,
|
|
||||||
};
|
|
||||||
debug!("Got details");
|
|
||||||
|
|
||||||
Ok(opt)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn store_variant_details<I: Identifier>(
|
pub(crate) async fn store_details<I: Identifier>(
|
||||||
&self,
|
&self,
|
||||||
identifier: &I,
|
identifier: &I,
|
||||||
filename: String,
|
|
||||||
details: &Details,
|
details: &Details,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error> {
|
||||||
where
|
match self.inner.repo {
|
||||||
Error: From<I::Error>,
|
Repo::Sled(ref sled_repo) => Ok(sled_repo.relate_details(identifier, details).await?),
|
||||||
{
|
}
|
||||||
let key = self.details_key(identifier, &filename)?;
|
|
||||||
let details_tree = self.inner.details_tree.clone();
|
|
||||||
let details_value = serde_json::to_vec(details)?;
|
|
||||||
|
|
||||||
debug!("Storing details");
|
|
||||||
web::block(move || details_tree.insert(key, details_value)).await??;
|
|
||||||
debug!("Stored details");
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get a list of aliases for a given file
|
|
||||||
pub(crate) async fn aliases_by_filename(&self, filename: String) -> Result<Vec<String>, Error> {
|
|
||||||
let fname_tree = self.inner.filename_tree.clone();
|
|
||||||
let hash = web::block(move || fname_tree.get(filename.as_bytes()))
|
|
||||||
.await??
|
|
||||||
.ok_or(UploadError::MissingAlias)?;
|
|
||||||
|
|
||||||
self.aliases_by_hash(&hash).await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get a list of aliases for a given alias
|
/// Get a list of aliases for a given alias
|
||||||
pub(crate) async fn aliases_by_alias(&self, alias: String) -> Result<Vec<String>, Error> {
|
pub(crate) async fn aliases_by_alias(&self, alias: &Alias) -> Result<Vec<Alias>, Error> {
|
||||||
let alias_tree = self.inner.alias_tree.clone();
|
match self.inner.repo {
|
||||||
let hash = web::block(move || alias_tree.get(alias.as_bytes()))
|
Repo::Sled(ref sled_repo) => {
|
||||||
.await??
|
let hash = sled_repo.hash(alias).await?;
|
||||||
.ok_or(UploadError::MissingFilename)?;
|
Ok(sled_repo.aliases(hash).await?)
|
||||||
|
}
|
||||||
self.aliases_by_hash(&hash).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn aliases_by_hash(&self, hash: &sled::IVec) -> Result<Vec<String>, Error> {
|
|
||||||
let (start, end) = alias_key_bounds(hash);
|
|
||||||
let main_tree = self.inner.main_tree.clone();
|
|
||||||
let aliases = web::block(move || {
|
|
||||||
main_tree
|
|
||||||
.range(start..end)
|
|
||||||
.values()
|
|
||||||
.collect::<Result<Vec<_>, _>>()
|
|
||||||
})
|
|
||||||
.await??;
|
|
||||||
|
|
||||||
debug!("Got {} aliases for hash", aliases.len());
|
|
||||||
let aliases = aliases
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|s| String::from_utf8(s.to_vec()).ok())
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
for alias in aliases.iter() {
|
|
||||||
debug!("{}", alias);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(aliases)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete an alias without a delete token
|
/// Delete an alias without a delete token
|
||||||
pub(crate) async fn delete_without_token<S: Store + 'static>(
|
pub(crate) async fn delete_without_token<S: Store + 'static>(
|
||||||
&self,
|
&self,
|
||||||
store: S,
|
store: S,
|
||||||
alias: String,
|
alias: Alias,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
let token_key = delete_key(&alias);
|
let token = match self.inner.repo {
|
||||||
let alias_tree = self.inner.alias_tree.clone();
|
Repo::Sled(ref sled_repo) => sled_repo.delete_token(&alias).await?,
|
||||||
let token = web::block(move || alias_tree.get(token_key.as_bytes()))
|
};
|
||||||
.await??
|
|
||||||
.ok_or(UploadError::MissingAlias)?;
|
|
||||||
|
|
||||||
self.delete(store, alias, String::from_utf8(token.to_vec())?)
|
self.delete(store, alias, token).await
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete the alias, and the file & variants if no more aliases exist
|
/// Delete the alias, and the file & variants if no more aliases exist
|
||||||
|
@ -428,57 +256,24 @@ impl UploadManager {
|
||||||
pub(crate) async fn delete<S: Store + 'static>(
|
pub(crate) async fn delete<S: Store + 'static>(
|
||||||
&self,
|
&self,
|
||||||
store: S,
|
store: S,
|
||||||
alias: String,
|
alias: Alias,
|
||||||
token: String,
|
token: DeleteToken,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
use sled::Transactional;
|
let hash = match self.inner.repo {
|
||||||
let main_tree = self.inner.main_tree.clone();
|
Repo::Sled(ref sled_repo) => {
|
||||||
let alias_tree = self.inner.alias_tree.clone();
|
let saved_delete_token = sled_repo.delete_token(&alias).await?;
|
||||||
|
if saved_delete_token != token {
|
||||||
let span = Span::current();
|
return Err(UploadError::InvalidToken.into());
|
||||||
let alias2 = alias.clone();
|
|
||||||
let hash = web::block(move || {
|
|
||||||
[&main_tree, &alias_tree].transaction(|v| {
|
|
||||||
let entered = span.enter();
|
|
||||||
let main_tree = &v[0];
|
|
||||||
let alias_tree = &v[1];
|
|
||||||
|
|
||||||
// -- GET TOKEN --
|
|
||||||
debug!("Deleting alias -> delete-token mapping");
|
|
||||||
let existing_token = alias_tree
|
|
||||||
.remove(delete_key(&alias2).as_bytes())?
|
|
||||||
.ok_or_else(|| trans_upload_error(UploadError::MissingAlias))?;
|
|
||||||
|
|
||||||
// Bail if invalid token
|
|
||||||
if existing_token != token {
|
|
||||||
warn!("Invalid delete token");
|
|
||||||
return Err(trans_upload_error(UploadError::InvalidToken));
|
|
||||||
}
|
}
|
||||||
|
let hash = sled_repo.hash(&alias).await?;
|
||||||
// -- GET ID FOR HASH TREE CLEANUP --
|
AliasRepo::cleanup(sled_repo, &alias).await?;
|
||||||
debug!("Deleting alias -> id mapping");
|
sled_repo.remove_alias(hash.clone(), &alias).await?;
|
||||||
let id = alias_tree
|
hash.to_vec()
|
||||||
.remove(alias_id_key(&alias2).as_bytes())?
|
}
|
||||||
.ok_or_else(|| trans_upload_error(UploadError::MissingAlias))?;
|
};
|
||||||
let id = String::from_utf8(id.to_vec()).map_err(trans_utf8_error)?;
|
|
||||||
|
|
||||||
// -- GET HASH FOR HASH TREE CLEANUP --
|
|
||||||
debug!("Deleting alias -> hash mapping");
|
|
||||||
let hash = alias_tree
|
|
||||||
.remove(alias2.as_bytes())?
|
|
||||||
.ok_or_else(|| trans_upload_error(UploadError::MissingAlias))?;
|
|
||||||
|
|
||||||
// -- REMOVE HASH TREE ELEMENT --
|
|
||||||
debug!("Deleting hash -> alias mapping");
|
|
||||||
main_tree.remove(alias_key(&hash, &id))?;
|
|
||||||
drop(entered);
|
|
||||||
Ok(hash)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.await??;
|
|
||||||
|
|
||||||
self.check_delete_files(store, hash).await
|
self.check_delete_files(store, hash).await
|
||||||
}
|
}
|
||||||
|
@ -486,206 +281,163 @@ impl UploadManager {
|
||||||
async fn check_delete_files<S: Store + 'static>(
|
async fn check_delete_files<S: Store + 'static>(
|
||||||
&self,
|
&self,
|
||||||
store: S,
|
store: S,
|
||||||
hash: sled::IVec,
|
hash: Vec<u8>,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
// -- CHECK IF ANY OTHER ALIASES EXIST --
|
match self.inner.repo {
|
||||||
let main_tree = self.inner.main_tree.clone();
|
Repo::Sled(ref sled_repo) => {
|
||||||
let (start, end) = alias_key_bounds(&hash);
|
let hash: <SledRepo as HashRepo>::Bytes = hash.into();
|
||||||
debug!("Checking for additional aliases referencing hash");
|
|
||||||
let any_aliases = web::block(move || {
|
|
||||||
Ok(main_tree.range(start..end).next().is_some()) as Result<bool, Error>
|
|
||||||
})
|
|
||||||
.await??;
|
|
||||||
|
|
||||||
// Bail if there are existing aliases
|
let aliases = sled_repo.aliases(hash.clone()).await?;
|
||||||
if any_aliases {
|
|
||||||
debug!("Other aliases reference file, not removing from disk");
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
// -- DELETE HASH ENTRY --
|
if !aliases.is_empty() {
|
||||||
let main_tree = self.inner.main_tree.clone();
|
return Ok(());
|
||||||
let hash2 = hash.clone();
|
|
||||||
debug!("Deleting hash -> filename mapping");
|
|
||||||
let filename = web::block(move || main_tree.remove(&hash2))
|
|
||||||
.await??
|
|
||||||
.ok_or(UploadError::MissingFile)?;
|
|
||||||
|
|
||||||
// -- DELETE FILES --
|
|
||||||
let this = self.clone();
|
|
||||||
let cleanup_span = tracing::info_span!(
|
|
||||||
parent: None,
|
|
||||||
"Cleanup",
|
|
||||||
filename = &tracing::field::display(String::from_utf8_lossy(&filename)),
|
|
||||||
);
|
|
||||||
cleanup_span.follows_from(Span::current());
|
|
||||||
debug!("Spawning cleanup task");
|
|
||||||
actix_rt::spawn(
|
|
||||||
async move {
|
|
||||||
if let Err(e) = this
|
|
||||||
.cleanup_files(store, FilenameIVec::new(filename.clone()))
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
error!("Error removing files from fs, {}", e);
|
|
||||||
}
|
}
|
||||||
info!(
|
|
||||||
"Files deleted for {:?}",
|
let variant_idents = sled_repo
|
||||||
String::from_utf8(filename.to_vec())
|
.variants::<S::Identifier>(hash.clone())
|
||||||
|
.await?
|
||||||
|
.into_iter()
|
||||||
|
.map(|(_, v)| v)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let main_ident = sled_repo.identifier(hash.clone()).await?;
|
||||||
|
let motion_ident = sled_repo.motion_identifier(hash.clone()).await?;
|
||||||
|
|
||||||
|
let repo = sled_repo.clone();
|
||||||
|
|
||||||
|
HashRepo::cleanup(sled_repo, hash).await?;
|
||||||
|
|
||||||
|
let cleanup_span = tracing::info_span!("Cleaning files");
|
||||||
|
cleanup_span.follows_from(Span::current());
|
||||||
|
|
||||||
|
actix_rt::spawn(
|
||||||
|
async move {
|
||||||
|
let mut errors = Vec::new();
|
||||||
|
|
||||||
|
for identifier in variant_idents
|
||||||
|
.iter()
|
||||||
|
.chain(&[main_ident])
|
||||||
|
.chain(motion_ident.iter())
|
||||||
|
{
|
||||||
|
debug!("Deleting {:?}", identifier);
|
||||||
|
if let Err(e) = store.remove(identifier).await {
|
||||||
|
let e: Error = e.into();
|
||||||
|
errors.push(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(e) = IdentifierRepo::cleanup(&repo, identifier).await {
|
||||||
|
let e: Error = e.into();
|
||||||
|
errors.push(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !errors.is_empty() {
|
||||||
|
let span = tracing::error_span!("Error deleting files");
|
||||||
|
span.in_scope(|| {
|
||||||
|
for error in errors {
|
||||||
|
error!("{}", error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.instrument(cleanup_span),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
.instrument(cleanup_span),
|
}
|
||||||
);
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch the real on-disk filename given an alias
|
|
||||||
#[instrument(skip(self))]
|
|
||||||
pub(crate) async fn from_alias(&self, alias: String) -> Result<String, Error> {
|
|
||||||
let tree = self.inner.alias_tree.clone();
|
|
||||||
debug!("Getting hash from alias");
|
|
||||||
let hash = web::block(move || tree.get(alias.as_bytes()))
|
|
||||||
.await??
|
|
||||||
.ok_or(UploadError::MissingAlias)?;
|
|
||||||
|
|
||||||
let main_tree = self.inner.main_tree.clone();
|
|
||||||
debug!("Getting filename from hash");
|
|
||||||
let filename = web::block(move || main_tree.get(hash))
|
|
||||||
.await??
|
|
||||||
.ok_or(UploadError::MissingFile)?;
|
|
||||||
|
|
||||||
let filename = String::from_utf8(filename.to_vec())?;
|
|
||||||
|
|
||||||
Ok(filename)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn session<S: Store + Clone + 'static>(&self, store: S) -> UploadManagerSession<S>
|
pub(crate) fn session<S: Store + Clone + 'static>(&self, store: S) -> UploadManagerSession<S>
|
||||||
where
|
where
|
||||||
Error: From<S::Error>,
|
Error: From<S::Error>,
|
||||||
{
|
{
|
||||||
UploadManagerSession::new(self.clone(), store)
|
UploadManagerSession::new(self.clone(), store)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find image variants and remove them from the DB and the disk
|
|
||||||
#[instrument(skip(self))]
|
|
||||||
async fn cleanup_files<S: Store>(&self, store: S, filename: FilenameIVec) -> Result<(), Error>
|
|
||||||
where
|
|
||||||
Error: From<S::Error>,
|
|
||||||
{
|
|
||||||
let filename = filename.inner;
|
|
||||||
|
|
||||||
let filename2 = filename.clone();
|
|
||||||
let identifier_tree = self.inner.identifier_tree.clone();
|
|
||||||
let identifier = web::block(move || identifier_tree.remove(filename2)).await??;
|
|
||||||
|
|
||||||
let mut errors = Vec::new();
|
|
||||||
if let Some(identifier) = identifier {
|
|
||||||
let identifier = S::Identifier::from_bytes(identifier.to_vec())?;
|
|
||||||
debug!("Deleting {:?}", identifier);
|
|
||||||
if let Err(e) = store.remove(&identifier).await {
|
|
||||||
errors.push(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let filename2 = filename.clone();
|
|
||||||
let fname_tree = self.inner.filename_tree.clone();
|
|
||||||
debug!("Deleting filename -> hash mapping");
|
|
||||||
web::block(move || fname_tree.remove(filename2)).await??;
|
|
||||||
|
|
||||||
let path_prefix = filename.clone();
|
|
||||||
let identifier_tree = self.inner.identifier_tree.clone();
|
|
||||||
debug!("Fetching file variants");
|
|
||||||
let identifiers = web::block(move || {
|
|
||||||
identifier_tree
|
|
||||||
.scan_prefix(path_prefix)
|
|
||||||
.values()
|
|
||||||
.collect::<Result<Vec<sled::IVec>, sled::Error>>()
|
|
||||||
})
|
|
||||||
.await??;
|
|
||||||
|
|
||||||
debug!("{} files prepared for deletion", identifiers.len());
|
|
||||||
|
|
||||||
for id in identifiers {
|
|
||||||
let identifier = S::Identifier::from_bytes(id.to_vec())?;
|
|
||||||
|
|
||||||
debug!("Deleting {:?}", identifier);
|
|
||||||
if let Err(e) = store.remove(&identifier).await {
|
|
||||||
errors.push(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let path_prefix = filename.clone();
|
|
||||||
let identifier_tree = self.inner.identifier_tree.clone();
|
|
||||||
debug!("Deleting path info");
|
|
||||||
web::block(move || {
|
|
||||||
for res in identifier_tree.scan_prefix(path_prefix).keys() {
|
|
||||||
let key = res?;
|
|
||||||
identifier_tree.remove(key)?;
|
|
||||||
}
|
|
||||||
Ok(()) as Result<(), Error>
|
|
||||||
})
|
|
||||||
.await??;
|
|
||||||
|
|
||||||
for error in errors {
|
|
||||||
error!("Error deleting files, {}", error);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn variant_key(
|
|
||||||
&self,
|
|
||||||
variant_process_path: &std::path::Path,
|
|
||||||
filename: &str,
|
|
||||||
) -> Result<Vec<u8>, Error> {
|
|
||||||
let path_string = variant_process_path
|
|
||||||
.to_str()
|
|
||||||
.ok_or(UploadError::Path)?
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
let vec = format!("{}/{}", filename, path_string).as_bytes().to_vec();
|
|
||||||
Ok(vec)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn details_key<I: Identifier>(&self, identifier: &I, filename: &str) -> Result<Vec<u8>, Error>
|
|
||||||
where
|
|
||||||
Error: From<I::Error>,
|
|
||||||
{
|
|
||||||
let mut vec = filename.as_bytes().to_vec();
|
|
||||||
vec.extend(b"/");
|
|
||||||
vec.extend(&identifier.to_bytes()?);
|
|
||||||
|
|
||||||
Ok(vec)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FilenameIVec {
|
async fn migrate_file<S1, S2>(
|
||||||
fn new(inner: sled::IVec) -> Self {
|
from: &S1,
|
||||||
FilenameIVec { inner }
|
to: &S2,
|
||||||
}
|
identifier: &S1::Identifier,
|
||||||
}
|
) -> Result<S2::Identifier, Error>
|
||||||
|
|
||||||
fn trans_upload_error(
|
|
||||||
upload_error: UploadError,
|
|
||||||
) -> sled::transaction::ConflictableTransactionError<Error> {
|
|
||||||
trans_err(upload_error)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn trans_utf8_error(e: FromUtf8Error) -> sled::transaction::ConflictableTransactionError<Error> {
|
|
||||||
trans_err(e)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn trans_err<E>(e: E) -> sled::transaction::ConflictableTransactionError<Error>
|
|
||||||
where
|
where
|
||||||
Error: From<E>,
|
S1: Store,
|
||||||
|
S2: Store,
|
||||||
|
Error: From<S1::Error> + From<S2::Error>,
|
||||||
{
|
{
|
||||||
sled::transaction::ConflictableTransactionError::Abort(e.into())
|
let stream = from.to_stream(identifier, None, None).await?;
|
||||||
|
futures_util::pin_mut!(stream);
|
||||||
|
let mut reader = tokio_util::io::StreamReader::new(stream);
|
||||||
|
|
||||||
|
let new_identifier = to.save_async_read(&mut reader).await?;
|
||||||
|
|
||||||
|
Ok(new_identifier)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delete_key(alias: &str) -> String {
|
async fn migrate_details<R, I1, I2>(repo: &R, from: I1, to: &I2) -> Result<(), Error>
|
||||||
format!("{}/delete", alias)
|
where
|
||||||
|
R: IdentifierRepo,
|
||||||
|
I1: Identifier,
|
||||||
|
I2: Identifier,
|
||||||
|
Error: From<<R as IdentifierRepo>::Error>,
|
||||||
|
{
|
||||||
|
if let Some(details) = repo.details(&from).await? {
|
||||||
|
repo.relate_details(to, &details).await?;
|
||||||
|
repo.cleanup(&from).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn do_migrate_store<R, S1, S2>(repo: &R, from: S1, to: S2) -> Result<(), Error>
|
||||||
|
where
|
||||||
|
S1: Store,
|
||||||
|
S2: Store,
|
||||||
|
Error: From<S1::Error> + From<S2::Error>,
|
||||||
|
R: IdentifierRepo + HashRepo + SettingsRepo,
|
||||||
|
Error: From<<R as IdentifierRepo>::Error>,
|
||||||
|
Error: From<<R as HashRepo>::Error>,
|
||||||
|
Error: From<<R as SettingsRepo>::Error>,
|
||||||
|
{
|
||||||
|
let stream = repo.hashes().await;
|
||||||
|
let mut stream = Box::pin(stream);
|
||||||
|
|
||||||
|
while let Some(hash) = stream.next().await {
|
||||||
|
let hash = hash?;
|
||||||
|
if let Some(identifier) = repo
|
||||||
|
.motion_identifier(hash.as_ref().to_vec().into())
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
let new_identifier = migrate_file(&from, &to, &identifier).await?;
|
||||||
|
migrate_details(repo, identifier, &new_identifier).await?;
|
||||||
|
repo.relate_motion_identifier(hash.as_ref().to_vec().into(), &new_identifier)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (variant, identifier) in repo.variants(hash.as_ref().to_vec().into()).await? {
|
||||||
|
let new_identifier = migrate_file(&from, &to, &identifier).await?;
|
||||||
|
migrate_details(repo, identifier, &new_identifier).await?;
|
||||||
|
repo.relate_variant_identifier(hash.as_ref().to_vec().into(), variant, &new_identifier)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let identifier = repo.identifier(hash.as_ref().to_vec().into()).await?;
|
||||||
|
let new_identifier = migrate_file(&from, &to, &identifier).await?;
|
||||||
|
migrate_details(repo, identifier, &new_identifier).await?;
|
||||||
|
repo.relate_identifier(hash.as_ref().to_vec().into(), &new_identifier)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
repo.set(STORE_MIGRATION_PROGRESS, hash.as_ref().to_vec().into())
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// clean up the migration key to avoid interfering with future migrations
|
||||||
|
repo.remove(STORE_MIGRATION_PROGRESS).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for UploadManager {
|
impl std::fmt::Debug for UploadManager {
|
||||||
|
@ -693,9 +445,3 @@ impl std::fmt::Debug for UploadManager {
|
||||||
f.debug_struct("UploadManager").finish()
|
f.debug_struct("UploadManager").finish()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for FilenameIVec {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
||||||
write!(f, "{:?}", String::from_utf8(self.inner.to_vec()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,19 +1,17 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
error::{Error, UploadError},
|
error::{Error, UploadError},
|
||||||
magick::ValidInputType,
|
magick::ValidInputType,
|
||||||
migrate::{alias_id_key, alias_key},
|
repo::{Alias, AliasRepo, AlreadyExists, DeleteToken, HashRepo, IdentifierRepo, Repo},
|
||||||
store::Store,
|
store::Store,
|
||||||
upload_manager::{
|
upload_manager::{
|
||||||
delete_key,
|
|
||||||
hasher::{Hash, Hasher},
|
hasher::{Hash, Hasher},
|
||||||
UploadManager,
|
UploadManager,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
use actix_web::web;
|
use actix_web::web;
|
||||||
use futures_util::stream::{Stream, StreamExt};
|
use futures_util::stream::{Stream, StreamExt};
|
||||||
use tracing::{debug, instrument, warn, Span};
|
use tracing::{debug, instrument, Span};
|
||||||
use tracing_futures::Instrument;
|
use tracing_futures::Instrument;
|
||||||
use uuid::Uuid;
|
|
||||||
|
|
||||||
pub(crate) struct UploadManagerSession<S: Store + Clone + 'static>
|
pub(crate) struct UploadManagerSession<S: Store + Clone + 'static>
|
||||||
where
|
where
|
||||||
|
@ -21,7 +19,7 @@ where
|
||||||
{
|
{
|
||||||
store: S,
|
store: S,
|
||||||
manager: UploadManager,
|
manager: UploadManager,
|
||||||
alias: Option<String>,
|
alias: Option<Alias>,
|
||||||
finished: bool,
|
finished: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -42,19 +40,8 @@ where
|
||||||
self.finished = true;
|
self.finished = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn alias(&self) -> Option<&str> {
|
pub(crate) fn alias(&self) -> Option<&Alias> {
|
||||||
self.alias.as_deref()
|
self.alias.as_ref()
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
enum Dup {
|
|
||||||
Exists,
|
|
||||||
New,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Dup {
|
|
||||||
fn exists(&self) -> bool {
|
|
||||||
matches!(self, Dup::Exists)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -79,20 +66,23 @@ where
|
||||||
actix_rt::spawn(
|
actix_rt::spawn(
|
||||||
async move {
|
async move {
|
||||||
// undo alias -> hash mapping
|
// undo alias -> hash mapping
|
||||||
debug!("Remove alias -> hash mapping");
|
match manager.inner.repo {
|
||||||
if let Ok(Some(hash)) = manager.inner.alias_tree.remove(&alias) {
|
Repo::Sled(ref sled_repo) => {
|
||||||
// undo alias -> id mapping
|
if let Ok(hash) = sled_repo.hash(&alias).await {
|
||||||
debug!("Remove alias -> id mapping");
|
debug!("Clean alias repo");
|
||||||
let key = alias_id_key(&alias);
|
let _ = AliasRepo::cleanup(sled_repo, &alias).await;
|
||||||
if let Ok(Some(id)) = manager.inner.alias_tree.remove(&key) {
|
|
||||||
// undo hash/id -> alias mapping
|
|
||||||
debug!("Remove hash/id -> alias mapping");
|
|
||||||
let id = String::from_utf8_lossy(&id);
|
|
||||||
let key = alias_key(&hash, &id);
|
|
||||||
let _ = manager.inner.main_tree.remove(&key);
|
|
||||||
}
|
|
||||||
|
|
||||||
let _ = manager.check_delete_files(store, hash).await;
|
if let Ok(identifier) = sled_repo.identifier(hash.clone()).await {
|
||||||
|
debug!("Clean identifier repo");
|
||||||
|
let _ = IdentifierRepo::cleanup(sled_repo, &identifier).await;
|
||||||
|
|
||||||
|
debug!("Remove stored files");
|
||||||
|
let _ = store.remove(&identifier).await;
|
||||||
|
}
|
||||||
|
debug!("Clean hash repo");
|
||||||
|
let _ = HashRepo::cleanup(sled_repo, hash).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.instrument(cleanup_span),
|
.instrument(cleanup_span),
|
||||||
|
@ -107,42 +97,30 @@ where
|
||||||
{
|
{
|
||||||
/// Generate a delete token for an alias
|
/// Generate a delete token for an alias
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn delete_token(&self) -> Result<String, Error> {
|
pub(crate) async fn delete_token(&self) -> Result<DeleteToken, Error> {
|
||||||
let alias = self.alias.clone().ok_or(UploadError::MissingAlias)?;
|
let alias = self.alias.clone().ok_or(UploadError::MissingAlias)?;
|
||||||
|
|
||||||
debug!("Generating delete token");
|
debug!("Generating delete token");
|
||||||
let s: String = Uuid::new_v4().to_string();
|
let delete_token = DeleteToken::generate();
|
||||||
let delete_token = s.clone();
|
|
||||||
|
|
||||||
debug!("Saving delete token");
|
debug!("Saving delete token");
|
||||||
let alias_tree = self.manager.inner.alias_tree.clone();
|
match self.manager.inner.repo {
|
||||||
let key = delete_key(&alias);
|
Repo::Sled(ref sled_repo) => {
|
||||||
let res = web::block(move || {
|
let res = sled_repo.relate_delete_token(&alias, &delete_token).await?;
|
||||||
alias_tree.compare_and_swap(
|
|
||||||
key.as_bytes(),
|
|
||||||
None as Option<sled::IVec>,
|
|
||||||
Some(s.as_bytes()),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.await??;
|
|
||||||
|
|
||||||
if let Err(sled::CompareAndSwapError {
|
Ok(if res.is_err() {
|
||||||
current: Some(ivec),
|
let delete_token = sled_repo.delete_token(&alias).await?;
|
||||||
..
|
debug!("Returning existing delete token, {:?}", delete_token);
|
||||||
}) = res
|
delete_token
|
||||||
{
|
} else {
|
||||||
let s = String::from_utf8(ivec.to_vec())?;
|
debug!("Returning new delete token, {:?}", delete_token);
|
||||||
|
delete_token
|
||||||
debug!("Returning existing delete token, {}", s);
|
})
|
||||||
return Ok(s);
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("Returning new delete token, {}", delete_token);
|
|
||||||
Ok(delete_token)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Upload the file while preserving the filename, optionally validating the uploaded image
|
/// Import the file, discarding bytes if it's already present, or saving if it's new
|
||||||
#[instrument(skip(self, stream))]
|
|
||||||
pub(crate) async fn import(
|
pub(crate) async fn import(
|
||||||
mut self,
|
mut self,
|
||||||
alias: String,
|
alias: String,
|
||||||
|
@ -158,7 +136,7 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("Validating bytes");
|
debug!("Validating bytes");
|
||||||
let (content_type, validated_reader) = crate::validate::validate_image_bytes(
|
let (_, validated_reader) = crate::validate::validate_image_bytes(
|
||||||
bytes_mut.freeze(),
|
bytes_mut.freeze(),
|
||||||
self.manager.inner.format,
|
self.manager.inner.format,
|
||||||
validate,
|
validate,
|
||||||
|
@ -167,20 +145,14 @@ where
|
||||||
|
|
||||||
let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
|
let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
|
||||||
|
|
||||||
let filename = self.next_file(content_type).await?;
|
let identifier = self.store.save_async_read(&mut hasher_reader).await?;
|
||||||
|
|
||||||
let identifier = self
|
|
||||||
.store
|
|
||||||
.save_async_read(&mut hasher_reader, &filename)
|
|
||||||
.await?;
|
|
||||||
let hash = hasher_reader.finalize_reset().await?;
|
let hash = hasher_reader.finalize_reset().await?;
|
||||||
|
|
||||||
debug!("Storing alias");
|
debug!("Adding alias");
|
||||||
self.alias = Some(alias.clone());
|
self.add_existing_alias(&hash, alias).await?;
|
||||||
self.add_existing_alias(&hash, &alias).await?;
|
|
||||||
|
|
||||||
debug!("Saving file");
|
debug!("Saving file");
|
||||||
self.save_upload(&identifier, hash, filename).await?;
|
self.save_upload(&identifier, hash).await?;
|
||||||
|
|
||||||
// Return alias to file
|
// Return alias to file
|
||||||
Ok(self)
|
Ok(self)
|
||||||
|
@ -210,106 +182,65 @@ where
|
||||||
|
|
||||||
let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
|
let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
|
||||||
|
|
||||||
let filename = self.next_file(input_type).await?;
|
let identifier = self.store.save_async_read(&mut hasher_reader).await?;
|
||||||
|
|
||||||
let identifier = self
|
|
||||||
.store
|
|
||||||
.save_async_read(&mut hasher_reader, &filename)
|
|
||||||
.await?;
|
|
||||||
let hash = hasher_reader.finalize_reset().await?;
|
let hash = hasher_reader.finalize_reset().await?;
|
||||||
|
|
||||||
debug!("Adding alias");
|
debug!("Adding alias");
|
||||||
self.add_alias(&hash, input_type).await?;
|
self.add_alias(&hash, input_type).await?;
|
||||||
|
|
||||||
debug!("Saving file");
|
debug!("Saving file");
|
||||||
self.save_upload(&identifier, hash, filename).await?;
|
self.save_upload(&identifier, hash).await?;
|
||||||
|
|
||||||
// Return alias to file
|
// Return alias to file
|
||||||
Ok(self)
|
Ok(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
// check duplicates & store image if new
|
// check duplicates & store image if new
|
||||||
async fn save_upload(
|
#[instrument(skip(self, hash))]
|
||||||
&self,
|
async fn save_upload(&self, identifier: &S::Identifier, hash: Hash) -> Result<(), Error> {
|
||||||
identifier: &S::Identifier,
|
let res = self.check_duplicate(&hash).await?;
|
||||||
hash: Hash,
|
|
||||||
filename: String,
|
|
||||||
) -> Result<(), Error> {
|
|
||||||
let dup = self.check_duplicate(hash, filename.clone()).await?;
|
|
||||||
|
|
||||||
// bail early with alias to existing file if this is a duplicate
|
// bail early with alias to existing file if this is a duplicate
|
||||||
if dup.exists() {
|
if res.is_err() {
|
||||||
debug!("Duplicate exists, removing file");
|
debug!("Duplicate exists, removing file");
|
||||||
|
|
||||||
self.store.remove(identifier).await?;
|
self.store.remove(identifier).await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
self.manager.store_identifier(filename, identifier).await?;
|
self.manager
|
||||||
|
.store_identifier(hash.into_inner(), identifier)
|
||||||
|
.await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
// check for an already-uploaded image with this hash, returning the path to the target file
|
// check for an already-uploaded image with this hash, returning the path to the target file
|
||||||
#[instrument(skip(self, hash))]
|
#[instrument(skip(self, hash))]
|
||||||
async fn check_duplicate(&self, hash: Hash, filename: String) -> Result<Dup, Error> {
|
async fn check_duplicate(&self, hash: &Hash) -> Result<Result<(), AlreadyExists>, Error> {
|
||||||
let main_tree = self.manager.inner.main_tree.clone();
|
let hash = hash.as_slice().to_vec();
|
||||||
|
|
||||||
let filename2 = filename.clone();
|
match self.manager.inner.repo {
|
||||||
let hash2 = hash.as_slice().to_vec();
|
Repo::Sled(ref sled_repo) => Ok(HashRepo::create(sled_repo, hash.into()).await?),
|
||||||
debug!("Inserting filename for hash");
|
|
||||||
let res = web::block(move || {
|
|
||||||
main_tree.compare_and_swap(
|
|
||||||
hash2,
|
|
||||||
None as Option<sled::IVec>,
|
|
||||||
Some(filename2.as_bytes()),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.await??;
|
|
||||||
|
|
||||||
if let Err(sled::CompareAndSwapError {
|
|
||||||
current: Some(ivec),
|
|
||||||
..
|
|
||||||
}) = res
|
|
||||||
{
|
|
||||||
let name = String::from_utf8(ivec.to_vec())?;
|
|
||||||
debug!("Filename exists for hash, {}", name);
|
|
||||||
return Ok(Dup::Exists);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let fname_tree = self.manager.inner.filename_tree.clone();
|
|
||||||
debug!("Saving filename -> hash relation");
|
|
||||||
web::block(move || fname_tree.insert(filename, hash.into_inner())).await??;
|
|
||||||
|
|
||||||
Ok(Dup::New)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// generate a short filename that isn't already in-use
|
// Add an alias from an existing filename
|
||||||
#[instrument(skip(self, input_type))]
|
async fn add_existing_alias(&mut self, hash: &Hash, filename: String) -> Result<(), Error> {
|
||||||
async fn next_file(&self, input_type: ValidInputType) -> Result<String, Error> {
|
let alias = Alias::from_existing(&filename);
|
||||||
loop {
|
|
||||||
debug!("Filename generation loop");
|
|
||||||
let filename = file_name(Uuid::new_v4(), input_type);
|
|
||||||
|
|
||||||
let identifier_tree = self.manager.inner.identifier_tree.clone();
|
match self.manager.inner.repo {
|
||||||
let filename2 = filename.clone();
|
Repo::Sled(ref sled_repo) => {
|
||||||
let filename_exists = web::block(move || identifier_tree.get(filename2.as_bytes()))
|
AliasRepo::create(sled_repo, &alias)
|
||||||
.await??
|
.await?
|
||||||
.is_some();
|
.map_err(|_| UploadError::DuplicateAlias)?;
|
||||||
|
self.alias = Some(alias.clone());
|
||||||
|
|
||||||
if !filename_exists {
|
let hash = hash.as_slice().to_vec();
|
||||||
return Ok(filename);
|
sled_repo.relate_hash(&alias, hash.clone().into()).await?;
|
||||||
|
sled_repo.relate_alias(hash.into(), &alias).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("Filename exists, trying again");
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
#[instrument(skip(self, hash, alias))]
|
|
||||||
async fn add_existing_alias(&self, hash: &Hash, alias: &str) -> Result<(), Error> {
|
|
||||||
self.save_alias_hash_mapping(hash, alias).await??;
|
|
||||||
|
|
||||||
self.store_hash_id_alias_mapping(hash, alias).await?;
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -319,96 +250,25 @@ where
|
||||||
// This will help if multiple 'users' upload the same file, and one of them wants to delete it
|
// This will help if multiple 'users' upload the same file, and one of them wants to delete it
|
||||||
#[instrument(skip(self, hash, input_type))]
|
#[instrument(skip(self, hash, input_type))]
|
||||||
async fn add_alias(&mut self, hash: &Hash, input_type: ValidInputType) -> Result<(), Error> {
|
async fn add_alias(&mut self, hash: &Hash, input_type: ValidInputType) -> Result<(), Error> {
|
||||||
let alias = self.next_alias(hash, input_type).await?;
|
|
||||||
|
|
||||||
self.store_hash_id_alias_mapping(hash, &alias).await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add a pre-defined alias to an existin file
|
|
||||||
//
|
|
||||||
// DANGER: this can cause BAD BAD BAD conflicts if the same alias is used for multiple files
|
|
||||||
#[instrument(skip(self, hash))]
|
|
||||||
async fn store_hash_id_alias_mapping(&self, hash: &Hash, alias: &str) -> Result<(), Error> {
|
|
||||||
let alias = alias.to_string();
|
|
||||||
loop {
|
|
||||||
debug!("hash -> alias save loop");
|
|
||||||
let db = self.manager.inner.db.clone();
|
|
||||||
let id = web::block(move || db.generate_id()).await??.to_string();
|
|
||||||
|
|
||||||
let alias_tree = self.manager.inner.alias_tree.clone();
|
|
||||||
let key = alias_id_key(&alias);
|
|
||||||
let id2 = id.clone();
|
|
||||||
debug!("Saving alias -> id mapping");
|
|
||||||
web::block(move || alias_tree.insert(key.as_bytes(), id2.as_bytes())).await??;
|
|
||||||
|
|
||||||
let key = alias_key(hash.as_slice(), &id);
|
|
||||||
let main_tree = self.manager.inner.main_tree.clone();
|
|
||||||
let alias2 = alias.clone();
|
|
||||||
debug!("Saving hash/id -> alias mapping");
|
|
||||||
let res = web::block(move || {
|
|
||||||
main_tree.compare_and_swap(key, None as Option<sled::IVec>, Some(alias2.as_bytes()))
|
|
||||||
})
|
|
||||||
.await??;
|
|
||||||
|
|
||||||
if res.is_ok() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Id exists, trying again");
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate an alias to the file
|
|
||||||
#[instrument(skip(self, hash, input_type))]
|
|
||||||
async fn next_alias(
|
|
||||||
&mut self,
|
|
||||||
hash: &Hash,
|
|
||||||
input_type: ValidInputType,
|
|
||||||
) -> Result<String, Error> {
|
|
||||||
loop {
|
loop {
|
||||||
debug!("Alias gen loop");
|
debug!("Alias gen loop");
|
||||||
let alias = file_name(Uuid::new_v4(), input_type);
|
let alias = Alias::generate(input_type.as_ext().to_string());
|
||||||
self.alias = Some(alias.clone());
|
|
||||||
|
|
||||||
let res = self.save_alias_hash_mapping(hash, &alias).await?;
|
match self.manager.inner.repo {
|
||||||
|
Repo::Sled(ref sled_repo) => {
|
||||||
|
let res = AliasRepo::create(sled_repo, &alias).await?;
|
||||||
|
|
||||||
|
if res.is_ok() {
|
||||||
|
self.alias = Some(alias.clone());
|
||||||
|
let hash = hash.as_slice().to_vec();
|
||||||
|
sled_repo.relate_hash(&alias, hash.clone().into()).await?;
|
||||||
|
sled_repo.relate_alias(hash.into(), &alias).await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
if res.is_ok() {
|
|
||||||
return Ok(alias);
|
|
||||||
}
|
|
||||||
debug!("Alias exists, regenning");
|
debug!("Alias exists, regenning");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save an alias to the database
|
|
||||||
#[instrument(skip(self, hash))]
|
|
||||||
async fn save_alias_hash_mapping(
|
|
||||||
&self,
|
|
||||||
hash: &Hash,
|
|
||||||
alias: &str,
|
|
||||||
) -> Result<Result<(), Error>, Error> {
|
|
||||||
let tree = self.manager.inner.alias_tree.clone();
|
|
||||||
let vec = hash.as_slice().to_vec();
|
|
||||||
let alias = alias.to_string();
|
|
||||||
|
|
||||||
debug!("Saving alias -> hash mapping");
|
|
||||||
let res = web::block(move || {
|
|
||||||
tree.compare_and_swap(alias.as_bytes(), None as Option<sled::IVec>, Some(vec))
|
|
||||||
})
|
|
||||||
.await??;
|
|
||||||
|
|
||||||
if res.is_err() {
|
|
||||||
warn!("Duplicate alias");
|
|
||||||
return Ok(Err(UploadError::DuplicateAlias.into()));
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Ok(()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn file_name(name: Uuid, input_type: ValidInputType) -> String {
|
|
||||||
format!("{}{}", name, input_type.as_ext())
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue