mirror of
https://git.asonix.dog/asonix/pict-rs
synced 2024-12-22 19:31:35 +00:00
Make pict-rs generic over file storage
This commit is contained in:
parent
f67aeb92fa
commit
48557bc2ea
18 changed files with 1088 additions and 761 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
@ -1155,6 +1155,7 @@ dependencies = [
|
||||||
"actix-server",
|
"actix-server",
|
||||||
"actix-web",
|
"actix-web",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"async-trait",
|
||||||
"awc",
|
"awc",
|
||||||
"base64",
|
"base64",
|
||||||
"dashmap",
|
"dashmap",
|
||||||
|
|
|
@ -19,6 +19,7 @@ actix-rt = "2.2.0"
|
||||||
actix-server = "2.0.0-beta.6"
|
actix-server = "2.0.0-beta.6"
|
||||||
actix-web = { version = "4.0.0-beta.10", default-features = false }
|
actix-web = { version = "4.0.0-beta.10", default-features = false }
|
||||||
anyhow = "1.0"
|
anyhow = "1.0"
|
||||||
|
async-trait = "0.1.51"
|
||||||
awc = { version = "3.0.0-beta.9", default-features = false, features = ["rustls"] }
|
awc = { version = "3.0.0-beta.9", default-features = false, features = ["rustls"] }
|
||||||
base64 = "0.13.0"
|
base64 = "0.13.0"
|
||||||
dashmap = "4.0.2"
|
dashmap = "4.0.2"
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
use std::{collections::HashSet, net::SocketAddr, path::PathBuf};
|
use std::{collections::HashSet, net::SocketAddr, path::PathBuf};
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
|
use crate::magick::ValidInputType;
|
||||||
|
|
||||||
#[derive(Clone, Debug, structopt::StructOpt)]
|
#[derive(Clone, Debug, structopt::StructOpt)]
|
||||||
pub(crate) struct Config {
|
pub(crate) struct Config {
|
||||||
#[structopt(
|
#[structopt(
|
||||||
|
@ -130,7 +132,7 @@ impl Config {
|
||||||
#[error("Invalid format supplied, {0}")]
|
#[error("Invalid format supplied, {0}")]
|
||||||
pub(crate) struct FormatError(String);
|
pub(crate) struct FormatError(String);
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Copy, Clone, Debug)]
|
||||||
pub(crate) enum Format {
|
pub(crate) enum Format {
|
||||||
Jpeg,
|
Jpeg,
|
||||||
Png,
|
Png,
|
||||||
|
@ -153,6 +155,14 @@ impl Format {
|
||||||
Format::Webp => "WEBP",
|
Format::Webp => "WEBP",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn to_hint(&self) -> Option<ValidInputType> {
|
||||||
|
match self {
|
||||||
|
Format::Jpeg => Some(ValidInputType::Jpeg),
|
||||||
|
Format::Png => Some(ValidInputType::Png),
|
||||||
|
Format::Webp => Some(ValidInputType::Webp),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::str::FromStr for Format {
|
impl std::str::FromStr for Format {
|
||||||
|
|
|
@ -69,6 +69,9 @@ pub(crate) enum UploadError {
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
StripPrefix(#[from] std::path::StripPrefixError),
|
StripPrefix(#[from] std::path::StripPrefixError),
|
||||||
|
|
||||||
|
#[error(transparent)]
|
||||||
|
FileStore(#[from] crate::store::file_store::FileError),
|
||||||
|
|
||||||
#[error("Provided process path is invalid")]
|
#[error("Provided process path is invalid")]
|
||||||
ParsePath,
|
ParsePath,
|
||||||
|
|
||||||
|
@ -114,18 +117,12 @@ pub(crate) enum UploadError {
|
||||||
#[error("Tried to save an image with an already-taken name")]
|
#[error("Tried to save an image with an already-taken name")]
|
||||||
DuplicateAlias,
|
DuplicateAlias,
|
||||||
|
|
||||||
#[error("Tried to create file, but file already exists")]
|
|
||||||
FileExists,
|
|
||||||
|
|
||||||
#[error("{0}")]
|
#[error("{0}")]
|
||||||
Json(#[from] serde_json::Error),
|
Json(#[from] serde_json::Error),
|
||||||
|
|
||||||
#[error("Range header not satisfiable")]
|
#[error("Range header not satisfiable")]
|
||||||
Range,
|
Range,
|
||||||
|
|
||||||
#[error("Command failed")]
|
|
||||||
Status,
|
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
Limit(#[from] super::LimitError),
|
Limit(#[from] super::LimitError),
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,9 @@
|
||||||
use crate::{
|
use crate::{error::Error, process::Process, store::Store};
|
||||||
error::{Error, UploadError},
|
|
||||||
process::Process,
|
|
||||||
};
|
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
use tokio::{io::AsyncRead, process::Command};
|
use tokio::io::AsyncRead;
|
||||||
use tracing::instrument;
|
use tracing::instrument;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
pub(crate) enum InputFormat {
|
pub(crate) enum InputFormat {
|
||||||
Gif,
|
Gif,
|
||||||
Mp4,
|
Mp4,
|
||||||
|
@ -71,48 +69,29 @@ pub(crate) fn to_mp4_bytes(
|
||||||
Ok(process.bytes_read(input).unwrap())
|
Ok(process.bytes_read(input).unwrap())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Create video thumbnail", skip(from, to))]
|
#[instrument(name = "Create video thumbnail")]
|
||||||
pub(crate) async fn thumbnail<P1, P2>(
|
pub(crate) async fn thumbnail<S: Store>(
|
||||||
from: P1,
|
store: S,
|
||||||
to: P2,
|
from: S::Identifier,
|
||||||
|
input_format: InputFormat,
|
||||||
format: ThumbnailFormat,
|
format: ThumbnailFormat,
|
||||||
) -> Result<(), Error>
|
) -> Result<impl AsyncRead + Unpin, Error> {
|
||||||
where
|
let process = Process::run(
|
||||||
P1: AsRef<std::path::Path>,
|
"ffmpeg",
|
||||||
P2: AsRef<std::path::Path>,
|
&[
|
||||||
{
|
"-f",
|
||||||
let command = "ffmpeg";
|
input_format.as_format(),
|
||||||
let first_arg = "-i";
|
"-i",
|
||||||
let args = [
|
"pipe:",
|
||||||
"-vframes",
|
"-vframes",
|
||||||
"1",
|
"1",
|
||||||
"-codec",
|
"-codec",
|
||||||
format.as_codec(),
|
format.as_codec(),
|
||||||
"-f",
|
"-f",
|
||||||
format.as_format(),
|
format.as_format(),
|
||||||
];
|
"pipe:",
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
|
||||||
tracing::info!(
|
Ok(process.store_read(store, from).unwrap())
|
||||||
"Spawning command: {} {} {:?} {:?} {:?}",
|
|
||||||
command,
|
|
||||||
first_arg,
|
|
||||||
from.as_ref(),
|
|
||||||
args,
|
|
||||||
to.as_ref()
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut child = Command::new(command)
|
|
||||||
.arg(first_arg)
|
|
||||||
.arg(from.as_ref())
|
|
||||||
.args(args)
|
|
||||||
.arg(to.as_ref())
|
|
||||||
.spawn()?;
|
|
||||||
|
|
||||||
let status = child.wait().await?;
|
|
||||||
|
|
||||||
if !status.success() {
|
|
||||||
return Err(UploadError::Status.into());
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@ use crate::{
|
||||||
config::Format,
|
config::Format,
|
||||||
error::{Error, UploadError},
|
error::{Error, UploadError},
|
||||||
process::Process,
|
process::Process,
|
||||||
|
store::Store,
|
||||||
};
|
};
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
use tokio::{
|
use tokio::{
|
||||||
|
@ -10,6 +11,15 @@ use tokio::{
|
||||||
};
|
};
|
||||||
use tracing::instrument;
|
use tracing::instrument;
|
||||||
|
|
||||||
|
pub(crate) fn details_hint(filename: &str) -> Option<ValidInputType> {
|
||||||
|
if filename.ends_with(".mp4") {
|
||||||
|
Some(ValidInputType::Mp4)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
pub(crate) enum ValidInputType {
|
pub(crate) enum ValidInputType {
|
||||||
Mp4,
|
Mp4,
|
||||||
Gif,
|
Gif,
|
||||||
|
@ -18,6 +28,18 @@ pub(crate) enum ValidInputType {
|
||||||
Webp,
|
Webp,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl ValidInputType {
|
||||||
|
fn to_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Mp4 => "MP4",
|
||||||
|
Self::Gif => "GIF",
|
||||||
|
Self::Png => "PNG",
|
||||||
|
Self::Jpeg => "JPEG",
|
||||||
|
Self::Webp => "WEBP",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(crate) struct Details {
|
pub(crate) struct Details {
|
||||||
pub(crate) mime_type: mime::Mime,
|
pub(crate) mime_type: mime::Mime,
|
||||||
|
@ -32,10 +54,19 @@ pub(crate) fn clear_metadata_bytes_read(input: Bytes) -> std::io::Result<impl As
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Getting details from input bytes", skip(input))]
|
#[instrument(name = "Getting details from input bytes", skip(input))]
|
||||||
pub(crate) async fn details_bytes(input: Bytes) -> Result<Details, Error> {
|
pub(crate) async fn details_bytes(
|
||||||
|
input: Bytes,
|
||||||
|
hint: Option<ValidInputType>,
|
||||||
|
) -> Result<Details, Error> {
|
||||||
|
let last_arg = if let Some(expected_format) = hint {
|
||||||
|
format!("{}:-", expected_format.to_str())
|
||||||
|
} else {
|
||||||
|
"-".to_owned()
|
||||||
|
};
|
||||||
|
|
||||||
let process = Process::run(
|
let process = Process::run(
|
||||||
"magick",
|
"magick",
|
||||||
&["identify", "-ping", "-format", "%w %h | %m\n", "-"],
|
&["identify", "-ping", "-format", "%w %h | %m\n", &last_arg],
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
let mut reader = process.bytes_read(input).unwrap();
|
let mut reader = process.bytes_read(input).unwrap();
|
||||||
|
@ -64,22 +95,28 @@ pub(crate) fn convert_bytes_read(
|
||||||
Ok(process.bytes_read(input).unwrap())
|
Ok(process.bytes_read(input).unwrap())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn details<P>(file: P) -> Result<Details, Error>
|
pub(crate) async fn details_store<S: Store>(
|
||||||
where
|
store: S,
|
||||||
P: AsRef<std::path::Path>,
|
identifier: S::Identifier,
|
||||||
{
|
expected_format: Option<ValidInputType>,
|
||||||
let command = "magick";
|
) -> Result<Details, Error> {
|
||||||
let args = ["identify", "-ping", "-format", "%w %h | %m\n"];
|
let last_arg = if let Some(expected_format) = expected_format {
|
||||||
let last_arg = file.as_ref();
|
format!("{}:-", expected_format.to_str())
|
||||||
|
} else {
|
||||||
|
"-".to_owned()
|
||||||
|
};
|
||||||
|
|
||||||
tracing::info!("Spawning command: {} {:?} {:?}", command, args, last_arg);
|
let process = Process::run(
|
||||||
let output = Command::new(command)
|
"magick",
|
||||||
.args(args)
|
&["identify", "-ping", "-format", "%w %h | %m\n", &last_arg],
|
||||||
.arg(last_arg)
|
)?;
|
||||||
.output()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let s = String::from_utf8_lossy(&output.stdout);
|
let mut reader = process.store_read(store, identifier).unwrap();
|
||||||
|
|
||||||
|
let mut output = Vec::new();
|
||||||
|
reader.read_to_end(&mut output).await?;
|
||||||
|
|
||||||
|
let s = String::from_utf8_lossy(&output);
|
||||||
|
|
||||||
parse_details(s)
|
parse_details(s)
|
||||||
}
|
}
|
||||||
|
@ -137,12 +174,13 @@ fn parse_details(s: std::borrow::Cow<'_, str>) -> Result<Details, Error> {
|
||||||
|
|
||||||
#[instrument(name = "Getting input type from bytes", skip(input))]
|
#[instrument(name = "Getting input type from bytes", skip(input))]
|
||||||
pub(crate) async fn input_type_bytes(input: Bytes) -> Result<ValidInputType, Error> {
|
pub(crate) async fn input_type_bytes(input: Bytes) -> Result<ValidInputType, Error> {
|
||||||
details_bytes(input).await?.validate_input()
|
details_bytes(input, None).await?.validate_input()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Spawning process command", skip(input))]
|
#[instrument(name = "Spawning process command")]
|
||||||
pub(crate) fn process_image_file_read(
|
pub(crate) fn process_image_store_read<S: Store>(
|
||||||
input: crate::file::File,
|
store: S,
|
||||||
|
identifier: S::Identifier,
|
||||||
args: Vec<String>,
|
args: Vec<String>,
|
||||||
format: Format,
|
format: Format,
|
||||||
) -> std::io::Result<impl AsyncRead + Unpin> {
|
) -> std::io::Result<impl AsyncRead + Unpin> {
|
||||||
|
@ -157,7 +195,7 @@ pub(crate) fn process_image_file_read(
|
||||||
.arg(last_arg),
|
.arg(last_arg),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
Ok(process.file_read(input).unwrap())
|
Ok(process.store_read(store, identifier).unwrap())
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Details {
|
impl Details {
|
||||||
|
|
439
src/main.rs
439
src/main.rs
|
@ -21,7 +21,6 @@ use tracing::{debug, error, info, instrument, Span};
|
||||||
use tracing_actix_web::TracingLogger;
|
use tracing_actix_web::TracingLogger;
|
||||||
use tracing_awc::Propagate;
|
use tracing_awc::Propagate;
|
||||||
use tracing_futures::Instrument;
|
use tracing_futures::Instrument;
|
||||||
use uuid::Uuid;
|
|
||||||
|
|
||||||
mod concurrent_processor;
|
mod concurrent_processor;
|
||||||
mod config;
|
mod config;
|
||||||
|
@ -29,25 +28,29 @@ mod either;
|
||||||
mod error;
|
mod error;
|
||||||
mod exiftool;
|
mod exiftool;
|
||||||
mod ffmpeg;
|
mod ffmpeg;
|
||||||
mod file;
|
|
||||||
mod init_tracing;
|
mod init_tracing;
|
||||||
mod magick;
|
mod magick;
|
||||||
|
mod map_error;
|
||||||
mod middleware;
|
mod middleware;
|
||||||
mod migrate;
|
mod migrate;
|
||||||
mod process;
|
mod process;
|
||||||
mod processor;
|
mod processor;
|
||||||
mod range;
|
mod range;
|
||||||
|
mod store;
|
||||||
mod upload_manager;
|
mod upload_manager;
|
||||||
mod validate;
|
mod validate;
|
||||||
|
|
||||||
|
use crate::{magick::details_hint, store::file_store::FileStore};
|
||||||
|
|
||||||
use self::{
|
use self::{
|
||||||
concurrent_processor::CancelSafeProcessor,
|
concurrent_processor::CancelSafeProcessor,
|
||||||
config::{Config, Format},
|
config::{Config, Format},
|
||||||
either::Either,
|
either::Either,
|
||||||
error::{Error, UploadError},
|
error::{Error, UploadError},
|
||||||
file::CrateError,
|
|
||||||
init_tracing::init_tracing,
|
init_tracing::init_tracing,
|
||||||
middleware::{Deadline, Internal},
|
middleware::{Deadline, Internal},
|
||||||
|
migrate::LatestDb,
|
||||||
|
store::Store,
|
||||||
upload_manager::{Details, UploadManager, UploadManagerSession},
|
upload_manager::{Details, UploadManager, UploadManagerSession},
|
||||||
validate::{image_webp, video_mp4},
|
validate::{image_webp, video_mp4},
|
||||||
};
|
};
|
||||||
|
@ -57,119 +60,19 @@ const MINUTES: u32 = 60;
|
||||||
const HOURS: u32 = 60 * MINUTES;
|
const HOURS: u32 = 60 * MINUTES;
|
||||||
const DAYS: u32 = 24 * HOURS;
|
const DAYS: u32 = 24 * HOURS;
|
||||||
|
|
||||||
static TMP_DIR: Lazy<PathBuf> = Lazy::new(|| {
|
|
||||||
let tmp_nonce = Uuid::new_v4();
|
|
||||||
|
|
||||||
let mut path = std::env::temp_dir();
|
|
||||||
path.push(format!("pict-rs-{}", tmp_nonce));
|
|
||||||
path
|
|
||||||
});
|
|
||||||
static CONFIG: Lazy<Config> = Lazy::new(Config::from_args);
|
static CONFIG: Lazy<Config> = Lazy::new(Config::from_args);
|
||||||
static PROCESS_SEMAPHORE: Lazy<Semaphore> =
|
static PROCESS_SEMAPHORE: Lazy<Semaphore> =
|
||||||
Lazy::new(|| Semaphore::new(num_cpus::get().saturating_sub(1).max(1)));
|
Lazy::new(|| Semaphore::new(num_cpus::get().saturating_sub(1).max(1)));
|
||||||
|
|
||||||
// try moving a file
|
|
||||||
#[instrument(name = "Moving file")]
|
|
||||||
async fn safe_move_file(from: PathBuf, to: PathBuf) -> Result<(), Error> {
|
|
||||||
if let Some(path) = to.parent() {
|
|
||||||
debug!("Creating directory {:?}", path);
|
|
||||||
tokio::fs::create_dir_all(path).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Checking if {:?} already exists", to);
|
|
||||||
if let Err(e) = tokio::fs::metadata(&to).await {
|
|
||||||
if e.kind() != std::io::ErrorKind::NotFound {
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return Err(UploadError::FileExists.into());
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Moving {:?} to {:?}", from, to);
|
|
||||||
tokio::fs::copy(&from, to).await?;
|
|
||||||
tokio::fs::remove_file(from).await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn safe_create_parent<P>(path: P) -> Result<(), Error>
|
|
||||||
where
|
|
||||||
P: AsRef<std::path::Path>,
|
|
||||||
{
|
|
||||||
if let Some(path) = path.as_ref().parent() {
|
|
||||||
debug!("Creating directory {:?}", path);
|
|
||||||
tokio::fs::create_dir_all(path).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try writing to a file
|
|
||||||
#[instrument(name = "Saving file", skip(bytes))]
|
|
||||||
async fn safe_save_file(path: PathBuf, bytes: web::Bytes) -> Result<(), Error> {
|
|
||||||
if let Some(path) = path.parent() {
|
|
||||||
// create the directory for the file
|
|
||||||
debug!("Creating directory {:?}", path);
|
|
||||||
tokio::fs::create_dir_all(path).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only write the file if it doesn't already exist
|
|
||||||
debug!("Checking if {:?} already exists", path);
|
|
||||||
if let Err(e) = tokio::fs::metadata(&path).await {
|
|
||||||
if e.kind() != std::io::ErrorKind::NotFound {
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Open the file for writing
|
|
||||||
debug!("Creating {:?}", path);
|
|
||||||
let mut file = crate::file::File::create(&path).await?;
|
|
||||||
|
|
||||||
// try writing
|
|
||||||
debug!("Writing to {:?}", path);
|
|
||||||
if let Err(e) = file.write_from_bytes(bytes).await {
|
|
||||||
error!("Error writing {:?}, {}", path, e);
|
|
||||||
// remove file if writing failed before completion
|
|
||||||
tokio::fs::remove_file(path).await?;
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
debug!("{:?} written", path);
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn tmp_file() -> PathBuf {
|
|
||||||
let s: String = Uuid::new_v4().to_string();
|
|
||||||
|
|
||||||
let name = format!("{}.tmp", s);
|
|
||||||
|
|
||||||
let mut path = TMP_DIR.clone();
|
|
||||||
path.push(&name);
|
|
||||||
|
|
||||||
path
|
|
||||||
}
|
|
||||||
|
|
||||||
fn to_ext(mime: mime::Mime) -> Result<&'static str, Error> {
|
|
||||||
if mime == mime::IMAGE_PNG {
|
|
||||||
Ok(".png")
|
|
||||||
} else if mime == mime::IMAGE_JPEG {
|
|
||||||
Ok(".jpg")
|
|
||||||
} else if mime == video_mp4() {
|
|
||||||
Ok(".mp4")
|
|
||||||
} else if mime == image_webp() {
|
|
||||||
Ok(".webp")
|
|
||||||
} else {
|
|
||||||
Err(UploadError::UnsupportedFormat.into())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Handle responding to succesful uploads
|
/// Handle responding to succesful uploads
|
||||||
#[instrument(name = "Uploaded files", skip(value, manager))]
|
#[instrument(name = "Uploaded files", skip(value, manager))]
|
||||||
async fn upload(
|
async fn upload<S: Store>(
|
||||||
value: Value<UploadManagerSession>,
|
value: Value<UploadManagerSession<S>>,
|
||||||
manager: web::Data<UploadManager>,
|
manager: web::Data<UploadManager<S>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let images = value
|
let images = value
|
||||||
.map()
|
.map()
|
||||||
.and_then(|mut m| m.remove("images"))
|
.and_then(|mut m| m.remove("images"))
|
||||||
|
@ -187,19 +90,23 @@ async fn upload(
|
||||||
let delete_token = image.result.delete_token().await?;
|
let delete_token = image.result.delete_token().await?;
|
||||||
|
|
||||||
let name = manager.from_alias(alias.to_owned()).await?;
|
let name = manager.from_alias(alias.to_owned()).await?;
|
||||||
let path = manager.path_from_filename(name.clone()).await?;
|
let identifier = manager.identifier_from_filename(name.clone()).await?;
|
||||||
|
|
||||||
let details = manager.variant_details(path.clone(), name.clone()).await?;
|
let details = manager
|
||||||
|
.variant_details(identifier.clone(), name.clone())
|
||||||
|
.await?;
|
||||||
|
|
||||||
let details = if let Some(details) = details {
|
let details = if let Some(details) = details {
|
||||||
debug!("details exist");
|
debug!("details exist");
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
debug!("generating new details from {:?}", path);
|
debug!("generating new details from {:?}", identifier);
|
||||||
let new_details = Details::from_path(path.clone()).await?;
|
let hint = details_hint(&name);
|
||||||
debug!("storing details for {:?} {}", path, name);
|
let new_details =
|
||||||
|
Details::from_store(manager.store().clone(), identifier.clone(), hint).await?;
|
||||||
|
debug!("storing details for {:?} {}", identifier, name);
|
||||||
manager
|
manager
|
||||||
.store_variant_details(path, name, &new_details)
|
.store_variant_details(identifier, name, &new_details)
|
||||||
.await?;
|
.await?;
|
||||||
debug!("stored");
|
debug!("stored");
|
||||||
new_details
|
new_details
|
||||||
|
@ -282,11 +189,14 @@ where
|
||||||
|
|
||||||
/// download an image from a URL
|
/// download an image from a URL
|
||||||
#[instrument(name = "Downloading file", skip(client, manager))]
|
#[instrument(name = "Downloading file", skip(client, manager))]
|
||||||
async fn download(
|
async fn download<S: Store>(
|
||||||
client: web::Data<Client>,
|
client: web::Data<Client>,
|
||||||
manager: web::Data<UploadManager>,
|
manager: web::Data<UploadManager<S>>,
|
||||||
query: web::Query<UrlQuery>,
|
query: web::Query<UrlQuery>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let res = client.get(&query.url).propagate().send().await?;
|
let res = client.get(&query.url).propagate().send().await?;
|
||||||
|
|
||||||
if !res.status().is_success() {
|
if !res.status().is_success() {
|
||||||
|
@ -294,7 +204,7 @@ async fn download(
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut stream = Limit::new(
|
let mut stream = Limit::new(
|
||||||
CrateError::new(res),
|
map_error::map_crate_error(res),
|
||||||
(CONFIG.max_file_size() * MEGABYTES) as u64,
|
(CONFIG.max_file_size() * MEGABYTES) as u64,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -308,16 +218,20 @@ async fn download(
|
||||||
let delete_token = session.delete_token().await?;
|
let delete_token = session.delete_token().await?;
|
||||||
|
|
||||||
let name = manager.from_alias(alias.to_owned()).await?;
|
let name = manager.from_alias(alias.to_owned()).await?;
|
||||||
let path = manager.path_from_filename(name.clone()).await?;
|
let identifier = manager.identifier_from_filename(name.clone()).await?;
|
||||||
|
|
||||||
let details = manager.variant_details(path.clone(), name.clone()).await?;
|
let details = manager
|
||||||
|
.variant_details(identifier.clone(), name.clone())
|
||||||
|
.await?;
|
||||||
|
|
||||||
let details = if let Some(details) = details {
|
let details = if let Some(details) = details {
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
let new_details = Details::from_path(path.clone()).await?;
|
let hint = details_hint(&name);
|
||||||
|
let new_details =
|
||||||
|
Details::from_store(manager.store().clone(), identifier.clone(), hint).await?;
|
||||||
manager
|
manager
|
||||||
.store_variant_details(path, name, &new_details)
|
.store_variant_details(identifier, name, &new_details)
|
||||||
.await?;
|
.await?;
|
||||||
new_details
|
new_details
|
||||||
};
|
};
|
||||||
|
@ -335,10 +249,13 @@ async fn download(
|
||||||
|
|
||||||
/// Delete aliases and files
|
/// Delete aliases and files
|
||||||
#[instrument(name = "Deleting file", skip(manager))]
|
#[instrument(name = "Deleting file", skip(manager))]
|
||||||
async fn delete(
|
async fn delete<S: Store>(
|
||||||
manager: web::Data<UploadManager>,
|
manager: web::Data<UploadManager<S>>,
|
||||||
path_entries: web::Path<(String, String)>,
|
path_entries: web::Path<(String, String)>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let (alias, token) = path_entries.into_inner();
|
let (alias, token) = path_entries.into_inner();
|
||||||
|
|
||||||
manager.delete(token, alias).await?;
|
manager.delete(token, alias).await?;
|
||||||
|
@ -348,12 +265,15 @@ async fn delete(
|
||||||
|
|
||||||
type ProcessQuery = Vec<(String, String)>;
|
type ProcessQuery = Vec<(String, String)>;
|
||||||
|
|
||||||
async fn prepare_process(
|
async fn prepare_process<S: Store>(
|
||||||
query: web::Query<ProcessQuery>,
|
query: web::Query<ProcessQuery>,
|
||||||
ext: &str,
|
ext: &str,
|
||||||
manager: &UploadManager,
|
manager: &UploadManager<S>,
|
||||||
filters: &Option<HashSet<String>>,
|
filters: &Option<HashSet<String>>,
|
||||||
) -> Result<(Format, String, PathBuf, Vec<String>), Error> {
|
) -> Result<(Format, String, PathBuf, Vec<String>), Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let (alias, operations) =
|
let (alias, operations) =
|
||||||
query
|
query
|
||||||
.into_inner()
|
.into_inner()
|
||||||
|
@ -394,21 +314,24 @@ async fn prepare_process(
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Fetching derived details", skip(manager, filters))]
|
#[instrument(name = "Fetching derived details", skip(manager, filters))]
|
||||||
async fn process_details(
|
async fn process_details<S: Store>(
|
||||||
query: web::Query<ProcessQuery>,
|
query: web::Query<ProcessQuery>,
|
||||||
ext: web::Path<String>,
|
ext: web::Path<String>,
|
||||||
manager: web::Data<UploadManager>,
|
manager: web::Data<UploadManager<S>>,
|
||||||
filters: web::Data<Option<HashSet<String>>>,
|
filters: web::Data<Option<HashSet<String>>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let (_, name, thumbnail_path, _) =
|
let (_, name, thumbnail_path, _) =
|
||||||
prepare_process(query, ext.as_str(), &manager, &filters).await?;
|
prepare_process(query, ext.as_str(), &manager, &filters).await?;
|
||||||
|
|
||||||
let real_path = manager
|
let identifier = manager
|
||||||
.variant_path(&thumbnail_path, &name)
|
.variant_identifier(&thumbnail_path, &name)
|
||||||
.await?
|
.await?
|
||||||
.ok_or(UploadError::MissingAlias)?;
|
.ok_or(UploadError::MissingAlias)?;
|
||||||
|
|
||||||
let details = manager.variant_details(real_path, name).await?;
|
let details = manager.variant_details(identifier, name).await?;
|
||||||
|
|
||||||
let details = details.ok_or(UploadError::NoFiles)?;
|
let details = details.ok_or(UploadError::NoFiles)?;
|
||||||
|
|
||||||
|
@ -417,55 +340,42 @@ async fn process_details(
|
||||||
|
|
||||||
/// Process files
|
/// Process files
|
||||||
#[instrument(name = "Serving processed image", skip(manager, filters))]
|
#[instrument(name = "Serving processed image", skip(manager, filters))]
|
||||||
async fn process(
|
async fn process<S: Store + 'static>(
|
||||||
range: Option<range::RangeHeader>,
|
range: Option<range::RangeHeader>,
|
||||||
query: web::Query<ProcessQuery>,
|
query: web::Query<ProcessQuery>,
|
||||||
ext: web::Path<String>,
|
ext: web::Path<String>,
|
||||||
manager: web::Data<UploadManager>,
|
manager: web::Data<UploadManager<S>>,
|
||||||
filters: web::Data<Option<HashSet<String>>>,
|
filters: web::Data<Option<HashSet<String>>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let (format, name, thumbnail_path, thumbnail_args) =
|
let (format, name, thumbnail_path, thumbnail_args) =
|
||||||
prepare_process(query, ext.as_str(), &manager, &filters).await?;
|
prepare_process(query, ext.as_str(), &manager, &filters).await?;
|
||||||
|
|
||||||
let real_path_opt = manager.variant_path(&thumbnail_path, &name).await?;
|
let identifier_opt = manager.variant_identifier(&thumbnail_path, &name).await?;
|
||||||
|
|
||||||
// If the thumbnail doesn't exist, we need to create it
|
if let Some(identifier) = identifier_opt {
|
||||||
let real_path_opt = if let Some(real_path) = real_path_opt {
|
|
||||||
if let Err(e) = tokio::fs::metadata(&real_path)
|
|
||||||
.instrument(tracing::info_span!("Get thumbnail metadata"))
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
if e.kind() != std::io::ErrorKind::NotFound {
|
|
||||||
error!("Error looking up processed image, {}", e);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
Some(real_path)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
if let Some(real_path) = real_path_opt {
|
|
||||||
let details_opt = manager
|
let details_opt = manager
|
||||||
.variant_details(real_path.clone(), name.clone())
|
.variant_details(identifier.clone(), name.clone())
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let details = if let Some(details) = details_opt {
|
let details = if let Some(details) = details_opt {
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
let details = Details::from_path(real_path.clone()).await?;
|
let hint = details_hint(&name);
|
||||||
|
let details =
|
||||||
|
Details::from_store(manager.store().clone(), identifier.clone(), hint).await?;
|
||||||
manager
|
manager
|
||||||
.store_variant_details(real_path.clone(), name, &details)
|
.store_variant_details(identifier.clone(), name, &details)
|
||||||
.await?;
|
.await?;
|
||||||
details
|
details
|
||||||
};
|
};
|
||||||
|
|
||||||
return ranged_file_resp(real_path, range, details).await;
|
return ranged_file_resp(manager.store().clone(), identifier, range, details).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let original_path = manager.still_path_from_filename(name.clone()).await?;
|
let identifier = manager.still_identifier_from_filename(name.clone()).await?;
|
||||||
|
|
||||||
let thumbnail_path2 = thumbnail_path.clone();
|
let thumbnail_path2 = thumbnail_path.clone();
|
||||||
let process_fut = async {
|
let process_fut = async {
|
||||||
|
@ -473,10 +383,12 @@ async fn process(
|
||||||
|
|
||||||
let permit = PROCESS_SEMAPHORE.acquire().await?;
|
let permit = PROCESS_SEMAPHORE.acquire().await?;
|
||||||
|
|
||||||
let file = crate::file::File::open(original_path.clone()).await?;
|
let mut processed_reader = crate::magick::process_image_store_read(
|
||||||
|
manager.store().clone(),
|
||||||
let mut processed_reader =
|
identifier,
|
||||||
crate::magick::process_image_file_read(file, thumbnail_args, format)?;
|
thumbnail_args,
|
||||||
|
format,
|
||||||
|
)?;
|
||||||
|
|
||||||
let mut vec = Vec::new();
|
let mut vec = Vec::new();
|
||||||
processed_reader.read_to_end(&mut vec).await?;
|
processed_reader.read_to_end(&mut vec).await?;
|
||||||
|
@ -484,7 +396,7 @@ async fn process(
|
||||||
|
|
||||||
drop(permit);
|
drop(permit);
|
||||||
|
|
||||||
let details = Details::from_bytes(bytes.clone()).await?;
|
let details = Details::from_bytes(bytes.clone(), format.to_hint()).await?;
|
||||||
|
|
||||||
let save_span = tracing::info_span!(
|
let save_span = tracing::info_span!(
|
||||||
parent: None,
|
parent: None,
|
||||||
|
@ -497,27 +409,22 @@ async fn process(
|
||||||
let bytes2 = bytes.clone();
|
let bytes2 = bytes.clone();
|
||||||
actix_rt::spawn(
|
actix_rt::spawn(
|
||||||
async move {
|
async move {
|
||||||
let real_path = match manager.next_directory() {
|
let identifier = match manager.store().save_bytes(bytes2).await {
|
||||||
Ok(real_path) => real_path.join(&name),
|
Ok(identifier) => identifier,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("Failed to generate directory path: {}", e);
|
tracing::warn!("Failed to generate directory path: {}", e);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Err(e) = safe_save_file(real_path.clone(), bytes2).await {
|
|
||||||
tracing::warn!("Error saving thumbnail: {}", e);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if let Err(e) = manager
|
if let Err(e) = manager
|
||||||
.store_variant_details(real_path.clone(), name.clone(), &details2)
|
.store_variant_details(identifier.clone(), name.clone(), &details2)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
tracing::warn!("Error saving variant details: {}", e);
|
tracing::warn!("Error saving variant details: {}", e);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if let Err(e) = manager
|
if let Err(e) = manager
|
||||||
.store_variant(Some(&thumbnail_path), &real_path, &name)
|
.store_variant(Some(&thumbnail_path), &identifier, &name)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
tracing::warn!("Error saving variant info: {}", e);
|
tracing::warn!("Error saving variant info: {}", e);
|
||||||
|
@ -569,21 +476,28 @@ async fn process(
|
||||||
|
|
||||||
/// Fetch file details
|
/// Fetch file details
|
||||||
#[instrument(name = "Fetching details", skip(manager))]
|
#[instrument(name = "Fetching details", skip(manager))]
|
||||||
async fn details(
|
async fn details<S: Store>(
|
||||||
alias: web::Path<String>,
|
alias: web::Path<String>,
|
||||||
manager: web::Data<UploadManager>,
|
manager: web::Data<UploadManager<S>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let name = manager.from_alias(alias.into_inner()).await?;
|
let name = manager.from_alias(alias.into_inner()).await?;
|
||||||
let path = manager.path_from_filename(name.clone()).await?;
|
let identifier = manager.identifier_from_filename(name.clone()).await?;
|
||||||
|
|
||||||
let details = manager.variant_details(path.clone(), name.clone()).await?;
|
let details = manager
|
||||||
|
.variant_details(identifier.clone(), name.clone())
|
||||||
|
.await?;
|
||||||
|
|
||||||
let details = if let Some(details) = details {
|
let details = if let Some(details) = details {
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
let new_details = Details::from_path(path.clone()).await?;
|
let hint = details_hint(&name);
|
||||||
|
let new_details =
|
||||||
|
Details::from_store(manager.store().clone(), identifier.clone(), hint).await?;
|
||||||
manager
|
manager
|
||||||
.store_variant_details(path.clone(), name, &new_details)
|
.store_variant_details(identifier, name, &new_details)
|
||||||
.await?;
|
.await?;
|
||||||
new_details
|
new_details
|
||||||
};
|
};
|
||||||
|
@ -593,34 +507,45 @@ async fn details(
|
||||||
|
|
||||||
/// Serve files
|
/// Serve files
|
||||||
#[instrument(name = "Serving file", skip(manager))]
|
#[instrument(name = "Serving file", skip(manager))]
|
||||||
async fn serve(
|
async fn serve<S: Store>(
|
||||||
range: Option<range::RangeHeader>,
|
range: Option<range::RangeHeader>,
|
||||||
alias: web::Path<String>,
|
alias: web::Path<String>,
|
||||||
manager: web::Data<UploadManager>,
|
manager: web::Data<UploadManager<S>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let name = manager.from_alias(alias.into_inner()).await?;
|
let name = manager.from_alias(alias.into_inner()).await?;
|
||||||
let path = manager.path_from_filename(name.clone()).await?;
|
let identifier = manager.identifier_from_filename(name.clone()).await?;
|
||||||
|
|
||||||
let details = manager.variant_details(path.clone(), name.clone()).await?;
|
let details = manager
|
||||||
|
.variant_details(identifier.clone(), name.clone())
|
||||||
|
.await?;
|
||||||
|
|
||||||
let details = if let Some(details) = details {
|
let details = if let Some(details) = details {
|
||||||
details
|
details
|
||||||
} else {
|
} else {
|
||||||
let details = Details::from_path(path.clone()).await?;
|
let hint = details_hint(&name);
|
||||||
|
let details =
|
||||||
|
Details::from_store(manager.store().clone(), identifier.clone(), hint).await?;
|
||||||
manager
|
manager
|
||||||
.store_variant_details(path.clone(), name, &details)
|
.store_variant_details(identifier.clone(), name, &details)
|
||||||
.await?;
|
.await?;
|
||||||
details
|
details
|
||||||
};
|
};
|
||||||
|
|
||||||
ranged_file_resp(path, range, details).await
|
ranged_file_resp(manager.store().clone(), identifier, range, details).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn ranged_file_resp(
|
async fn ranged_file_resp<S: Store>(
|
||||||
path: PathBuf,
|
store: S,
|
||||||
|
identifier: S::Identifier,
|
||||||
range: Option<range::RangeHeader>,
|
range: Option<range::RangeHeader>,
|
||||||
details: Details,
|
details: Details,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let (builder, stream) = match range {
|
let (builder, stream) = match range {
|
||||||
//Range header exists - return as ranged
|
//Range header exists - return as ranged
|
||||||
Some(range_header) => {
|
Some(range_header) => {
|
||||||
|
@ -631,24 +556,27 @@ async fn ranged_file_resp(
|
||||||
if range_header.is_empty() {
|
if range_header.is_empty() {
|
||||||
return Err(UploadError::Range.into());
|
return Err(UploadError::Range.into());
|
||||||
} else if range_header.len() == 1 {
|
} else if range_header.len() == 1 {
|
||||||
let file = crate::file::File::open(path).await?;
|
let len = store.len(&identifier).await?;
|
||||||
|
|
||||||
let meta = file.metadata().await?;
|
|
||||||
|
|
||||||
let range = range_header.ranges().next().unwrap();
|
let range = range_header.ranges().next().unwrap();
|
||||||
|
|
||||||
let mut builder = HttpResponse::PartialContent();
|
let mut builder = HttpResponse::PartialContent();
|
||||||
builder.insert_header(range.to_content_range(meta.len()));
|
builder.insert_header(range.to_content_range(len));
|
||||||
|
|
||||||
(builder, Either::left(range.chop_file(file).await?))
|
(
|
||||||
|
builder,
|
||||||
|
Either::left(map_error::map_crate_error(
|
||||||
|
range.chop_store(store, identifier).await?,
|
||||||
|
)),
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
return Err(UploadError::Range.into());
|
return Err(UploadError::Range.into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
//No Range header in the request - return the entire document
|
//No Range header in the request - return the entire document
|
||||||
None => {
|
None => {
|
||||||
let file = crate::file::File::open(path).await?;
|
let stream =
|
||||||
let stream = file.read_to_stream(None, None).await?;
|
map_error::map_crate_error(store.to_stream(&identifier, None, None).await?);
|
||||||
(HttpResponse::Ok(), Either::right(stream))
|
(HttpResponse::Ok(), Either::right(stream))
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -696,10 +624,13 @@ enum FileOrAlias {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Purging file", skip(upload_manager))]
|
#[instrument(name = "Purging file", skip(upload_manager))]
|
||||||
async fn purge(
|
async fn purge<S: Store>(
|
||||||
query: web::Query<FileOrAlias>,
|
query: web::Query<FileOrAlias>,
|
||||||
upload_manager: web::Data<UploadManager>,
|
upload_manager: web::Data<UploadManager<S>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let aliases = match query.into_inner() {
|
let aliases = match query.into_inner() {
|
||||||
FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
|
FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
|
||||||
FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
|
FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
|
||||||
|
@ -718,10 +649,13 @@ async fn purge(
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Fetching aliases", skip(upload_manager))]
|
#[instrument(name = "Fetching aliases", skip(upload_manager))]
|
||||||
async fn aliases(
|
async fn aliases<S: Store>(
|
||||||
query: web::Query<FileOrAlias>,
|
query: web::Query<FileOrAlias>,
|
||||||
upload_manager: web::Data<UploadManager>,
|
upload_manager: web::Data<UploadManager<S>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let aliases = match query.into_inner() {
|
let aliases = match query.into_inner() {
|
||||||
FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
|
FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
|
||||||
FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
|
FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
|
||||||
|
@ -739,10 +673,13 @@ struct ByAlias {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(name = "Fetching filename", skip(upload_manager))]
|
#[instrument(name = "Fetching filename", skip(upload_manager))]
|
||||||
async fn filename_by_alias(
|
async fn filename_by_alias<S: Store>(
|
||||||
query: web::Query<ByAlias>,
|
query: web::Query<ByAlias>,
|
||||||
upload_manager: web::Data<UploadManager>,
|
upload_manager: web::Data<UploadManager<S>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
let filename = upload_manager.from_alias(query.into_inner().alias).await?;
|
let filename = upload_manager.from_alias(query.into_inner().alias).await?;
|
||||||
|
|
||||||
Ok(HttpResponse::Ok().json(&serde_json::json!({
|
Ok(HttpResponse::Ok().json(&serde_json::json!({
|
||||||
|
@ -751,12 +688,17 @@ async fn filename_by_alias(
|
||||||
})))
|
})))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[actix_rt::main]
|
fn transform_error(error: actix_form_data::Error) -> actix_web::Error {
|
||||||
async fn main() -> anyhow::Result<()> {
|
let error: Error = error.into();
|
||||||
let manager = UploadManager::new(CONFIG.data_dir(), CONFIG.format()).await?;
|
let error: actix_web::Error = error.into();
|
||||||
|
error
|
||||||
init_tracing("pict-rs", CONFIG.opentelemetry_url())?;
|
}
|
||||||
|
|
||||||
|
async fn launch<S: Store>(manager: UploadManager<S>) -> anyhow::Result<()>
|
||||||
|
where
|
||||||
|
S::Error: Unpin,
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
// Create a new Multipart Form validator
|
// Create a new Multipart Form validator
|
||||||
//
|
//
|
||||||
// This form is expecting a single array field, 'images' with at most 10 files in it
|
// This form is expecting a single array field, 'images' with at most 10 files in it
|
||||||
|
@ -764,7 +706,7 @@ async fn main() -> anyhow::Result<()> {
|
||||||
let form = Form::new()
|
let form = Form::new()
|
||||||
.max_files(10)
|
.max_files(10)
|
||||||
.max_file_size(CONFIG.max_file_size() * MEGABYTES)
|
.max_file_size(CONFIG.max_file_size() * MEGABYTES)
|
||||||
.transform_error(|e| Error::from(e).into())
|
.transform_error(transform_error)
|
||||||
.field(
|
.field(
|
||||||
"images",
|
"images",
|
||||||
Field::array(Field::file(move |filename, _, stream| {
|
Field::array(Field::file(move |filename, _, stream| {
|
||||||
|
@ -775,7 +717,10 @@ async fn main() -> anyhow::Result<()> {
|
||||||
async move {
|
async move {
|
||||||
let permit = PROCESS_SEMAPHORE.acquire().await?;
|
let permit = PROCESS_SEMAPHORE.acquire().await?;
|
||||||
|
|
||||||
let res = manager.session().upload(stream).await;
|
let res = manager
|
||||||
|
.session()
|
||||||
|
.upload(map_error::map_crate_error(stream))
|
||||||
|
.await;
|
||||||
|
|
||||||
drop(permit);
|
drop(permit);
|
||||||
res
|
res
|
||||||
|
@ -792,7 +737,7 @@ async fn main() -> anyhow::Result<()> {
|
||||||
let import_form = Form::new()
|
let import_form = Form::new()
|
||||||
.max_files(10)
|
.max_files(10)
|
||||||
.max_file_size(CONFIG.max_file_size() * MEGABYTES)
|
.max_file_size(CONFIG.max_file_size() * MEGABYTES)
|
||||||
.transform_error(|e| Error::from(e).into())
|
.transform_error(transform_error)
|
||||||
.field(
|
.field(
|
||||||
"images",
|
"images",
|
||||||
Field::array(Field::file(move |filename, content_type, stream| {
|
Field::array(Field::file(move |filename, content_type, stream| {
|
||||||
|
@ -805,7 +750,12 @@ async fn main() -> anyhow::Result<()> {
|
||||||
|
|
||||||
let res = manager
|
let res = manager
|
||||||
.session()
|
.session()
|
||||||
.import(filename, content_type, validate_imports, stream)
|
.import(
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
validate_imports,
|
||||||
|
map_error::map_crate_error(stream),
|
||||||
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
drop(permit);
|
drop(permit);
|
||||||
|
@ -832,24 +782,25 @@ async fn main() -> anyhow::Result<()> {
|
||||||
web::resource("")
|
web::resource("")
|
||||||
.guard(guard::Post())
|
.guard(guard::Post())
|
||||||
.wrap(form.clone())
|
.wrap(form.clone())
|
||||||
.route(web::post().to(upload)),
|
.route(web::post().to(upload::<S>)),
|
||||||
)
|
)
|
||||||
.service(web::resource("/download").route(web::get().to(download)))
|
.service(web::resource("/download").route(web::get().to(download::<S>)))
|
||||||
.service(
|
.service(
|
||||||
web::resource("/delete/{delete_token}/{filename}")
|
web::resource("/delete/{delete_token}/{filename}")
|
||||||
.route(web::delete().to(delete))
|
.route(web::delete().to(delete::<S>))
|
||||||
.route(web::get().to(delete)),
|
.route(web::get().to(delete::<S>)),
|
||||||
)
|
)
|
||||||
.service(web::resource("/original/{filename}").route(web::get().to(serve)))
|
.service(web::resource("/original/{filename}").route(web::get().to(serve::<S>)))
|
||||||
.service(web::resource("/process.{ext}").route(web::get().to(process)))
|
.service(web::resource("/process.{ext}").route(web::get().to(process::<S>)))
|
||||||
.service(
|
.service(
|
||||||
web::scope("/details")
|
web::scope("/details")
|
||||||
.service(
|
.service(
|
||||||
web::resource("/original/{filename}").route(web::get().to(details)),
|
web::resource("/original/{filename}")
|
||||||
|
.route(web::get().to(details::<S>)),
|
||||||
)
|
)
|
||||||
.service(
|
.service(
|
||||||
web::resource("/process.{ext}")
|
web::resource("/process.{ext}")
|
||||||
.route(web::get().to(process_details)),
|
.route(web::get().to(process_details::<S>)),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
@ -859,20 +810,34 @@ async fn main() -> anyhow::Result<()> {
|
||||||
.service(
|
.service(
|
||||||
web::resource("/import")
|
web::resource("/import")
|
||||||
.wrap(import_form.clone())
|
.wrap(import_form.clone())
|
||||||
.route(web::post().to(upload)),
|
.route(web::post().to(upload::<S>)),
|
||||||
)
|
)
|
||||||
.service(web::resource("/purge").route(web::post().to(purge)))
|
.service(web::resource("/purge").route(web::post().to(purge::<S>)))
|
||||||
.service(web::resource("/aliases").route(web::get().to(aliases)))
|
.service(web::resource("/aliases").route(web::get().to(aliases::<S>)))
|
||||||
.service(web::resource("/filename").route(web::get().to(filename_by_alias))),
|
.service(
|
||||||
|
web::resource("/filename").route(web::get().to(filename_by_alias::<S>)),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.bind(CONFIG.bind_address())?
|
.bind(CONFIG.bind_address())?
|
||||||
.run()
|
.run()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if tokio::fs::metadata(&*TMP_DIR).await.is_ok() {
|
|
||||||
tokio::fs::remove_dir_all(&*TMP_DIR).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[actix_rt::main]
|
||||||
|
async fn main() -> anyhow::Result<()> {
|
||||||
|
init_tracing("pict-rs", CONFIG.opentelemetry_url())?;
|
||||||
|
|
||||||
|
let root_dir = CONFIG.data_dir();
|
||||||
|
let db = LatestDb::exists(root_dir.clone()).migrate()?;
|
||||||
|
|
||||||
|
let store = FileStore::build(root_dir, &db)?;
|
||||||
|
|
||||||
|
let manager = UploadManager::new(store, db, CONFIG.format()).await?;
|
||||||
|
|
||||||
|
// TODO: move restructure to FileStore
|
||||||
|
manager.restructure().await?;
|
||||||
|
launch(manager).await
|
||||||
|
}
|
||||||
|
|
43
src/map_error.rs
Normal file
43
src/map_error.rs
Normal file
|
@ -0,0 +1,43 @@
|
||||||
|
use crate::error::Error;
|
||||||
|
use futures_util::stream::Stream;
|
||||||
|
use std::{
|
||||||
|
marker::PhantomData,
|
||||||
|
pin::Pin,
|
||||||
|
task::{Context, Poll},
|
||||||
|
};
|
||||||
|
|
||||||
|
pin_project_lite::pin_project! {
|
||||||
|
pub(super) struct MapError<E, S> {
|
||||||
|
#[pin]
|
||||||
|
inner: S,
|
||||||
|
|
||||||
|
_error: PhantomData<E>,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn map_crate_error<S>(inner: S) -> MapError<Error, S> {
|
||||||
|
map_error(inner)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn map_error<S, E>(inner: S) -> MapError<E, S> {
|
||||||
|
MapError {
|
||||||
|
inner,
|
||||||
|
_error: PhantomData,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T, StreamErr, E, S> Stream for MapError<E, S>
|
||||||
|
where
|
||||||
|
S: Stream<Item = Result<T, StreamErr>>,
|
||||||
|
E: From<StreamErr>,
|
||||||
|
{
|
||||||
|
type Item = Result<T, E>;
|
||||||
|
|
||||||
|
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||||
|
let this = self.as_mut().project();
|
||||||
|
|
||||||
|
this.inner
|
||||||
|
.poll_next(cx)
|
||||||
|
.map(|opt| opt.map(|res| res.map_err(Into::into)))
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,3 +1,4 @@
|
||||||
|
use crate::store::Store;
|
||||||
use actix_rt::task::JoinHandle;
|
use actix_rt::task::JoinHandle;
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
use std::{
|
use std::{
|
||||||
|
@ -66,7 +67,7 @@ impl Process {
|
||||||
let mut stdin = self.child.stdin.take()?;
|
let mut stdin = self.child.stdin.take()?;
|
||||||
let stdout = self.child.stdout.take()?;
|
let stdout = self.child.stdout.take()?;
|
||||||
|
|
||||||
let (tx, rx) = channel();
|
let (tx, rx) = channel::<std::io::Error>();
|
||||||
|
|
||||||
let span = self.spawn_span();
|
let span = self.spawn_span();
|
||||||
let mut child = self.child;
|
let mut child = self.child;
|
||||||
|
@ -102,9 +103,10 @@ impl Process {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn file_read(
|
pub(crate) fn store_read<S: Store>(
|
||||||
mut self,
|
mut self,
|
||||||
mut input_file: crate::file::File,
|
store: S,
|
||||||
|
identifier: S::Identifier,
|
||||||
) -> Option<impl AsyncRead + Unpin> {
|
) -> Option<impl AsyncRead + Unpin> {
|
||||||
let mut stdin = self.child.stdin.take()?;
|
let mut stdin = self.child.stdin.take()?;
|
||||||
let stdout = self.child.stdout.take()?;
|
let stdout = self.child.stdout.take()?;
|
||||||
|
@ -115,7 +117,7 @@ impl Process {
|
||||||
let mut child = self.child;
|
let mut child = self.child;
|
||||||
let handle = actix_rt::spawn(
|
let handle = actix_rt::spawn(
|
||||||
async move {
|
async move {
|
||||||
if let Err(e) = input_file.read_to_async_write(&mut stdin).await {
|
if let Err(e) = store.read_into(&identifier, &mut stdin).await {
|
||||||
let _ = tx.send(e);
|
let _ = tx.send(e);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,6 +14,50 @@ pub(crate) trait Processor {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct Identity;
|
pub(crate) struct Identity;
|
||||||
|
pub(crate) struct Thumbnail(usize);
|
||||||
|
pub(crate) struct Resize(usize);
|
||||||
|
pub(crate) struct Crop(usize, usize);
|
||||||
|
pub(crate) struct Blur(f64);
|
||||||
|
|
||||||
|
#[instrument]
|
||||||
|
pub(crate) fn build_chain(
|
||||||
|
args: &[(String, String)],
|
||||||
|
filename: String,
|
||||||
|
) -> Result<(PathBuf, Vec<String>), Error> {
|
||||||
|
fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, UploadError> {
|
||||||
|
if key == P::NAME {
|
||||||
|
return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! parse {
|
||||||
|
($inner:expr, $x:ident, $k:expr, $v:expr) => {{
|
||||||
|
if let Some(processor) = parse::<$x>($k, $v)? {
|
||||||
|
return Ok((processor.path($inner.0), processor.command($inner.1)));
|
||||||
|
};
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
|
||||||
|
let (path, args) =
|
||||||
|
args.into_iter()
|
||||||
|
.fold(Ok((PathBuf::default(), vec![])), |inner, (name, value)| {
|
||||||
|
if let Ok(inner) = inner {
|
||||||
|
parse!(inner, Identity, name, value);
|
||||||
|
parse!(inner, Thumbnail, name, value);
|
||||||
|
parse!(inner, Resize, name, value);
|
||||||
|
parse!(inner, Crop, name, value);
|
||||||
|
parse!(inner, Blur, name, value);
|
||||||
|
|
||||||
|
Err(Error::from(UploadError::ParsePath))
|
||||||
|
} else {
|
||||||
|
inner
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok((path.join(filename), args))
|
||||||
|
}
|
||||||
|
|
||||||
impl Processor for Identity {
|
impl Processor for Identity {
|
||||||
const NAME: &'static str = "identity";
|
const NAME: &'static str = "identity";
|
||||||
|
@ -34,8 +78,6 @@ impl Processor for Identity {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct Thumbnail(usize);
|
|
||||||
|
|
||||||
impl Processor for Thumbnail {
|
impl Processor for Thumbnail {
|
||||||
const NAME: &'static str = "thumbnail";
|
const NAME: &'static str = "thumbnail";
|
||||||
|
|
||||||
|
@ -60,8 +102,6 @@ impl Processor for Thumbnail {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct Resize(usize);
|
|
||||||
|
|
||||||
impl Processor for Resize {
|
impl Processor for Resize {
|
||||||
const NAME: &'static str = "resize";
|
const NAME: &'static str = "resize";
|
||||||
|
|
||||||
|
@ -91,8 +131,6 @@ impl Processor for Resize {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct Crop(usize, usize);
|
|
||||||
|
|
||||||
impl Processor for Crop {
|
impl Processor for Crop {
|
||||||
const NAME: &'static str = "crop";
|
const NAME: &'static str = "crop";
|
||||||
|
|
||||||
|
@ -133,8 +171,6 @@ impl Processor for Crop {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct Blur(f64);
|
|
||||||
|
|
||||||
impl Processor for Blur {
|
impl Processor for Blur {
|
||||||
const NAME: &'static str = "blur";
|
const NAME: &'static str = "blur";
|
||||||
|
|
||||||
|
@ -155,43 +191,3 @@ impl Processor for Blur {
|
||||||
args
|
args
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument]
|
|
||||||
pub(crate) fn build_chain(
|
|
||||||
args: &[(String, String)],
|
|
||||||
filename: String,
|
|
||||||
) -> Result<(PathBuf, Vec<String>), Error> {
|
|
||||||
fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, UploadError> {
|
|
||||||
if key == P::NAME {
|
|
||||||
return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?));
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(None)
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! parse {
|
|
||||||
($inner:expr, $x:ident, $k:expr, $v:expr) => {{
|
|
||||||
if let Some(processor) = parse::<$x>($k, $v)? {
|
|
||||||
return Ok((processor.path($inner.0), processor.command($inner.1)));
|
|
||||||
};
|
|
||||||
}};
|
|
||||||
}
|
|
||||||
|
|
||||||
let (path, args) =
|
|
||||||
args.into_iter()
|
|
||||||
.fold(Ok((PathBuf::default(), vec![])), |inner, (name, value)| {
|
|
||||||
if let Ok(inner) = inner {
|
|
||||||
parse!(inner, Identity, name, value);
|
|
||||||
parse!(inner, Thumbnail, name, value);
|
|
||||||
parse!(inner, Resize, name, value);
|
|
||||||
parse!(inner, Crop, name, value);
|
|
||||||
parse!(inner, Blur, name, value);
|
|
||||||
|
|
||||||
Err(Error::from(UploadError::ParsePath))
|
|
||||||
} else {
|
|
||||||
inner
|
|
||||||
}
|
|
||||||
})?;
|
|
||||||
|
|
||||||
Ok((path.join(filename), args))
|
|
||||||
}
|
|
||||||
|
|
28
src/range.rs
28
src/range.rs
|
@ -1,4 +1,7 @@
|
||||||
use crate::error::{Error, UploadError};
|
use crate::{
|
||||||
|
error::{Error, UploadError},
|
||||||
|
store::Store,
|
||||||
|
};
|
||||||
use actix_web::{
|
use actix_web::{
|
||||||
dev::Payload,
|
dev::Payload,
|
||||||
http::{
|
http::{
|
||||||
|
@ -52,17 +55,22 @@ impl Range {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn chop_file(
|
pub(crate) async fn chop_store<S: Store>(
|
||||||
&self,
|
&self,
|
||||||
file: crate::file::File,
|
store: S,
|
||||||
) -> Result<impl Stream<Item = Result<Bytes, Error>>, Error> {
|
identifier: S::Identifier,
|
||||||
|
) -> Result<impl Stream<Item = std::io::Result<Bytes>>, Error>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
match self {
|
match self {
|
||||||
Range::Start(start) => file.read_to_stream(Some(*start), None).await,
|
Range::Start(start) => Ok(store.to_stream(&identifier, Some(*start), None).await?),
|
||||||
Range::SuffixLength(from_start) => file.read_to_stream(None, Some(*from_start)).await,
|
Range::SuffixLength(from_start) => Ok(store
|
||||||
Range::Segment(start, end) => {
|
.to_stream(&identifier, None, Some(*from_start))
|
||||||
file.read_to_stream(Some(*start), Some(end.saturating_sub(*start)))
|
.await?),
|
||||||
.await
|
Range::Segment(start, end) => Ok(store
|
||||||
}
|
.to_stream(&identifier, Some(*start), Some(end.saturating_sub(*start)))
|
||||||
|
.await?),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
52
src/store.rs
Normal file
52
src/store.rs
Normal file
|
@ -0,0 +1,52 @@
|
||||||
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
use actix_web::web::Bytes;
|
||||||
|
use futures_util::stream::Stream;
|
||||||
|
use tokio::io::{AsyncRead, AsyncWrite};
|
||||||
|
|
||||||
|
pub(crate) mod file_store;
|
||||||
|
|
||||||
|
pub(crate) trait Identifier: Send + Sync + Clone + Debug {
|
||||||
|
type Error: std::error::Error;
|
||||||
|
|
||||||
|
fn to_bytes(&self) -> Result<Vec<u8>, Self::Error>;
|
||||||
|
|
||||||
|
fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error>
|
||||||
|
where
|
||||||
|
Self: Sized;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait(?Send)]
|
||||||
|
pub(crate) trait Store: Send + Sync + Clone + Debug + 'static {
|
||||||
|
type Error: std::error::Error;
|
||||||
|
type Identifier: Identifier<Error = Self::Error>;
|
||||||
|
type Stream: Stream<Item = std::io::Result<Bytes>>;
|
||||||
|
|
||||||
|
async fn save_async_read<Reader>(
|
||||||
|
&self,
|
||||||
|
reader: &mut Reader,
|
||||||
|
) -> Result<Self::Identifier, Self::Error>
|
||||||
|
where
|
||||||
|
Reader: AsyncRead + Unpin;
|
||||||
|
|
||||||
|
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error>;
|
||||||
|
|
||||||
|
async fn to_stream(
|
||||||
|
&self,
|
||||||
|
identifier: &Self::Identifier,
|
||||||
|
from_start: Option<u64>,
|
||||||
|
len: Option<u64>,
|
||||||
|
) -> Result<Self::Stream, Self::Error>;
|
||||||
|
|
||||||
|
async fn read_into<Writer>(
|
||||||
|
&self,
|
||||||
|
identifier: &Self::Identifier,
|
||||||
|
writer: &mut Writer,
|
||||||
|
) -> Result<(), std::io::Error>
|
||||||
|
where
|
||||||
|
Writer: AsyncWrite + Unpin;
|
||||||
|
|
||||||
|
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Self::Error>;
|
||||||
|
|
||||||
|
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error>;
|
||||||
|
}
|
303
src/store/file_store.rs
Normal file
303
src/store/file_store.rs
Normal file
|
@ -0,0 +1,303 @@
|
||||||
|
use crate::store::Store;
|
||||||
|
use actix_web::web::Bytes;
|
||||||
|
use futures_util::stream::Stream;
|
||||||
|
use std::{
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
pin::Pin,
|
||||||
|
};
|
||||||
|
use storage_path_generator::Generator;
|
||||||
|
use tokio::io::{AsyncRead, AsyncWrite};
|
||||||
|
use tracing::{debug, error, instrument};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
mod file;
|
||||||
|
mod file_id;
|
||||||
|
mod restructure;
|
||||||
|
use file::File;
|
||||||
|
pub(crate) use file_id::FileId;
|
||||||
|
|
||||||
|
// - Settings Tree
|
||||||
|
// - last-path -> last generated path
|
||||||
|
// - fs-restructure-01-complete -> bool
|
||||||
|
|
||||||
|
const GENERATOR_KEY: &'static [u8] = b"last-path";
|
||||||
|
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub(crate) enum FileError {
|
||||||
|
#[error(transparent)]
|
||||||
|
Sled(#[from] sled::Error),
|
||||||
|
|
||||||
|
#[error(transparent)]
|
||||||
|
Io(#[from] std::io::Error),
|
||||||
|
|
||||||
|
#[error(transparent)]
|
||||||
|
PathGenerator(#[from] storage_path_generator::PathError),
|
||||||
|
|
||||||
|
#[error("Error formatting file store identifier")]
|
||||||
|
IdError,
|
||||||
|
|
||||||
|
#[error("Mailformed file store identifier")]
|
||||||
|
PrefixError,
|
||||||
|
|
||||||
|
#[error("Tried to save over existing file")]
|
||||||
|
FileExists,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct FileStore {
|
||||||
|
path_gen: Generator,
|
||||||
|
root_dir: PathBuf,
|
||||||
|
settings_tree: sled::Tree,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait(?Send)]
|
||||||
|
impl Store for FileStore {
|
||||||
|
type Error = FileError;
|
||||||
|
type Identifier = FileId;
|
||||||
|
type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
|
||||||
|
|
||||||
|
async fn save_async_read<Reader>(
|
||||||
|
&self,
|
||||||
|
reader: &mut Reader,
|
||||||
|
) -> Result<Self::Identifier, Self::Error>
|
||||||
|
where
|
||||||
|
Reader: AsyncRead + Unpin,
|
||||||
|
{
|
||||||
|
let path = self.next_file()?;
|
||||||
|
|
||||||
|
if let Err(e) = self.safe_save_reader(&path, reader).await {
|
||||||
|
self.safe_remove_file(&path).await?;
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.file_id_from_path(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
|
||||||
|
let path = self.next_file()?;
|
||||||
|
|
||||||
|
if let Err(e) = self.safe_save_bytes(&path, bytes).await {
|
||||||
|
self.safe_remove_file(&path).await?;
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.file_id_from_path(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn to_stream(
|
||||||
|
&self,
|
||||||
|
identifier: &Self::Identifier,
|
||||||
|
from_start: Option<u64>,
|
||||||
|
len: Option<u64>,
|
||||||
|
) -> Result<Self::Stream, Self::Error> {
|
||||||
|
let path = self.path_from_file_id(identifier);
|
||||||
|
|
||||||
|
let stream = File::open(path)
|
||||||
|
.await?
|
||||||
|
.read_to_stream(from_start, len)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(Box::pin(stream))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read_into<Writer>(
|
||||||
|
&self,
|
||||||
|
identifier: &Self::Identifier,
|
||||||
|
writer: &mut Writer,
|
||||||
|
) -> Result<(), std::io::Error>
|
||||||
|
where
|
||||||
|
Writer: AsyncWrite + Unpin,
|
||||||
|
{
|
||||||
|
let path = self.path_from_file_id(identifier);
|
||||||
|
|
||||||
|
File::open(&path).await?.read_to_async_write(writer).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Self::Error> {
|
||||||
|
let path = self.path_from_file_id(identifier);
|
||||||
|
|
||||||
|
let len = tokio::fs::metadata(path).await?.len();
|
||||||
|
|
||||||
|
Ok(len)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error> {
|
||||||
|
let path = self.path_from_file_id(identifier);
|
||||||
|
|
||||||
|
self.safe_remove_file(path).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileStore {
|
||||||
|
pub fn build(root_dir: PathBuf, db: &sled::Db) -> Result<Self, FileError> {
|
||||||
|
let settings_tree = db.open_tree("settings")?;
|
||||||
|
|
||||||
|
let path_gen = init_generator(&settings_tree)?;
|
||||||
|
|
||||||
|
Ok(FileStore {
|
||||||
|
root_dir,
|
||||||
|
path_gen,
|
||||||
|
settings_tree,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn next_directory(&self) -> Result<PathBuf, FileError> {
|
||||||
|
let path = self.path_gen.next();
|
||||||
|
|
||||||
|
self.settings_tree
|
||||||
|
.insert(GENERATOR_KEY, path.to_be_bytes())?;
|
||||||
|
|
||||||
|
let mut target_path = self.root_dir.join("files");
|
||||||
|
for dir in path.to_strings() {
|
||||||
|
target_path.push(dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(target_path)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn next_file(&self) -> Result<PathBuf, FileError> {
|
||||||
|
let target_path = self.next_directory()?;
|
||||||
|
|
||||||
|
let filename = Uuid::new_v4().to_string();
|
||||||
|
|
||||||
|
Ok(target_path.join(filename))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn safe_remove_file<P: AsRef<Path>>(&self, path: P) -> Result<(), FileError> {
|
||||||
|
tokio::fs::remove_file(&path).await?;
|
||||||
|
self.try_remove_parents(path.as_ref()).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn try_remove_parents(&self, mut path: &Path) {
|
||||||
|
while let Some(parent) = path.parent() {
|
||||||
|
if parent.ends_with(&self.root_dir) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if tokio::fs::remove_dir(parent).await.is_err() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
path = parent;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try writing to a file
|
||||||
|
#[instrument(name = "Saving file", skip(bytes), fields(path = tracing::field::debug(&path.as_ref())))]
|
||||||
|
async fn safe_save_bytes<P: AsRef<Path>>(
|
||||||
|
&self,
|
||||||
|
path: P,
|
||||||
|
bytes: Bytes,
|
||||||
|
) -> Result<(), FileError> {
|
||||||
|
safe_create_parent(&path).await?;
|
||||||
|
|
||||||
|
// Only write the file if it doesn't already exist
|
||||||
|
debug!("Checking if {:?} already exists", path.as_ref());
|
||||||
|
if let Err(e) = tokio::fs::metadata(&path).await {
|
||||||
|
if e.kind() != std::io::ErrorKind::NotFound {
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open the file for writing
|
||||||
|
debug!("Creating {:?}", path.as_ref());
|
||||||
|
let mut file = File::create(&path).await?;
|
||||||
|
|
||||||
|
// try writing
|
||||||
|
debug!("Writing to {:?}", path.as_ref());
|
||||||
|
if let Err(e) = file.write_from_bytes(bytes).await {
|
||||||
|
error!("Error writing {:?}, {}", path.as_ref(), e);
|
||||||
|
// remove file if writing failed before completion
|
||||||
|
self.safe_remove_file(path).await?;
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
debug!("{:?} written", path.as_ref());
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(input), fields(to = tracing::field::debug(&to.as_ref())))]
|
||||||
|
async fn safe_save_reader<P: AsRef<Path>>(
|
||||||
|
&self,
|
||||||
|
to: P,
|
||||||
|
input: &mut (impl AsyncRead + Unpin + ?Sized),
|
||||||
|
) -> Result<(), FileError> {
|
||||||
|
safe_create_parent(&to).await?;
|
||||||
|
|
||||||
|
debug!("Checking if {:?} already exists", to.as_ref());
|
||||||
|
if let Err(e) = tokio::fs::metadata(&to).await {
|
||||||
|
if e.kind() != std::io::ErrorKind::NotFound {
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return Err(FileError::FileExists);
|
||||||
|
}
|
||||||
|
|
||||||
|
debug!("Writing stream to {:?}", to.as_ref());
|
||||||
|
|
||||||
|
let mut file = File::create(to).await?;
|
||||||
|
|
||||||
|
file.write_from_async_read(input).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// try moving a file
|
||||||
|
#[instrument(name = "Moving file", fields(from = tracing::field::debug(&from.as_ref()), to = tracing::field::debug(&to.as_ref())))]
|
||||||
|
pub(crate) async fn safe_move_file<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||||
|
&self,
|
||||||
|
from: P,
|
||||||
|
to: Q,
|
||||||
|
) -> Result<(), FileError> {
|
||||||
|
safe_create_parent(&to).await?;
|
||||||
|
|
||||||
|
debug!("Checking if {:?} already exists", to.as_ref());
|
||||||
|
if let Err(e) = tokio::fs::metadata(&to).await {
|
||||||
|
if e.kind() != std::io::ErrorKind::NotFound {
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return Err(FileError::FileExists);
|
||||||
|
}
|
||||||
|
|
||||||
|
debug!("Moving {:?} to {:?}", from.as_ref(), to.as_ref());
|
||||||
|
tokio::fs::copy(&from, &to).await?;
|
||||||
|
self.safe_remove_file(from).await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn safe_create_parent<P: AsRef<Path>>(path: P) -> Result<(), FileError> {
|
||||||
|
if let Some(path) = path.as_ref().parent() {
|
||||||
|
debug!("Creating directory {:?}", path);
|
||||||
|
tokio::fs::create_dir_all(path).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn init_generator(settings: &sled::Tree) -> Result<Generator, FileError> {
|
||||||
|
if let Some(ivec) = settings.get(GENERATOR_KEY)? {
|
||||||
|
Ok(Generator::from_existing(
|
||||||
|
storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
Ok(Generator::new())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for FileStore {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("FileStore")
|
||||||
|
.field("path_gen", &self.path_gen)
|
||||||
|
.field("root_dir", &self.root_dir)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,50 +1,15 @@
|
||||||
use futures_util::stream::Stream;
|
|
||||||
use std::{
|
|
||||||
pin::Pin,
|
|
||||||
task::{Context, Poll},
|
|
||||||
};
|
|
||||||
|
|
||||||
#[cfg(feature = "io-uring")]
|
#[cfg(feature = "io-uring")]
|
||||||
pub(crate) use io_uring::File;
|
pub(crate) use io_uring::File;
|
||||||
|
|
||||||
#[cfg(not(feature = "io-uring"))]
|
#[cfg(not(feature = "io-uring"))]
|
||||||
pub(crate) use tokio_file::File;
|
pub(crate) use tokio_file::File;
|
||||||
|
|
||||||
pin_project_lite::pin_project! {
|
|
||||||
pub(super) struct CrateError<S> {
|
|
||||||
#[pin]
|
|
||||||
inner: S
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<S> CrateError<S> {
|
|
||||||
pub(super) fn new(inner: S) -> Self {
|
|
||||||
CrateError { inner }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T, E, S> Stream for CrateError<S>
|
|
||||||
where
|
|
||||||
S: Stream<Item = Result<T, E>>,
|
|
||||||
crate::error::Error: From<E>,
|
|
||||||
{
|
|
||||||
type Item = Result<T, crate::error::Error>;
|
|
||||||
|
|
||||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
|
||||||
let this = self.as_mut().project();
|
|
||||||
|
|
||||||
this.inner
|
|
||||||
.poll_next(cx)
|
|
||||||
.map(|opt| opt.map(|res| res.map_err(Into::into)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(feature = "io-uring"))]
|
#[cfg(not(feature = "io-uring"))]
|
||||||
mod tokio_file {
|
mod tokio_file {
|
||||||
use crate::Either;
|
use crate::{store::file_store::FileError, Either};
|
||||||
use actix_web::web::{Bytes, BytesMut};
|
use actix_web::web::{Bytes, BytesMut};
|
||||||
use futures_util::stream::Stream;
|
use futures_util::stream::Stream;
|
||||||
use std::{fs::Metadata, io::SeekFrom, path::Path};
|
use std::{io::SeekFrom, path::Path};
|
||||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
|
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
|
||||||
use tokio_util::codec::{BytesCodec, FramedRead};
|
use tokio_util::codec::{BytesCodec, FramedRead};
|
||||||
|
|
||||||
|
@ -65,20 +30,13 @@ mod tokio_file {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn metadata(&self) -> std::io::Result<Metadata> {
|
pub(crate) async fn write_from_bytes(&mut self, mut bytes: Bytes) -> std::io::Result<()> {
|
||||||
self.inner.metadata().await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn write_from_bytes<'a>(
|
|
||||||
&'a mut self,
|
|
||||||
mut bytes: Bytes,
|
|
||||||
) -> std::io::Result<()> {
|
|
||||||
self.inner.write_all_buf(&mut bytes).await?;
|
self.inner.write_all_buf(&mut bytes).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn write_from_async_read<'a, R>(
|
pub(crate) async fn write_from_async_read<R>(
|
||||||
&'a mut self,
|
&mut self,
|
||||||
mut reader: R,
|
mut reader: R,
|
||||||
) -> std::io::Result<()>
|
) -> std::io::Result<()>
|
||||||
where
|
where
|
||||||
|
@ -88,12 +46,9 @@ mod tokio_file {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn read_to_async_write<'a, W>(
|
pub(crate) async fn read_to_async_write<W>(&mut self, writer: &mut W) -> std::io::Result<()>
|
||||||
&'a mut self,
|
|
||||||
writer: &'a mut W,
|
|
||||||
) -> std::io::Result<()>
|
|
||||||
where
|
where
|
||||||
W: AsyncWrite + Unpin,
|
W: AsyncWrite + Unpin + ?Sized,
|
||||||
{
|
{
|
||||||
tokio::io::copy(&mut self.inner, writer).await?;
|
tokio::io::copy(&mut self.inner, writer).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
|
@ -103,8 +58,7 @@ mod tokio_file {
|
||||||
mut self,
|
mut self,
|
||||||
from_start: Option<u64>,
|
from_start: Option<u64>,
|
||||||
len: Option<u64>,
|
len: Option<u64>,
|
||||||
) -> Result<impl Stream<Item = Result<Bytes, crate::error::Error>>, crate::error::Error>
|
) -> Result<impl Stream<Item = std::io::Result<Bytes>>, FileError> {
|
||||||
{
|
|
||||||
let obj = match (from_start, len) {
|
let obj = match (from_start, len) {
|
||||||
(Some(lower), Some(upper)) => {
|
(Some(lower), Some(upper)) => {
|
||||||
self.inner.seek(SeekFrom::Start(lower)).await?;
|
self.inner.seek(SeekFrom::Start(lower)).await?;
|
||||||
|
@ -118,10 +72,7 @@ mod tokio_file {
|
||||||
(None, None) => Either::right(self.inner),
|
(None, None) => Either::right(self.inner),
|
||||||
};
|
};
|
||||||
|
|
||||||
Ok(super::CrateError::new(BytesFreezer::new(FramedRead::new(
|
Ok(BytesFreezer::new(FramedRead::new(obj, BytesCodec::new())))
|
||||||
obj,
|
|
||||||
BytesCodec::new(),
|
|
||||||
))))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -159,6 +110,7 @@ mod tokio_file {
|
||||||
|
|
||||||
#[cfg(feature = "io-uring")]
|
#[cfg(feature = "io-uring")]
|
||||||
mod io_uring {
|
mod io_uring {
|
||||||
|
use crate::store::file_store::FileError;
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
use futures_util::stream::Stream;
|
use futures_util::stream::Stream;
|
||||||
use std::{
|
use std::{
|
||||||
|
@ -182,7 +134,7 @@ mod io_uring {
|
||||||
|
|
||||||
impl File {
|
impl File {
|
||||||
pub(crate) async fn open(path: impl AsRef<Path>) -> std::io::Result<Self> {
|
pub(crate) async fn open(path: impl AsRef<Path>) -> std::io::Result<Self> {
|
||||||
tracing::info!("Opening io-uring file");
|
tracing::info!("Opening io-uring file: {:?}", path.as_ref());
|
||||||
Ok(File {
|
Ok(File {
|
||||||
path: path.as_ref().to_owned(),
|
path: path.as_ref().to_owned(),
|
||||||
inner: tokio_uring::fs::File::open(path).await?,
|
inner: tokio_uring::fs::File::open(path).await?,
|
||||||
|
@ -190,21 +142,18 @@ mod io_uring {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn create(path: impl AsRef<Path>) -> std::io::Result<Self> {
|
pub(crate) async fn create(path: impl AsRef<Path>) -> std::io::Result<Self> {
|
||||||
tracing::info!("Creating io-uring file");
|
tracing::info!("Creating io-uring file: {:?}", path.as_ref());
|
||||||
Ok(File {
|
Ok(File {
|
||||||
path: path.as_ref().to_owned(),
|
path: path.as_ref().to_owned(),
|
||||||
inner: tokio_uring::fs::File::create(path).await?,
|
inner: tokio_uring::fs::File::create(path).await?,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn metadata(&self) -> std::io::Result<Metadata> {
|
async fn metadata(&self) -> std::io::Result<Metadata> {
|
||||||
tokio::fs::metadata(&self.path).await
|
tokio::fs::metadata(&self.path).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn write_from_bytes<'a>(
|
pub(crate) async fn write_from_bytes(&mut self, bytes: Bytes) -> std::io::Result<()> {
|
||||||
&'a mut self,
|
|
||||||
bytes: Bytes,
|
|
||||||
) -> std::io::Result<()> {
|
|
||||||
let mut buf = bytes.to_vec();
|
let mut buf = bytes.to_vec();
|
||||||
let len: u64 = buf.len().try_into().unwrap();
|
let len: u64 = buf.len().try_into().unwrap();
|
||||||
|
|
||||||
|
@ -233,8 +182,8 @@ mod io_uring {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn write_from_async_read<'a, R>(
|
pub(crate) async fn write_from_async_read<R>(
|
||||||
&'a mut self,
|
&mut self,
|
||||||
mut reader: R,
|
mut reader: R,
|
||||||
) -> std::io::Result<()>
|
) -> std::io::Result<()>
|
||||||
where
|
where
|
||||||
|
@ -283,12 +232,9 @@ mod io_uring {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn read_to_async_write<'a, W>(
|
pub(crate) async fn read_to_async_write<W>(&mut self, writer: &mut W) -> std::io::Result<()>
|
||||||
&'a mut self,
|
|
||||||
writer: &mut W,
|
|
||||||
) -> std::io::Result<()>
|
|
||||||
where
|
where
|
||||||
W: AsyncWrite + Unpin,
|
W: AsyncWrite + Unpin + ?Sized,
|
||||||
{
|
{
|
||||||
let metadata = self.metadata().await?;
|
let metadata = self.metadata().await?;
|
||||||
let size = metadata.len();
|
let size = metadata.len();
|
||||||
|
@ -323,20 +269,17 @@ mod io_uring {
|
||||||
self,
|
self,
|
||||||
from_start: Option<u64>,
|
from_start: Option<u64>,
|
||||||
len: Option<u64>,
|
len: Option<u64>,
|
||||||
) -> Result<impl Stream<Item = Result<Bytes, crate::error::Error>>, crate::error::Error>
|
) -> Result<impl Stream<Item = std::io::Result<Bytes>>, FileError> {
|
||||||
{
|
|
||||||
let size = self.metadata().await?.len();
|
let size = self.metadata().await?.len();
|
||||||
|
|
||||||
let cursor = from_start.unwrap_or(0);
|
let cursor = from_start.unwrap_or(0);
|
||||||
let size = len.unwrap_or(size - cursor) + cursor;
|
let size = len.unwrap_or(size - cursor) + cursor;
|
||||||
|
|
||||||
Ok(super::CrateError {
|
Ok(BytesStream {
|
||||||
inner: BytesStream {
|
state: ReadFileState::File { file: Some(self) },
|
||||||
state: ReadFileState::File { file: Some(self) },
|
size,
|
||||||
size,
|
cursor,
|
||||||
cursor,
|
callback: read_file,
|
||||||
callback: read_file,
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
48
src/store/file_store/file_id.rs
Normal file
48
src/store/file_store/file_id.rs
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
use crate::store::{
|
||||||
|
file_store::{FileError, FileStore},
|
||||||
|
Identifier,
|
||||||
|
};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub(crate) struct FileId(PathBuf);
|
||||||
|
|
||||||
|
impl Identifier for FileId {
|
||||||
|
type Error = FileError;
|
||||||
|
|
||||||
|
fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> {
|
||||||
|
let vec = self
|
||||||
|
.0
|
||||||
|
.to_str()
|
||||||
|
.ok_or(FileError::IdError)?
|
||||||
|
.as_bytes()
|
||||||
|
.to_vec();
|
||||||
|
|
||||||
|
Ok(vec)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error>
|
||||||
|
where
|
||||||
|
Self: Sized,
|
||||||
|
{
|
||||||
|
let string = String::from_utf8(bytes).map_err(|_| FileError::IdError)?;
|
||||||
|
|
||||||
|
let id = FileId(PathBuf::from(string));
|
||||||
|
|
||||||
|
Ok(id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileStore {
|
||||||
|
pub(super) fn file_id_from_path(&self, path: PathBuf) -> Result<FileId, FileError> {
|
||||||
|
let stripped = path
|
||||||
|
.strip_prefix(&self.root_dir)
|
||||||
|
.map_err(|_| FileError::PrefixError)?;
|
||||||
|
|
||||||
|
Ok(FileId(stripped.to_path_buf()))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn path_from_file_id(&self, file_id: &FileId) -> PathBuf {
|
||||||
|
self.root_dir.join(&file_id.0)
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,6 +1,6 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
error::{Error, UploadError},
|
error::{Error, UploadError},
|
||||||
safe_move_file,
|
store::file_store::FileStore,
|
||||||
upload_manager::UploadManager,
|
upload_manager::UploadManager,
|
||||||
};
|
};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
@ -8,24 +8,22 @@ use std::path::{Path, PathBuf};
|
||||||
const RESTRUCTURE_COMPLETE: &'static [u8] = b"fs-restructure-01-complete";
|
const RESTRUCTURE_COMPLETE: &'static [u8] = b"fs-restructure-01-complete";
|
||||||
const DETAILS: &'static [u8] = b"details";
|
const DETAILS: &'static [u8] = b"details";
|
||||||
|
|
||||||
impl UploadManager {
|
impl UploadManager<FileStore> {
|
||||||
#[tracing::instrument(skip(self))]
|
#[tracing::instrument(skip(self))]
|
||||||
pub(super) async fn restructure(&self) -> Result<(), Error> {
|
pub(crate) async fn restructure(&self) -> Result<(), Error> {
|
||||||
if self.restructure_complete()? {
|
if self.restructure_complete()? {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
for res in self.inner.filename_tree.iter() {
|
for res in self.inner().filename_tree.iter() {
|
||||||
let (filename, hash) = res?;
|
let (filename, hash) = res?;
|
||||||
let filename = String::from_utf8(filename.to_vec())?;
|
let filename = String::from_utf8(filename.to_vec())?;
|
||||||
tracing::info!("Migrating {}", filename);
|
tracing::info!("Migrating {}", filename);
|
||||||
|
|
||||||
let mut file_path = self.inner.root_dir.join("files");
|
let file_path = self.store().root_dir.join("files").join(&filename);
|
||||||
file_path.push(filename.clone());
|
|
||||||
|
|
||||||
if tokio::fs::metadata(&file_path).await.is_ok() {
|
if tokio::fs::metadata(&file_path).await.is_ok() {
|
||||||
let mut target_path = self.next_directory()?;
|
let target_path = self.store().next_directory()?.join(&filename);
|
||||||
target_path.push(filename.clone());
|
|
||||||
|
|
||||||
let target_path_bytes = self
|
let target_path_bytes = self
|
||||||
.generalize_path(&target_path)?
|
.generalize_path(&target_path)?
|
||||||
|
@ -34,24 +32,23 @@ impl UploadManager {
|
||||||
.as_bytes()
|
.as_bytes()
|
||||||
.to_vec();
|
.to_vec();
|
||||||
|
|
||||||
self.inner
|
self.inner()
|
||||||
.path_tree
|
.identifier_tree
|
||||||
.insert(filename.as_bytes(), target_path_bytes)?;
|
.insert(filename.as_bytes(), target_path_bytes)?;
|
||||||
|
|
||||||
safe_move_file(file_path, target_path).await?;
|
self.store().safe_move_file(file_path, target_path).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let (start, end) = variant_key_bounds(&hash);
|
let (start, end) = variant_key_bounds(&hash);
|
||||||
|
|
||||||
for res in self.inner.main_tree.range(start..end) {
|
for res in self.inner().main_tree.range(start..end) {
|
||||||
let (hash_variant_key, variant_path_or_details) = res?;
|
let (hash_variant_key, variant_path_or_details) = res?;
|
||||||
|
|
||||||
if !hash_variant_key.ends_with(DETAILS) {
|
if !hash_variant_key.ends_with(DETAILS) {
|
||||||
let variant_path =
|
let variant_path =
|
||||||
PathBuf::from(String::from_utf8(variant_path_or_details.to_vec())?);
|
PathBuf::from(String::from_utf8(variant_path_or_details.to_vec())?);
|
||||||
if tokio::fs::metadata(&variant_path).await.is_ok() {
|
if tokio::fs::metadata(&variant_path).await.is_ok() {
|
||||||
let mut target_path = self.next_directory()?;
|
let target_path = self.store().next_directory()?.join(&filename);
|
||||||
target_path.push(filename.clone());
|
|
||||||
|
|
||||||
let relative_target_path_bytes = self
|
let relative_target_path_bytes = self
|
||||||
.generalize_path(&target_path)?
|
.generalize_path(&target_path)?
|
||||||
|
@ -62,16 +59,18 @@ impl UploadManager {
|
||||||
|
|
||||||
let variant_key = self.migrate_variant_key(&variant_path, &filename)?;
|
let variant_key = self.migrate_variant_key(&variant_path, &filename)?;
|
||||||
|
|
||||||
self.inner
|
self.inner()
|
||||||
.path_tree
|
.identifier_tree
|
||||||
.insert(variant_key, relative_target_path_bytes)?;
|
.insert(variant_key, relative_target_path_bytes)?;
|
||||||
|
|
||||||
safe_move_file(variant_path.clone(), target_path).await?;
|
self.store()
|
||||||
self.try_remove_parents(&variant_path).await?;
|
.safe_move_file(variant_path.clone(), target_path)
|
||||||
|
.await?;
|
||||||
|
self.store().try_remove_parents(&variant_path).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.inner.main_tree.remove(hash_variant_key)?;
|
self.inner().main_tree.remove(hash_variant_key)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,22 +80,22 @@ impl UploadManager {
|
||||||
|
|
||||||
fn restructure_complete(&self) -> Result<bool, Error> {
|
fn restructure_complete(&self) -> Result<bool, Error> {
|
||||||
Ok(self
|
Ok(self
|
||||||
.inner
|
.store()
|
||||||
.settings_tree
|
.settings_tree
|
||||||
.get(RESTRUCTURE_COMPLETE)?
|
.get(RESTRUCTURE_COMPLETE)?
|
||||||
.is_some())
|
.is_some())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn mark_restructure_complete(&self) -> Result<(), Error> {
|
fn mark_restructure_complete(&self) -> Result<(), Error> {
|
||||||
self.inner
|
self.store()
|
||||||
.settings_tree
|
.settings_tree
|
||||||
.insert(RESTRUCTURE_COMPLETE, b"true")?;
|
.insert(RESTRUCTURE_COMPLETE, b"true")?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn generalize_path<'a>(&self, path: &'a Path) -> Result<&'a Path, Error> {
|
fn generalize_path<'a>(&self, path: &'a Path) -> Result<&'a Path, Error> {
|
||||||
Ok(path.strip_prefix(&self.inner.root_dir)?)
|
Ok(path.strip_prefix(&self.store().root_dir)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn migrate_variant_key(
|
fn migrate_variant_key(
|
|
@ -1,22 +1,22 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
config::Format,
|
config::Format,
|
||||||
error::{Error, UploadError},
|
error::{Error, UploadError},
|
||||||
ffmpeg::ThumbnailFormat,
|
ffmpeg::{InputFormat, ThumbnailFormat},
|
||||||
migrate::{alias_id_key, alias_key, alias_key_bounds, LatestDb},
|
magick::{details_hint, ValidInputType},
|
||||||
|
migrate::{alias_id_key, alias_key, alias_key_bounds},
|
||||||
|
store::{Identifier, Store},
|
||||||
};
|
};
|
||||||
use actix_web::web;
|
use actix_web::web;
|
||||||
use sha2::Digest;
|
use sha2::Digest;
|
||||||
use std::{
|
use std::{
|
||||||
ops::{Deref, DerefMut},
|
ops::{Deref, DerefMut},
|
||||||
path::PathBuf,
|
string::FromUtf8Error,
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
use storage_path_generator::{Generator, Path};
|
|
||||||
use tracing::{debug, error, info, instrument, warn, Span};
|
use tracing::{debug, error, info, instrument, warn, Span};
|
||||||
use tracing_futures::Instrument;
|
use tracing_futures::Instrument;
|
||||||
|
|
||||||
mod hasher;
|
mod hasher;
|
||||||
mod restructure;
|
|
||||||
mod session;
|
mod session;
|
||||||
|
|
||||||
pub(super) use session::UploadManagerSession;
|
pub(super) use session::UploadManagerSession;
|
||||||
|
@ -35,33 +35,26 @@ pub(super) use session::UploadManagerSession;
|
||||||
// - Filename Tree
|
// - Filename Tree
|
||||||
// - filename -> hash
|
// - filename -> hash
|
||||||
// - Details Tree
|
// - Details Tree
|
||||||
// - filename / relative path -> details
|
// - filename / S::Identifier -> details
|
||||||
// - Path Tree
|
// - Path Tree
|
||||||
// - filename -> relative path
|
// - filename -> S::Identifier
|
||||||
// - filename / relative variant path -> relative variant path
|
// - filename / variant path -> S::Identifier
|
||||||
// - filename / motion -> relative motion path
|
// - filename / motion -> S::Identifier
|
||||||
// - Settings Tree
|
|
||||||
// - last-path -> last generated path
|
|
||||||
// - fs-restructure-01-complete -> bool
|
|
||||||
|
|
||||||
const GENERATOR_KEY: &'static [u8] = b"last-path";
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct UploadManager {
|
pub(crate) struct UploadManager<S> {
|
||||||
inner: Arc<UploadManagerInner>,
|
inner: Arc<UploadManagerInner>,
|
||||||
|
store: S,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct UploadManagerInner {
|
pub(crate) struct UploadManagerInner {
|
||||||
format: Option<Format>,
|
format: Option<Format>,
|
||||||
hasher: sha2::Sha256,
|
hasher: sha2::Sha256,
|
||||||
root_dir: PathBuf,
|
pub(crate) alias_tree: sled::Tree,
|
||||||
alias_tree: sled::Tree,
|
pub(crate) filename_tree: sled::Tree,
|
||||||
filename_tree: sled::Tree,
|
pub(crate) main_tree: sled::Tree,
|
||||||
main_tree: sled::Tree,
|
|
||||||
details_tree: sled::Tree,
|
details_tree: sled::Tree,
|
||||||
path_tree: sled::Tree,
|
pub(crate) identifier_tree: sled::Tree,
|
||||||
settings_tree: sled::Tree,
|
|
||||||
path_gen: Generator,
|
|
||||||
db: sled::Db,
|
db: sled::Db,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -82,88 +75,85 @@ struct FilenameIVec {
|
||||||
inner: sled::IVec,
|
inner: sled::IVec,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl UploadManager {
|
impl<S> UploadManager<S>
|
||||||
|
where
|
||||||
|
S: Store + 'static,
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
/// Create a new UploadManager
|
/// Create a new UploadManager
|
||||||
pub(crate) async fn new(root_dir: PathBuf, format: Option<Format>) -> Result<Self, Error> {
|
pub(crate) async fn new(store: S, db: sled::Db, format: Option<Format>) -> Result<Self, Error> {
|
||||||
let root_clone = root_dir.clone();
|
|
||||||
// sled automatically creates it's own directories
|
|
||||||
let db = web::block(move || LatestDb::exists(root_clone).migrate()).await??;
|
|
||||||
|
|
||||||
// Ensure file dir exists
|
|
||||||
tokio::fs::create_dir_all(&root_dir).await?;
|
|
||||||
|
|
||||||
let settings_tree = db.open_tree("settings")?;
|
|
||||||
|
|
||||||
let path_gen = init_generator(&settings_tree)?;
|
|
||||||
|
|
||||||
let manager = UploadManager {
|
let manager = UploadManager {
|
||||||
inner: Arc::new(UploadManagerInner {
|
inner: Arc::new(UploadManagerInner {
|
||||||
format,
|
format,
|
||||||
hasher: sha2::Sha256::new(),
|
hasher: sha2::Sha256::new(),
|
||||||
root_dir,
|
|
||||||
alias_tree: db.open_tree("alias")?,
|
alias_tree: db.open_tree("alias")?,
|
||||||
filename_tree: db.open_tree("filename")?,
|
filename_tree: db.open_tree("filename")?,
|
||||||
details_tree: db.open_tree("details")?,
|
details_tree: db.open_tree("details")?,
|
||||||
main_tree: db.open_tree("main")?,
|
main_tree: db.open_tree("main")?,
|
||||||
path_tree: db.open_tree("path")?,
|
identifier_tree: db.open_tree("path")?,
|
||||||
settings_tree,
|
|
||||||
path_gen,
|
|
||||||
db,
|
db,
|
||||||
}),
|
}),
|
||||||
|
store,
|
||||||
};
|
};
|
||||||
|
|
||||||
manager.restructure().await?;
|
|
||||||
|
|
||||||
Ok(manager)
|
Ok(manager)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn still_path_from_filename(
|
pub(crate) fn store(&self) -> &S {
|
||||||
&self,
|
&self.store
|
||||||
filename: String,
|
|
||||||
) -> Result<PathBuf, Error> {
|
|
||||||
let path = self.path_from_filename(filename.clone()).await?;
|
|
||||||
let details =
|
|
||||||
if let Some(details) = self.variant_details(path.clone(), filename.clone()).await? {
|
|
||||||
details
|
|
||||||
} else {
|
|
||||||
Details::from_path(&path).await?
|
|
||||||
};
|
|
||||||
|
|
||||||
if !details.is_motion() {
|
|
||||||
return Ok(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(motion_path) = self.motion_path(&filename).await? {
|
|
||||||
return Ok(motion_path);
|
|
||||||
}
|
|
||||||
|
|
||||||
let jpeg_path = self.next_directory()?.join(&filename);
|
|
||||||
crate::safe_create_parent(&jpeg_path).await?;
|
|
||||||
|
|
||||||
let permit = crate::PROCESS_SEMAPHORE.acquire().await;
|
|
||||||
let res = crate::ffmpeg::thumbnail(&path, &jpeg_path, ThumbnailFormat::Jpeg).await;
|
|
||||||
drop(permit);
|
|
||||||
|
|
||||||
if let Err(e) = res {
|
|
||||||
error!("transcode error: {:?}", e);
|
|
||||||
self.remove_path(&jpeg_path).await?;
|
|
||||||
return Err(e);
|
|
||||||
}
|
|
||||||
|
|
||||||
self.store_motion_path(&filename, &jpeg_path).await?;
|
|
||||||
Ok(jpeg_path)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn motion_path(&self, filename: &str) -> Result<Option<PathBuf>, Error> {
|
pub(crate) fn inner(&self) -> &UploadManagerInner {
|
||||||
let path_tree = self.inner.path_tree.clone();
|
&self.inner
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn still_identifier_from_filename(
|
||||||
|
&self,
|
||||||
|
filename: String,
|
||||||
|
) -> Result<S::Identifier, Error> {
|
||||||
|
let identifier = self.identifier_from_filename(filename.clone()).await?;
|
||||||
|
let details = if let Some(details) = self
|
||||||
|
.variant_details(identifier.clone(), filename.clone())
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
details
|
||||||
|
} else {
|
||||||
|
let hint = details_hint(&filename);
|
||||||
|
Details::from_store(self.store.clone(), identifier.clone(), hint).await?
|
||||||
|
};
|
||||||
|
|
||||||
|
if !details.is_motion() {
|
||||||
|
return Ok(identifier);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(motion_identifier) = self.motion_identifier(&filename).await? {
|
||||||
|
return Ok(motion_identifier);
|
||||||
|
}
|
||||||
|
|
||||||
|
let permit = crate::PROCESS_SEMAPHORE.acquire().await;
|
||||||
|
let mut reader = crate::ffmpeg::thumbnail(
|
||||||
|
self.store.clone(),
|
||||||
|
identifier,
|
||||||
|
InputFormat::Mp4,
|
||||||
|
ThumbnailFormat::Jpeg,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let motion_identifier = self.store.save_async_read(&mut reader).await?;
|
||||||
|
drop(permit);
|
||||||
|
|
||||||
|
self.store_motion_path(&filename, &motion_identifier)
|
||||||
|
.await?;
|
||||||
|
Ok(motion_identifier)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn motion_identifier(&self, filename: &str) -> Result<Option<S::Identifier>, Error> {
|
||||||
|
let identifier_tree = self.inner.identifier_tree.clone();
|
||||||
let motion_key = format!("{}/motion", filename);
|
let motion_key = format!("{}/motion", filename);
|
||||||
|
|
||||||
let opt = web::block(move || path_tree.get(motion_key.as_bytes())).await??;
|
let opt = web::block(move || identifier_tree.get(motion_key.as_bytes())).await??;
|
||||||
|
|
||||||
if let Some(ivec) = opt {
|
if let Some(ivec) = opt {
|
||||||
return Ok(Some(
|
return Ok(Some(S::Identifier::from_bytes(ivec.to_vec())?));
|
||||||
self.inner.root_dir.join(String::from_utf8(ivec.to_vec())?),
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(None)
|
Ok(None)
|
||||||
|
@ -172,59 +162,57 @@ impl UploadManager {
|
||||||
async fn store_motion_path(
|
async fn store_motion_path(
|
||||||
&self,
|
&self,
|
||||||
filename: &str,
|
filename: &str,
|
||||||
path: impl AsRef<std::path::Path>,
|
identifier: &S::Identifier,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
let path_bytes = self
|
let identifier_bytes = identifier.to_bytes()?;
|
||||||
.generalize_path(path.as_ref())?
|
|
||||||
.to_str()
|
|
||||||
.ok_or(UploadError::Path)?
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec();
|
|
||||||
let motion_key = format!("{}/motion", filename);
|
let motion_key = format!("{}/motion", filename);
|
||||||
let path_tree = self.inner.path_tree.clone();
|
let identifier_tree = self.inner.identifier_tree.clone();
|
||||||
|
|
||||||
web::block(move || path_tree.insert(motion_key.as_bytes(), path_bytes)).await??;
|
web::block(move || identifier_tree.insert(motion_key.as_bytes(), identifier_bytes))
|
||||||
|
.await??;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn path_from_filename(&self, filename: String) -> Result<PathBuf, Error> {
|
pub(crate) async fn identifier_from_filename(
|
||||||
let path_tree = self.inner.path_tree.clone();
|
&self,
|
||||||
let path_ivec = web::block(move || path_tree.get(filename.as_bytes()))
|
filename: String,
|
||||||
|
) -> Result<S::Identifier, Error> {
|
||||||
|
let identifier_tree = self.inner.identifier_tree.clone();
|
||||||
|
let path_ivec = web::block(move || identifier_tree.get(filename.as_bytes()))
|
||||||
.await??
|
.await??
|
||||||
.ok_or(UploadError::MissingFile)?;
|
.ok_or(UploadError::MissingFile)?;
|
||||||
|
|
||||||
let relative = PathBuf::from(String::from_utf8(path_ivec.to_vec())?);
|
let identifier = S::Identifier::from_bytes(path_ivec.to_vec())?;
|
||||||
|
|
||||||
Ok(self.inner.root_dir.join(relative))
|
Ok(identifier)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
async fn store_path(&self, filename: String, path: &std::path::Path) -> Result<(), Error> {
|
async fn store_identifier(
|
||||||
let path_bytes = self
|
&self,
|
||||||
.generalize_path(path)?
|
filename: String,
|
||||||
.to_str()
|
identifier: &S::Identifier,
|
||||||
.ok_or(UploadError::Path)?
|
) -> Result<(), Error> {
|
||||||
.as_bytes()
|
let identifier_bytes = identifier.to_bytes()?;
|
||||||
.to_vec();
|
let identifier_tree = self.inner.identifier_tree.clone();
|
||||||
let path_tree = self.inner.path_tree.clone();
|
web::block(move || identifier_tree.insert(filename.as_bytes(), identifier_bytes)).await??;
|
||||||
web::block(move || path_tree.insert(filename.as_bytes(), path_bytes)).await??;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn variant_path(
|
pub(crate) async fn variant_identifier(
|
||||||
&self,
|
&self,
|
||||||
process_path: &std::path::Path,
|
process_path: &std::path::Path,
|
||||||
filename: &str,
|
filename: &str,
|
||||||
) -> Result<Option<PathBuf>, Error> {
|
) -> Result<Option<S::Identifier>, Error> {
|
||||||
let key = self.variant_key(process_path, filename)?;
|
let key = self.variant_key(process_path, filename)?;
|
||||||
let path_tree = self.inner.path_tree.clone();
|
let identifier_tree = self.inner.identifier_tree.clone();
|
||||||
let path_opt = web::block(move || path_tree.get(key)).await??;
|
let path_opt = web::block(move || identifier_tree.get(key)).await??;
|
||||||
|
|
||||||
if let Some(path_ivec) = path_opt {
|
if let Some(ivec) = path_opt {
|
||||||
let relative = PathBuf::from(String::from_utf8(path_ivec.to_vec())?);
|
let identifier = S::Identifier::from_bytes(ivec.to_vec())?;
|
||||||
Ok(Some(self.inner.root_dir.join(relative)))
|
Ok(Some(identifier))
|
||||||
} else {
|
} else {
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
@ -235,22 +223,22 @@ impl UploadManager {
|
||||||
pub(crate) async fn store_variant(
|
pub(crate) async fn store_variant(
|
||||||
&self,
|
&self,
|
||||||
variant_process_path: Option<&std::path::Path>,
|
variant_process_path: Option<&std::path::Path>,
|
||||||
real_path: &std::path::Path,
|
identifier: &S::Identifier,
|
||||||
filename: &str,
|
filename: &str,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
let path_bytes = self
|
let key = if let Some(path) = variant_process_path {
|
||||||
.generalize_path(real_path)?
|
self.variant_key(path, filename)?
|
||||||
.to_str()
|
} else {
|
||||||
.ok_or(UploadError::Path)?
|
let mut vec = filename.as_bytes().to_vec();
|
||||||
.as_bytes()
|
vec.extend(b"/");
|
||||||
.to_vec();
|
vec.extend(&identifier.to_bytes()?);
|
||||||
|
vec
|
||||||
let variant_path = variant_process_path.unwrap_or(real_path);
|
};
|
||||||
let key = self.variant_key(variant_path, filename)?;
|
let identifier_tree = self.inner.identifier_tree.clone();
|
||||||
let path_tree = self.inner.path_tree.clone();
|
let identifier_bytes = identifier.to_bytes()?;
|
||||||
|
|
||||||
debug!("Storing variant");
|
debug!("Storing variant");
|
||||||
web::block(move || path_tree.insert(key, path_bytes)).await??;
|
web::block(move || identifier_tree.insert(key, identifier_bytes)).await??;
|
||||||
debug!("Stored variant");
|
debug!("Stored variant");
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
@ -260,10 +248,10 @@ impl UploadManager {
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn variant_details(
|
pub(crate) async fn variant_details(
|
||||||
&self,
|
&self,
|
||||||
path: PathBuf,
|
identifier: S::Identifier,
|
||||||
filename: String,
|
filename: String,
|
||||||
) -> Result<Option<Details>, Error> {
|
) -> Result<Option<Details>, Error> {
|
||||||
let key = self.details_key(&path, &filename)?;
|
let key = self.details_key(identifier, &filename)?;
|
||||||
let details_tree = self.inner.details_tree.clone();
|
let details_tree = self.inner.details_tree.clone();
|
||||||
|
|
||||||
debug!("Getting details");
|
debug!("Getting details");
|
||||||
|
@ -282,11 +270,11 @@ impl UploadManager {
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn store_variant_details(
|
pub(crate) async fn store_variant_details(
|
||||||
&self,
|
&self,
|
||||||
path: PathBuf,
|
identifier: S::Identifier,
|
||||||
filename: String,
|
filename: String,
|
||||||
details: &Details,
|
details: &Details,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
let key = self.details_key(&path, &filename)?;
|
let key = self.details_key(identifier, &filename)?;
|
||||||
let details_tree = self.inner.details_tree.clone();
|
let details_tree = self.inner.details_tree.clone();
|
||||||
let details_value = serde_json::to_vec(details)?;
|
let details_value = serde_json::to_vec(details)?;
|
||||||
|
|
||||||
|
@ -317,21 +305,6 @@ impl UploadManager {
|
||||||
self.aliases_by_hash(&hash).await
|
self.aliases_by_hash(&hash).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn next_directory(&self) -> Result<PathBuf, Error> {
|
|
||||||
let path = self.inner.path_gen.next();
|
|
||||||
|
|
||||||
self.inner
|
|
||||||
.settings_tree
|
|
||||||
.insert(GENERATOR_KEY, path.to_be_bytes())?;
|
|
||||||
|
|
||||||
let mut target_path = self.inner.root_dir.join("files");
|
|
||||||
for dir in path.to_strings() {
|
|
||||||
target_path.push(dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(target_path)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn aliases_by_hash(&self, hash: &sled::IVec) -> Result<Vec<String>, Error> {
|
async fn aliases_by_hash(&self, hash: &sled::IVec) -> Result<Vec<String>, Error> {
|
||||||
let (start, end) = alias_key_bounds(hash);
|
let (start, end) = alias_key_bounds(hash);
|
||||||
let main_tree = self.inner.main_tree.clone();
|
let main_tree = self.inner.main_tree.clone();
|
||||||
|
@ -386,26 +359,26 @@ impl UploadManager {
|
||||||
debug!("Deleting alias -> delete-token mapping");
|
debug!("Deleting alias -> delete-token mapping");
|
||||||
let existing_token = alias_tree
|
let existing_token = alias_tree
|
||||||
.remove(delete_key(&alias2).as_bytes())?
|
.remove(delete_key(&alias2).as_bytes())?
|
||||||
.ok_or_else(|| trans_err(UploadError::MissingAlias))?;
|
.ok_or_else(|| trans_upload_error(UploadError::MissingAlias))?;
|
||||||
|
|
||||||
// Bail if invalid token
|
// Bail if invalid token
|
||||||
if existing_token != token {
|
if existing_token != token {
|
||||||
warn!("Invalid delete token");
|
warn!("Invalid delete token");
|
||||||
return Err(trans_err(UploadError::InvalidToken));
|
return Err(trans_upload_error(UploadError::InvalidToken));
|
||||||
}
|
}
|
||||||
|
|
||||||
// -- GET ID FOR HASH TREE CLEANUP --
|
// -- GET ID FOR HASH TREE CLEANUP --
|
||||||
debug!("Deleting alias -> id mapping");
|
debug!("Deleting alias -> id mapping");
|
||||||
let id = alias_tree
|
let id = alias_tree
|
||||||
.remove(alias_id_key(&alias2).as_bytes())?
|
.remove(alias_id_key(&alias2).as_bytes())?
|
||||||
.ok_or_else(|| trans_err(UploadError::MissingAlias))?;
|
.ok_or_else(|| trans_upload_error(UploadError::MissingAlias))?;
|
||||||
let id = String::from_utf8(id.to_vec()).map_err(trans_err)?;
|
let id = String::from_utf8(id.to_vec()).map_err(trans_utf8_error)?;
|
||||||
|
|
||||||
// -- GET HASH FOR HASH TREE CLEANUP --
|
// -- GET HASH FOR HASH TREE CLEANUP --
|
||||||
debug!("Deleting alias -> hash mapping");
|
debug!("Deleting alias -> hash mapping");
|
||||||
let hash = alias_tree
|
let hash = alias_tree
|
||||||
.remove(alias2.as_bytes())?
|
.remove(alias2.as_bytes())?
|
||||||
.ok_or_else(|| trans_err(UploadError::MissingAlias))?;
|
.ok_or_else(|| trans_upload_error(UploadError::MissingAlias))?;
|
||||||
|
|
||||||
// -- REMOVE HASH TREE ELEMENT --
|
// -- REMOVE HASH TREE ELEMENT --
|
||||||
debug!("Deleting hash -> alias mapping");
|
debug!("Deleting hash -> alias mapping");
|
||||||
|
@ -491,7 +464,7 @@ impl UploadManager {
|
||||||
Ok(filename)
|
Ok(filename)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn session(&self) -> UploadManagerSession {
|
pub(crate) fn session(&self) -> UploadManagerSession<S> {
|
||||||
UploadManagerSession::new(self.clone())
|
UploadManagerSession::new(self.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -501,14 +474,14 @@ impl UploadManager {
|
||||||
let filename = filename.inner;
|
let filename = filename.inner;
|
||||||
|
|
||||||
let filename2 = filename.clone();
|
let filename2 = filename.clone();
|
||||||
let path_tree = self.inner.path_tree.clone();
|
let identifier_tree = self.inner.identifier_tree.clone();
|
||||||
let path = web::block(move || path_tree.remove(filename2)).await??;
|
let identifier = web::block(move || identifier_tree.remove(filename2)).await??;
|
||||||
|
|
||||||
let mut errors = Vec::new();
|
let mut errors = Vec::new();
|
||||||
if let Some(path) = path {
|
if let Some(identifier) = identifier {
|
||||||
let path = self.inner.root_dir.join(String::from_utf8(path.to_vec())?);
|
let identifier = S::Identifier::from_bytes(identifier.to_vec())?;
|
||||||
debug!("Deleting {:?}", path);
|
debug!("Deleting {:?}", identifier);
|
||||||
if let Err(e) = self.remove_path(&path).await {
|
if let Err(e) = self.store.remove(&identifier).await {
|
||||||
errors.push(e.into());
|
errors.push(e.into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -519,36 +492,34 @@ impl UploadManager {
|
||||||
web::block(move || fname_tree.remove(filename2)).await??;
|
web::block(move || fname_tree.remove(filename2)).await??;
|
||||||
|
|
||||||
let path_prefix = filename.clone();
|
let path_prefix = filename.clone();
|
||||||
let path_tree = self.inner.path_tree.clone();
|
let identifier_tree = self.inner.identifier_tree.clone();
|
||||||
debug!("Fetching file variants");
|
debug!("Fetching file variants");
|
||||||
let paths = web::block(move || {
|
let identifiers = web::block(move || {
|
||||||
path_tree
|
identifier_tree
|
||||||
.scan_prefix(path_prefix)
|
.scan_prefix(path_prefix)
|
||||||
.values()
|
.values()
|
||||||
.collect::<Result<Vec<sled::IVec>, sled::Error>>()
|
.collect::<Result<Vec<sled::IVec>, sled::Error>>()
|
||||||
})
|
})
|
||||||
.await??;
|
.await??;
|
||||||
|
|
||||||
debug!("{} files prepared for deletion", paths.len());
|
debug!("{} files prepared for deletion", identifiers.len());
|
||||||
|
|
||||||
for path in paths {
|
for id in identifiers {
|
||||||
let path = self
|
let identifier = S::Identifier::from_bytes(id.to_vec())?;
|
||||||
.inner
|
|
||||||
.root_dir
|
debug!("Deleting {:?}", identifier);
|
||||||
.join(String::from_utf8_lossy(&path).as_ref());
|
if let Err(e) = self.store.remove(&identifier).await {
|
||||||
debug!("Deleting {:?}", path);
|
|
||||||
if let Err(e) = self.remove_path(&path).await {
|
|
||||||
errors.push(e);
|
errors.push(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let path_prefix = filename.clone();
|
let path_prefix = filename.clone();
|
||||||
let path_tree = self.inner.path_tree.clone();
|
let identifier_tree = self.inner.identifier_tree.clone();
|
||||||
debug!("Deleting path info");
|
debug!("Deleting path info");
|
||||||
web::block(move || {
|
web::block(move || {
|
||||||
for res in path_tree.scan_prefix(path_prefix).keys() {
|
for res in identifier_tree.scan_prefix(path_prefix).keys() {
|
||||||
let key = res?;
|
let key = res?;
|
||||||
path_tree.remove(key)?;
|
identifier_tree.remove(key)?;
|
||||||
}
|
}
|
||||||
Ok(()) as Result<(), Error>
|
Ok(()) as Result<(), Error>
|
||||||
})
|
})
|
||||||
|
@ -560,30 +531,7 @@ impl UploadManager {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn try_remove_parents(&self, mut path: &std::path::Path) -> Result<(), Error> {
|
pub(crate) fn variant_key(
|
||||||
let root = self.inner.root_dir.join("files");
|
|
||||||
|
|
||||||
while let Some(parent) = path.parent() {
|
|
||||||
if parent.ends_with(&root) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if tokio::fs::remove_dir(parent).await.is_err() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
path = parent;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn remove_path(&self, path: &std::path::Path) -> Result<(), Error> {
|
|
||||||
tokio::fs::remove_file(path).await?;
|
|
||||||
self.try_remove_parents(path).await
|
|
||||||
}
|
|
||||||
|
|
||||||
fn variant_key(
|
|
||||||
&self,
|
&self,
|
||||||
variant_process_path: &std::path::Path,
|
variant_process_path: &std::path::Path,
|
||||||
filename: &str,
|
filename: &str,
|
||||||
|
@ -597,15 +545,11 @@ impl UploadManager {
|
||||||
Ok(vec)
|
Ok(vec)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn details_key(
|
fn details_key(&self, identifier: S::Identifier, filename: &str) -> Result<Vec<u8>, Error> {
|
||||||
&self,
|
let mut vec = filename.as_bytes().to_vec();
|
||||||
variant_path: &std::path::Path,
|
vec.extend(b"/");
|
||||||
filename: &str,
|
vec.extend(&identifier.to_bytes()?);
|
||||||
) -> Result<Vec<u8>, Error> {
|
|
||||||
let path = self.generalize_path(variant_path)?;
|
|
||||||
let path_string = path.to_str().ok_or(UploadError::Path)?.to_string();
|
|
||||||
|
|
||||||
let vec = format!("{}/{}", filename, path_string).as_bytes().to_vec();
|
|
||||||
Ok(vec)
|
Ok(vec)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -623,8 +567,11 @@ impl Details {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument("Details from bytes", skip(input))]
|
#[tracing::instrument("Details from bytes", skip(input))]
|
||||||
pub(crate) async fn from_bytes(input: web::Bytes) -> Result<Self, Error> {
|
pub(crate) async fn from_bytes(
|
||||||
let details = crate::magick::details_bytes(input).await?;
|
input: web::Bytes,
|
||||||
|
hint: Option<ValidInputType>,
|
||||||
|
) -> Result<Self, Error> {
|
||||||
|
let details = crate::magick::details_bytes(input, hint).await?;
|
||||||
|
|
||||||
Ok(Details::now(
|
Ok(Details::now(
|
||||||
details.width,
|
details.width,
|
||||||
|
@ -633,12 +580,13 @@ impl Details {
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument("Details from path", fields(path = &tracing::field::debug(&path.as_ref())))]
|
#[tracing::instrument("Details from store")]
|
||||||
pub(crate) async fn from_path<P>(path: P) -> Result<Self, Error>
|
pub(crate) async fn from_store<S: Store>(
|
||||||
where
|
store: S,
|
||||||
P: AsRef<std::path::Path>,
|
identifier: S::Identifier,
|
||||||
{
|
expected_format: Option<ValidInputType>,
|
||||||
let details = crate::magick::details(&path).await?;
|
) -> Result<Self, Error> {
|
||||||
|
let details = crate::magick::details_store(store, identifier, expected_format).await?;
|
||||||
|
|
||||||
Ok(Details::now(
|
Ok(Details::now(
|
||||||
details.width,
|
details.width,
|
||||||
|
@ -671,14 +619,14 @@ impl FilenameIVec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn init_generator(settings: &sled::Tree) -> Result<Generator, Error> {
|
fn trans_upload_error(
|
||||||
if let Some(ivec) = settings.get(GENERATOR_KEY)? {
|
upload_error: UploadError,
|
||||||
Ok(Generator::from_existing(Path::from_be_bytes(
|
) -> sled::transaction::ConflictableTransactionError<Error> {
|
||||||
ivec.to_vec(),
|
trans_err(upload_error)
|
||||||
)?))
|
}
|
||||||
} else {
|
|
||||||
Ok(Generator::new())
|
fn trans_utf8_error(e: FromUtf8Error) -> sled::transaction::ConflictableTransactionError<Error> {
|
||||||
}
|
trans_err(e)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn trans_err<E>(e: E) -> sled::transaction::ConflictableTransactionError<Error>
|
fn trans_err<E>(e: E) -> sled::transaction::ConflictableTransactionError<Error>
|
||||||
|
@ -706,7 +654,7 @@ impl<T> DerefMut for Serde<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for UploadManager {
|
impl<S> std::fmt::Debug for UploadManager<S> {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||||
f.debug_struct("UploadManager").finish()
|
f.debug_struct("UploadManager").finish()
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
error::{Error, UploadError},
|
error::{Error, UploadError},
|
||||||
migrate::{alias_id_key, alias_key},
|
migrate::{alias_id_key, alias_key},
|
||||||
to_ext,
|
store::Store,
|
||||||
upload_manager::{
|
upload_manager::{
|
||||||
delete_key,
|
delete_key,
|
||||||
hasher::{Hash, Hasher},
|
hasher::{Hash, Hasher},
|
||||||
|
@ -10,20 +10,24 @@ use crate::{
|
||||||
};
|
};
|
||||||
use actix_web::web;
|
use actix_web::web;
|
||||||
use futures_util::stream::{Stream, StreamExt};
|
use futures_util::stream::{Stream, StreamExt};
|
||||||
use std::path::PathBuf;
|
|
||||||
use tokio::io::AsyncRead;
|
|
||||||
use tracing::{debug, instrument, warn, Span};
|
use tracing::{debug, instrument, warn, Span};
|
||||||
use tracing_futures::Instrument;
|
use tracing_futures::Instrument;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
pub(crate) struct UploadManagerSession {
|
pub(crate) struct UploadManagerSession<S: Store>
|
||||||
manager: UploadManager,
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
|
manager: UploadManager<S>,
|
||||||
alias: Option<String>,
|
alias: Option<String>,
|
||||||
finished: bool,
|
finished: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl UploadManagerSession {
|
impl<S: Store> UploadManagerSession<S>
|
||||||
pub(super) fn new(manager: UploadManager) -> Self {
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
|
pub(super) fn new(manager: UploadManager<S>) -> Self {
|
||||||
UploadManagerSession {
|
UploadManagerSession {
|
||||||
manager,
|
manager,
|
||||||
alias: None,
|
alias: None,
|
||||||
|
@ -51,7 +55,10 @@ impl Dup {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for UploadManagerSession {
|
impl<S: Store> Drop for UploadManagerSession<S>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if self.finished {
|
if self.finished {
|
||||||
return;
|
return;
|
||||||
|
@ -90,7 +97,10 @@ impl Drop for UploadManagerSession {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl UploadManagerSession {
|
impl<S: Store> UploadManagerSession<S>
|
||||||
|
where
|
||||||
|
Error: From<S::Error>,
|
||||||
|
{
|
||||||
/// Generate a delete token for an alias
|
/// Generate a delete token for an alias
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(crate) async fn delete_token(&self) -> Result<String, Error> {
|
pub(crate) async fn delete_token(&self) -> Result<String, Error> {
|
||||||
|
@ -129,17 +139,13 @@ impl UploadManagerSession {
|
||||||
|
|
||||||
/// Upload the file while preserving the filename, optionally validating the uploaded image
|
/// Upload the file while preserving the filename, optionally validating the uploaded image
|
||||||
#[instrument(skip(self, stream))]
|
#[instrument(skip(self, stream))]
|
||||||
pub(crate) async fn import<E>(
|
pub(crate) async fn import(
|
||||||
mut self,
|
mut self,
|
||||||
alias: String,
|
alias: String,
|
||||||
content_type: mime::Mime,
|
content_type: mime::Mime,
|
||||||
validate: bool,
|
validate: bool,
|
||||||
mut stream: impl Stream<Item = Result<web::Bytes, E>> + Unpin,
|
mut stream: impl Stream<Item = Result<web::Bytes, Error>> + Unpin,
|
||||||
) -> Result<Self, Error>
|
) -> Result<Self, Error> {
|
||||||
where
|
|
||||||
Error: From<E>,
|
|
||||||
E: Unpin + 'static,
|
|
||||||
{
|
|
||||||
let mut bytes_mut = actix_web::web::BytesMut::new();
|
let mut bytes_mut = actix_web::web::BytesMut::new();
|
||||||
|
|
||||||
debug!("Reading stream to memory");
|
debug!("Reading stream to memory");
|
||||||
|
@ -158,8 +164,11 @@ impl UploadManagerSession {
|
||||||
|
|
||||||
let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
|
let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
|
||||||
|
|
||||||
let tmpfile = crate::tmp_file();
|
let identifier = self
|
||||||
safe_save_reader(tmpfile.clone(), &mut hasher_reader).await?;
|
.manager
|
||||||
|
.store
|
||||||
|
.save_async_read(&mut hasher_reader)
|
||||||
|
.await?;
|
||||||
let hash = hasher_reader.finalize_reset().await?;
|
let hash = hasher_reader.finalize_reset().await?;
|
||||||
|
|
||||||
debug!("Storing alias");
|
debug!("Storing alias");
|
||||||
|
@ -167,7 +176,7 @@ impl UploadManagerSession {
|
||||||
self.add_existing_alias(&hash, &alias).await?;
|
self.add_existing_alias(&hash, &alias).await?;
|
||||||
|
|
||||||
debug!("Saving file");
|
debug!("Saving file");
|
||||||
self.save_upload(tmpfile, hash, content_type).await?;
|
self.save_upload(&identifier, hash, content_type).await?;
|
||||||
|
|
||||||
// Return alias to file
|
// Return alias to file
|
||||||
Ok(self)
|
Ok(self)
|
||||||
|
@ -175,13 +184,10 @@ impl UploadManagerSession {
|
||||||
|
|
||||||
/// Upload the file, discarding bytes if it's already present, or saving if it's new
|
/// Upload the file, discarding bytes if it's already present, or saving if it's new
|
||||||
#[instrument(skip(self, stream))]
|
#[instrument(skip(self, stream))]
|
||||||
pub(crate) async fn upload<E>(
|
pub(crate) async fn upload(
|
||||||
mut self,
|
mut self,
|
||||||
mut stream: impl Stream<Item = Result<web::Bytes, E>> + Unpin,
|
mut stream: impl Stream<Item = Result<web::Bytes, Error>> + Unpin,
|
||||||
) -> Result<Self, Error>
|
) -> Result<Self, Error> {
|
||||||
where
|
|
||||||
Error: From<E>,
|
|
||||||
{
|
|
||||||
let mut bytes_mut = actix_web::web::BytesMut::new();
|
let mut bytes_mut = actix_web::web::BytesMut::new();
|
||||||
|
|
||||||
debug!("Reading stream to memory");
|
debug!("Reading stream to memory");
|
||||||
|
@ -200,15 +206,18 @@ impl UploadManagerSession {
|
||||||
|
|
||||||
let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
|
let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
|
||||||
|
|
||||||
let tmpfile = crate::tmp_file();
|
let identifier = self
|
||||||
safe_save_reader(tmpfile.clone(), &mut hasher_reader).await?;
|
.manager
|
||||||
|
.store
|
||||||
|
.save_async_read(&mut hasher_reader)
|
||||||
|
.await?;
|
||||||
let hash = hasher_reader.finalize_reset().await?;
|
let hash = hasher_reader.finalize_reset().await?;
|
||||||
|
|
||||||
debug!("Adding alias");
|
debug!("Adding alias");
|
||||||
self.add_alias(&hash, content_type.clone()).await?;
|
self.add_alias(&hash, content_type.clone()).await?;
|
||||||
|
|
||||||
debug!("Saving file");
|
debug!("Saving file");
|
||||||
self.save_upload(tmpfile, hash, content_type).await?;
|
self.save_upload(&identifier, hash, content_type).await?;
|
||||||
|
|
||||||
// Return alias to file
|
// Return alias to file
|
||||||
Ok(self)
|
Ok(self)
|
||||||
|
@ -217,7 +226,7 @@ impl UploadManagerSession {
|
||||||
// check duplicates & store image if new
|
// check duplicates & store image if new
|
||||||
async fn save_upload(
|
async fn save_upload(
|
||||||
&self,
|
&self,
|
||||||
tmpfile: PathBuf,
|
identifier: &S::Identifier,
|
||||||
hash: Hash,
|
hash: Hash,
|
||||||
content_type: mime::Mime,
|
content_type: mime::Mime,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
|
@ -225,15 +234,13 @@ impl UploadManagerSession {
|
||||||
|
|
||||||
// bail early with alias to existing file if this is a duplicate
|
// bail early with alias to existing file if this is a duplicate
|
||||||
if dup.exists() {
|
if dup.exists() {
|
||||||
debug!("Duplicate exists, not saving file");
|
debug!("Duplicate exists, removing file");
|
||||||
|
|
||||||
|
self.manager.store.remove(&identifier).await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
// -- WRITE NEW FILE --
|
self.manager.store_identifier(name, &identifier).await?;
|
||||||
let real_path = self.manager.next_directory()?.join(&name);
|
|
||||||
|
|
||||||
self.manager.store_path(name, &real_path).await?;
|
|
||||||
crate::safe_move_file(tmpfile, real_path).await?;
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -248,6 +255,7 @@ impl UploadManagerSession {
|
||||||
let main_tree = self.manager.inner.main_tree.clone();
|
let main_tree = self.manager.inner.main_tree.clone();
|
||||||
|
|
||||||
let filename = self.next_file(content_type).await?;
|
let filename = self.next_file(content_type).await?;
|
||||||
|
|
||||||
let filename2 = filename.clone();
|
let filename2 = filename.clone();
|
||||||
let hash2 = hash.as_slice().to_vec();
|
let hash2 = hash.as_slice().to_vec();
|
||||||
debug!("Inserting filename for hash");
|
debug!("Inserting filename for hash");
|
||||||
|
@ -283,13 +291,11 @@ impl UploadManagerSession {
|
||||||
async fn next_file(&self, content_type: mime::Mime) -> Result<String, Error> {
|
async fn next_file(&self, content_type: mime::Mime) -> Result<String, Error> {
|
||||||
loop {
|
loop {
|
||||||
debug!("Filename generation loop");
|
debug!("Filename generation loop");
|
||||||
let s: String = Uuid::new_v4().to_string();
|
let filename = file_name(Uuid::new_v4(), content_type.clone())?;
|
||||||
|
|
||||||
let filename = file_name(s, content_type.clone())?;
|
let identifier_tree = self.manager.inner.identifier_tree.clone();
|
||||||
|
|
||||||
let path_tree = self.manager.inner.path_tree.clone();
|
|
||||||
let filename2 = filename.clone();
|
let filename2 = filename.clone();
|
||||||
let filename_exists = web::block(move || path_tree.get(filename2.as_bytes()))
|
let filename_exists = web::block(move || identifier_tree.get(filename2.as_bytes()))
|
||||||
.await??
|
.await??
|
||||||
.is_some();
|
.is_some();
|
||||||
|
|
||||||
|
@ -363,8 +369,7 @@ impl UploadManagerSession {
|
||||||
async fn next_alias(&mut self, hash: &Hash, content_type: mime::Mime) -> Result<String, Error> {
|
async fn next_alias(&mut self, hash: &Hash, content_type: mime::Mime) -> Result<String, Error> {
|
||||||
loop {
|
loop {
|
||||||
debug!("Alias gen loop");
|
debug!("Alias gen loop");
|
||||||
let s: String = Uuid::new_v4().to_string();
|
let alias = file_name(Uuid::new_v4(), content_type.clone())?;
|
||||||
let alias = file_name(s, content_type.clone())?;
|
|
||||||
self.alias = Some(alias.clone());
|
self.alias = Some(alias.clone());
|
||||||
|
|
||||||
let res = self.save_alias_hash_mapping(hash, &alias).await?;
|
let res = self.save_alias_hash_mapping(hash, &alias).await?;
|
||||||
|
@ -402,31 +407,20 @@ impl UploadManagerSession {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn file_name(name: String, content_type: mime::Mime) -> Result<String, Error> {
|
fn file_name(name: Uuid, content_type: mime::Mime) -> Result<String, Error> {
|
||||||
Ok(format!("{}{}", name, to_ext(content_type)?))
|
Ok(format!("{}{}", name, to_ext(content_type)?))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(input))]
|
fn to_ext(mime: mime::Mime) -> Result<&'static str, Error> {
|
||||||
async fn safe_save_reader(to: PathBuf, input: &mut (impl AsyncRead + Unpin)) -> Result<(), Error> {
|
if mime == mime::IMAGE_PNG {
|
||||||
if let Some(path) = to.parent() {
|
Ok(".png")
|
||||||
debug!("Creating directory {:?}", path);
|
} else if mime == mime::IMAGE_JPEG {
|
||||||
tokio::fs::create_dir_all(path.to_owned()).await?;
|
Ok(".jpg")
|
||||||
}
|
} else if mime == crate::video_mp4() {
|
||||||
|
Ok(".mp4")
|
||||||
debug!("Checking if {:?} already exists", to);
|
} else if mime == crate::image_webp() {
|
||||||
if let Err(e) = tokio::fs::metadata(to.clone()).await {
|
Ok(".webp")
|
||||||
if e.kind() != std::io::ErrorKind::NotFound {
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
return Err(UploadError::FileExists.into());
|
Err(UploadError::UnsupportedFormat.into())
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("Writing stream to {:?}", to);
|
|
||||||
|
|
||||||
let mut file = crate::file::File::create(to).await?;
|
|
||||||
|
|
||||||
file.write_from_async_read(input).await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue