2020-06-06 21:02:26 +00:00
|
|
|
use actix_form_data::{Field, Form, Value};
|
2021-01-14 16:26:17 +00:00
|
|
|
use actix_web::{
|
|
|
|
guard,
|
2022-03-01 17:23:15 +00:00
|
|
|
http::header::{CacheControl, CacheDirective, LastModified, Range, ACCEPT_RANGES},
|
2021-04-17 19:59:42 +00:00
|
|
|
web, App, HttpResponse, HttpResponseBuilder, HttpServer,
|
2021-01-14 16:26:17 +00:00
|
|
|
};
|
2021-03-10 02:51:03 +00:00
|
|
|
use awc::Client;
|
2021-11-03 17:04:59 +00:00
|
|
|
use futures_util::{
|
|
|
|
stream::{empty, once},
|
|
|
|
Stream,
|
|
|
|
};
|
2022-03-25 23:47:50 +00:00
|
|
|
use once_cell::sync::Lazy;
|
2021-09-06 00:05:58 +00:00
|
|
|
use std::{
|
|
|
|
collections::HashSet,
|
2021-10-21 00:28:40 +00:00
|
|
|
future::ready,
|
2021-09-06 00:05:58 +00:00
|
|
|
path::PathBuf,
|
|
|
|
pin::Pin,
|
|
|
|
task::{Context, Poll},
|
|
|
|
time::SystemTime,
|
|
|
|
};
|
2021-10-21 00:28:40 +00:00
|
|
|
use tokio::{io::AsyncReadExt, sync::Semaphore};
|
|
|
|
use tracing::{debug, error, info, instrument, Span};
|
2021-09-14 01:22:42 +00:00
|
|
|
use tracing_actix_web::TracingLogger;
|
2021-12-03 20:21:31 +00:00
|
|
|
use tracing_awc::Tracing;
|
2021-09-16 22:51:20 +00:00
|
|
|
use tracing_futures::Instrument;
|
2020-06-06 21:41:17 +00:00
|
|
|
|
2021-10-21 00:28:40 +00:00
|
|
|
mod concurrent_processor;
|
2020-06-07 00:54:06 +00:00
|
|
|
mod config;
|
2022-03-24 22:09:15 +00:00
|
|
|
mod details;
|
2021-10-14 00:06:53 +00:00
|
|
|
mod either;
|
2020-06-06 21:41:17 +00:00
|
|
|
mod error;
|
2021-08-31 16:02:30 +00:00
|
|
|
mod exiftool;
|
2021-08-26 02:46:11 +00:00
|
|
|
mod ffmpeg;
|
2021-10-23 19:14:12 +00:00
|
|
|
mod file;
|
2021-10-21 00:28:40 +00:00
|
|
|
mod init_tracing;
|
2021-08-28 22:15:14 +00:00
|
|
|
mod magick;
|
2021-10-23 04:48:56 +00:00
|
|
|
mod map_error;
|
2020-06-14 02:28:06 +00:00
|
|
|
mod middleware;
|
2020-07-11 20:40:40 +00:00
|
|
|
mod migrate;
|
2021-10-21 00:28:40 +00:00
|
|
|
mod process;
|
2020-06-07 17:51:45 +00:00
|
|
|
mod processor;
|
2021-01-18 23:11:32 +00:00
|
|
|
mod range;
|
2022-03-24 18:16:41 +00:00
|
|
|
mod repo;
|
2021-10-28 04:06:03 +00:00
|
|
|
mod serde_str;
|
2021-10-23 04:48:56 +00:00
|
|
|
mod store;
|
2021-10-23 19:14:12 +00:00
|
|
|
mod tmp_file;
|
2020-06-06 21:41:17 +00:00
|
|
|
mod upload_manager;
|
2020-06-11 16:46:00 +00:00
|
|
|
mod validate;
|
2020-06-06 21:41:17 +00:00
|
|
|
|
2020-06-14 02:28:06 +00:00
|
|
|
use self::{
|
2021-10-21 00:28:40 +00:00
|
|
|
concurrent_processor::CancelSafeProcessor,
|
2022-03-25 23:47:50 +00:00
|
|
|
config::{CommandConfig, Config, Format, RequiredFilesystemStorage, RequiredObjectStorage},
|
2022-03-24 22:09:15 +00:00
|
|
|
details::Details,
|
2021-10-14 00:06:53 +00:00
|
|
|
either::Either,
|
2021-09-14 01:22:42 +00:00
|
|
|
error::{Error, UploadError},
|
2021-10-21 00:28:40 +00:00
|
|
|
init_tracing::init_tracing,
|
2022-03-25 23:47:50 +00:00
|
|
|
magick::details_hint,
|
2021-09-14 01:22:42 +00:00
|
|
|
middleware::{Deadline, Internal},
|
2021-10-23 04:48:56 +00:00
|
|
|
migrate::LatestDb,
|
2022-03-26 21:49:23 +00:00
|
|
|
repo::{Alias, DeleteToken, Repo},
|
2022-03-25 23:47:50 +00:00
|
|
|
store::{file_store::FileStore, object_store::ObjectStore, Store},
|
2022-03-24 22:09:15 +00:00
|
|
|
upload_manager::{UploadManager, UploadManagerSession},
|
2020-06-14 02:28:06 +00:00
|
|
|
};
|
2020-06-06 21:02:26 +00:00
|
|
|
|
|
|
|
const MEGABYTES: usize = 1024 * 1024;
|
2020-10-02 20:38:52 +00:00
|
|
|
const MINUTES: u32 = 60;
|
|
|
|
const HOURS: u32 = 60 * MINUTES;
|
|
|
|
const DAYS: u32 = 24 * HOURS;
|
2020-06-06 21:02:26 +00:00
|
|
|
|
2021-10-28 04:06:03 +00:00
|
|
|
static CONFIG: Lazy<Config> = Lazy::new(|| Config::build().unwrap());
|
2021-09-12 15:42:44 +00:00
|
|
|
static PROCESS_SEMAPHORE: Lazy<Semaphore> =
|
|
|
|
Lazy::new(|| Semaphore::new(num_cpus::get().saturating_sub(1).max(1)));
|
2021-09-05 22:15:30 +00:00
|
|
|
|
2020-06-06 21:41:17 +00:00
|
|
|
/// Handle responding to succesful uploads
|
2021-09-14 01:22:42 +00:00
|
|
|
#[instrument(name = "Uploaded files", skip(value, manager))]
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn upload<S: Store>(
|
|
|
|
value: Value<UploadManagerSession<S>>,
|
2021-11-01 02:11:35 +00:00
|
|
|
manager: web::Data<UploadManager>,
|
|
|
|
store: web::Data<S>,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
2020-06-06 21:02:26 +00:00
|
|
|
let images = value
|
|
|
|
.map()
|
|
|
|
.and_then(|mut m| m.remove("images"))
|
|
|
|
.and_then(|images| images.array())
|
|
|
|
.ok_or(UploadError::NoFiles)?;
|
|
|
|
|
|
|
|
let mut files = Vec::new();
|
2021-09-12 00:53:26 +00:00
|
|
|
let images = images
|
|
|
|
.into_iter()
|
|
|
|
.filter_map(|i| i.file())
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
for image in &images {
|
|
|
|
if let Some(alias) = image.result.alias() {
|
2020-12-26 02:39:19 +00:00
|
|
|
info!("Uploaded {} as {:?}", image.filename, alias);
|
2021-09-12 00:53:26 +00:00
|
|
|
let delete_token = image.result.delete_token().await?;
|
2020-12-26 02:39:19 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let identifier = manager.identifier_from_alias::<S>(alias).await?;
|
|
|
|
let details = manager.details(&identifier).await?;
|
2020-12-26 02:39:19 +00:00
|
|
|
|
|
|
|
let details = if let Some(details) = details {
|
2021-08-31 17:14:23 +00:00
|
|
|
debug!("details exist");
|
2020-12-26 02:39:19 +00:00
|
|
|
details
|
|
|
|
} else {
|
2021-10-23 04:48:56 +00:00
|
|
|
debug!("generating new details from {:?}", identifier);
|
2022-03-26 21:49:23 +00:00
|
|
|
let hint = details_hint(alias);
|
2021-10-23 04:48:56 +00:00
|
|
|
let new_details =
|
2021-11-01 02:11:35 +00:00
|
|
|
Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
2022-03-26 21:49:23 +00:00
|
|
|
debug!("storing details for {:?}", identifier);
|
|
|
|
manager.store_details(&identifier, &new_details).await?;
|
2021-08-31 17:14:23 +00:00
|
|
|
debug!("stored");
|
2020-12-26 02:39:19 +00:00
|
|
|
new_details
|
|
|
|
};
|
|
|
|
|
2020-06-07 00:29:15 +00:00
|
|
|
files.push(serde_json::json!({
|
2022-03-26 21:49:23 +00:00
|
|
|
"file": alias.to_string(),
|
|
|
|
"delete_token": delete_token.to_string(),
|
2020-12-26 02:39:19 +00:00
|
|
|
"details": details,
|
2020-06-07 00:29:15 +00:00
|
|
|
}));
|
2020-06-06 21:02:26 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-12 00:53:26 +00:00
|
|
|
for image in images {
|
|
|
|
image.result.succeed();
|
|
|
|
}
|
2021-02-10 22:57:42 +00:00
|
|
|
Ok(HttpResponse::Created().json(&serde_json::json!({
|
2020-06-07 15:59:58 +00:00
|
|
|
"msg": "ok",
|
|
|
|
"files": files
|
|
|
|
})))
|
|
|
|
}
|
|
|
|
|
2020-06-24 16:58:46 +00:00
|
|
|
#[derive(Debug, serde::Deserialize)]
|
|
|
|
struct UrlQuery {
|
|
|
|
url: String,
|
|
|
|
}
|
|
|
|
|
2021-10-20 23:58:32 +00:00
|
|
|
pin_project_lite::pin_project! {
|
|
|
|
struct Limit<S> {
|
|
|
|
#[pin]
|
|
|
|
inner: S,
|
|
|
|
|
|
|
|
count: u64,
|
|
|
|
limit: u64,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<S> Limit<S> {
|
|
|
|
fn new(inner: S, limit: u64) -> Self {
|
|
|
|
Limit {
|
|
|
|
inner,
|
|
|
|
count: 0,
|
|
|
|
limit,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
|
|
#[error("Resonse body larger than size limit")]
|
|
|
|
struct LimitError;
|
|
|
|
|
|
|
|
impl<S, E> Stream for Limit<S>
|
|
|
|
where
|
|
|
|
S: Stream<Item = Result<web::Bytes, E>>,
|
|
|
|
E: From<LimitError>,
|
|
|
|
{
|
|
|
|
type Item = Result<web::Bytes, E>;
|
|
|
|
|
|
|
|
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
|
|
|
let this = self.as_mut().project();
|
|
|
|
|
|
|
|
let limit = this.limit;
|
|
|
|
let count = this.count;
|
|
|
|
let inner = this.inner;
|
|
|
|
|
|
|
|
inner.poll_next(cx).map(|opt| {
|
|
|
|
opt.map(|res| match res {
|
|
|
|
Ok(bytes) => {
|
|
|
|
*count += bytes.len() as u64;
|
|
|
|
if *count > *limit {
|
|
|
|
return Err(LimitError.into());
|
|
|
|
}
|
|
|
|
Ok(bytes)
|
|
|
|
}
|
|
|
|
Err(e) => Err(e),
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-07 15:59:58 +00:00
|
|
|
/// download an image from a URL
|
2021-09-14 01:22:42 +00:00
|
|
|
#[instrument(name = "Downloading file", skip(client, manager))]
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn download<S: Store>(
|
2020-06-07 15:59:58 +00:00
|
|
|
client: web::Data<Client>,
|
2021-11-01 02:11:35 +00:00
|
|
|
manager: web::Data<UploadManager>,
|
|
|
|
store: web::Data<S>,
|
2020-06-07 15:59:58 +00:00
|
|
|
query: web::Query<UrlQuery>,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
2021-12-03 20:21:31 +00:00
|
|
|
let res = client.get(&query.url).send().await?;
|
2020-06-07 15:59:58 +00:00
|
|
|
|
|
|
|
if !res.status().is_success() {
|
2021-09-14 01:22:42 +00:00
|
|
|
return Err(UploadError::Download(res.status()).into());
|
2020-06-07 15:59:58 +00:00
|
|
|
}
|
|
|
|
|
2021-11-01 02:11:35 +00:00
|
|
|
let stream = Limit::new(
|
2021-10-23 04:48:56 +00:00
|
|
|
map_error::map_crate_error(res),
|
2021-10-20 23:58:32 +00:00
|
|
|
(CONFIG.max_file_size() * MEGABYTES) as u64,
|
|
|
|
);
|
2020-06-07 15:59:58 +00:00
|
|
|
|
2021-11-01 02:11:35 +00:00
|
|
|
futures_util::pin_mut!(stream);
|
2020-06-07 15:59:58 +00:00
|
|
|
|
2021-09-06 17:08:13 +00:00
|
|
|
let permit = PROCESS_SEMAPHORE.acquire().await?;
|
2021-11-01 02:11:35 +00:00
|
|
|
let session = manager.session((**store).clone()).upload(stream).await?;
|
2021-09-12 00:53:26 +00:00
|
|
|
let alias = session.alias().unwrap().to_owned();
|
2021-09-04 21:01:48 +00:00
|
|
|
drop(permit);
|
2021-09-12 00:53:26 +00:00
|
|
|
let delete_token = session.delete_token().await?;
|
2020-06-07 15:59:58 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let identifier = manager.identifier_from_alias::<S>(&alias).await?;
|
2021-01-05 06:28:17 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let details = manager.details(&identifier).await?;
|
2021-01-05 06:28:17 +00:00
|
|
|
|
|
|
|
let details = if let Some(details) = details {
|
|
|
|
details
|
|
|
|
} else {
|
2022-03-26 21:49:23 +00:00
|
|
|
let hint = details_hint(&alias);
|
2021-11-01 02:11:35 +00:00
|
|
|
let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
2022-03-26 21:49:23 +00:00
|
|
|
manager.store_details(&identifier, &new_details).await?;
|
2021-01-05 06:28:17 +00:00
|
|
|
new_details
|
|
|
|
};
|
|
|
|
|
2021-09-12 00:53:26 +00:00
|
|
|
session.succeed();
|
2021-02-10 22:57:42 +00:00
|
|
|
Ok(HttpResponse::Created().json(&serde_json::json!({
|
2020-06-07 15:59:58 +00:00
|
|
|
"msg": "ok",
|
|
|
|
"files": [{
|
2022-03-26 21:49:23 +00:00
|
|
|
"file": alias.to_string(),
|
|
|
|
"delete_token": delete_token.to_string(),
|
2021-01-05 06:28:17 +00:00
|
|
|
"details": details,
|
2020-06-07 15:59:58 +00:00
|
|
|
}]
|
|
|
|
})))
|
2020-06-06 21:02:26 +00:00
|
|
|
}
|
|
|
|
|
2020-06-15 02:41:45 +00:00
|
|
|
/// Delete aliases and files
|
2021-09-14 01:22:42 +00:00
|
|
|
#[instrument(name = "Deleting file", skip(manager))]
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn delete<S: Store>(
|
2021-11-01 02:11:35 +00:00
|
|
|
manager: web::Data<UploadManager>,
|
|
|
|
store: web::Data<S>,
|
2020-06-07 00:29:15 +00:00
|
|
|
path_entries: web::Path<(String, String)>,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
|
|
|
let (token, alias) = path_entries.into_inner();
|
2020-06-07 00:29:15 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let token = DeleteToken::from_existing(&token);
|
2022-03-27 01:45:12 +00:00
|
|
|
let alias = Alias::from_existing(&alias);
|
2022-03-26 21:49:23 +00:00
|
|
|
|
|
|
|
manager.delete((**store).clone(), alias, token).await?;
|
2020-06-07 00:29:15 +00:00
|
|
|
|
|
|
|
Ok(HttpResponse::NoContent().finish())
|
|
|
|
}
|
|
|
|
|
2020-06-24 16:58:46 +00:00
|
|
|
type ProcessQuery = Vec<(String, String)>;
|
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
fn prepare_process(
|
2020-06-24 16:58:46 +00:00
|
|
|
query: web::Query<ProcessQuery>,
|
2020-12-10 18:49:10 +00:00
|
|
|
ext: &str,
|
2021-10-19 04:51:04 +00:00
|
|
|
filters: &Option<HashSet<String>>,
|
2022-03-26 21:49:23 +00:00
|
|
|
) -> Result<(Format, Alias, PathBuf, Vec<String>), Error> {
|
2020-06-24 16:58:46 +00:00
|
|
|
let (alias, operations) =
|
|
|
|
query
|
|
|
|
.into_inner()
|
|
|
|
.into_iter()
|
|
|
|
.fold((String::new(), Vec::new()), |(s, mut acc), (k, v)| {
|
|
|
|
if k == "src" {
|
|
|
|
(v, acc)
|
|
|
|
} else {
|
|
|
|
acc.push((k, v));
|
|
|
|
(s, acc)
|
|
|
|
}
|
|
|
|
});
|
2020-06-06 21:41:17 +00:00
|
|
|
|
2021-09-12 15:42:44 +00:00
|
|
|
if alias.is_empty() {
|
2021-09-14 01:22:42 +00:00
|
|
|
return Err(UploadError::MissingFilename.into());
|
2020-06-24 16:58:46 +00:00
|
|
|
}
|
2020-06-06 21:02:26 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let alias = Alias::from_existing(&alias);
|
2020-06-06 21:41:17 +00:00
|
|
|
|
2021-10-19 04:51:04 +00:00
|
|
|
let operations = if let Some(filters) = filters.as_ref() {
|
2020-06-24 16:58:46 +00:00
|
|
|
operations
|
|
|
|
.into_iter()
|
2021-10-19 04:51:04 +00:00
|
|
|
.filter(|(k, _)| filters.contains(&k.to_lowercase()))
|
2020-06-24 16:58:46 +00:00
|
|
|
.collect()
|
|
|
|
} else {
|
|
|
|
operations
|
|
|
|
};
|
|
|
|
|
2020-11-01 20:23:56 +00:00
|
|
|
let format = ext
|
|
|
|
.parse::<Format>()
|
|
|
|
.map_err(|_| UploadError::UnsupportedFormat)?;
|
2021-10-21 01:13:39 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let (thumbnail_path, thumbnail_args) = self::processor::build_chain(&operations)?;
|
2020-06-06 21:02:26 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
Ok((format, alias, thumbnail_path, thumbnail_args))
|
2020-12-10 18:49:10 +00:00
|
|
|
}
|
|
|
|
|
2021-10-19 04:51:04 +00:00
|
|
|
#[instrument(name = "Fetching derived details", skip(manager, filters))]
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn process_details<S: Store>(
|
2020-12-10 18:49:10 +00:00
|
|
|
query: web::Query<ProcessQuery>,
|
|
|
|
ext: web::Path<String>,
|
2021-11-01 02:11:35 +00:00
|
|
|
manager: web::Data<UploadManager>,
|
|
|
|
store: web::Data<S>,
|
2021-10-19 04:51:04 +00:00
|
|
|
filters: web::Data<Option<HashSet<String>>>,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
2022-03-26 21:49:23 +00:00
|
|
|
let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
|
2020-12-10 18:49:10 +00:00
|
|
|
|
2021-10-23 04:48:56 +00:00
|
|
|
let identifier = manager
|
2022-03-26 21:49:23 +00:00
|
|
|
.variant_identifier::<S>(&alias, &thumbnail_path)
|
2021-10-19 04:37:11 +00:00
|
|
|
.await?
|
|
|
|
.ok_or(UploadError::MissingAlias)?;
|
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let details = manager.details(&identifier).await?;
|
2020-12-10 18:49:10 +00:00
|
|
|
|
|
|
|
let details = details.ok_or(UploadError::NoFiles)?;
|
|
|
|
|
2021-02-10 22:57:42 +00:00
|
|
|
Ok(HttpResponse::Ok().json(&details))
|
2020-12-10 18:49:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Process files
|
2021-10-19 04:51:04 +00:00
|
|
|
#[instrument(name = "Serving processed image", skip(manager, filters))]
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn process<S: Store + 'static>(
|
2022-03-01 17:23:15 +00:00
|
|
|
range: Option<web::Header<Range>>,
|
2020-12-10 18:49:10 +00:00
|
|
|
query: web::Query<ProcessQuery>,
|
|
|
|
ext: web::Path<String>,
|
2021-11-01 02:11:35 +00:00
|
|
|
manager: web::Data<UploadManager>,
|
|
|
|
store: web::Data<S>,
|
2021-10-19 04:51:04 +00:00
|
|
|
filters: web::Data<Option<HashSet<String>>>,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
2022-03-26 21:49:23 +00:00
|
|
|
let (format, alias, thumbnail_path, thumbnail_args) =
|
|
|
|
prepare_process(query, ext.as_str(), &filters)?;
|
2020-12-10 18:49:10 +00:00
|
|
|
|
2021-11-01 02:11:35 +00:00
|
|
|
let identifier_opt = manager
|
2022-03-26 21:49:23 +00:00
|
|
|
.variant_identifier::<S>(&alias, &thumbnail_path)
|
2021-11-01 02:11:35 +00:00
|
|
|
.await?;
|
2020-06-06 21:02:26 +00:00
|
|
|
|
2021-10-23 04:48:56 +00:00
|
|
|
if let Some(identifier) = identifier_opt {
|
2022-03-26 21:49:23 +00:00
|
|
|
let details_opt = manager.details(&identifier).await?;
|
2021-10-19 04:37:11 +00:00
|
|
|
|
2021-10-21 00:28:40 +00:00
|
|
|
let details = if let Some(details) = details_opt {
|
|
|
|
details
|
|
|
|
} else {
|
2022-03-26 21:49:23 +00:00
|
|
|
let hint = details_hint(&alias);
|
2021-11-01 02:11:35 +00:00
|
|
|
let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
2022-03-26 21:49:23 +00:00
|
|
|
manager.store_details(&identifier, &details).await?;
|
2021-10-21 00:28:40 +00:00
|
|
|
details
|
|
|
|
};
|
2021-10-19 04:37:11 +00:00
|
|
|
|
2021-11-01 02:11:35 +00:00
|
|
|
return ranged_file_resp(&**store, identifier, range, details).await;
|
2021-10-19 04:37:11 +00:00
|
|
|
}
|
|
|
|
|
2021-11-01 02:11:35 +00:00
|
|
|
let identifier = manager
|
2022-03-26 21:49:23 +00:00
|
|
|
.still_identifier_from_alias((**store).clone(), &alias)
|
2021-11-01 02:11:35 +00:00
|
|
|
.await?;
|
2021-10-19 04:37:11 +00:00
|
|
|
|
|
|
|
let thumbnail_path2 = thumbnail_path.clone();
|
|
|
|
let process_fut = async {
|
|
|
|
let thumbnail_path = thumbnail_path2;
|
2020-06-24 16:58:46 +00:00
|
|
|
|
2021-10-19 04:37:11 +00:00
|
|
|
let permit = PROCESS_SEMAPHORE.acquire().await?;
|
2020-06-06 21:02:26 +00:00
|
|
|
|
2021-10-23 04:48:56 +00:00
|
|
|
let mut processed_reader = crate::magick::process_image_store_read(
|
2021-11-01 02:11:35 +00:00
|
|
|
(**store).clone(),
|
2021-10-23 04:48:56 +00:00
|
|
|
identifier,
|
|
|
|
thumbnail_args,
|
|
|
|
format,
|
|
|
|
)?;
|
2021-09-04 21:01:48 +00:00
|
|
|
|
2021-10-19 04:37:11 +00:00
|
|
|
let mut vec = Vec::new();
|
|
|
|
processed_reader.read_to_end(&mut vec).await?;
|
|
|
|
let bytes = web::Bytes::from(vec);
|
2021-08-31 02:19:47 +00:00
|
|
|
|
2021-10-19 04:37:11 +00:00
|
|
|
drop(permit);
|
2021-09-05 22:15:30 +00:00
|
|
|
|
2021-10-28 04:06:03 +00:00
|
|
|
let details = Details::from_bytes(bytes.clone(), format.as_hint()).await?;
|
2021-09-05 22:15:30 +00:00
|
|
|
|
2021-10-19 04:37:11 +00:00
|
|
|
let save_span = tracing::info_span!(
|
|
|
|
parent: None,
|
|
|
|
"Saving variant information",
|
|
|
|
path = tracing::field::debug(&thumbnail_path),
|
2022-03-26 21:49:23 +00:00
|
|
|
name = tracing::field::display(&alias),
|
2021-10-19 04:37:11 +00:00
|
|
|
);
|
|
|
|
save_span.follows_from(Span::current());
|
|
|
|
let details2 = details.clone();
|
|
|
|
let bytes2 = bytes.clone();
|
2022-03-26 21:49:23 +00:00
|
|
|
let alias2 = alias.clone();
|
2021-10-19 04:37:11 +00:00
|
|
|
actix_rt::spawn(
|
|
|
|
async move {
|
2022-03-26 21:49:23 +00:00
|
|
|
let identifier = match store.save_bytes(bytes2).await {
|
2021-10-23 04:48:56 +00:00
|
|
|
Ok(identifier) => identifier,
|
2021-10-19 04:37:11 +00:00
|
|
|
Err(e) => {
|
|
|
|
tracing::warn!("Failed to generate directory path: {}", e);
|
2021-09-16 22:51:20 +00:00
|
|
|
return;
|
|
|
|
}
|
2021-10-19 04:37:11 +00:00
|
|
|
};
|
2022-03-26 21:49:23 +00:00
|
|
|
if let Err(e) = manager.store_details(&identifier, &details2).await {
|
2021-10-19 04:37:11 +00:00
|
|
|
tracing::warn!("Error saving variant details: {}", e);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if let Err(e) = manager
|
2022-03-26 21:49:23 +00:00
|
|
|
.store_variant(&alias2, &thumbnail_path, &identifier)
|
2021-10-19 04:37:11 +00:00
|
|
|
.await
|
|
|
|
{
|
|
|
|
tracing::warn!("Error saving variant info: {}", e);
|
2021-09-09 21:35:07 +00:00
|
|
|
}
|
|
|
|
}
|
2021-10-19 04:37:11 +00:00
|
|
|
.instrument(save_span),
|
|
|
|
);
|
2020-06-06 21:02:26 +00:00
|
|
|
|
2021-10-19 04:37:11 +00:00
|
|
|
Ok((details, bytes)) as Result<(Details, web::Bytes), Error>
|
2020-12-10 18:49:10 +00:00
|
|
|
};
|
2020-12-10 05:05:04 +00:00
|
|
|
|
2021-10-19 04:37:11 +00:00
|
|
|
let (details, bytes) = CancelSafeProcessor::new(thumbnail_path.clone(), process_fut).await?;
|
|
|
|
|
2022-03-01 17:23:15 +00:00
|
|
|
let (builder, stream) = if let Some(web::Header(range_header)) = range {
|
|
|
|
if let Some(range) = range::single_bytes_range(&range_header) {
|
|
|
|
let len = bytes.len() as u64;
|
|
|
|
|
|
|
|
if let Some(content_range) = range::to_content_range(range, len) {
|
2021-10-19 04:37:11 +00:00
|
|
|
let mut builder = HttpResponse::PartialContent();
|
|
|
|
builder.insert_header(content_range);
|
2022-03-01 17:23:15 +00:00
|
|
|
let stream = range::chop_bytes(range, bytes, len)?;
|
2021-10-19 04:37:11 +00:00
|
|
|
|
2021-11-03 17:04:59 +00:00
|
|
|
(builder, Either::left(Either::left(stream)))
|
2021-10-19 04:37:11 +00:00
|
|
|
} else {
|
2021-11-03 17:04:59 +00:00
|
|
|
(
|
|
|
|
HttpResponse::RangeNotSatisfiable(),
|
|
|
|
Either::left(Either::right(empty())),
|
|
|
|
)
|
2021-10-19 04:37:11 +00:00
|
|
|
}
|
2021-11-03 17:04:59 +00:00
|
|
|
} else {
|
|
|
|
return Err(UploadError::Range.into());
|
2021-10-19 04:37:11 +00:00
|
|
|
}
|
2021-11-03 17:04:59 +00:00
|
|
|
} else {
|
|
|
|
(HttpResponse::Ok(), Either::right(once(ready(Ok(bytes)))))
|
|
|
|
};
|
|
|
|
|
|
|
|
Ok(srv_response(
|
|
|
|
builder,
|
|
|
|
stream,
|
|
|
|
details.content_type(),
|
|
|
|
7 * DAYS,
|
|
|
|
details.system_time(),
|
|
|
|
))
|
2020-06-24 16:58:46 +00:00
|
|
|
}
|
|
|
|
|
2020-12-10 18:49:10 +00:00
|
|
|
/// Fetch file details
|
2021-09-14 01:22:42 +00:00
|
|
|
#[instrument(name = "Fetching details", skip(manager))]
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn details<S: Store>(
|
2020-12-10 18:49:10 +00:00
|
|
|
alias: web::Path<String>,
|
2021-11-01 02:11:35 +00:00
|
|
|
manager: web::Data<UploadManager>,
|
|
|
|
store: web::Data<S>,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
2022-03-26 21:49:23 +00:00
|
|
|
let alias = alias.into_inner();
|
|
|
|
let alias = Alias::from_existing(&alias);
|
|
|
|
|
|
|
|
let identifier = manager.identifier_from_alias::<S>(&alias).await?;
|
2020-12-10 18:49:10 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let details = manager.details(&identifier).await?;
|
2020-12-10 18:49:10 +00:00
|
|
|
|
|
|
|
let details = if let Some(details) = details {
|
|
|
|
details
|
|
|
|
} else {
|
2022-03-26 21:49:23 +00:00
|
|
|
let hint = details_hint(&alias);
|
2021-11-01 02:11:35 +00:00
|
|
|
let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
2022-03-26 21:49:23 +00:00
|
|
|
manager.store_details(&identifier, &new_details).await?;
|
2020-12-10 18:49:10 +00:00
|
|
|
new_details
|
|
|
|
};
|
|
|
|
|
2021-02-10 22:57:42 +00:00
|
|
|
Ok(HttpResponse::Ok().json(&details))
|
2020-12-10 18:49:10 +00:00
|
|
|
}
|
|
|
|
|
2020-06-24 16:58:46 +00:00
|
|
|
/// Serve files
|
2021-09-14 01:22:42 +00:00
|
|
|
#[instrument(name = "Serving file", skip(manager))]
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn serve<S: Store>(
|
2022-03-01 17:23:15 +00:00
|
|
|
range: Option<web::Header<Range>>,
|
2020-06-24 16:58:46 +00:00
|
|
|
alias: web::Path<String>,
|
2021-11-01 02:11:35 +00:00
|
|
|
manager: web::Data<UploadManager>,
|
|
|
|
store: web::Data<S>,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
2022-03-26 21:49:23 +00:00
|
|
|
let alias = alias.into_inner();
|
|
|
|
let alias = Alias::from_existing(&alias);
|
|
|
|
let identifier = manager.identifier_from_alias::<S>(&alias).await?;
|
2020-12-10 05:05:04 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let details = manager.details(&identifier).await?;
|
2020-12-10 05:05:04 +00:00
|
|
|
|
2020-12-10 18:49:10 +00:00
|
|
|
let details = if let Some(details) = details {
|
|
|
|
details
|
|
|
|
} else {
|
2022-03-26 21:49:23 +00:00
|
|
|
let hint = details_hint(&alias);
|
2021-11-01 02:11:35 +00:00
|
|
|
let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
2022-03-26 21:49:23 +00:00
|
|
|
manager.store_details(&identifier, &details).await?;
|
2020-12-10 18:49:10 +00:00
|
|
|
details
|
|
|
|
};
|
2020-06-24 16:58:46 +00:00
|
|
|
|
2021-11-01 02:11:35 +00:00
|
|
|
ranged_file_resp(&**store, identifier, range, details).await
|
2021-01-14 13:52:11 +00:00
|
|
|
}
|
2021-01-14 01:08:04 +00:00
|
|
|
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn ranged_file_resp<S: Store>(
|
2021-11-01 02:11:35 +00:00
|
|
|
store: &S,
|
2021-10-23 04:48:56 +00:00
|
|
|
identifier: S::Identifier,
|
2022-03-01 17:23:15 +00:00
|
|
|
range: Option<web::Header<Range>>,
|
2021-01-14 16:26:17 +00:00
|
|
|
details: Details,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
2022-03-01 17:23:15 +00:00
|
|
|
let (builder, stream) = if let Some(web::Header(range_header)) = range {
|
2021-01-14 13:52:11 +00:00
|
|
|
//Range header exists - return as ranged
|
2022-03-01 17:23:15 +00:00
|
|
|
if let Some(range) = range::single_bytes_range(&range_header) {
|
2021-11-03 17:04:59 +00:00
|
|
|
let len = store.len(&identifier).await?;
|
2021-01-18 23:11:32 +00:00
|
|
|
|
2022-03-01 17:23:15 +00:00
|
|
|
if let Some(content_range) = range::to_content_range(range, len) {
|
2021-01-18 23:11:32 +00:00
|
|
|
let mut builder = HttpResponse::PartialContent();
|
2021-11-03 17:04:59 +00:00
|
|
|
builder.insert_header(content_range);
|
2021-10-23 04:48:56 +00:00
|
|
|
(
|
|
|
|
builder,
|
2021-11-03 17:04:59 +00:00
|
|
|
Either::left(Either::left(map_error::map_crate_error(
|
2022-03-01 17:23:15 +00:00
|
|
|
range::chop_store(range, store, &identifier, len).await?,
|
2021-11-03 17:04:59 +00:00
|
|
|
))),
|
2021-10-23 04:48:56 +00:00
|
|
|
)
|
2021-01-18 23:11:32 +00:00
|
|
|
} else {
|
2021-11-03 17:04:59 +00:00
|
|
|
(
|
|
|
|
HttpResponse::RangeNotSatisfiable(),
|
|
|
|
Either::left(Either::right(empty())),
|
|
|
|
)
|
2021-01-18 23:11:32 +00:00
|
|
|
}
|
2021-11-03 17:04:59 +00:00
|
|
|
} else {
|
|
|
|
return Err(UploadError::Range.into());
|
2021-01-14 01:08:04 +00:00
|
|
|
}
|
2021-11-03 17:04:59 +00:00
|
|
|
} else {
|
2021-01-14 01:08:04 +00:00
|
|
|
//No Range header in the request - return the entire document
|
2021-11-03 17:04:59 +00:00
|
|
|
let stream = map_error::map_crate_error(store.to_stream(&identifier, None, None).await?);
|
|
|
|
(HttpResponse::Ok(), Either::right(stream))
|
2021-01-14 01:08:04 +00:00
|
|
|
};
|
2021-01-18 23:11:32 +00:00
|
|
|
|
|
|
|
Ok(srv_response(
|
|
|
|
builder,
|
2021-10-20 23:58:32 +00:00
|
|
|
stream,
|
2021-01-18 23:11:32 +00:00
|
|
|
details.content_type(),
|
|
|
|
7 * DAYS,
|
|
|
|
details.system_time(),
|
|
|
|
))
|
2020-06-06 21:02:26 +00:00
|
|
|
}
|
|
|
|
|
2020-06-06 21:41:17 +00:00
|
|
|
// A helper method to produce responses with proper cache headers
|
2020-10-02 20:38:52 +00:00
|
|
|
fn srv_response<S, E>(
|
2021-01-18 23:11:32 +00:00
|
|
|
mut builder: HttpResponseBuilder,
|
2020-10-02 20:38:52 +00:00
|
|
|
stream: S,
|
|
|
|
ext: mime::Mime,
|
|
|
|
expires: u32,
|
|
|
|
modified: SystemTime,
|
|
|
|
) -> HttpResponse
|
2020-06-06 21:02:26 +00:00
|
|
|
where
|
2021-10-20 23:58:32 +00:00
|
|
|
S: Stream<Item = Result<web::Bytes, E>> + 'static,
|
2021-06-19 19:39:41 +00:00
|
|
|
E: std::error::Error + 'static,
|
2021-01-18 23:11:32 +00:00
|
|
|
actix_web::Error: From<E>,
|
2021-01-14 01:08:04 +00:00
|
|
|
{
|
2021-01-18 23:11:32 +00:00
|
|
|
builder
|
2021-02-10 22:57:42 +00:00
|
|
|
.insert_header(LastModified(modified.into()))
|
|
|
|
.insert_header(CacheControl(vec![
|
2021-01-14 01:08:04 +00:00
|
|
|
CacheDirective::Public,
|
|
|
|
CacheDirective::MaxAge(expires),
|
|
|
|
CacheDirective::Extension("immutable".to_owned(), None),
|
|
|
|
]))
|
2021-02-10 22:57:42 +00:00
|
|
|
.insert_header((ACCEPT_RANGES, "bytes"))
|
2020-06-06 21:02:26 +00:00
|
|
|
.content_type(ext.to_string())
|
2021-11-23 18:29:42 +00:00
|
|
|
.streaming(stream)
|
2020-06-06 21:02:26 +00:00
|
|
|
}
|
|
|
|
|
2020-07-11 22:22:45 +00:00
|
|
|
#[derive(Debug, serde::Deserialize)]
|
2022-03-26 21:49:23 +00:00
|
|
|
struct AliasQuery {
|
|
|
|
alias: String,
|
2020-07-11 22:22:45 +00:00
|
|
|
}
|
|
|
|
|
2021-09-14 01:22:42 +00:00
|
|
|
#[instrument(name = "Purging file", skip(upload_manager))]
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn purge<S: Store>(
|
2022-03-26 21:49:23 +00:00
|
|
|
query: web::Query<AliasQuery>,
|
2021-11-01 02:11:35 +00:00
|
|
|
upload_manager: web::Data<UploadManager>,
|
|
|
|
store: web::Data<S>,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
2022-03-26 21:49:23 +00:00
|
|
|
let alias = Alias::from_existing(&query.alias);
|
|
|
|
let aliases = upload_manager.aliases_by_alias(&alias).await?;
|
2020-07-11 22:22:45 +00:00
|
|
|
|
|
|
|
for alias in aliases.iter() {
|
|
|
|
upload_manager
|
2021-11-01 02:11:35 +00:00
|
|
|
.delete_without_token((**store).clone(), alias.to_owned())
|
2020-07-11 22:22:45 +00:00
|
|
|
.await?;
|
|
|
|
}
|
|
|
|
|
2021-02-10 22:57:42 +00:00
|
|
|
Ok(HttpResponse::Ok().json(&serde_json::json!({
|
2020-07-11 22:22:45 +00:00
|
|
|
"msg": "ok",
|
2022-03-26 21:49:23 +00:00
|
|
|
"aliases": aliases.iter().map(|a| a.to_string()).collect::<Vec<_>>()
|
2020-07-11 22:22:45 +00:00
|
|
|
})))
|
|
|
|
}
|
|
|
|
|
2021-09-14 01:22:42 +00:00
|
|
|
#[instrument(name = "Fetching aliases", skip(upload_manager))]
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn aliases<S: Store>(
|
2022-03-26 21:49:23 +00:00
|
|
|
query: web::Query<AliasQuery>,
|
2021-11-01 02:11:35 +00:00
|
|
|
upload_manager: web::Data<UploadManager>,
|
|
|
|
store: web::Data<S>,
|
2022-03-27 01:45:12 +00:00
|
|
|
) -> Result<HttpResponse, Error> {
|
2022-03-26 21:49:23 +00:00
|
|
|
let alias = Alias::from_existing(&query.alias);
|
|
|
|
let aliases = upload_manager.aliases_by_alias(&alias).await?;
|
2020-07-11 22:22:45 +00:00
|
|
|
|
2021-02-10 22:57:42 +00:00
|
|
|
Ok(HttpResponse::Ok().json(&serde_json::json!({
|
2020-07-11 22:22:45 +00:00
|
|
|
"msg": "ok",
|
2022-03-26 21:49:23 +00:00
|
|
|
"aliases": aliases.iter().map(|a| a.to_string()).collect::<Vec<_>>()
|
2020-07-11 22:22:45 +00:00
|
|
|
})))
|
|
|
|
}
|
|
|
|
|
2021-10-23 04:48:56 +00:00
|
|
|
fn transform_error(error: actix_form_data::Error) -> actix_web::Error {
|
|
|
|
let error: Error = error.into();
|
|
|
|
let error: actix_web::Error = error.into();
|
|
|
|
error
|
|
|
|
}
|
2021-10-21 00:28:40 +00:00
|
|
|
|
2021-11-02 22:21:00 +00:00
|
|
|
fn build_client() -> awc::Client {
|
|
|
|
Client::builder()
|
2021-12-03 20:21:31 +00:00
|
|
|
.wrap(Tracing)
|
2021-12-19 02:24:22 +00:00
|
|
|
.add_default_header(("User-Agent", "pict-rs v0.3.0-main"))
|
2021-11-02 22:21:00 +00:00
|
|
|
.finish()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn build_reqwest_client() -> reqwest::Result<reqwest::Client> {
|
|
|
|
reqwest::Client::builder()
|
|
|
|
.user_agent("pict-rs v0.3.0-main")
|
|
|
|
.build()
|
|
|
|
}
|
|
|
|
|
2022-03-27 01:45:12 +00:00
|
|
|
async fn launch<S: Store + Clone + 'static>(
|
|
|
|
manager: UploadManager,
|
|
|
|
store: S,
|
|
|
|
) -> anyhow::Result<()> {
|
2020-06-06 21:41:17 +00:00
|
|
|
// Create a new Multipart Form validator
|
|
|
|
//
|
|
|
|
// This form is expecting a single array field, 'images' with at most 10 files in it
|
2020-06-06 21:02:26 +00:00
|
|
|
let manager2 = manager.clone();
|
2021-11-01 02:11:35 +00:00
|
|
|
let store2 = store.clone();
|
2020-06-06 21:02:26 +00:00
|
|
|
let form = Form::new()
|
|
|
|
.max_files(10)
|
2020-06-14 15:07:31 +00:00
|
|
|
.max_file_size(CONFIG.max_file_size() * MEGABYTES)
|
2021-10-23 04:48:56 +00:00
|
|
|
.transform_error(transform_error)
|
2020-06-06 21:02:26 +00:00
|
|
|
.field(
|
|
|
|
"images",
|
2020-06-14 18:56:42 +00:00
|
|
|
Field::array(Field::file(move |filename, _, stream| {
|
2021-11-01 02:11:35 +00:00
|
|
|
let store = store2.clone();
|
2020-06-06 21:02:26 +00:00
|
|
|
let manager = manager2.clone();
|
|
|
|
|
2021-09-16 22:51:20 +00:00
|
|
|
let span = tracing::info_span!("file-upload", ?filename);
|
2020-06-14 18:56:42 +00:00
|
|
|
|
2021-09-16 22:51:20 +00:00
|
|
|
async move {
|
2021-09-06 17:08:13 +00:00
|
|
|
let permit = PROCESS_SEMAPHORE.acquire().await?;
|
2021-09-04 21:01:48 +00:00
|
|
|
|
2021-10-23 04:48:56 +00:00
|
|
|
let res = manager
|
2021-11-01 02:11:35 +00:00
|
|
|
.session(store)
|
2021-10-23 04:48:56 +00:00
|
|
|
.upload(map_error::map_crate_error(stream))
|
|
|
|
.await;
|
2021-09-04 21:01:48 +00:00
|
|
|
|
|
|
|
drop(permit);
|
2020-06-14 18:56:42 +00:00
|
|
|
res
|
2020-06-07 15:59:58 +00:00
|
|
|
}
|
2021-09-16 22:51:20 +00:00
|
|
|
.instrument(span)
|
2020-06-06 21:02:26 +00:00
|
|
|
})),
|
|
|
|
);
|
|
|
|
|
2020-06-11 00:26:54 +00:00
|
|
|
// Create a new Multipart Form validator for internal imports
|
|
|
|
//
|
|
|
|
// This form is expecting a single array field, 'images' with at most 10 files in it
|
2020-06-14 15:07:31 +00:00
|
|
|
let validate_imports = CONFIG.validate_imports();
|
2020-06-11 00:26:54 +00:00
|
|
|
let manager2 = manager.clone();
|
2021-11-01 02:11:35 +00:00
|
|
|
let store2 = store.clone();
|
2020-06-11 00:26:54 +00:00
|
|
|
let import_form = Form::new()
|
|
|
|
.max_files(10)
|
2020-06-14 15:07:31 +00:00
|
|
|
.max_file_size(CONFIG.max_file_size() * MEGABYTES)
|
2021-10-23 04:48:56 +00:00
|
|
|
.transform_error(transform_error)
|
2020-06-11 00:26:54 +00:00
|
|
|
.field(
|
|
|
|
"images",
|
2021-10-23 17:35:07 +00:00
|
|
|
Field::array(Field::file(move |filename, _, stream| {
|
2021-11-01 02:11:35 +00:00
|
|
|
let store = store2.clone();
|
2021-11-02 22:21:00 +00:00
|
|
|
let manager = manager2.clone();
|
2020-06-11 00:26:54 +00:00
|
|
|
|
2021-09-16 22:51:20 +00:00
|
|
|
let span = tracing::info_span!("file-import", ?filename);
|
2020-06-14 18:56:42 +00:00
|
|
|
|
2021-09-16 22:51:20 +00:00
|
|
|
async move {
|
2021-09-06 17:08:13 +00:00
|
|
|
let permit = PROCESS_SEMAPHORE.acquire().await?;
|
2021-09-04 21:01:48 +00:00
|
|
|
|
2020-06-14 18:56:42 +00:00
|
|
|
let res = manager
|
2021-11-01 02:11:35 +00:00
|
|
|
.session(store)
|
2021-10-23 04:48:56 +00:00
|
|
|
.import(
|
|
|
|
filename,
|
|
|
|
validate_imports,
|
|
|
|
map_error::map_crate_error(stream),
|
|
|
|
)
|
2021-09-12 00:53:26 +00:00
|
|
|
.await;
|
2021-09-04 21:01:48 +00:00
|
|
|
|
|
|
|
drop(permit);
|
2020-06-14 18:56:42 +00:00
|
|
|
res
|
2020-06-11 00:26:54 +00:00
|
|
|
}
|
2021-09-16 22:51:20 +00:00
|
|
|
.instrument(span)
|
2020-06-11 00:26:54 +00:00
|
|
|
})),
|
|
|
|
);
|
|
|
|
|
2020-06-06 21:02:26 +00:00
|
|
|
HttpServer::new(move || {
|
|
|
|
App::new()
|
2021-09-18 21:34:23 +00:00
|
|
|
.wrap(TracingLogger::default())
|
2021-09-11 20:31:00 +00:00
|
|
|
.wrap(Deadline)
|
2021-11-01 02:11:35 +00:00
|
|
|
.app_data(web::Data::new(store.clone()))
|
2021-06-26 18:10:28 +00:00
|
|
|
.app_data(web::Data::new(manager.clone()))
|
2021-11-02 22:21:00 +00:00
|
|
|
.app_data(web::Data::new(build_client()))
|
2021-10-19 04:51:04 +00:00
|
|
|
.app_data(web::Data::new(CONFIG.allowed_filters()))
|
2020-06-06 21:02:26 +00:00
|
|
|
.service(
|
|
|
|
web::scope("/image")
|
|
|
|
.service(
|
|
|
|
web::resource("")
|
|
|
|
.guard(guard::Post())
|
|
|
|
.wrap(form.clone())
|
2021-10-23 04:48:56 +00:00
|
|
|
.route(web::post().to(upload::<S>)),
|
2020-06-06 21:02:26 +00:00
|
|
|
)
|
2021-10-23 04:48:56 +00:00
|
|
|
.service(web::resource("/download").route(web::get().to(download::<S>)))
|
2020-06-07 00:29:15 +00:00
|
|
|
.service(
|
|
|
|
web::resource("/delete/{delete_token}/{filename}")
|
2021-10-23 04:48:56 +00:00
|
|
|
.route(web::delete().to(delete::<S>))
|
|
|
|
.route(web::get().to(delete::<S>)),
|
2020-06-07 00:29:15 +00:00
|
|
|
)
|
2021-10-23 04:48:56 +00:00
|
|
|
.service(web::resource("/original/{filename}").route(web::get().to(serve::<S>)))
|
|
|
|
.service(web::resource("/process.{ext}").route(web::get().to(process::<S>)))
|
2020-12-10 18:49:10 +00:00
|
|
|
.service(
|
|
|
|
web::scope("/details")
|
|
|
|
.service(
|
2021-10-23 04:48:56 +00:00
|
|
|
web::resource("/original/{filename}")
|
|
|
|
.route(web::get().to(details::<S>)),
|
2020-12-10 18:49:10 +00:00
|
|
|
)
|
|
|
|
.service(
|
|
|
|
web::resource("/process.{ext}")
|
2021-10-23 04:48:56 +00:00
|
|
|
.route(web::get().to(process_details::<S>)),
|
2020-12-10 18:49:10 +00:00
|
|
|
),
|
|
|
|
),
|
2020-06-06 21:02:26 +00:00
|
|
|
)
|
2020-06-11 00:26:54 +00:00
|
|
|
.service(
|
2020-07-11 21:28:49 +00:00
|
|
|
web::scope("/internal")
|
|
|
|
.wrap(Internal(CONFIG.api_key().map(|s| s.to_owned())))
|
|
|
|
.service(
|
|
|
|
web::resource("/import")
|
|
|
|
.wrap(import_form.clone())
|
2021-10-23 04:48:56 +00:00
|
|
|
.route(web::post().to(upload::<S>)),
|
2020-07-11 22:22:45 +00:00
|
|
|
)
|
2021-10-23 04:48:56 +00:00
|
|
|
.service(web::resource("/purge").route(web::post().to(purge::<S>)))
|
2022-03-26 21:49:23 +00:00
|
|
|
.service(web::resource("/aliases").route(web::get().to(aliases::<S>))),
|
2020-06-11 00:26:54 +00:00
|
|
|
)
|
2020-06-06 21:02:26 +00:00
|
|
|
})
|
2020-06-14 15:07:31 +00:00
|
|
|
.bind(CONFIG.bind_address())?
|
2020-06-06 21:02:26 +00:00
|
|
|
.run()
|
|
|
|
.await?;
|
|
|
|
|
2021-10-23 19:14:12 +00:00
|
|
|
crate::tmp_file::remove_tmp_dir().await?;
|
|
|
|
|
2020-06-06 21:02:26 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
2021-10-23 04:48:56 +00:00
|
|
|
|
2021-11-01 02:11:35 +00:00
|
|
|
async fn migrate_inner<S1>(
|
|
|
|
manager: &UploadManager,
|
2022-03-26 21:49:23 +00:00
|
|
|
repo: &Repo,
|
2021-11-01 02:11:35 +00:00
|
|
|
from: S1,
|
2022-03-25 23:47:50 +00:00
|
|
|
to: &config::Storage,
|
2021-11-01 02:11:35 +00:00
|
|
|
) -> anyhow::Result<()>
|
|
|
|
where
|
|
|
|
S1: Store,
|
|
|
|
{
|
|
|
|
match to {
|
2022-03-25 23:47:50 +00:00
|
|
|
config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
|
2022-03-26 21:49:23 +00:00
|
|
|
let to = FileStore::build(path.clone(), repo.clone()).await?;
|
2021-11-01 02:11:35 +00:00
|
|
|
manager.migrate_store::<S1, FileStore>(from, to).await?;
|
|
|
|
}
|
2022-03-25 23:47:50 +00:00
|
|
|
config::Storage::ObjectStorage(RequiredObjectStorage {
|
2021-11-01 02:11:35 +00:00
|
|
|
bucket_name,
|
|
|
|
region,
|
|
|
|
access_key,
|
|
|
|
secret_key,
|
|
|
|
security_token,
|
|
|
|
session_token,
|
2022-03-25 23:47:50 +00:00
|
|
|
}) => {
|
2021-11-01 02:11:35 +00:00
|
|
|
let to = ObjectStore::build(
|
|
|
|
bucket_name,
|
2022-03-25 23:47:50 +00:00
|
|
|
region.clone(),
|
2021-11-01 02:11:35 +00:00
|
|
|
access_key.clone(),
|
|
|
|
secret_key.clone(),
|
|
|
|
security_token.clone(),
|
|
|
|
session_token.clone(),
|
2022-03-26 21:49:23 +00:00
|
|
|
repo.clone(),
|
2021-11-02 22:21:00 +00:00
|
|
|
build_reqwest_client()?,
|
2022-03-26 21:49:23 +00:00
|
|
|
)
|
|
|
|
.await?;
|
2021-11-01 02:11:35 +00:00
|
|
|
|
|
|
|
manager.migrate_store::<S1, ObjectStore>(from, to).await?;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2021-10-23 04:48:56 +00:00
|
|
|
#[actix_rt::main]
|
|
|
|
async fn main() -> anyhow::Result<()> {
|
2022-03-22 02:55:25 +00:00
|
|
|
init_tracing(
|
|
|
|
"pict-rs",
|
|
|
|
CONFIG.opentelemetry_url(),
|
|
|
|
CONFIG.console_buffer_capacity(),
|
|
|
|
)?;
|
2021-10-23 04:48:56 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let repo = Repo::open(CONFIG.repo())?;
|
2021-11-01 02:11:35 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let db = LatestDb::exists(CONFIG.data_dir()).migrate()?;
|
2022-03-25 23:47:50 +00:00
|
|
|
repo.from_db(db).await?;
|
2021-11-01 02:11:35 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
let manager = UploadManager::new(repo.clone(), CONFIG.format()).await?;
|
2021-11-01 02:11:35 +00:00
|
|
|
|
2022-03-25 23:47:50 +00:00
|
|
|
match CONFIG.command()? {
|
|
|
|
CommandConfig::Run => (),
|
2022-03-27 01:45:12 +00:00
|
|
|
CommandConfig::Dump { path } => {
|
|
|
|
let configuration = toml::to_string_pretty(&*CONFIG)?;
|
|
|
|
tokio::fs::write(path, configuration).await?;
|
|
|
|
return Ok(());
|
|
|
|
}
|
2022-03-25 23:47:50 +00:00
|
|
|
CommandConfig::MigrateRepo { to: _ } => {
|
|
|
|
unimplemented!("Repo migrations are currently unsupported")
|
|
|
|
}
|
|
|
|
CommandConfig::MigrateStore { to } => {
|
|
|
|
let from = CONFIG.store()?;
|
2021-11-01 02:11:35 +00:00
|
|
|
|
2022-03-25 23:47:50 +00:00
|
|
|
match from {
|
|
|
|
config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
|
2022-03-26 21:49:23 +00:00
|
|
|
let from = FileStore::build(path.clone(), repo.clone()).await?;
|
|
|
|
migrate_inner(&manager, &repo, from, &to).await?;
|
2022-03-25 23:47:50 +00:00
|
|
|
}
|
|
|
|
config::Storage::ObjectStorage(RequiredObjectStorage {
|
2021-11-01 02:11:35 +00:00
|
|
|
bucket_name,
|
2022-03-25 23:47:50 +00:00
|
|
|
region,
|
|
|
|
access_key,
|
|
|
|
secret_key,
|
|
|
|
security_token,
|
|
|
|
session_token,
|
|
|
|
}) => {
|
|
|
|
let from = ObjectStore::build(
|
|
|
|
&bucket_name,
|
|
|
|
region,
|
|
|
|
access_key,
|
|
|
|
secret_key,
|
|
|
|
security_token,
|
|
|
|
session_token,
|
2022-03-26 21:49:23 +00:00
|
|
|
repo.clone(),
|
2022-03-25 23:47:50 +00:00
|
|
|
build_reqwest_client()?,
|
2022-03-26 21:49:23 +00:00
|
|
|
)
|
|
|
|
.await?;
|
2022-03-25 23:47:50 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
migrate_inner(&manager, &repo, from, &to).await?;
|
2022-03-25 23:47:50 +00:00
|
|
|
}
|
2021-11-01 02:11:35 +00:00
|
|
|
}
|
|
|
|
|
2022-03-25 23:47:50 +00:00
|
|
|
return Ok(());
|
|
|
|
}
|
2021-11-01 02:11:35 +00:00
|
|
|
}
|
2021-10-23 04:48:56 +00:00
|
|
|
|
2022-03-25 23:47:50 +00:00
|
|
|
match CONFIG.store()? {
|
|
|
|
config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
|
2022-03-26 21:49:23 +00:00
|
|
|
let store = FileStore::build(path, repo).await?;
|
2021-11-01 02:11:35 +00:00
|
|
|
launch(manager, store).await
|
2021-10-28 04:06:03 +00:00
|
|
|
}
|
2022-03-25 23:47:50 +00:00
|
|
|
config::Storage::ObjectStorage(RequiredObjectStorage {
|
2021-10-28 04:06:03 +00:00
|
|
|
bucket_name,
|
|
|
|
region,
|
|
|
|
access_key,
|
|
|
|
secret_key,
|
|
|
|
security_token,
|
|
|
|
session_token,
|
2022-03-25 23:47:50 +00:00
|
|
|
}) => {
|
|
|
|
let store = ObjectStore::build(
|
|
|
|
&bucket_name,
|
|
|
|
region,
|
|
|
|
access_key,
|
|
|
|
secret_key,
|
|
|
|
security_token,
|
|
|
|
session_token,
|
2022-03-26 21:49:23 +00:00
|
|
|
repo,
|
2021-11-02 22:21:00 +00:00
|
|
|
build_reqwest_client()?,
|
2022-03-26 21:49:23 +00:00
|
|
|
)
|
|
|
|
.await?;
|
2021-10-28 04:06:03 +00:00
|
|
|
|
2021-11-01 02:11:35 +00:00
|
|
|
launch(manager, store).await
|
2021-10-28 04:06:03 +00:00
|
|
|
}
|
|
|
|
}
|
2021-10-23 04:48:56 +00:00
|
|
|
}
|