pict-rs/src/main.rs

909 lines
28 KiB
Rust
Raw Normal View History

use actix_form_data::{Field, Form, Value};
2021-01-14 16:26:17 +00:00
use actix_web::{
guard,
http::header::{CacheControl, CacheDirective, LastModified, ACCEPT_RANGES},
2021-04-17 19:59:42 +00:00
web, App, HttpResponse, HttpResponseBuilder, HttpServer,
2021-01-14 16:26:17 +00:00
};
2021-03-10 02:51:03 +00:00
use awc::Client;
2021-10-14 00:06:53 +00:00
use futures_util::{stream::once, Stream};
2021-09-06 17:08:13 +00:00
use once_cell::sync::Lazy;
use std::{
collections::HashSet,
2021-10-21 00:28:40 +00:00
future::ready,
path::PathBuf,
pin::Pin,
task::{Context, Poll},
time::SystemTime,
};
2020-06-07 00:54:06 +00:00
use structopt::StructOpt;
2021-10-21 00:28:40 +00:00
use tokio::{io::AsyncReadExt, sync::Semaphore};
use tracing::{debug, error, info, instrument, Span};
2021-09-14 01:22:42 +00:00
use tracing_actix_web::TracingLogger;
2021-09-25 22:09:55 +00:00
use tracing_awc::Propagate;
2021-09-16 22:51:20 +00:00
use tracing_futures::Instrument;
2021-10-19 01:29:06 +00:00
use uuid::Uuid;
2020-06-06 21:41:17 +00:00
2021-10-21 00:28:40 +00:00
mod concurrent_processor;
2020-06-07 00:54:06 +00:00
mod config;
2021-10-14 00:06:53 +00:00
mod either;
2020-06-06 21:41:17 +00:00
mod error;
2021-08-31 16:02:30 +00:00
mod exiftool;
mod ffmpeg;
2021-10-13 04:16:31 +00:00
mod file;
2021-10-21 00:28:40 +00:00
mod init_tracing;
mod magick;
mod middleware;
2020-07-11 20:40:40 +00:00
mod migrate;
2021-10-21 00:28:40 +00:00
mod process;
mod processor;
mod range;
2020-06-06 21:41:17 +00:00
mod upload_manager;
2020-06-11 16:46:00 +00:00
mod validate;
2020-06-06 21:41:17 +00:00
use self::{
2021-10-21 00:28:40 +00:00
concurrent_processor::CancelSafeProcessor,
config::{Config, Format},
2021-10-14 00:06:53 +00:00
either::Either,
2021-09-14 01:22:42 +00:00
error::{Error, UploadError},
2021-10-20 23:58:32 +00:00
file::CrateError,
2021-10-21 00:28:40 +00:00
init_tracing::init_tracing,
2021-09-14 01:22:42 +00:00
middleware::{Deadline, Internal},
2021-09-12 00:53:26 +00:00
upload_manager::{Details, UploadManager, UploadManagerSession},
2020-06-24 16:58:46 +00:00
validate::{image_webp, video_mp4},
};
const MEGABYTES: usize = 1024 * 1024;
const MINUTES: u32 = 60;
const HOURS: u32 = 60 * MINUTES;
const DAYS: u32 = 24 * HOURS;
static TMP_DIR: Lazy<PathBuf> = Lazy::new(|| {
2021-10-19 01:29:06 +00:00
let tmp_nonce = Uuid::new_v4();
let mut path = std::env::temp_dir();
path.push(format!("pict-rs-{}", tmp_nonce));
path
});
static CONFIG: Lazy<Config> = Lazy::new(Config::from_args);
2021-09-12 15:42:44 +00:00
static PROCESS_SEMAPHORE: Lazy<Semaphore> =
Lazy::new(|| Semaphore::new(num_cpus::get().saturating_sub(1).max(1)));
2020-06-24 16:58:46 +00:00
// try moving a file
2021-09-14 01:22:42 +00:00
#[instrument(name = "Moving file")]
async fn safe_move_file(from: PathBuf, to: PathBuf) -> Result<(), Error> {
2020-06-24 16:58:46 +00:00
if let Some(path) = to.parent() {
debug!("Creating directory {:?}", path);
2021-09-04 19:20:31 +00:00
tokio::fs::create_dir_all(path).await?;
2020-06-24 16:58:46 +00:00
}
debug!("Checking if {:?} already exists", to);
2021-09-04 19:20:31 +00:00
if let Err(e) = tokio::fs::metadata(&to).await {
if e.kind() != std::io::ErrorKind::NotFound {
2020-06-24 16:58:46 +00:00
return Err(e.into());
}
} else {
2021-09-14 01:22:42 +00:00
return Err(UploadError::FileExists.into());
2020-06-24 16:58:46 +00:00
}
debug!("Moving {:?} to {:?}", from, to);
2021-09-04 19:20:31 +00:00
tokio::fs::copy(&from, to).await?;
tokio::fs::remove_file(from).await?;
2020-06-24 16:58:46 +00:00
Ok(())
}
2021-09-14 01:22:42 +00:00
async fn safe_create_parent<P>(path: P) -> Result<(), Error>
where
P: AsRef<std::path::Path>,
{
if let Some(path) = path.as_ref().parent() {
debug!("Creating directory {:?}", path);
2021-09-04 19:20:31 +00:00
tokio::fs::create_dir_all(path).await?;
}
Ok(())
}
2020-06-06 21:41:17 +00:00
// Try writing to a file
2021-09-14 01:22:42 +00:00
#[instrument(name = "Saving file", skip(bytes))]
2021-10-14 00:06:53 +00:00
async fn safe_save_file(path: PathBuf, bytes: web::Bytes) -> Result<(), Error> {
2020-06-06 21:41:17 +00:00
if let Some(path) = path.parent() {
// create the directory for the file
2020-06-14 15:07:31 +00:00
debug!("Creating directory {:?}", path);
2021-09-04 19:20:31 +00:00
tokio::fs::create_dir_all(path).await?;
}
2020-06-06 21:41:17 +00:00
// Only write the file if it doesn't already exist
2020-06-14 15:07:31 +00:00
debug!("Checking if {:?} already exists", path);
2021-09-04 19:20:31 +00:00
if let Err(e) = tokio::fs::metadata(&path).await {
if e.kind() != std::io::ErrorKind::NotFound {
return Err(e.into());
}
2020-06-06 21:41:17 +00:00
} else {
return Ok(());
}
2020-06-06 21:41:17 +00:00
// Open the file for writing
2020-06-14 15:07:31 +00:00
debug!("Creating {:?}", path);
2021-10-13 04:16:31 +00:00
let mut file = crate::file::File::create(&path).await?;
2020-06-06 21:41:17 +00:00
// try writing
2020-06-14 15:07:31 +00:00
debug!("Writing to {:?}", path);
2021-10-14 00:06:53 +00:00
if let Err(e) = file.write_from_bytes(bytes).await {
2020-06-14 15:07:31 +00:00
error!("Error writing {:?}, {}", path, e);
2020-06-06 21:41:17 +00:00
// remove file if writing failed before completion
2021-09-04 19:20:31 +00:00
tokio::fs::remove_file(path).await?;
2020-06-06 21:41:17 +00:00
return Err(e.into());
}
2020-06-14 15:07:31 +00:00
debug!("{:?} written", path);
2020-06-06 21:41:17 +00:00
Ok(())
}
2020-06-24 16:58:46 +00:00
pub(crate) fn tmp_file() -> PathBuf {
2021-10-19 01:29:06 +00:00
let s: String = Uuid::new_v4().to_string();
2020-06-24 16:58:46 +00:00
let name = format!("{}.tmp", s);
let mut path = TMP_DIR.clone();
2020-06-24 16:58:46 +00:00
path.push(&name);
path
}
2021-09-14 01:22:42 +00:00
fn to_ext(mime: mime::Mime) -> Result<&'static str, Error> {
if mime == mime::IMAGE_PNG {
2020-06-24 16:58:46 +00:00
Ok(".png")
} else if mime == mime::IMAGE_JPEG {
2020-06-24 16:58:46 +00:00
Ok(".jpg")
} else if mime == video_mp4() {
Ok(".mp4")
} else if mime == image_webp() {
Ok(".webp")
} else {
2021-09-14 01:22:42 +00:00
Err(UploadError::UnsupportedFormat.into())
2020-06-24 16:58:46 +00:00
}
}
2020-06-06 21:41:17 +00:00
/// Handle responding to succesful uploads
2021-09-14 01:22:42 +00:00
#[instrument(name = "Uploaded files", skip(value, manager))]
2020-06-07 00:29:15 +00:00
async fn upload(
2021-09-12 00:53:26 +00:00
value: Value<UploadManagerSession>,
2020-06-07 00:29:15 +00:00
manager: web::Data<UploadManager>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
let images = value
.map()
.and_then(|mut m| m.remove("images"))
.and_then(|images| images.array())
.ok_or(UploadError::NoFiles)?;
let mut files = Vec::new();
2021-09-12 00:53:26 +00:00
let images = images
.into_iter()
.filter_map(|i| i.file())
.collect::<Vec<_>>();
for image in &images {
if let Some(alias) = image.result.alias() {
info!("Uploaded {} as {:?}", image.filename, alias);
2021-09-12 00:53:26 +00:00
let delete_token = image.result.delete_token().await?;
let name = manager.from_alias(alias.to_owned()).await?;
2021-10-19 04:37:11 +00:00
let path = manager.path_from_filename(name.clone()).await?;
let details = manager.variant_details(path.clone(), name.clone()).await?;
let details = if let Some(details) = details {
2021-08-31 17:14:23 +00:00
debug!("details exist");
details
} else {
2021-08-31 17:14:23 +00:00
debug!("generating new details from {:?}", path);
let new_details = Details::from_path(path.clone()).await?;
2021-08-31 17:14:23 +00:00
debug!("storing details for {:?} {}", path, name);
manager
.store_variant_details(path, name, &new_details)
.await?;
2021-08-31 17:14:23 +00:00
debug!("stored");
new_details
};
2020-06-07 00:29:15 +00:00
files.push(serde_json::json!({
"file": alias,
"delete_token": delete_token,
"details": details,
2020-06-07 00:29:15 +00:00
}));
}
}
2021-09-12 00:53:26 +00:00
for image in images {
image.result.succeed();
}
2021-02-10 22:57:42 +00:00
Ok(HttpResponse::Created().json(&serde_json::json!({
"msg": "ok",
"files": files
})))
}
2020-06-24 16:58:46 +00:00
#[derive(Debug, serde::Deserialize)]
struct UrlQuery {
url: String,
}
2021-10-20 23:58:32 +00:00
pin_project_lite::pin_project! {
struct Limit<S> {
#[pin]
inner: S,
count: u64,
limit: u64,
}
}
impl<S> Limit<S> {
fn new(inner: S, limit: u64) -> Self {
Limit {
inner,
count: 0,
limit,
}
}
}
#[derive(Debug, thiserror::Error)]
#[error("Resonse body larger than size limit")]
struct LimitError;
impl<S, E> Stream for Limit<S>
where
S: Stream<Item = Result<web::Bytes, E>>,
E: From<LimitError>,
{
type Item = Result<web::Bytes, E>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.as_mut().project();
let limit = this.limit;
let count = this.count;
let inner = this.inner;
inner.poll_next(cx).map(|opt| {
opt.map(|res| match res {
Ok(bytes) => {
*count += bytes.len() as u64;
if *count > *limit {
return Err(LimitError.into());
}
Ok(bytes)
}
Err(e) => Err(e),
})
})
}
}
/// download an image from a URL
2021-09-14 01:22:42 +00:00
#[instrument(name = "Downloading file", skip(client, manager))]
async fn download(
client: web::Data<Client>,
manager: web::Data<UploadManager>,
query: web::Query<UrlQuery>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
2021-10-20 23:58:32 +00:00
let res = client.get(&query.url).propagate().send().await?;
if !res.status().is_success() {
2021-09-14 01:22:42 +00:00
return Err(UploadError::Download(res.status()).into());
}
2021-10-20 23:58:32 +00:00
let mut stream = Limit::new(
CrateError::new(res),
(CONFIG.max_file_size() * MEGABYTES) as u64,
);
2021-10-20 23:58:32 +00:00
// SAFETY: stream is shadowed, so original cannot not be moved
let stream = unsafe { Pin::new_unchecked(&mut stream) };
2021-09-06 17:08:13 +00:00
let permit = PROCESS_SEMAPHORE.acquire().await?;
2021-09-12 00:53:26 +00:00
let session = manager.session().upload(stream).await?;
let alias = session.alias().unwrap().to_owned();
drop(permit);
2021-09-12 00:53:26 +00:00
let delete_token = session.delete_token().await?;
let name = manager.from_alias(alias.to_owned()).await?;
2021-10-19 04:37:11 +00:00
let path = manager.path_from_filename(name.clone()).await?;
let details = manager.variant_details(path.clone(), name.clone()).await?;
let details = if let Some(details) = details {
details
} else {
let new_details = Details::from_path(path.clone()).await?;
manager
.store_variant_details(path, name, &new_details)
.await?;
new_details
};
2021-09-12 00:53:26 +00:00
session.succeed();
2021-02-10 22:57:42 +00:00
Ok(HttpResponse::Created().json(&serde_json::json!({
"msg": "ok",
"files": [{
"file": alias,
"delete_token": delete_token,
"details": details,
}]
})))
}
2020-06-15 02:41:45 +00:00
/// Delete aliases and files
2021-09-14 01:22:42 +00:00
#[instrument(name = "Deleting file", skip(manager))]
2020-06-07 00:29:15 +00:00
async fn delete(
manager: web::Data<UploadManager>,
path_entries: web::Path<(String, String)>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
2020-06-07 00:29:15 +00:00
let (alias, token) = path_entries.into_inner();
manager.delete(token, alias).await?;
Ok(HttpResponse::NoContent().finish())
}
2020-06-24 16:58:46 +00:00
type ProcessQuery = Vec<(String, String)>;
async fn prepare_process(
2020-06-24 16:58:46 +00:00
query: web::Query<ProcessQuery>,
ext: &str,
manager: &UploadManager,
2021-10-19 04:51:04 +00:00
filters: &Option<HashSet<String>>,
2021-09-14 01:22:42 +00:00
) -> Result<(Format, String, PathBuf, Vec<String>), Error> {
2020-06-24 16:58:46 +00:00
let (alias, operations) =
query
.into_inner()
.into_iter()
.fold((String::new(), Vec::new()), |(s, mut acc), (k, v)| {
if k == "src" {
(v, acc)
} else {
acc.push((k, v));
(s, acc)
}
});
2020-06-06 21:41:17 +00:00
2021-09-12 15:42:44 +00:00
if alias.is_empty() {
2021-09-14 01:22:42 +00:00
return Err(UploadError::MissingFilename.into());
2020-06-24 16:58:46 +00:00
}
2020-06-06 22:43:33 +00:00
let name = manager.from_alias(alias).await?;
2020-06-06 21:41:17 +00:00
2021-10-19 04:51:04 +00:00
let operations = if let Some(filters) = filters.as_ref() {
2020-06-24 16:58:46 +00:00
operations
.into_iter()
2021-10-19 04:51:04 +00:00
.filter(|(k, _)| filters.contains(&k.to_lowercase()))
2020-06-24 16:58:46 +00:00
.collect()
} else {
operations
};
let chain = self::processor::build_chain(&operations);
let format = ext
.parse::<Format>()
.map_err(|_| UploadError::UnsupportedFormat)?;
2020-06-24 16:58:46 +00:00
let processed_name = format!("{}.{}", name, ext);
2021-10-19 04:37:11 +00:00
let thumbnail_path = self::processor::build_path(&chain, processed_name);
let thumbnail_args = self::processor::build_args(&chain);
Ok((format, name, thumbnail_path, thumbnail_args))
}
2021-10-19 04:51:04 +00:00
#[instrument(name = "Fetching derived details", skip(manager, filters))]
async fn process_details(
query: web::Query<ProcessQuery>,
ext: web::Path<String>,
manager: web::Data<UploadManager>,
2021-10-19 04:51:04 +00:00
filters: web::Data<Option<HashSet<String>>>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
let (_, name, thumbnail_path, _) =
2021-10-19 04:51:04 +00:00
prepare_process(query, ext.as_str(), &manager, &filters).await?;
2021-10-19 04:37:11 +00:00
let real_path = manager
.variant_path(&thumbnail_path, &name)
.await?
.ok_or(UploadError::MissingAlias)?;
let details = manager.variant_details(real_path, name).await?;
let details = details.ok_or(UploadError::NoFiles)?;
2021-02-10 22:57:42 +00:00
Ok(HttpResponse::Ok().json(&details))
}
/// Process files
2021-10-19 04:51:04 +00:00
#[instrument(name = "Serving processed image", skip(manager, filters))]
async fn process(
range: Option<range::RangeHeader>,
query: web::Query<ProcessQuery>,
ext: web::Path<String>,
manager: web::Data<UploadManager>,
2021-10-19 04:51:04 +00:00
filters: web::Data<Option<HashSet<String>>>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
let (format, name, thumbnail_path, thumbnail_args) =
2021-10-19 04:51:04 +00:00
prepare_process(query, ext.as_str(), &manager, &filters).await?;
2021-10-19 04:37:11 +00:00
let real_path_opt = manager.variant_path(&thumbnail_path, &name).await?;
2020-06-06 21:41:17 +00:00
// If the thumbnail doesn't exist, we need to create it
2021-10-21 00:28:40 +00:00
let real_path_opt = if let Some(real_path) = real_path_opt {
if let Err(e) = tokio::fs::metadata(&real_path)
2021-10-19 04:37:11 +00:00
.instrument(tracing::info_span!("Get thumbnail metadata"))
.await
{
if e.kind() != std::io::ErrorKind::NotFound {
error!("Error looking up processed image, {}", e);
return Err(e.into());
}
2021-10-21 00:28:40 +00:00
None
2021-10-19 04:37:11 +00:00
} else {
2021-10-21 00:28:40 +00:00
Some(real_path)
}
} else {
2021-10-21 00:28:40 +00:00
None
};
2021-10-21 00:28:40 +00:00
if let Some(real_path) = real_path_opt {
let details_opt = manager
.variant_details(real_path.clone(), name.clone())
.await?;
2021-10-19 04:37:11 +00:00
2021-10-21 00:28:40 +00:00
let details = if let Some(details) = details_opt {
details
} else {
let details = Details::from_path(real_path.clone()).await?;
manager
.store_variant_details(real_path.clone(), name, &details)
.await?;
details
};
2021-10-19 04:37:11 +00:00
2021-10-21 00:28:40 +00:00
return ranged_file_resp(real_path, range, details).await;
2021-10-19 04:37:11 +00:00
}
let mut original_path = manager.path_from_filename(name.clone()).await?;
let thumbnail_path2 = thumbnail_path.clone();
let process_fut = async {
let thumbnail_path = thumbnail_path2;
// Create and save a JPG for motion images (gif, mp4)
if let Some((updated_path, exists)) =
self::processor::prepare_image(original_path.clone()).await?
{
original_path = updated_path.clone();
if exists.is_new() {
// Save the transcoded file in another task
debug!("Spawning storage task");
let manager2 = manager.clone();
let name = name.clone();
let span = tracing::info_span!(
parent: None,
"Storing variant info",
path = &tracing::field::debug(&updated_path),
name = &tracing::field::display(&name),
);
span.follows_from(Span::current());
actix_rt::spawn(
async move {
if let Err(e) = manager2.store_variant(None, &updated_path, &name).await {
error!("Error storing variant, {}", e);
return;
}
2021-10-19 04:37:11 +00:00
}
.instrument(span),
);
2020-06-16 23:13:22 +00:00
}
2021-10-19 04:37:11 +00:00
}
2020-06-24 16:58:46 +00:00
2021-10-19 04:37:11 +00:00
let permit = PROCESS_SEMAPHORE.acquire().await?;
2021-10-19 04:37:11 +00:00
let file = crate::file::File::open(original_path.clone()).await?;
2021-10-19 04:37:11 +00:00
let mut processed_reader =
crate::magick::process_image_file_read(file, thumbnail_args, format)?;
2021-10-19 04:37:11 +00:00
let mut vec = Vec::new();
processed_reader.read_to_end(&mut vec).await?;
let bytes = web::Bytes::from(vec);
2021-10-19 04:37:11 +00:00
drop(permit);
2021-10-19 04:37:11 +00:00
let details = Details::from_bytes(bytes.clone()).await?;
2021-10-19 04:37:11 +00:00
let save_span = tracing::info_span!(
parent: None,
"Saving variant information",
path = tracing::field::debug(&thumbnail_path),
name = tracing::field::display(&name),
);
save_span.follows_from(Span::current());
let details2 = details.clone();
let bytes2 = bytes.clone();
actix_rt::spawn(
async move {
let real_path = match manager.next_directory() {
Ok(real_path) => real_path.join(&name),
Err(e) => {
tracing::warn!("Failed to generate directory path: {}", e);
2021-09-16 22:51:20 +00:00
return;
}
2021-10-19 04:37:11 +00:00
};
2021-10-19 04:37:11 +00:00
if let Err(e) = safe_save_file(real_path.clone(), bytes2).await {
tracing::warn!("Error saving thumbnail: {}", e);
return;
}
2021-10-19 04:37:11 +00:00
if let Err(e) = manager
.store_variant_details(real_path.clone(), name.clone(), &details2)
.await
{
tracing::warn!("Error saving variant details: {}", e);
return;
}
if let Err(e) = manager
.store_variant(Some(&thumbnail_path), &real_path, &name)
.await
{
tracing::warn!("Error saving variant info: {}", e);
}
}
2021-10-19 04:37:11 +00:00
.instrument(save_span),
);
2021-10-19 04:37:11 +00:00
Ok((details, bytes)) as Result<(Details, web::Bytes), Error>
};
2021-10-19 04:37:11 +00:00
let (details, bytes) = CancelSafeProcessor::new(thumbnail_path.clone(), process_fut).await?;
match range {
Some(range_header) => {
if !range_header.is_bytes() {
return Err(UploadError::Range.into());
}
if range_header.is_empty() {
Err(UploadError::Range.into())
} else if range_header.len() == 1 {
let range = range_header.ranges().next().unwrap();
let content_range = range.to_content_range(bytes.len() as u64);
let stream = range.chop_bytes(bytes);
let mut builder = HttpResponse::PartialContent();
builder.insert_header(content_range);
Ok(srv_response(
builder,
stream,
details.content_type(),
7 * DAYS,
details.system_time(),
))
} else {
Err(UploadError::Range.into())
}
}
None => Ok(srv_response(
HttpResponse::Ok(),
once(ready(Ok(bytes) as Result<_, Error>)),
details.content_type(),
7 * DAYS,
details.system_time(),
)),
}
2020-06-24 16:58:46 +00:00
}
/// Fetch file details
2021-09-14 01:22:42 +00:00
#[instrument(name = "Fetching details", skip(manager))]
async fn details(
alias: web::Path<String>,
manager: web::Data<UploadManager>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
let name = manager.from_alias(alias.into_inner()).await?;
2021-10-19 04:37:11 +00:00
let path = manager.path_from_filename(name.clone()).await?;
let details = manager.variant_details(path.clone(), name.clone()).await?;
let details = if let Some(details) = details {
details
} else {
let new_details = Details::from_path(path.clone()).await?;
manager
.store_variant_details(path.clone(), name, &new_details)
.await?;
new_details
};
2021-02-10 22:57:42 +00:00
Ok(HttpResponse::Ok().json(&details))
}
2020-06-24 16:58:46 +00:00
/// Serve files
2021-09-14 01:22:42 +00:00
#[instrument(name = "Serving file", skip(manager))]
2020-06-24 16:58:46 +00:00
async fn serve(
range: Option<range::RangeHeader>,
2020-06-24 16:58:46 +00:00
alias: web::Path<String>,
manager: web::Data<UploadManager>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
2020-06-24 16:58:46 +00:00
let name = manager.from_alias(alias.into_inner()).await?;
2021-10-19 04:37:11 +00:00
let path = manager.path_from_filename(name.clone()).await?;
let details = manager.variant_details(path.clone(), name.clone()).await?;
let details = if let Some(details) = details {
details
} else {
let details = Details::from_path(path.clone()).await?;
manager
.store_variant_details(path.clone(), name, &details)
.await?;
details
};
2020-06-24 16:58:46 +00:00
ranged_file_resp(path, range, details).await
}
2021-01-14 01:08:04 +00:00
2021-01-14 16:26:17 +00:00
async fn ranged_file_resp(
path: PathBuf,
range: Option<range::RangeHeader>,
2021-01-14 16:26:17 +00:00
details: Details,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
let (builder, stream) = match range {
//Range header exists - return as ranged
Some(range_header) => {
if !range_header.is_bytes() {
2021-09-14 01:22:42 +00:00
return Err(UploadError::Range.into());
}
2021-01-14 01:08:04 +00:00
if range_header.is_empty() {
2021-09-14 01:22:42 +00:00
return Err(UploadError::Range.into());
} else if range_header.len() == 1 {
2021-10-13 04:16:31 +00:00
let file = crate::file::File::open(path).await?;
2021-09-04 19:20:31 +00:00
let meta = file.metadata().await?;
let range = range_header.ranges().next().unwrap();
let mut builder = HttpResponse::PartialContent();
2021-02-10 22:57:42 +00:00
builder.insert_header(range.to_content_range(meta.len()));
2021-01-14 01:08:04 +00:00
(builder, Either::left(range.chop_file(file).await?))
} else {
2021-09-14 01:22:42 +00:00
return Err(UploadError::Range.into());
}
2021-01-14 01:08:04 +00:00
}
//No Range header in the request - return the entire document
None => {
2021-10-13 04:16:31 +00:00
let file = crate::file::File::open(path).await?;
2021-10-14 00:06:53 +00:00
let stream = file.read_to_stream(None, None).await?;
(HttpResponse::Ok(), Either::right(stream))
2021-01-14 01:08:04 +00:00
}
};
Ok(srv_response(
builder,
2021-10-20 23:58:32 +00:00
stream,
details.content_type(),
7 * DAYS,
details.system_time(),
))
}
2020-06-06 21:41:17 +00:00
// A helper method to produce responses with proper cache headers
fn srv_response<S, E>(
mut builder: HttpResponseBuilder,
stream: S,
ext: mime::Mime,
expires: u32,
modified: SystemTime,
) -> HttpResponse
where
2021-10-20 23:58:32 +00:00
S: Stream<Item = Result<web::Bytes, E>> + 'static,
2021-06-19 19:39:41 +00:00
E: std::error::Error + 'static,
actix_web::Error: From<E>,
2021-01-14 01:08:04 +00:00
{
builder
2021-02-10 22:57:42 +00:00
.insert_header(LastModified(modified.into()))
.insert_header(CacheControl(vec![
2021-01-14 01:08:04 +00:00
CacheDirective::Public,
CacheDirective::MaxAge(expires),
CacheDirective::Extension("immutable".to_owned(), None),
]))
2021-02-10 22:57:42 +00:00
.insert_header((ACCEPT_RANGES, "bytes"))
.content_type(ext.to_string())
2021-10-20 23:58:32 +00:00
// TODO: remove pin when actix-web drops Unpin requirement
.streaming(Box::pin(stream))
}
#[derive(Debug, serde::Deserialize)]
#[serde(untagged)]
enum FileOrAlias {
File { file: String },
Alias { alias: String },
}
2021-09-14 01:22:42 +00:00
#[instrument(name = "Purging file", skip(upload_manager))]
async fn purge(
query: web::Query<FileOrAlias>,
upload_manager: web::Data<UploadManager>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
let aliases = match query.into_inner() {
FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
};
for alias in aliases.iter() {
upload_manager
.delete_without_token(alias.to_owned())
.await?;
}
2021-02-10 22:57:42 +00:00
Ok(HttpResponse::Ok().json(&serde_json::json!({
"msg": "ok",
"aliases": aliases
})))
}
2021-09-14 01:22:42 +00:00
#[instrument(name = "Fetching aliases", skip(upload_manager))]
async fn aliases(
query: web::Query<FileOrAlias>,
upload_manager: web::Data<UploadManager>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
let aliases = match query.into_inner() {
FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
};
2021-02-10 22:57:42 +00:00
Ok(HttpResponse::Ok().json(&serde_json::json!({
"msg": "ok",
"aliases": aliases,
})))
}
#[derive(Debug, serde::Deserialize)]
struct ByAlias {
alias: String,
}
2021-09-14 01:22:42 +00:00
#[instrument(name = "Fetching filename", skip(upload_manager))]
async fn filename_by_alias(
query: web::Query<ByAlias>,
upload_manager: web::Data<UploadManager>,
2021-09-14 01:22:42 +00:00
) -> Result<HttpResponse, Error> {
let filename = upload_manager.from_alias(query.into_inner().alias).await?;
2021-02-10 22:57:42 +00:00
Ok(HttpResponse::Ok().json(&serde_json::json!({
"msg": "ok",
"filename": filename,
})))
}
#[actix_rt::main]
2021-10-21 00:28:40 +00:00
async fn main() -> anyhow::Result<()> {
2020-06-14 15:07:31 +00:00
let manager = UploadManager::new(CONFIG.data_dir(), CONFIG.format()).await?;
2021-10-21 00:28:40 +00:00
init_tracing("pict-rs", CONFIG.opentelemetry_url())?;
2020-06-06 21:41:17 +00:00
// Create a new Multipart Form validator
//
// This form is expecting a single array field, 'images' with at most 10 files in it
let manager2 = manager.clone();
let form = Form::new()
.max_files(10)
2020-06-14 15:07:31 +00:00
.max_file_size(CONFIG.max_file_size() * MEGABYTES)
2021-09-14 01:22:42 +00:00
.transform_error(|e| Error::from(e).into())
.field(
"images",
2020-06-14 18:56:42 +00:00
Field::array(Field::file(move |filename, _, stream| {
let manager = manager2.clone();
2021-09-16 22:51:20 +00:00
let span = tracing::info_span!("file-upload", ?filename);
2020-06-14 18:56:42 +00:00
2021-09-16 22:51:20 +00:00
async move {
2021-09-06 17:08:13 +00:00
let permit = PROCESS_SEMAPHORE.acquire().await?;
2021-09-12 00:53:26 +00:00
let res = manager.session().upload(stream).await;
drop(permit);
2020-06-14 18:56:42 +00:00
res
}
2021-09-16 22:51:20 +00:00
.instrument(span)
})),
);
// Create a new Multipart Form validator for internal imports
//
// This form is expecting a single array field, 'images' with at most 10 files in it
2020-06-14 15:07:31 +00:00
let validate_imports = CONFIG.validate_imports();
let manager2 = manager.clone();
let import_form = Form::new()
.max_files(10)
2020-06-14 15:07:31 +00:00
.max_file_size(CONFIG.max_file_size() * MEGABYTES)
2021-09-14 01:22:42 +00:00
.transform_error(|e| Error::from(e).into())
.field(
"images",
Field::array(Field::file(move |filename, content_type, stream| {
let manager = manager2.clone();
2021-09-16 22:51:20 +00:00
let span = tracing::info_span!("file-import", ?filename);
2020-06-14 18:56:42 +00:00
2021-09-16 22:51:20 +00:00
async move {
2021-09-06 17:08:13 +00:00
let permit = PROCESS_SEMAPHORE.acquire().await?;
2020-06-14 18:56:42 +00:00
let res = manager
2021-09-12 00:53:26 +00:00
.session()
2020-06-14 15:07:31 +00:00
.import(filename, content_type, validate_imports, stream)
2021-09-12 00:53:26 +00:00
.await;
drop(permit);
2020-06-14 18:56:42 +00:00
res
}
2021-09-16 22:51:20 +00:00
.instrument(span)
})),
);
HttpServer::new(move || {
2020-09-14 15:52:18 +00:00
let client = Client::builder()
2021-09-18 21:34:23 +00:00
.header("User-Agent", "pict-rs v0.3.0-main")
.finish();
App::new()
2021-09-18 21:34:23 +00:00
.wrap(TracingLogger::default())
2021-09-11 20:31:00 +00:00
.wrap(Deadline)
2021-06-26 18:10:28 +00:00
.app_data(web::Data::new(manager.clone()))
.app_data(web::Data::new(client))
2021-10-19 04:51:04 +00:00
.app_data(web::Data::new(CONFIG.allowed_filters()))
.service(
web::scope("/image")
.service(
web::resource("")
.guard(guard::Post())
.wrap(form.clone())
.route(web::post().to(upload)),
)
.service(web::resource("/download").route(web::get().to(download)))
2020-06-07 00:29:15 +00:00
.service(
web::resource("/delete/{delete_token}/{filename}")
2020-06-09 22:37:48 +00:00
.route(web::delete().to(delete))
.route(web::get().to(delete)),
2020-06-07 00:29:15 +00:00
)
2020-06-24 16:58:46 +00:00
.service(web::resource("/original/{filename}").route(web::get().to(serve)))
.service(web::resource("/process.{ext}").route(web::get().to(process)))
.service(
web::scope("/details")
.service(
web::resource("/original/{filename}").route(web::get().to(details)),
)
.service(
web::resource("/process.{ext}")
.route(web::get().to(process_details)),
),
),
)
.service(
2020-07-11 21:28:49 +00:00
web::scope("/internal")
.wrap(Internal(CONFIG.api_key().map(|s| s.to_owned())))
.service(
web::resource("/import")
.wrap(import_form.clone())
.route(web::post().to(upload)),
)
.service(web::resource("/purge").route(web::post().to(purge)))
.service(web::resource("/aliases").route(web::get().to(aliases)))
.service(web::resource("/filename").route(web::get().to(filename_by_alias))),
)
})
2020-06-14 15:07:31 +00:00
.bind(CONFIG.bind_address())?
.run()
.await?;
2021-09-04 19:20:31 +00:00
if tokio::fs::metadata(&*TMP_DIR).await.is_ok() {
tokio::fs::remove_dir_all(&*TMP_DIR).await?;
}
Ok(())
}