From f67aeb92fa4fc8c30d078e675db1b1a1d2eaf20b Mon Sep 17 00:00:00 2001 From: "Aode (Lion)" Date: Wed, 20 Oct 2021 21:36:18 -0500 Subject: [PATCH] Keep better track of gif/mp4 thumbnail --- src/error.rs | 9 ++-- src/main.rs | 33 +----------- src/processor.rs | 104 +++++++++----------------------------- src/upload_manager/mod.rs | 104 +++++++++++++++++++++++++++++++++++++- 4 files changed, 132 insertions(+), 118 deletions(-) diff --git a/src/error.rs b/src/error.rs index cec2cc2..2d6c4df 100644 --- a/src/error.rs +++ b/src/error.rs @@ -6,12 +6,6 @@ pub(crate) struct Error { kind: UploadError, } -impl Error { - pub(crate) fn kind(&self) -> &UploadError { - &self.kind - } -} - impl std::fmt::Debug for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}\n", self.kind) @@ -75,6 +69,9 @@ pub(crate) enum UploadError { #[error(transparent)] StripPrefix(#[from] std::path::StripPrefixError), + #[error("Provided process path is invalid")] + ParsePath, + #[error("Failed to acquire the semaphore")] Semaphore, diff --git a/src/main.rs b/src/main.rs index e574e46..3ad8655 100644 --- a/src/main.rs +++ b/src/main.rs @@ -388,7 +388,7 @@ async fn prepare_process( let processed_name = format!("{}.{}", name, ext); let (thumbnail_path, thumbnail_args) = - self::processor::build_chain(&operations, processed_name); + self::processor::build_chain(&operations, processed_name)?; Ok((format, name, thumbnail_path, thumbnail_args)) } @@ -465,40 +465,11 @@ async fn process( return ranged_file_resp(real_path, range, details).await; } - let mut original_path = manager.path_from_filename(name.clone()).await?; + let original_path = manager.still_path_from_filename(name.clone()).await?; let thumbnail_path2 = thumbnail_path.clone(); let process_fut = async { let thumbnail_path = thumbnail_path2; - // Create and save a JPG for motion images (gif, mp4) - if let Some((updated_path, exists)) = - self::processor::prepare_image(original_path.clone()).await? - { - original_path = updated_path.clone(); - - if exists.is_new() { - // Save the transcoded file in another task - debug!("Spawning storage task"); - let manager2 = manager.clone(); - let name = name.clone(); - let span = tracing::info_span!( - parent: None, - "Storing variant info", - path = &tracing::field::debug(&updated_path), - name = &tracing::field::display(&name), - ); - span.follows_from(Span::current()); - actix_rt::spawn( - async move { - if let Err(e) = manager2.store_variant(None, &updated_path, &name).await { - error!("Error storing variant, {}", e); - return; - } - } - .instrument(span), - ); - } - } let permit = PROCESS_SEMAPHORE.acquire().await?; diff --git a/src/processor.rs b/src/processor.rs index d72dcea..bac220e 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -1,13 +1,6 @@ -use crate::{ - error::{Error, UploadError}, - ffmpeg::ThumbnailFormat, -}; -use std::path::{Path, PathBuf}; -use tracing::{debug, error, instrument}; - -fn ptos(path: &Path) -> Result { - Ok(path.to_str().ok_or(UploadError::Path)?.to_owned()) -} +use crate::error::{Error, UploadError}; +use std::path::PathBuf; +use tracing::instrument; pub(crate) trait Processor { const NAME: &'static str; @@ -164,88 +157,41 @@ impl Processor for Blur { } #[instrument] -pub(crate) fn build_chain(args: &[(String, String)], filename: String) -> (PathBuf, Vec) { - fn parse(key: &str, value: &str) -> Option

{ +pub(crate) fn build_chain( + args: &[(String, String)], + filename: String, +) -> Result<(PathBuf, Vec), Error> { + fn parse(key: &str, value: &str) -> Result, UploadError> { if key == P::NAME { - return P::parse(key, value); + return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?)); } - None + Ok(None) } macro_rules! parse { ($inner:expr, $x:ident, $k:expr, $v:expr) => {{ - if let Some(processor) = parse::<$x>($k, $v) { - return (processor.path($inner.0), processor.command($inner.1)); + if let Some(processor) = parse::<$x>($k, $v)? { + return Ok((processor.path($inner.0), processor.command($inner.1))); }; }}; } let (path, args) = args.into_iter() - .fold((PathBuf::default(), vec![]), |inner, (name, value)| { - parse!(inner, Identity, name, value); - parse!(inner, Thumbnail, name, value); - parse!(inner, Resize, name, value); - parse!(inner, Crop, name, value); - parse!(inner, Blur, name, value); + .fold(Ok((PathBuf::default(), vec![])), |inner, (name, value)| { + if let Ok(inner) = inner { + parse!(inner, Identity, name, value); + parse!(inner, Thumbnail, name, value); + parse!(inner, Resize, name, value); + parse!(inner, Crop, name, value); + parse!(inner, Blur, name, value); - debug!("Skipping {}: {}, invalid", name, value); + Err(Error::from(UploadError::ParsePath)) + } else { + inner + } + })?; - inner - }); - - (path.join(filename), args) -} - -fn is_motion(s: &str) -> bool { - s.ends_with(".gif") || s.ends_with(".mp4") -} - -pub(crate) enum Exists { - Exists, - New, -} - -impl Exists { - pub(crate) fn is_new(&self) -> bool { - matches!(self, Exists::New) - } -} - -pub(crate) async fn prepare_image( - original_file: PathBuf, -) -> Result, Error> { - let original_path_str = ptos(&original_file)?; - let jpg_path = format!("{}.jpg", original_path_str); - let jpg_path = PathBuf::from(jpg_path); - - if tokio::fs::metadata(&jpg_path).await.is_ok() { - return Ok(Some((jpg_path, Exists::Exists))); - } - - if is_motion(&original_path_str) { - let orig_path = original_path_str.clone(); - - let tmpfile = crate::tmp_file(); - crate::safe_create_parent(&tmpfile).await?; - - let res = crate::ffmpeg::thumbnail(orig_path, &tmpfile, ThumbnailFormat::Jpeg).await; - - if let Err(e) = res { - error!("transcode error: {:?}", e); - tokio::fs::remove_file(&tmpfile).await?; - return Err(e); - } - - return match crate::safe_move_file(tmpfile, jpg_path.clone()).await { - Err(e) if matches!(e.kind(), UploadError::FileExists) => { - Ok(Some((jpg_path, Exists::Exists))) - } - Err(e) => Err(e), - _ => Ok(Some((jpg_path, Exists::New))), - }; - } - - Ok(None) + Ok((path.join(filename), args)) } diff --git a/src/upload_manager/mod.rs b/src/upload_manager/mod.rs index e1d34a8..b8bac4a 100644 --- a/src/upload_manager/mod.rs +++ b/src/upload_manager/mod.rs @@ -1,11 +1,16 @@ use crate::{ config::Format, error::{Error, UploadError}, + ffmpeg::ThumbnailFormat, migrate::{alias_id_key, alias_key, alias_key_bounds, LatestDb}, }; use actix_web::web; use sha2::Digest; -use std::{path::PathBuf, sync::Arc}; +use std::{ + ops::{Deref, DerefMut}, + path::PathBuf, + sync::Arc, +}; use storage_path_generator::{Generator, Path}; use tracing::{debug, error, info, instrument, warn, Span}; use tracing_futures::Instrument; @@ -34,6 +39,7 @@ pub(super) use session::UploadManagerSession; // - Path Tree // - filename -> relative path // - filename / relative variant path -> relative variant path +// - filename / motion -> relative motion path // - Settings Tree // - last-path -> last generated path // - fs-restructure-01-complete -> bool @@ -111,6 +117,76 @@ impl UploadManager { Ok(manager) } + pub(crate) async fn still_path_from_filename( + &self, + filename: String, + ) -> Result { + let path = self.path_from_filename(filename.clone()).await?; + let details = + if let Some(details) = self.variant_details(path.clone(), filename.clone()).await? { + details + } else { + Details::from_path(&path).await? + }; + + if !details.is_motion() { + return Ok(path); + } + + if let Some(motion_path) = self.motion_path(&filename).await? { + return Ok(motion_path); + } + + let jpeg_path = self.next_directory()?.join(&filename); + crate::safe_create_parent(&jpeg_path).await?; + + let permit = crate::PROCESS_SEMAPHORE.acquire().await; + let res = crate::ffmpeg::thumbnail(&path, &jpeg_path, ThumbnailFormat::Jpeg).await; + drop(permit); + + if let Err(e) = res { + error!("transcode error: {:?}", e); + self.remove_path(&jpeg_path).await?; + return Err(e); + } + + self.store_motion_path(&filename, &jpeg_path).await?; + Ok(jpeg_path) + } + + async fn motion_path(&self, filename: &str) -> Result, Error> { + let path_tree = self.inner.path_tree.clone(); + let motion_key = format!("{}/motion", filename); + + let opt = web::block(move || path_tree.get(motion_key.as_bytes())).await??; + + if let Some(ivec) = opt { + return Ok(Some( + self.inner.root_dir.join(String::from_utf8(ivec.to_vec())?), + )); + } + + Ok(None) + } + + async fn store_motion_path( + &self, + filename: &str, + path: impl AsRef, + ) -> Result<(), Error> { + let path_bytes = self + .generalize_path(path.as_ref())? + .to_str() + .ok_or(UploadError::Path)? + .as_bytes() + .to_vec(); + let motion_key = format!("{}/motion", filename); + let path_tree = self.inner.path_tree.clone(); + + web::block(move || path_tree.insert(motion_key.as_bytes(), path_bytes)).await??; + Ok(()) + } + #[instrument(skip(self))] pub(crate) async fn path_from_filename(&self, filename: String) -> Result { let path_tree = self.inner.path_tree.clone(); @@ -125,7 +201,12 @@ impl UploadManager { #[instrument(skip(self))] async fn store_path(&self, filename: String, path: &std::path::Path) -> Result<(), Error> { - let path_bytes = path.to_str().ok_or(UploadError::Path)?.as_bytes().to_vec(); + let path_bytes = self + .generalize_path(path)? + .to_str() + .ok_or(UploadError::Path)? + .as_bytes() + .to_vec(); let path_tree = self.inner.path_tree.clone(); web::block(move || path_tree.insert(filename.as_bytes(), path_bytes)).await??; Ok(()) @@ -536,6 +617,11 @@ impl Serde { } impl Details { + fn is_motion(&self) -> bool { + self.content_type.type_() == "video" + || self.content_type.type_() == "image" && self.content_type.subtype() == "gif" + } + #[tracing::instrument("Details from bytes", skip(input))] pub(crate) async fn from_bytes(input: web::Bytes) -> Result { let details = crate::magick::details_bytes(input).await?; @@ -606,6 +692,20 @@ fn delete_key(alias: &str) -> String { format!("{}/delete", alias) } +impl Deref for Serde { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Serde { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + impl std::fmt::Debug for UploadManager { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.debug_struct("UploadManager").finish()