Fix restructure, upload

src/main.rs

@@ -300,8 +300,7 @@ async fn upload(
             let delete_token = image.result.delete_token().await?;
 
             let name = manager.from_alias(alias.to_owned()).await?;
-            let mut path = manager.image_dir();
+            let path = manager.path_from_filename(name.clone()).await?;
-            path.push(name.clone());
 
             let details = manager.variant_details(path.clone(), name.clone()).await?;
 
@@ -365,8 +364,7 @@ async fn download(
     let delete_token = session.delete_token().await?;
 
     let name = manager.from_alias(alias.to_owned()).await?;
-    let mut path = manager.image_dir();
+    let path = manager.path_from_filename(name.clone()).await?;
-    path.push(name.clone());
 
     let details = manager.variant_details(path.clone(), name.clone()).await?;
 
@@ -446,8 +444,7 @@ async fn prepare_process(
         .parse::<Format>()
         .map_err(|_| UploadError::UnsupportedFormat)?;
     let processed_name = format!("{}.{}", name, ext);
-    let base = manager.image_dir();
+    let thumbnail_path = self::processor::build_path(&chain, processed_name);
-    let thumbnail_path = self::processor::build_path(base, &chain, processed_name);
     let thumbnail_args = self::processor::build_args(&chain);
 
     Ok((format, name, thumbnail_path, thumbnail_args))
@@ -463,7 +460,12 @@ async fn process_details(
     let (_, name, thumbnail_path, _) =
         prepare_process(query, ext.as_str(), &manager, &whitelist).await?;
 
-    let details = manager.variant_details(thumbnail_path, name).await?;
+    let real_path = manager
+        .variant_path(&thumbnail_path, &name)
+        .await?
+        .ok_or(UploadError::MissingAlias)?;
+
+    let details = manager.variant_details(real_path, name).await?;
 
     let details = details.ok_or(UploadError::NoFiles)?;
 
@@ -482,8 +484,11 @@ async fn process(
     let (format, name, thumbnail_path, thumbnail_args) =
         prepare_process(query, ext.as_str(), &manager, &whitelist).await?;
 
+    let real_path_opt = manager.variant_path(&thumbnail_path, &name).await?;
+
     // If the thumbnail doesn't exist, we need to create it
-    let thumbnail_exists = if let Err(e) = tokio::fs::metadata(&thumbnail_path)
+    let thumbnail_exists = if let Some(real_path) = &real_path_opt {
+        if let Err(e) = tokio::fs::metadata(real_path)
             .instrument(tracing::info_span!("Get thumbnail metadata"))
             .await
         {
@@ -494,15 +499,32 @@ async fn process(
             false
         } else {
             true
+        }
+    } else {
+        false
     };
 
-    let details = manager
+    if thumbnail_exists {
-        .variant_details(thumbnail_path.clone(), name.clone())
+        if let Some(real_path) = real_path_opt {
+            let details_opt = manager
+                .variant_details(real_path.clone(), name.clone())
                 .await?;
 
-    if !thumbnail_exists || details.is_none() {
+            let details = if let Some(details) = details_opt {
-        let mut original_path = manager.image_dir();
+                details
-        original_path.push(name.clone());
+            } else {
+                let details = Details::from_path(real_path.clone()).await?;
+                manager
+                    .store_variant_details(real_path.clone(), name, &details)
+                    .await?;
+                details
+            };
+
+            return ranged_file_resp(real_path, range, details).await;
+        }
+    }
+
+    let mut original_path = manager.path_from_filename(name.clone()).await?;
 
     let thumbnail_path2 = thumbnail_path.clone();
     let process_fut = async {
@@ -527,7 +549,7 @@ async fn process(
                 span.follows_from(Span::current());
                 actix_rt::spawn(
                     async move {
-                            if let Err(e) = manager2.store_variant(updated_path, name).await {
+                        if let Err(e) = manager2.store_variant(None, &updated_path, &name).await {
                             error!("Error storing variant, {}", e);
                             return;
                         }
@@ -550,11 +572,7 @@ async fn process(
 
         drop(permit);
 
-            let details = if let Some(details) = details {
+        let details = Details::from_bytes(bytes.clone()).await?;
-                details
-            } else {
-                Details::from_bytes(bytes.clone()).await?
-            };
 
         let save_span = tracing::info_span!(
             parent: None,
@@ -567,18 +585,29 @@ async fn process(
         let bytes2 = bytes.clone();
         actix_rt::spawn(
             async move {
-                    if let Err(e) = safe_save_file(thumbnail_path.clone(), bytes2).await {
+                let real_path = match manager.next_directory() {
+                    Ok(real_path) => real_path.join(&name),
+                    Err(e) => {
+                        tracing::warn!("Failed to generate directory path: {}", e);
+                        return;
+                    }
+                };
+
+                if let Err(e) = safe_save_file(real_path.clone(), bytes2).await {
                     tracing::warn!("Error saving thumbnail: {}", e);
                     return;
                 }
                 if let Err(e) = manager
-                        .store_variant_details(thumbnail_path.clone(), name.clone(), &details2)
+                    .store_variant_details(real_path.clone(), name.clone(), &details2)
                     .await
                 {
                     tracing::warn!("Error saving variant details: {}", e);
                     return;
                 }
-                    if let Err(e) = manager.store_variant(thumbnail_path, name.clone()).await {
+                if let Err(e) = manager
+                    .store_variant(Some(&thumbnail_path), &real_path, &name)
+                    .await
+                {
                     tracing::warn!("Error saving variant info: {}", e);
                 }
             }
@@ -588,10 +617,9 @@ async fn process(
         Ok((details, bytes)) as Result<(Details, web::Bytes), Error>
     };
 
-        let (details, bytes) =
+    let (details, bytes) = CancelSafeProcessor::new(thumbnail_path.clone(), process_fut).await?;
-            CancelSafeProcessor::new(thumbnail_path.clone(), process_fut).await?;
 
-        return match range {
+    match range {
         Some(range_header) => {
             if !range_header.is_bytes() {
                 return Err(UploadError::Range.into());
@@ -624,20 +652,7 @@ async fn process(
             7 * DAYS,
             details.system_time(),
         )),
-        };
     }
-
-    let details = if let Some(details) = details {
-        details
-    } else {
-        let details = Details::from_path(thumbnail_path.clone()).await?;
-        manager
-            .store_variant_details(thumbnail_path.clone(), name, &details)
-            .await?;
-        details
-    };
-
-    ranged_file_resp(thumbnail_path, range, details).await
 }
 
 /// Fetch file details
@@ -647,8 +662,7 @@ async fn details(
     manager: web::Data<UploadManager>,
 ) -> Result<HttpResponse, Error> {
     let name = manager.from_alias(alias.into_inner()).await?;
-    let mut path = manager.image_dir();
+    let path = manager.path_from_filename(name.clone()).await?;
-    path.push(name.clone());
 
     let details = manager.variant_details(path.clone(), name.clone()).await?;
 
@@ -673,8 +687,7 @@ async fn serve(
     manager: web::Data<UploadManager>,
 ) -> Result<HttpResponse, Error> {
     let name = manager.from_alias(alias.into_inner()).await?;
-    let mut path = manager.image_dir();
+    let path = manager.path_from_filename(name.clone()).await?;
-    path.push(name.clone());
 
     let details = manager.variant_details(path.clone(), name.clone()).await?;
 

src/processor.rs

@@ -266,11 +266,11 @@ pub(crate) fn build_chain(args: &[(String, String)]) -> ProcessChain {
     ProcessChain { inner }
 }
 
-pub(crate) fn build_path(base: PathBuf, chain: &ProcessChain, filename: String) -> PathBuf {
+pub(crate) fn build_path(chain: &ProcessChain, filename: String) -> PathBuf {
     let mut path = chain
         .inner
         .iter()
-        .fold(base, |acc, processor| processor.path(acc));
+        .fold(PathBuf::default(), |acc, processor| processor.path(acc));
 
     path.push(filename);
     path

src/upload_manager/mod.rs

@@ -48,7 +48,7 @@ pub struct UploadManager {
 struct UploadManagerInner {
     format: Option<Format>,
     hasher: sha2::Sha256,
-    image_dir: PathBuf,
+    root_dir: PathBuf,
     alias_tree: sled::Tree,
     filename_tree: sled::Tree,
     main_tree: sled::Tree,
@@ -77,19 +77,12 @@ struct FilenameIVec {
 }
 
 impl UploadManager {
-    /// Get the image directory
-    pub(crate) fn image_dir(&self) -> PathBuf {
-        self.inner.image_dir.clone()
-    }
-
     /// Create a new UploadManager
-    pub(crate) async fn new(mut root_dir: PathBuf, format: Option<Format>) -> Result<Self, Error> {
+    pub(crate) async fn new(root_dir: PathBuf, format: Option<Format>) -> Result<Self, Error> {
         let root_clone = root_dir.clone();
         // sled automatically creates it's own directories
         let db = web::block(move || LatestDb::exists(root_clone).migrate()).await??;
 
-        root_dir.push("files");
-
         // Ensure file dir exists
         tokio::fs::create_dir_all(&root_dir).await?;
 
@@ -101,7 +94,7 @@ impl UploadManager {
             inner: Arc::new(UploadManagerInner {
                 format,
                 hasher: sha2::Sha256::new(),
-                image_dir: root_dir,
+                root_dir,
                 alias_tree: db.open_tree("alias")?,
                 filename_tree: db.open_tree("filename")?,
                 details_tree: db.open_tree("details")?,
@@ -118,17 +111,61 @@ impl UploadManager {
         Ok(manager)
     }
 
+    #[instrument(skip(self))]
+    pub(crate) async fn path_from_filename(&self, filename: String) -> Result<PathBuf, Error> {
+        let path_tree = self.inner.path_tree.clone();
+        let path_ivec = web::block(move || path_tree.get(filename.as_bytes()))
+            .await??
+            .ok_or(UploadError::MissingFile)?;
+
+        let relative = PathBuf::from(String::from_utf8(path_ivec.to_vec())?);
+
+        Ok(self.inner.root_dir.join(relative))
+    }
+
+    #[instrument(skip(self))]
+    async fn store_path(&self, filename: String, path: &std::path::Path) -> Result<(), Error> {
+        let path_bytes = path.to_str().ok_or(UploadError::Path)?.as_bytes().to_vec();
+        let path_tree = self.inner.path_tree.clone();
+        web::block(move || path_tree.insert(filename.as_bytes(), path_bytes)).await??;
+        Ok(())
+    }
+
+    #[instrument(skip(self))]
+    pub(crate) async fn variant_path(
+        &self,
+        process_path: &std::path::Path,
+        filename: &str,
+    ) -> Result<Option<PathBuf>, Error> {
+        let key = self.variant_key(process_path, filename)?;
+        let path_tree = self.inner.path_tree.clone();
+        let path_opt = web::block(move || path_tree.get(key)).await??;
+
+        if let Some(path_ivec) = path_opt {
+            let relative = PathBuf::from(String::from_utf8(path_ivec.to_vec())?);
+            Ok(Some(self.inner.root_dir.join(relative)))
+        } else {
+            Ok(None)
+        }
+    }
+
     /// Store the path to a generated image variant so we can easily clean it up later
     #[instrument(skip(self))]
-    pub(crate) async fn store_variant(&self, path: PathBuf, filename: String) -> Result<(), Error> {
+    pub(crate) async fn store_variant(
+        &self,
+        variant_process_path: Option<&std::path::Path>,
+        real_path: &std::path::Path,
+        filename: &str,
+    ) -> Result<(), Error> {
         let path_bytes = self
-            .generalize_path(&path)?
+            .generalize_path(real_path)?
             .to_str()
             .ok_or(UploadError::Path)?
             .as_bytes()
             .to_vec();
 
-        let key = self.variant_key(&path, &filename)?;
+        let variant_path = variant_process_path.unwrap_or(real_path);
+        let key = self.variant_key(variant_path, filename)?;
         let path_tree = self.inner.path_tree.clone();
 
         debug!("Storing variant");
@@ -199,14 +236,14 @@ impl UploadManager {
         self.aliases_by_hash(&hash).await
     }
 
-    fn next_directory(&self) -> Result<PathBuf, Error> {
+    pub(crate) fn next_directory(&self) -> Result<PathBuf, Error> {
         let path = self.inner.path_gen.next();
 
         self.inner
             .settings_tree
             .insert(GENERATOR_KEY, path.to_be_bytes())?;
 
-        let mut target_path = self.image_dir();
+        let mut target_path = self.inner.root_dir.join("files");
         for dir in path.to_strings() {
             target_path.push(dir)
         }
@@ -381,15 +418,19 @@ impl UploadManager {
     #[instrument(skip(self))]
     async fn cleanup_files(&self, filename: FilenameIVec) -> Result<(), Error> {
         let filename = filename.inner;
-        let mut path = self.image_dir();
+
-        let fname = String::from_utf8(filename.to_vec())?;
+        let filename2 = filename.clone();
-        path.push(fname);
+        let path_tree = self.inner.path_tree.clone();
+        let path = web::block(move || path_tree.remove(filename2)).await??;
 
         let mut errors = Vec::new();
+        if let Some(path) = path {
+            let path = self.inner.root_dir.join(String::from_utf8(path.to_vec())?);
             debug!("Deleting {:?}", path);
-        if let Err(e) = tokio::fs::remove_file(path).await {
+            if let Err(e) = self.remove_path(&path).await {
                 errors.push(e.into());
             }
+        }
 
         let filename2 = filename.clone();
         let fname_tree = self.inner.filename_tree.clone();
@@ -410,9 +451,12 @@ impl UploadManager {
         debug!("{} files prepared for deletion", paths.len());
 
         for path in paths {
-            let s = String::from_utf8_lossy(&path);
+            let path = self
-            debug!("Deleting {}", s);
+                .inner
-            if let Err(e) = remove_path(path).await {
+                .root_dir
+                .join(String::from_utf8_lossy(&path).as_ref());
+            debug!("Deleting {:?}", path);
+            if let Err(e) = self.remove_path(&path).await {
                 errors.push(e);
             }
         }
@@ -434,6 +478,55 @@ impl UploadManager {
         }
         Ok(())
     }
+
+    async fn try_remove_parents(&self, mut path: &std::path::Path) -> Result<(), Error> {
+        let root = self.inner.root_dir.join("files");
+
+        while let Some(parent) = path.parent() {
+            if parent.ends_with(&root) {
+                break;
+            }
+
+            if tokio::fs::remove_dir(parent).await.is_err() {
+                break;
+            }
+
+            path = parent;
+        }
+
+        Ok(())
+    }
+
+    async fn remove_path(&self, path: &std::path::Path) -> Result<(), Error> {
+        tokio::fs::remove_file(path).await?;
+        self.try_remove_parents(path).await
+    }
+
+    fn variant_key(
+        &self,
+        variant_process_path: &std::path::Path,
+        filename: &str,
+    ) -> Result<Vec<u8>, Error> {
+        let path_string = variant_process_path
+            .to_str()
+            .ok_or(UploadError::Path)?
+            .to_string();
+
+        let vec = format!("{}/{}", filename, path_string).as_bytes().to_vec();
+        Ok(vec)
+    }
+
+    fn details_key(
+        &self,
+        variant_path: &std::path::Path,
+        filename: &str,
+    ) -> Result<Vec<u8>, Error> {
+        let path = self.generalize_path(variant_path)?;
+        let path_string = path.to_str().ok_or(UploadError::Path)?.to_string();
+
+        let vec = format!("{}/{}", filename, path_string).as_bytes().to_vec();
+        Ok(vec)
+    }
 }
 
 impl<T> Serde<T> {
@@ -502,12 +595,6 @@ fn init_generator(settings: &sled::Tree) -> Result<Generator, Error> {
     }
 }
 
-async fn remove_path(path: sled::IVec) -> Result<(), Error> {
-    let path_string = String::from_utf8(path.to_vec())?;
-    tokio::fs::remove_file(path_string).await?;
-    Ok(())
-}
-
 fn trans_err<E>(e: E) -> sled::transaction::ConflictableTransactionError<Error>
 where
     Error: From<E>,

src/upload_manager/restructure.rs

@@ -20,7 +20,7 @@ impl UploadManager {
             let filename = String::from_utf8(filename.to_vec())?;
             tracing::info!("Migrating {}", filename);
 
-            let mut file_path = self.image_dir();
+            let mut file_path = self.inner.root_dir.join("files");
             file_path.push(filename.clone());
 
             if tokio::fs::metadata(&file_path).await.is_ok() {
@@ -46,18 +46,7 @@ impl UploadManager {
             for res in self.inner.main_tree.range(start..end) {
                 let (hash_variant_key, variant_path_or_details) = res?;
 
-                if hash_variant_key.ends_with(DETAILS) {
+                if !hash_variant_key.ends_with(DETAILS) {
-                    let details = variant_path_or_details;
-
-                    let start_index = hash.len() + 1;
-                    let end_index = hash_variant_key.len() - DETAILS.len();
-                    let path_bytes = &hash_variant_key[start_index..end_index];
-
-                    let variant_path = PathBuf::from(String::from_utf8(path_bytes.to_vec())?);
-                    let key = self.details_key(&variant_path, &filename)?;
-
-                    self.inner.details_tree.insert(key, details)?;
-                } else {
                     let variant_path =
                         PathBuf::from(String::from_utf8(variant_path_or_details.to_vec())?);
                     if tokio::fs::metadata(&variant_path).await.is_ok() {
@@ -71,13 +60,14 @@ impl UploadManager {
                             .as_bytes()
                             .to_vec();
 
-                        let variant_key = self.variant_key(&target_path, &filename)?;
+                        let variant_key = self.migrate_variant_key(&variant_path, &filename)?;
 
                         self.inner
                             .path_tree
                             .insert(variant_key, relative_target_path_bytes)?;
 
-                        safe_move_file(variant_path, target_path).await?;
+                        safe_move_file(variant_path.clone(), target_path).await?;
+                        self.try_remove_parents(&variant_path).await?;
                     }
                 }
 
@@ -90,7 +80,7 @@ impl UploadManager {
     }
 
     fn restructure_complete(&self) -> Result<bool, Error> {
-        Ok(!self
+        Ok(self
             .inner
             .settings_tree
             .get(RESTRUCTURE_COMPLETE)?
@@ -106,31 +96,19 @@ impl UploadManager {
     }
 
     pub(super) fn generalize_path<'a>(&self, path: &'a Path) -> Result<&'a Path, Error> {
-        Ok(path.strip_prefix(&self.inner.image_dir)?)
+        Ok(path.strip_prefix(&self.inner.root_dir)?)
     }
 
-    pub(super) fn details_key(
+    fn migrate_variant_key(
         &self,
-        variant_path: &Path,
+        variant_process_path: &Path,
         filename: &str,
     ) -> Result<Vec<u8>, Error> {
-        let path = self.generalize_path(variant_path)?;
+        let path = self
-        let path_string = path.to_str().ok_or(UploadError::Path)?.to_string();
+            .generalize_path(variant_process_path)?
+            .strip_prefix("files")?;
 
-        let vec = format!("{}/{}", filename, path_string).as_bytes().to_vec();
+        self.variant_key(path, filename)
-        Ok(vec)
-    }
-
-    pub(super) fn variant_key(
-        &self,
-        variant_path: &Path,
-        filename: &str,
-    ) -> Result<Vec<u8>, Error> {
-        let path = self.generalize_path(variant_path)?;
-        let path_string = path.to_str().ok_or(UploadError::Path)?.to_string();
-
-        let vec = format!("{}/{}", filename, path_string).as_bytes().to_vec();
-        Ok(vec)
     }
 }
 

src/upload_manager/session.rs

@@ -229,9 +229,9 @@ impl UploadManagerSession {
         }
 
         // -- WRITE NEW FILE --
-        let mut real_path = self.manager.image_dir();
+        let real_path = self.manager.next_directory()?.join(&name);
-        real_path.push(name);
 
+        self.manager.store_path(name, &real_path).await?;
         crate::safe_move_file(tmpfile, real_path).await?;
 
         Ok(())
@@ -280,23 +280,21 @@ impl UploadManagerSession {
     // generate a short filename that isn't already in-use
     #[instrument(skip(self, content_type))]
     async fn next_file(&self, content_type: mime::Mime) -> Result<String, Error> {
-        let image_dir = self.manager.image_dir();
         loop {
             debug!("Filename generation loop");
-            let mut path = image_dir.clone();
             let s: String = Uuid::new_v4().to_string();
 
             let filename = file_name(s, content_type.clone())?;
 
-            path.push(filename.clone());
+            let path_tree = self.manager.inner.path_tree.clone();
+            let filename2 = filename.clone();
+            let filename_exists = web::block(move || path_tree.get(filename2.as_bytes()))
+                .await??
+                .is_some();
 
-            if let Err(e) = tokio::fs::metadata(path).await {
+            if !filename_exists {
-                if e.kind() == std::io::ErrorKind::NotFound {
-                    debug!("Generated unused filename {}", filename);
                 return Ok(filename);
             }
-                return Err(e.into());
-            }
 
             debug!("Filename exists, trying again");
         }