From a0c99d05ebd4e55c94ecf272eec171df799f6226 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Thu, 24 Mar 2022 13:16:41 -0500
Subject: [PATCH 01/37] Create initial trait representation for database

---
 src/main.rs |  1 +
 src/repo.rs | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)
 create mode 100644 src/repo.rs

diff --git a/src/main.rs b/src/main.rs
index 4b2a059..6ed8cc1 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -39,6 +39,7 @@ mod migrate;
 mod process;
 mod processor;
 mod range;
+mod repo;
 mod serde_str;
 mod store;
 mod tmp_file;
diff --git a/src/repo.rs b/src/repo.rs
new file mode 100644
index 0000000..8da18f6
--- /dev/null
+++ b/src/repo.rs
@@ -0,0 +1,74 @@
+use crate::{store::Identifier, upload_manager::Details};
+use futures_util::Stream;
+use uuid::Uuid;
+
+pub(crate) struct Alias {
+    id: Uuid,
+    extension: String,
+}
+pub(crate) struct DeleteToken {
+    id: Uuid,
+}
+pub(crate) struct AlreadyExists;
+
+#[async_trait::async_trait]
+pub(crate) trait SettingsRepo {
+    type Error: std::error::Error;
+
+    async fn set(&self, key: &'static [u8], value: Vec<u8>) -> Result<(), Self::Error>;
+    async fn get(&self, key: &'static [u8]) -> Result<Option<Vec<u8>>, Self::Error>;
+    async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error>;
+}
+
+#[async_trait::async_trait]
+pub(crate) trait IdentifierRepo<I: Identifier> {
+    type Hash: AsRef<[u8]>;
+    type Error: std::error::Error;
+
+    async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error>;
+    async fn details(&self, identifier: I) -> Result<Option<Details>, Self::Error>;
+
+    async fn relate_hash(&self, identifier: I, hash: Self::Hash) -> Result<(), Self::Error>;
+    async fn hash(&self, identifier: I) -> Result<Self::Hash, Self::Error>;
+
+    async fn cleanup(&self, identifier: I) -> Result<(), Self::Error>;
+}
+
+#[async_trait::async_trait]
+pub(crate) trait HashRepo {
+    type Hash: AsRef<[u8]>;
+    type Error: std::error::Error;
+    type Stream: Stream<Item = Result<Self::Hash, Self::Error>>;
+
+    async fn hashes(&self) -> Self::Stream;
+
+    async fn create(&self, hash: Self::Hash) -> Result<Result<(), AlreadyExists>, Self::Error>;
+
+    async fn relate_alias(&self, hash: Self::Hash, alias: Alias) -> Result<(), Self::Error>;
+    async fn remove_alias(&self, hash: Self::Hash, alias: Alias) -> Result<(), Self::Error>;
+    async fn aliases(&self, hash: Self::Hash) -> Result<Vec<Alias>, Self::Error>;
+
+    async fn cleanup(&self, hash: Self::Hash) -> Result<(), Self::Error>;
+}
+
+#[async_trait::async_trait]
+pub(crate) trait AliasRepo<I: Identifier> {
+    type Hash: AsRef<[u8]>;
+    type Error: std::error::Error;
+
+    async fn create(&self, alias: Alias) -> Result<Result<(), AlreadyExists>, Self::Error>;
+
+    async fn create_delete_token(
+        &self,
+        alias: Alias,
+    ) -> Result<Result<DeleteToken, AlreadyExists>, Self::Error>;
+    async fn delete_token(&self, alias: Alias) -> Result<DeleteToken, Self::Error>;
+
+    async fn relate_hash(&self, alias: Alias, hash: Self::Hash) -> Result<(), Self::Error>;
+    async fn hash(&self, alias: Alias) -> Result<Self::Hash, Self::Error>;
+
+    async fn relate_identifier(&self, alias: Alias, identifier: I) -> Result<(), Self::Error>;
+    async fn identifier(&self, alias: Alias) -> Result<I, Self::Error>;
+
+    async fn cleanup(&self, alias: Alias) -> Result<(), Self::Error>;
+}

From d6567fbbbd8aee510455ac5a9bba3ed618839fdc Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Thu, 24 Mar 2022 17:09:15 -0500
Subject: [PATCH 02/37] Implement sled repo

---
 src/concurrent_processor.rs |   2 +-
 src/details.rs              |  66 ++++++
 src/main.rs                 |   4 +-
 src/repo.rs                 |  83 +++++---
 src/repo/sled.rs            | 398 ++++++++++++++++++++++++++++++++++++
 src/upload_manager.rs       |  68 +-----
 6 files changed, 530 insertions(+), 91 deletions(-)
 create mode 100644 src/details.rs
 create mode 100644 src/repo/sled.rs

diff --git a/src/concurrent_processor.rs b/src/concurrent_processor.rs
index abaeaca..52eb27e 100644
--- a/src/concurrent_processor.rs
+++ b/src/concurrent_processor.rs
@@ -1,6 +1,6 @@
 use crate::{
+    details::Details,
     error::{Error, UploadError},
-    upload_manager::Details,
 };
 use actix_web::web;
 use dashmap::{mapref::entry::Entry, DashMap};
diff --git a/src/details.rs b/src/details.rs
new file mode 100644
index 0000000..0730689
--- /dev/null
+++ b/src/details.rs
@@ -0,0 +1,66 @@
+use crate::{error::Error, magick::ValidInputType, serde_str::Serde, store::Store};
+use actix_web::web;
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Details {
+    width: usize,
+    height: usize,
+    content_type: Serde<mime::Mime>,
+    created_at: time::OffsetDateTime,
+}
+
+impl Details {
+    pub(crate) fn is_motion(&self) -> bool {
+        self.content_type.type_() == "video"
+            || self.content_type.type_() == "image" && self.content_type.subtype() == "gif"
+    }
+
+    #[tracing::instrument("Details from bytes", skip(input))]
+    pub(crate) async fn from_bytes(
+        input: web::Bytes,
+        hint: Option<ValidInputType>,
+    ) -> Result<Self, Error> {
+        let details = crate::magick::details_bytes(input, hint).await?;
+
+        Ok(Details::now(
+            details.width,
+            details.height,
+            details.mime_type,
+        ))
+    }
+
+    #[tracing::instrument("Details from store")]
+    pub(crate) async fn from_store<S: Store>(
+        store: S,
+        identifier: S::Identifier,
+        expected_format: Option<ValidInputType>,
+    ) -> Result<Self, Error>
+    where
+        Error: From<S::Error>,
+    {
+        let details = crate::magick::details_store(store, identifier, expected_format).await?;
+
+        Ok(Details::now(
+            details.width,
+            details.height,
+            details.mime_type,
+        ))
+    }
+
+    pub(crate) fn now(width: usize, height: usize, content_type: mime::Mime) -> Self {
+        Details {
+            width,
+            height,
+            content_type: Serde::new(content_type),
+            created_at: time::OffsetDateTime::now_utc(),
+        }
+    }
+
+    pub(crate) fn content_type(&self) -> mime::Mime {
+        (*self.content_type).clone()
+    }
+
+    pub(crate) fn system_time(&self) -> std::time::SystemTime {
+        self.created_at.into()
+    }
+}
diff --git a/src/main.rs b/src/main.rs
index 6ed8cc1..6013799 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -26,6 +26,7 @@ use tracing_futures::Instrument;
 
 mod concurrent_processor;
 mod config;
+mod details;
 mod either;
 mod error;
 mod exiftool;
@@ -51,13 +52,14 @@ use crate::{magick::details_hint, store::file_store::FileStore};
 use self::{
     concurrent_processor::CancelSafeProcessor,
     config::{Config, Format, Migrate},
+    details::Details,
     either::Either,
     error::{Error, UploadError},
     init_tracing::init_tracing,
     middleware::{Deadline, Internal},
     migrate::LatestDb,
     store::Store,
-    upload_manager::{Details, UploadManager, UploadManagerSession},
+    upload_manager::{UploadManager, UploadManagerSession},
 };
 
 const MEGABYTES: usize = 1024 * 1024;
diff --git a/src/repo.rs b/src/repo.rs
index 8da18f6..25e9be3 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -1,7 +1,9 @@
-use crate::{store::Identifier, upload_manager::Details};
+use crate::{details::Details, store::Identifier};
 use futures_util::Stream;
 use uuid::Uuid;
 
+pub(crate) mod sled;
+
 pub(crate) struct Alias {
     id: Uuid,
     extension: String,
@@ -11,64 +13,99 @@ pub(crate) struct DeleteToken {
 }
 pub(crate) struct AlreadyExists;
 
+impl Alias {
+    fn to_bytes(&self) -> Vec<u8> {
+        let mut v = self.id.as_bytes().to_vec();
+
+        v.extend_from_slice(self.extension.as_bytes());
+
+        v
+    }
+
+    fn from_slice(bytes: &[u8]) -> Option<Self> {
+        if bytes.len() > 16 {
+            let id = Uuid::from_slice(&bytes[0..16]).expect("Already checked length");
+            let extension = String::from_utf8_lossy(&bytes[16..]).to_string();
+
+            Some(Self { id, extension })
+        } else {
+            None
+        }
+    }
+}
+
+impl DeleteToken {
+    fn to_bytes(&self) -> Vec<u8> {
+        self.id.as_bytes().to_vec()
+    }
+
+    fn from_slice(bytes: &[u8]) -> Option<Self> {
+        Some(DeleteToken {
+            id: Uuid::from_slice(bytes).ok()?,
+        })
+    }
+}
+
 #[async_trait::async_trait]
 pub(crate) trait SettingsRepo {
+    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
     type Error: std::error::Error;
 
-    async fn set(&self, key: &'static [u8], value: Vec<u8>) -> Result<(), Self::Error>;
-    async fn get(&self, key: &'static [u8]) -> Result<Option<Vec<u8>>, Self::Error>;
+    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Self::Error>;
+    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Self::Error>;
     async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error>;
 }
 
 #[async_trait::async_trait]
 pub(crate) trait IdentifierRepo<I: Identifier> {
-    type Hash: AsRef<[u8]>;
+    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
     type Error: std::error::Error;
 
     async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error>;
     async fn details(&self, identifier: I) -> Result<Option<Details>, Self::Error>;
 
-    async fn relate_hash(&self, identifier: I, hash: Self::Hash) -> Result<(), Self::Error>;
-    async fn hash(&self, identifier: I) -> Result<Self::Hash, Self::Error>;
+    async fn relate_hash(&self, identifier: I, hash: Self::Bytes) -> Result<(), Self::Error>;
+    async fn hash(&self, identifier: I) -> Result<Self::Bytes, Self::Error>;
 
     async fn cleanup(&self, identifier: I) -> Result<(), Self::Error>;
 }
 
 #[async_trait::async_trait]
-pub(crate) trait HashRepo {
-    type Hash: AsRef<[u8]>;
+pub(crate) trait HashRepo<I: Identifier> {
+    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
     type Error: std::error::Error;
-    type Stream: Stream<Item = Result<Self::Hash, Self::Error>>;
+    type Stream: Stream<Item = Result<Self::Bytes, Self::Error>>;
 
     async fn hashes(&self) -> Self::Stream;
 
-    async fn create(&self, hash: Self::Hash) -> Result<Result<(), AlreadyExists>, Self::Error>;
+    async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Self::Error>;
 
-    async fn relate_alias(&self, hash: Self::Hash, alias: Alias) -> Result<(), Self::Error>;
-    async fn remove_alias(&self, hash: Self::Hash, alias: Alias) -> Result<(), Self::Error>;
-    async fn aliases(&self, hash: Self::Hash) -> Result<Vec<Alias>, Self::Error>;
+    async fn relate_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
+    async fn remove_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
+    async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Self::Error>;
 
-    async fn cleanup(&self, hash: Self::Hash) -> Result<(), Self::Error>;
+    async fn relate_identifier(&self, hash: Self::Bytes, identifier: I) -> Result<(), Self::Error>;
+    async fn identifier(&self, hash: Self::Bytes) -> Result<I, Self::Error>;
+
+    async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error>;
 }
 
 #[async_trait::async_trait]
-pub(crate) trait AliasRepo<I: Identifier> {
-    type Hash: AsRef<[u8]>;
+pub(crate) trait AliasRepo {
+    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
     type Error: std::error::Error;
 
     async fn create(&self, alias: Alias) -> Result<Result<(), AlreadyExists>, Self::Error>;
 
-    async fn create_delete_token(
+    async fn relate_delete_token(
         &self,
         alias: Alias,
-    ) -> Result<Result<DeleteToken, AlreadyExists>, Self::Error>;
+        delete_token: DeleteToken,
+    ) -> Result<Result<(), AlreadyExists>, Self::Error>;
     async fn delete_token(&self, alias: Alias) -> Result<DeleteToken, Self::Error>;
 
-    async fn relate_hash(&self, alias: Alias, hash: Self::Hash) -> Result<(), Self::Error>;
-    async fn hash(&self, alias: Alias) -> Result<Self::Hash, Self::Error>;
-
-    async fn relate_identifier(&self, alias: Alias, identifier: I) -> Result<(), Self::Error>;
-    async fn identifier(&self, alias: Alias) -> Result<I, Self::Error>;
+    async fn relate_hash(&self, alias: Alias, hash: Self::Bytes) -> Result<(), Self::Error>;
+    async fn hash(&self, alias: Alias) -> Result<Self::Bytes, Self::Error>;
 
     async fn cleanup(&self, alias: Alias) -> Result<(), Self::Error>;
 }
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
new file mode 100644
index 0000000..b543202
--- /dev/null
+++ b/src/repo/sled.rs
@@ -0,0 +1,398 @@
+use super::{
+    Alias, AliasRepo, AlreadyExists, DeleteToken, Details, HashRepo, Identifier, IdentifierRepo,
+    SettingsRepo,
+};
+use sled::{Db, IVec, Tree};
+
+macro_rules! b {
+    ($self:ident.$ident:ident, $expr:expr) => {{
+        let $ident = $self.$ident.clone();
+
+        actix_rt::task::spawn_blocking(move || $expr).await??
+    }};
+}
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum Error {
+    #[error("Error in database")]
+    Sled(#[from] sled::Error),
+
+    #[error("Invalid identifier")]
+    Identifier(#[source] Box<dyn std::error::Error + Send>),
+
+    #[error("Invalid details json")]
+    Details(#[from] serde_json::Error),
+
+    #[error("Required field was not present")]
+    Missing,
+
+    #[error("Operation panicked")]
+    Panic,
+}
+
+pub(crate) struct SledRepo {
+    settings: Tree,
+    identifier_hashes: Tree,
+    identifier_details: Tree,
+    hashes: Tree,
+    hash_aliases: Tree,
+    hash_identifiers: Tree,
+    aliases: Tree,
+    alias_hashes: Tree,
+    alias_delete_tokens: Tree,
+    _db: Db,
+}
+
+impl SledRepo {
+    pub(crate) fn new(db: Db) -> Result<Self, Error> {
+        Ok(SledRepo {
+            settings: db.open_tree("pict-rs-settings-tree")?,
+            identifier_hashes: db.open_tree("pict-rs-identifier-hashes-tree")?,
+            identifier_details: db.open_tree("pict-rs-identifier-details-tree")?,
+            hashes: db.open_tree("pict-rs-hashes-tree")?,
+            hash_aliases: db.open_tree("pict-rs-hash-aliases-tree")?,
+            hash_identifiers: db.open_tree("pict-rs-hash-identifiers-tree")?,
+            aliases: db.open_tree("pict-rs-aliases-tree")?,
+            alias_hashes: db.open_tree("pict-rs-alias-hashes-tree")?,
+            alias_delete_tokens: db.open_tree("pict-rs-alias-delete-tokens-tree")?,
+            _db: db,
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl SettingsRepo for SledRepo {
+    type Bytes = IVec;
+    type Error = Error;
+
+    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Self::Error> {
+        b!(self.settings, settings.insert(key, value));
+
+        Ok(())
+    }
+
+    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Self::Error> {
+        let opt = b!(self.settings, settings.get(key));
+
+        Ok(opt)
+    }
+
+    async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error> {
+        b!(self.settings, settings.remove(key));
+
+        Ok(())
+    }
+}
+
+fn identifier_bytes<I>(identifier: &I) -> Result<Vec<u8>, Error>
+where
+    I: Identifier,
+    I::Error: Send + 'static,
+{
+    identifier
+        .to_bytes()
+        .map_err(|e| Error::Identifier(Box::new(e)))
+}
+
+#[async_trait::async_trait]
+impl<I> IdentifierRepo<I> for SledRepo
+where
+    I: Identifier + 'static,
+    I::Error: Send + 'static,
+{
+    type Bytes = IVec;
+    type Error = Error;
+
+    async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error> {
+        let key = identifier_bytes(&identifier)?;
+
+        let details = serde_json::to_vec(&details)?;
+
+        b!(
+            self.identifier_details,
+            identifier_details.insert(key, details)
+        );
+
+        Ok(())
+    }
+
+    async fn details(&self, identifier: I) -> Result<Option<Details>, Self::Error> {
+        let key = identifier_bytes(&identifier)?;
+
+        let opt = b!(self.identifier_details, identifier_details.get(key));
+
+        if let Some(ivec) = opt {
+            Ok(Some(serde_json::from_slice(&ivec)?))
+        } else {
+            Ok(None)
+        }
+    }
+
+    async fn relate_hash(&self, identifier: I, hash: Self::Bytes) -> Result<(), Self::Error> {
+        let key = identifier_bytes(&identifier)?;
+
+        b!(self.identifier_hashes, identifier_hashes.insert(key, hash));
+
+        Ok(())
+    }
+
+    async fn hash(&self, identifier: I) -> Result<Self::Bytes, Self::Error> {
+        let key = identifier_bytes(&identifier)?;
+
+        let opt = b!(self.identifier_hashes, identifier_hashes.get(key));
+
+        opt.ok_or(Error::Missing)
+    }
+
+    async fn cleanup(&self, identifier: I) -> Result<(), Self::Error> {
+        let key = identifier_bytes(&identifier)?;
+
+        let key2 = key.clone();
+        b!(self.identifier_hashes, identifier_hashes.remove(key2));
+        b!(self.identifier_details, identifier_details.remove(key));
+
+        Ok(())
+    }
+}
+
+type BoxIterator<'a, T> = Box<dyn std::iter::Iterator<Item = T> + Send + 'a>;
+
+type HashIterator = BoxIterator<'static, Result<IVec, sled::Error>>;
+
+type StreamItem = Result<IVec, Error>;
+
+type NextFutResult = Result<(HashIterator, Option<StreamItem>), Error>;
+
+pub(crate) struct HashStream {
+    hashes: Option<HashIterator>,
+    next_fut: Option<futures_util::future::LocalBoxFuture<'static, NextFutResult>>,
+}
+
+impl futures_util::Stream for HashStream {
+    type Item = StreamItem;
+
+    fn poll_next(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        let this = self.get_mut();
+
+        if let Some(mut fut) = this.next_fut.take() {
+            match fut.as_mut().poll(cx) {
+                std::task::Poll::Ready(Ok((iter, opt))) => {
+                    this.hashes = Some(iter);
+                    std::task::Poll::Ready(opt)
+                }
+                std::task::Poll::Ready(Err(e)) => std::task::Poll::Ready(Some(Err(e))),
+                std::task::Poll::Pending => {
+                    this.next_fut = Some(fut);
+                    std::task::Poll::Pending
+                }
+            }
+        } else if let Some(mut iter) = this.hashes.take() {
+            let fut = Box::pin(async move {
+                actix_rt::task::spawn_blocking(move || {
+                    let opt = iter.next().map(|res| res.map_err(Error::from));
+
+                    (iter, opt)
+                })
+                .await
+                .map_err(Error::from)
+            });
+
+            this.next_fut = Some(fut);
+            std::pin::Pin::new(this).poll_next(cx)
+        } else {
+            std::task::Poll::Ready(None)
+        }
+    }
+}
+
+fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> {
+    let mut v = hash.to_vec();
+    v.append(&mut alias.to_bytes());
+    v
+}
+
+#[async_trait::async_trait]
+impl<I> HashRepo<I> for SledRepo
+where
+    I: Identifier + 'static,
+    I::Error: Send + 'static,
+{
+    type Bytes = IVec;
+    type Error = Error;
+    type Stream = HashStream;
+
+    async fn hashes(&self) -> Self::Stream {
+        let iter = self.hashes.iter().keys();
+
+        HashStream {
+            hashes: Some(Box::new(iter)),
+            next_fut: None,
+        }
+    }
+
+    async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Self::Error> {
+        let res = b!(self.hashes, {
+            let hash2 = hash.clone();
+            hashes.compare_and_swap(hash, None as Option<Self::Bytes>, Some(hash2))
+        });
+
+        Ok(res.map_err(|_| AlreadyExists))
+    }
+
+    async fn relate_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error> {
+        let key = hash_alias_key(&hash, &alias);
+
+        b!(
+            self.hash_aliases,
+            hash_aliases.insert(key, alias.to_bytes())
+        );
+
+        Ok(())
+    }
+
+    async fn remove_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error> {
+        let key = hash_alias_key(&hash, &alias);
+
+        b!(self.hash_aliases, hash_aliases.remove(key));
+
+        Ok(())
+    }
+
+    async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Self::Error> {
+        let v = b!(self.hash_aliases, {
+            Ok(hash_aliases
+                .scan_prefix(hash)
+                .values()
+                .filter_map(Result::ok)
+                .filter_map(|ivec| Alias::from_slice(&ivec))
+                .collect::<Vec<_>>()) as Result<_, Error>
+        });
+
+        Ok(v)
+    }
+
+    async fn relate_identifier(&self, hash: Self::Bytes, identifier: I) -> Result<(), Self::Error> {
+        let bytes = identifier_bytes(&identifier)?;
+
+        b!(self.hash_identifiers, hash_identifiers.insert(hash, bytes));
+
+        Ok(())
+    }
+
+    async fn identifier(&self, hash: Self::Bytes) -> Result<I, Self::Error> {
+        let opt = b!(self.hash_identifiers, hash_identifiers.get(hash));
+
+        opt.ok_or(Error::Missing).and_then(|ivec| {
+            I::from_bytes(ivec.to_vec()).map_err(|e| Error::Identifier(Box::new(e)))
+        })
+    }
+
+    async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error> {
+        let hash2 = hash.clone();
+        b!(self.hashes, hashes.remove(hash2));
+
+        let hash2 = hash.clone();
+        b!(self.hash_identifiers, hash_identifiers.remove(hash2));
+
+        let aliases = HashRepo::<I>::aliases(self, hash.clone()).await?;
+
+        b!(self.hash_aliases, {
+            for alias in aliases {
+                let key = hash_alias_key(&hash, &alias);
+
+                let _ = hash_aliases.remove(key);
+            }
+            Ok(()) as Result<(), Error>
+        });
+
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl AliasRepo for SledRepo {
+    type Bytes = sled::IVec;
+    type Error = Error;
+
+    async fn create(&self, alias: Alias) -> Result<Result<(), AlreadyExists>, Self::Error> {
+        let bytes = alias.to_bytes();
+        let bytes2 = bytes.clone();
+
+        let res = b!(
+            self.aliases,
+            aliases.compare_and_swap(bytes, None as Option<Self::Bytes>, Some(bytes2))
+        );
+
+        Ok(res.map_err(|_| AlreadyExists))
+    }
+
+    async fn relate_delete_token(
+        &self,
+        alias: Alias,
+        delete_token: DeleteToken,
+    ) -> Result<Result<(), AlreadyExists>, Self::Error> {
+        let key = alias.to_bytes();
+        let token = delete_token.to_bytes();
+
+        let res = b!(
+            self.alias_delete_tokens,
+            alias_delete_tokens.compare_and_swap(key, None as Option<Self::Bytes>, Some(token))
+        );
+
+        Ok(res.map_err(|_| AlreadyExists))
+    }
+
+    async fn delete_token(&self, alias: Alias) -> Result<DeleteToken, Self::Error> {
+        let key = alias.to_bytes();
+
+        let opt = b!(self.alias_delete_tokens, alias_delete_tokens.get(key));
+
+        opt.and_then(|ivec| DeleteToken::from_slice(&ivec))
+            .ok_or(Error::Missing)
+    }
+
+    async fn relate_hash(&self, alias: Alias, hash: Self::Bytes) -> Result<(), Self::Error> {
+        let key = alias.to_bytes();
+
+        b!(self.alias_hashes, alias_hashes.insert(key, hash));
+
+        Ok(())
+    }
+
+    async fn hash(&self, alias: Alias) -> Result<Self::Bytes, Self::Error> {
+        let key = alias.to_bytes();
+
+        let opt = b!(self.alias_hashes, alias_hashes.get(key));
+
+        opt.ok_or(Error::Missing)
+    }
+
+    async fn cleanup(&self, alias: Alias) -> Result<(), Self::Error> {
+        let key = alias.to_bytes();
+
+        let key2 = key.clone();
+        b!(self.aliases, aliases.remove(key2));
+
+        let key2 = key.clone();
+        b!(self.alias_delete_tokens, alias_delete_tokens.remove(key2));
+
+        b!(self.alias_hashes, alias_hashes.remove(key));
+
+        Ok(())
+    }
+}
+
+impl std::fmt::Debug for SledRepo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SledRepo").finish()
+    }
+}
+
+impl From<actix_rt::task::JoinError> for Error {
+    fn from(_: actix_rt::task::JoinError) -> Self {
+        Error::Panic
+    }
+}
diff --git a/src/upload_manager.rs b/src/upload_manager.rs
index e934d31..7e1b7cf 100644
--- a/src/upload_manager.rs
+++ b/src/upload_manager.rs
@@ -1,10 +1,10 @@
 use crate::{
     config::Format,
+    details::Details,
     error::{Error, UploadError},
     ffmpeg::{InputFormat, ThumbnailFormat},
-    magick::{details_hint, ValidInputType},
+    magick::details_hint,
     migrate::{alias_id_key, alias_key, alias_key_bounds},
-    serde_str::Serde,
     store::{Identifier, Store},
 };
 use actix_web::web;
@@ -59,14 +59,6 @@ pub(crate) struct UploadManagerInner {
     db: sled::Db,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
-pub(crate) struct Details {
-    width: usize,
-    height: usize,
-    content_type: Serde<mime::Mime>,
-    created_at: time::OffsetDateTime,
-}
-
 struct FilenameIVec {
     inner: sled::IVec,
 }
@@ -669,62 +661,6 @@ impl UploadManager {
     }
 }
 
-impl Details {
-    fn is_motion(&self) -> bool {
-        self.content_type.type_() == "video"
-            || self.content_type.type_() == "image" && self.content_type.subtype() == "gif"
-    }
-
-    #[tracing::instrument("Details from bytes", skip(input))]
-    pub(crate) async fn from_bytes(
-        input: web::Bytes,
-        hint: Option<ValidInputType>,
-    ) -> Result<Self, Error> {
-        let details = crate::magick::details_bytes(input, hint).await?;
-
-        Ok(Details::now(
-            details.width,
-            details.height,
-            details.mime_type,
-        ))
-    }
-
-    #[tracing::instrument("Details from store")]
-    pub(crate) async fn from_store<S: Store>(
-        store: S,
-        identifier: S::Identifier,
-        expected_format: Option<ValidInputType>,
-    ) -> Result<Self, Error>
-    where
-        Error: From<S::Error>,
-    {
-        let details = crate::magick::details_store(store, identifier, expected_format).await?;
-
-        Ok(Details::now(
-            details.width,
-            details.height,
-            details.mime_type,
-        ))
-    }
-
-    fn now(width: usize, height: usize, content_type: mime::Mime) -> Self {
-        Details {
-            width,
-            height,
-            content_type: Serde::new(content_type),
-            created_at: time::OffsetDateTime::now_utc(),
-        }
-    }
-
-    pub(crate) fn content_type(&self) -> mime::Mime {
-        (*self.content_type).clone()
-    }
-
-    pub(crate) fn system_time(&self) -> std::time::SystemTime {
-        self.created_at.into()
-    }
-}
-
 impl FilenameIVec {
     fn new(inner: sled::IVec) -> Self {
         FilenameIVec { inner }

From 750ce4782e85af1bbe869ed2c055e08498da531f Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Thu, 24 Mar 2022 22:06:29 -0500
Subject: [PATCH 03/37] Rework configuration

---
 Cargo.lock          | 107 +++++++++-------
 Cargo.toml          |   2 +-
 src/config.rs       | 302 +++++++++++++++++++++++++++++++-------------
 src/migrate/repo.rs |   0
 src/repo.rs         |   2 +
 src/serde_str.rs    |   4 +
 6 files changed, 281 insertions(+), 136 deletions(-)
 create mode 100644 src/migrate/repo.rs

diff --git a/Cargo.lock b/Cargo.lock
index 15de7f6..a7a97e8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -470,17 +470,32 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clap"
-version = "2.34.0"
+version = "3.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+checksum = "d8c93436c21e4698bacadf42917db28b23017027a4deccb35dbe47a7e7840123"
 dependencies = [
- "ansi_term",
  "atty",
  "bitflags",
+ "clap_derive",
+ "indexmap",
+ "lazy_static",
+ "os_str_bytes",
  "strsim",
+ "termcolor",
  "textwrap",
- "unicode-width",
- "vec_map",
+]
+
+[[package]]
+name = "clap_derive"
+version = "3.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da95d038ede1a964ce99f49cbe27a7fb538d1da595e4b4f70b8c8f338d17bf16"
+dependencies = [
+ "heck 0.4.0",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
@@ -976,6 +991,12 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "heck"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -1459,6 +1480,15 @@ dependencies = [
  "hashbrown 0.12.0",
 ]
 
+[[package]]
+name = "os_str_bytes"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -1590,6 +1620,7 @@ dependencies = [
  "async-trait",
  "awc",
  "base64",
+ "clap",
  "config",
  "console-subscriber",
  "dashmap",
@@ -1607,7 +1638,6 @@ dependencies = [
  "sha2 0.10.2",
  "sled",
  "storage-path-generator",
- "structopt",
  "thiserror",
  "time",
  "tokio",
@@ -1713,7 +1743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
 dependencies = [
  "bytes",
- "heck",
+ "heck 0.3.3",
  "itertools",
  "lazy_static",
  "log",
@@ -2206,33 +2236,9 @@ dependencies = [
 
 [[package]]
 name = "strsim"
-version = "0.8.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
-
-[[package]]
-name = "structopt"
-version = "0.3.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"
-dependencies = [
- "clap",
- "lazy_static",
- "structopt-derive",
-]
-
-[[package]]
-name = "structopt-derive"
-version = "0.4.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
-dependencies = [
- "heck",
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn",
-]
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "subtle"
@@ -2266,14 +2272,20 @@ dependencies = [
 ]
 
 [[package]]
-name = "textwrap"
-version = "0.11.0"
+name = "termcolor"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
 dependencies = [
- "unicode-width",
+ "winapi-util",
 ]
 
+[[package]]
+name = "textwrap"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
+
 [[package]]
 name = "thiserror"
 version = "1.0.30"
@@ -2711,12 +2723,6 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
 
-[[package]]
-name = "unicode-width"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
-
 [[package]]
 name = "unicode-xid"
 version = "0.2.2"
@@ -2758,12 +2764,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
-[[package]]
-name = "vec_map"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
-
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -2914,6 +2914,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
diff --git a/Cargo.toml b/Cargo.toml
index cd13060..c70255e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -28,6 +28,7 @@ anyhow = "1.0"
 async-trait = "0.1.51"
 awc = { version = "3.0.0", default-features = false, features = ["rustls"] }
 base64 = "0.13.0"
+clap = { version = "3.1.6", features = ["derive"] }
 config = "0.12.0"
 console-subscriber = "0.1"
 dashmap = "5.1.0"
@@ -51,7 +52,6 @@ serde_json = "1.0"
 sha2 = "0.10.0"
 sled = { version = "0.34.7" }
 storage-path-generator = "0.1.0"
-structopt = "0.3.14"
 thiserror = "1.0"
 time = { version = "0.3.0", features = ["serde"] }
 tokio = { version = "1", features = ["full", "tracing"] }
diff --git a/src/config.rs b/src/config.rs
index be1e09c..8ff247f 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,18 +1,19 @@
+use crate::serde_str::Serde;
+use clap::{ArgEnum, Parser, Subcommand};
 use std::{collections::HashSet, net::SocketAddr, path::PathBuf};
-use structopt::StructOpt;
 use url::Url;
 
 use crate::magick::ValidInputType;
 
-#[derive(Clone, Debug, StructOpt)]
+#[derive(Clone, Debug, Parser)]
 pub(crate) struct Args {
-    #[structopt(short, long, help = "Path to the pict-rs configuration file")]
+    #[clap(short, long, help = "Path to the pict-rs configuration file")]
     config_file: Option<PathBuf>,
 
-    #[structopt(long, help = "Path to a file defining a store migration")]
-    migrate_file: Option<PathBuf>,
+    #[clap(subcommand)]
+    command: Command,
 
-    #[structopt(flatten)]
+    #[clap(flatten)]
     overrides: Overrides,
 }
 
@@ -20,10 +21,10 @@ fn is_false(b: &bool) -> bool {
     !b
 }
 
-#[derive(Clone, Debug, serde::Serialize, structopt::StructOpt)]
+#[derive(Clone, Debug, serde::Serialize, Parser)]
 #[serde(rename_all = "snake_case")]
 pub(crate) struct Overrides {
-    #[structopt(
+    #[clap(
         short,
         long,
         help = "Whether to skip validating images uploaded via the internal import API"
@@ -31,15 +32,15 @@ pub(crate) struct Overrides {
     #[serde(skip_serializing_if = "is_false")]
     skip_validate_imports: bool,
 
-    #[structopt(short, long, help = "The address and port the server binds to.")]
+    #[clap(short, long, help = "The address and port the server binds to.")]
     #[serde(skip_serializing_if = "Option::is_none")]
     addr: Option<SocketAddr>,
 
-    #[structopt(short, long, help = "The path to the data directory, e.g. data/")]
+    #[clap(short, long, help = "The path to the data directory, e.g. data/")]
     #[serde(skip_serializing_if = "Option::is_none")]
     path: Option<PathBuf>,
 
-    #[structopt(
+    #[clap(
         short,
         long,
         help = "An optional image format to convert all uploaded files into, supports 'jpg', 'png', and 'webp'"
@@ -47,7 +48,7 @@ pub(crate) struct Overrides {
     #[serde(skip_serializing_if = "Option::is_none")]
     image_format: Option<Format>,
 
-    #[structopt(
+    #[clap(
         short,
         long,
         help = "An optional list of filters to permit, supports 'identity', 'thumbnail', 'resize', 'crop', and 'blur'"
@@ -55,7 +56,7 @@ pub(crate) struct Overrides {
     #[serde(skip_serializing_if = "Option::is_none")]
     filters: Option<Vec<String>>,
 
-    #[structopt(
+    #[clap(
         short,
         long,
         help = "Specify the maximum allowed uploaded file size (in Megabytes)"
@@ -63,40 +64,40 @@ pub(crate) struct Overrides {
     #[serde(skip_serializing_if = "Option::is_none")]
     max_file_size: Option<usize>,
 
-    #[structopt(long, help = "Specify the maximum width in pixels allowed on an image")]
+    #[clap(long, help = "Specify the maximum width in pixels allowed on an image")]
     #[serde(skip_serializing_if = "Option::is_none")]
     max_image_width: Option<usize>,
 
-    #[structopt(long, help = "Specify the maximum width in pixels allowed on an image")]
+    #[clap(long, help = "Specify the maximum width in pixels allowed on an image")]
     #[serde(skip_serializing_if = "Option::is_none")]
     max_image_height: Option<usize>,
 
-    #[structopt(long, help = "Specify the maximum area in pixels allowed in an image")]
+    #[clap(long, help = "Specify the maximum area in pixels allowed in an image")]
     #[serde(skip_serializing_if = "Option::is_none")]
     max_image_area: Option<usize>,
 
-    #[structopt(
+    #[clap(
         long,
         help = "Specify the number of bytes sled is allowed to use for it's cache"
     )]
     #[serde(skip_serializing_if = "Option::is_none")]
     sled_cache_capacity: Option<u64>,
 
-    #[structopt(
+    #[clap(
         long,
         help = "Specify the number of events the console subscriber is allowed to buffer"
     )]
     #[serde(skip_serializing_if = "Option::is_none")]
     console_buffer_capacity: Option<usize>,
 
-    #[structopt(
+    #[clap(
         long,
         help = "An optional string to be checked on requests to privileged endpoints"
     )]
     #[serde(skip_serializing_if = "Option::is_none")]
     api_key: Option<String>,
 
-    #[structopt(
+    #[clap(
         short,
         long,
         help = "Enable OpenTelemetry Tracing exports to the given OpenTelemetry collector"
@@ -104,9 +105,41 @@ pub(crate) struct Overrides {
     #[serde(skip_serializing_if = "Option::is_none")]
     opentelemetry_url: Option<Url>,
 
-    #[structopt(subcommand)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    repo: Option<Repo>,
+
+    #[clap(flatten)]
+    sled_repo: SledRepo,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     store: Option<Store>,
+
+    #[clap(flatten)]
+    filesystem_storage: FilesystemStorage,
+
+    #[clap(flatten)]
+    object_storage: ObjectStorage,
+}
+
+impl ObjectStorage {
+    pub(crate) fn required(&self) -> Result<RequiredObjectStorage, RequiredError> {
+        Ok(RequiredObjectStorage {
+            bucket_name: self
+                .s3_store_bucket_name
+                .as_ref()
+                .cloned()
+                .ok_or(RequiredError)?,
+            region: self
+                .s3_store_region
+                .as_ref()
+                .cloned()
+                .map(Serde::into_inner)
+                .ok_or(RequiredError)?,
+            access_key: self.s3_store_access_key.as_ref().cloned(),
+            security_token: self.s3_store_security_token.as_ref().cloned(),
+            session_token: self.s3_store_session_token.as_ref().cloned(),
+        })
+    }
 }
 
 impl Overrides {
@@ -124,67 +157,99 @@ impl Overrides {
             && self.console_buffer_capacity.is_none()
             && self.api_key.is_none()
             && self.opentelemetry_url.is_none()
+            && self.repo.is_none()
             && self.store.is_none()
     }
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
-#[serde(rename_all = "snake_case")]
-pub(crate) struct Migrate {
-    from: Store,
-    to: Store,
-}
-
-impl Migrate {
-    pub(crate) fn from(&self) -> &Store {
-        &self.from
-    }
-
-    pub(crate) fn to(&self) -> &Store {
-        &self.to
-    }
-}
-
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, structopt::StructOpt)]
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Subcommand)]
 #[serde(rename_all = "snake_case")]
 #[serde(tag = "type")]
+pub(crate) enum Command {
+    Run,
+    MigrateStore { to: Store },
+    MigrateRepo { to: Repo },
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum Repo {
+    Sled,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct SledRepo {
+    // defaults to {config.path}
+    #[clap(long, help = "Path in which pict-rs will create it's 'repo' directory")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    sled_repo_path: Option<PathBuf>,
+
+    #[clap(
+        long,
+        help = "The number of bytes sled is allowed to use for it's in-memory cache"
+    )]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    sled_repo_cache_capacity: Option<u64>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
+#[serde(rename_all = "snake_case")]
 pub(crate) enum Store {
-    FileStore {
-        // defaults to {config.path}
-        #[structopt(
-            long,
-            help = "Path in which pict-rs will create it's 'files' directory"
-        )]
-        #[serde(skip_serializing_if = "Option::is_none")]
-        path: Option<PathBuf>,
-    },
+    Filesystem,
     #[cfg(feature = "object-storage")]
-    S3Store {
-        #[structopt(long, help = "Name of the bucket in which pict-rs will store images")]
-        bucket_name: String,
+    ObjectStorage,
+}
 
-        #[structopt(
-            long,
-            help = "Region in which the bucket exists, can be an http endpoint"
-        )]
-        region: crate::serde_str::Serde<s3::Region>,
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct FilesystemStorage {
+    // defaults to {config.path}
+    #[clap(
+        long,
+        help = "Path in which pict-rs will create it's 'files' directory"
+    )]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    filesystem_storage_path: Option<PathBuf>,
+}
 
-        #[serde(skip_serializing_if = "Option::is_none")]
-        #[structopt(long)]
-        access_key: Option<String>,
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct ObjectStorage {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[clap(long, help = "Name of the bucket in which pict-rs will store images")]
+    s3_store_bucket_name: Option<String>,
 
-        #[structopt(long)]
-        #[serde(skip_serializing_if = "Option::is_none")]
-        secret_key: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[clap(
+        long,
+        help = "Region in which the bucket exists, can be an http endpoint"
+    )]
+    s3_store_region: Option<Serde<s3::Region>>,
 
-        #[structopt(long)]
-        #[serde(skip_serializing_if = "Option::is_none")]
-        security_token: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[clap(long)]
+    s3_store_access_key: Option<String>,
 
-        #[structopt(long)]
-        #[serde(skip_serializing_if = "Option::is_none")]
-        session_token: Option<String>,
-    },
+    #[clap(long)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    s3_store_secret_key: Option<String>,
+
+    #[clap(long)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    s3_store_security_token: Option<String>,
+
+    #[clap(long)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    s3_store_session_token: Option<String>,
+}
+
+pub(crate) struct RequiredObjectStorage {
+    pub(crate) bucket_name: String,
+    pub(crate) region: s3::Region,
+    pub(crate) access_key: Option<String>,
+    pub(crate) security_token: Option<String>,
+    pub(crate) session_token: Option<String>,
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
@@ -203,7 +268,11 @@ pub(crate) struct Config {
     console_buffer_capacity: Option<usize>,
     api_key: Option<String>,
     opentelemetry_url: Option<Url>,
+    repo: Repo,
+    sled_repo: SledRepo,
     store: Store,
+    filesystem_storage: FilesystemStorage,
+    object_storage: ObjectStorage,
 }
 
 #[derive(serde::Serialize)]
@@ -216,9 +285,22 @@ pub(crate) struct Defaults {
     max_image_height: usize,
     max_image_area: usize,
     sled_cache_capacity: u64,
+    repo: Repo,
+    sled_repo: SledRepoDefaults,
     store: Store,
+    filesystem_store: FilesystemDefaults,
 }
 
+#[derive(serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct SledRepoDefaults {
+    sled_repo_cache_capacity: usize,
+}
+
+#[derive(serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct FilesystemDefaults {}
+
 impl Defaults {
     fn new() -> Self {
         Defaults {
@@ -229,23 +311,19 @@ impl Defaults {
             max_image_height: 10_000,
             max_image_area: 40_000_000,
             sled_cache_capacity: 1024 * 1024 * 64, // 16 times smaller than sled's default of 1GB
-            store: Store::FileStore { path: None },
+            repo: Repo::Sled,
+            sled_repo: SledRepoDefaults {
+                sled_repo_cache_capacity: 1024 * 1024 * 64,
+            },
+            store: Store::Filesystem,
+            filesystem_store: FilesystemDefaults {},
         }
     }
 }
 
 impl Config {
     pub(crate) fn build() -> anyhow::Result<Self> {
-        let args = Args::from_args();
-
-        if let Some(path) = args.migrate_file {
-            let migrate_config = config::Config::builder()
-                .add_source(config::File::from(path))
-                .build()?;
-            let migrate: Migrate = migrate_config.try_deserialize()?;
-
-            crate::MIGRATE.set(migrate).unwrap();
-        }
+        let args = Args::parse();
 
         let mut base_config =
             config::Config::builder().add_source(config::Config::try_from(&Defaults::new())?);
@@ -254,6 +332,8 @@ impl Config {
             base_config = base_config.add_source(config::File::from(path));
         };
 
+        // TODO: Command parsing
+
         if !args.overrides.is_default() {
             let merging = config::Config::try_from(&args.overrides)?;
 
@@ -272,6 +352,18 @@ impl Config {
         &self.store
     }
 
+    pub(crate) fn repo(&self) -> &Repo {
+        &self.repo
+    }
+
+    pub(crate) fn object_storage(&self) -> Result<RequiredObjectStorage, RequiredError> {
+        self.object_storage.required()
+    }
+
+    pub(crate) fn filesystem_storage_path(&self) -> Option<&PathBuf> {
+        self.filesystem_storage.filesystem_storage_path.as_ref()
+    }
+
     pub(crate) fn bind_address(&self) -> SocketAddr {
         self.addr
     }
@@ -329,7 +421,19 @@ impl Config {
 #[error("Invalid format supplied, {0}")]
 pub(crate) struct FormatError(String);
 
-#[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Debug, thiserror::Error)]
+#[error("Invalid store supplied, {0}")]
+pub(crate) struct StoreError(String);
+
+#[derive(Debug, thiserror::Error)]
+#[error("Invalid repo supplied, {0}")]
+pub(crate) struct RepoError(String);
+
+#[derive(Debug, thiserror::Error)]
+#[error("Missing required fields")]
+pub(crate) struct RequiredError;
+
+#[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
 #[serde(rename_all = "snake_case")]
 pub(crate) enum Format {
     Jpeg,
@@ -359,11 +463,37 @@ impl std::str::FromStr for Format {
     type Err = FormatError;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "png" => Ok(Format::Png),
-            "jpg" => Ok(Format::Jpeg),
-            "webp" => Ok(Format::Webp),
-            other => Err(FormatError(other.to_string())),
+        for variant in Self::value_variants() {
+            if variant.to_possible_value().unwrap().matches(s, false) {
+                return Ok(*variant);
+            }
         }
+        Err(FormatError(s.into()))
+    }
+}
+
+impl std::str::FromStr for Store {
+    type Err = StoreError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        for variant in Self::value_variants() {
+            if variant.to_possible_value().unwrap().matches(s, false) {
+                return Ok(*variant);
+            }
+        }
+        Err(StoreError(s.into()))
+    }
+}
+
+impl std::str::FromStr for Repo {
+    type Err = RepoError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        for variant in Self::value_variants() {
+            if variant.to_possible_value().unwrap().matches(s, false) {
+                return Ok(*variant);
+            }
+        }
+        Err(RepoError(s.into()))
     }
 }
diff --git a/src/migrate/repo.rs b/src/migrate/repo.rs
new file mode 100644
index 0000000..e69de29
diff --git a/src/repo.rs b/src/repo.rs
index 25e9be3..87d751a 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -8,9 +8,11 @@ pub(crate) struct Alias {
     id: Uuid,
     extension: String,
 }
+
 pub(crate) struct DeleteToken {
     id: Uuid,
 }
+
 pub(crate) struct AlreadyExists;
 
 impl Alias {
diff --git a/src/serde_str.rs b/src/serde_str.rs
index 53c6d1b..c255f5b 100644
--- a/src/serde_str.rs
+++ b/src/serde_str.rs
@@ -12,6 +12,10 @@ impl<T> Serde<T> {
     pub(crate) fn new(inner: T) -> Self {
         Serde { inner }
     }
+
+    pub(crate) fn into_inner(this: Self) -> T {
+        this.inner
+    }
 }
 
 impl<T> Deref for Serde<T> {

From 323016f994bc7bde857898f09cff011db40f1cb1 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Fri, 25 Mar 2022 18:47:50 -0500
Subject: [PATCH 04/37] Build out migration path

---
 Cargo.lock              |  20 +--
 Cargo.toml              |   7 +-
 src/config.rs           | 222 +++++++++++++++++++++----------
 src/error.rs            |   1 -
 src/main.rs             | 135 +++++++++----------
 src/migrate.rs          |  27 ++--
 src/migrate/repo.rs     |   0
 src/migrate/s034.rs     |   8 +-
 src/repo.rs             | 285 ++++++++++++++++++++++++++++++++--------
 src/repo/old.rs         | 171 ++++++++++++++++++++++++
 src/repo/sled.rs        | 134 +++++++++++++++----
 src/store.rs            |   1 -
 src/store/file_store.rs |   2 +-
 13 files changed, 755 insertions(+), 258 deletions(-)
 delete mode 100644 src/migrate/repo.rs
 create mode 100644 src/repo/old.rs

diff --git a/Cargo.lock b/Cargo.lock
index a7a97e8..76fc925 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1064,9 +1064,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "hyper"
-version = "0.14.17"
+version = "0.14.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "043f0e083e9901b6cc658a77d1eb86f4fc650bbb977a4337dd63192826aa85dd"
+checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -1245,9 +1245,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.14"
+version = "0.4.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"
 dependencies = [
  "cfg-if",
 ]
@@ -1820,9 +1820,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c"
+checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0"
 dependencies = [
  "bitflags",
 ]
@@ -2317,9 +2317,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.7"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d"
+checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd"
 dependencies = [
  "itoa",
  "libc",
@@ -2330,9 +2330,9 @@ dependencies = [
 
 [[package]]
 name = "time-macros"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25eb0ca3468fc0acc11828786797f6ef9aa1555e4a211a60d64cc8e4d1be47d6"
+checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
 
 [[package]]
 name = "tinyvec"
diff --git a/Cargo.toml b/Cargo.toml
index c70255e..478f4d1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,8 +10,7 @@ edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [features]
-default = ["object-storage"]
-object-storage = ["reqwest", "rust-s3"]
+default = []
 io-uring = [
   "actix-rt/io-uring",
   "actix-server/io-uring",
@@ -42,11 +41,11 @@ pin-project-lite = "0.2.7"
 reqwest = { version = "0.11.5", default-features = false, features = [
   "rustls-tls",
   "stream",
-], optional = true }
+] }
 rust-s3 = { version = "0.29.0", default-features = false, features = [
   "fail-on-err",
   "with-reqwest",
-], optional = true, git = "https://github.com/asonix/rust-s3", branch = "asonix/generic-client" }
+], git = "https://github.com/asonix/rust-s3", branch = "asonix/generic-client" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sha2 = "0.10.0"
diff --git a/src/config.rs b/src/config.rs
index 8ff247f..1c68b96 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -76,13 +76,6 @@ pub(crate) struct Overrides {
     #[serde(skip_serializing_if = "Option::is_none")]
     max_image_area: Option<usize>,
 
-    #[clap(
-        long,
-        help = "Specify the number of bytes sled is allowed to use for it's cache"
-    )]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    sled_cache_capacity: Option<u64>,
-
     #[clap(
         long,
         help = "Specify the number of events the console subscriber is allowed to buffer"
@@ -106,12 +99,22 @@ pub(crate) struct Overrides {
     opentelemetry_url: Option<Url>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[clap(
+        short = 'R',
+        long,
+        help = "Set the database implementation. Available options are 'sled'. Default is 'sled'"
+    )]
     repo: Option<Repo>,
 
     #[clap(flatten)]
-    sled_repo: SledRepo,
+    sled: Sled,
 
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[clap(
+        short = 'S',
+        long,
+        help = "Set the image store. Available options are 'object-storage' or 'filesystem'. Default is 'filesystem'"
+    )]
     store: Option<Store>,
 
     #[clap(flatten)]
@@ -125,19 +128,20 @@ impl ObjectStorage {
     pub(crate) fn required(&self) -> Result<RequiredObjectStorage, RequiredError> {
         Ok(RequiredObjectStorage {
             bucket_name: self
-                .s3_store_bucket_name
+                .object_store_bucket_name
                 .as_ref()
                 .cloned()
-                .ok_or(RequiredError)?,
+                .ok_or(RequiredError("object-store-bucket-name"))?,
             region: self
-                .s3_store_region
+                .object_store_region
                 .as_ref()
                 .cloned()
                 .map(Serde::into_inner)
-                .ok_or(RequiredError)?,
-            access_key: self.s3_store_access_key.as_ref().cloned(),
-            security_token: self.s3_store_security_token.as_ref().cloned(),
-            session_token: self.s3_store_session_token.as_ref().cloned(),
+                .ok_or(RequiredError("object-store-region"))?,
+            access_key: self.object_store_access_key.as_ref().cloned(),
+            secret_key: self.object_store_secret_key.as_ref().cloned(),
+            security_token: self.object_store_security_token.as_ref().cloned(),
+            session_token: self.object_store_session_token.as_ref().cloned(),
         })
     }
 }
@@ -153,7 +157,6 @@ impl Overrides {
             && self.max_image_width.is_none()
             && self.max_image_height.is_none()
             && self.max_image_area.is_none()
-            && self.sled_cache_capacity.is_none()
             && self.console_buffer_capacity.is_none()
             && self.api_key.is_none()
             && self.opentelemetry_url.is_none()
@@ -171,37 +174,47 @@ pub(crate) enum Command {
     MigrateRepo { to: Repo },
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
+pub(crate) enum CommandConfig {
+    Run,
+    MigrateStore {
+        to: Storage,
+    },
+    MigrateRepo {
+        #[allow(dead_code)]
+        to: Repository,
+    },
+}
+
+#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
 #[serde(rename_all = "snake_case")]
 pub(crate) enum Repo {
     Sled,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)]
+#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize, Parser)]
 #[serde(rename_all = "snake_case")]
-pub(crate) struct SledRepo {
+pub(crate) struct Sled {
     // defaults to {config.path}
     #[clap(long, help = "Path in which pict-rs will create it's 'repo' directory")]
     #[serde(skip_serializing_if = "Option::is_none")]
-    sled_repo_path: Option<PathBuf>,
+    pub(crate) sled_path: Option<PathBuf>,
 
     #[clap(
         long,
         help = "The number of bytes sled is allowed to use for it's in-memory cache"
     )]
     #[serde(skip_serializing_if = "Option::is_none")]
-    sled_repo_cache_capacity: Option<u64>,
+    pub(crate) sled_cache_capacity: Option<u64>,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
+#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
 #[serde(rename_all = "snake_case")]
 pub(crate) enum Store {
     Filesystem,
-    #[cfg(feature = "object-storage")]
     ObjectStorage,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)]
+#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize, Parser)]
 #[serde(rename_all = "snake_case")]
 pub(crate) struct FilesystemStorage {
     // defaults to {config.path}
@@ -210,51 +223,71 @@ pub(crate) struct FilesystemStorage {
         help = "Path in which pict-rs will create it's 'files' directory"
     )]
     #[serde(skip_serializing_if = "Option::is_none")]
-    filesystem_storage_path: Option<PathBuf>,
+    pub(crate) filesystem_storage_path: Option<PathBuf>,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)]
+#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize, Parser)]
 #[serde(rename_all = "snake_case")]
 pub(crate) struct ObjectStorage {
     #[serde(skip_serializing_if = "Option::is_none")]
     #[clap(long, help = "Name of the bucket in which pict-rs will store images")]
-    s3_store_bucket_name: Option<String>,
+    object_store_bucket_name: Option<String>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     #[clap(
         long,
         help = "Region in which the bucket exists, can be an http endpoint"
     )]
-    s3_store_region: Option<Serde<s3::Region>>,
+    object_store_region: Option<Serde<s3::Region>>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     #[clap(long)]
-    s3_store_access_key: Option<String>,
+    object_store_access_key: Option<String>,
 
     #[clap(long)]
     #[serde(skip_serializing_if = "Option::is_none")]
-    s3_store_secret_key: Option<String>,
+    object_store_secret_key: Option<String>,
 
     #[clap(long)]
     #[serde(skip_serializing_if = "Option::is_none")]
-    s3_store_security_token: Option<String>,
+    object_store_security_token: Option<String>,
 
     #[clap(long)]
     #[serde(skip_serializing_if = "Option::is_none")]
-    s3_store_session_token: Option<String>,
+    object_store_session_token: Option<String>,
+}
+
+pub(crate) struct RequiredSledRepo {
+    pub(crate) path: PathBuf,
+    pub(crate) cache_capacity: u64,
 }
 
 pub(crate) struct RequiredObjectStorage {
     pub(crate) bucket_name: String,
     pub(crate) region: s3::Region,
     pub(crate) access_key: Option<String>,
+    pub(crate) secret_key: Option<String>,
     pub(crate) security_token: Option<String>,
     pub(crate) session_token: Option<String>,
 }
 
+pub(crate) struct RequiredFilesystemStorage {
+    pub(crate) path: PathBuf,
+}
+
+pub(crate) enum Storage {
+    ObjectStorage(RequiredObjectStorage),
+    Filesystem(RequiredFilesystemStorage),
+}
+
+pub(crate) enum Repository {
+    Sled(RequiredSledRepo),
+}
+
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
 #[serde(rename_all = "snake_case")]
 pub(crate) struct Config {
+    command: Command,
     skip_validate_imports: bool,
     addr: SocketAddr,
     path: PathBuf,
@@ -264,59 +297,52 @@ pub(crate) struct Config {
     max_image_width: usize,
     max_image_height: usize,
     max_image_area: usize,
-    sled_cache_capacity: u64,
     console_buffer_capacity: Option<usize>,
     api_key: Option<String>,
     opentelemetry_url: Option<Url>,
     repo: Repo,
-    sled_repo: SledRepo,
+    sled: Option<Sled>,
     store: Store,
-    filesystem_storage: FilesystemStorage,
-    object_storage: ObjectStorage,
+    filesystem_storage: Option<FilesystemStorage>,
+    object_storage: Option<ObjectStorage>,
 }
 
 #[derive(serde::Serialize)]
 #[serde(rename_all = "snake_case")]
 pub(crate) struct Defaults {
+    command: Command,
     skip_validate_imports: bool,
     addr: SocketAddr,
     max_file_size: usize,
     max_image_width: usize,
     max_image_height: usize,
     max_image_area: usize,
-    sled_cache_capacity: u64,
     repo: Repo,
-    sled_repo: SledRepoDefaults,
+    sled: SledDefaults,
     store: Store,
-    filesystem_store: FilesystemDefaults,
 }
 
 #[derive(serde::Serialize)]
 #[serde(rename_all = "snake_case")]
-struct SledRepoDefaults {
-    sled_repo_cache_capacity: usize,
+struct SledDefaults {
+    sled_cache_capacity: usize,
 }
 
-#[derive(serde::Serialize)]
-#[serde(rename_all = "snake_case")]
-struct FilesystemDefaults {}
-
 impl Defaults {
     fn new() -> Self {
         Defaults {
+            command: Command::Run,
             skip_validate_imports: false,
             addr: ([0, 0, 0, 0], 8080).into(),
             max_file_size: 40,
             max_image_width: 10_000,
             max_image_height: 10_000,
             max_image_area: 40_000_000,
-            sled_cache_capacity: 1024 * 1024 * 64, // 16 times smaller than sled's default of 1GB
             repo: Repo::Sled,
-            sled_repo: SledRepoDefaults {
-                sled_repo_cache_capacity: 1024 * 1024 * 64,
+            sled: SledDefaults {
+                sled_cache_capacity: 1024 * 1024 * 64,
             },
             store: Store::Filesystem,
-            filesystem_store: FilesystemDefaults {},
         }
     }
 }
@@ -332,8 +358,6 @@ impl Config {
             base_config = base_config.add_source(config::File::from(path));
         };
 
-        // TODO: Command parsing
-
         if !args.overrides.is_default() {
             let merging = config::Config::try_from(&args.overrides)?;
 
@@ -348,20 +372,88 @@ impl Config {
         Ok(config)
     }
 
-    pub(crate) fn store(&self) -> &Store {
-        &self.store
+    pub(crate) fn command(&self) -> anyhow::Result<CommandConfig> {
+        Ok(match &self.command {
+            Command::Run => CommandConfig::Run,
+            Command::MigrateStore { to } => CommandConfig::MigrateStore {
+                to: match to {
+                    Store::ObjectStorage => Storage::ObjectStorage(
+                        self.object_storage
+                            .as_ref()
+                            .cloned()
+                            .unwrap_or_default()
+                            .required()?,
+                    ),
+                    Store::Filesystem => Storage::Filesystem(RequiredFilesystemStorage {
+                        path: self
+                            .filesystem_storage
+                            .as_ref()
+                            .and_then(|f| f.filesystem_storage_path.clone())
+                            .unwrap_or_else(|| {
+                                let mut path = self.path.clone();
+                                path.push("files");
+                                path
+                            }),
+                    }),
+                },
+            },
+            Command::MigrateRepo { to } => CommandConfig::MigrateRepo {
+                to: match to {
+                    Repo::Sled => {
+                        let sled = self.sled.as_ref().cloned().unwrap_or_default();
+
+                        Repository::Sled(RequiredSledRepo {
+                            path: sled.sled_path.unwrap_or_else(|| {
+                                let mut path = self.path.clone();
+                                path.push("sled-repo");
+                                path
+                            }),
+                            cache_capacity: sled.sled_cache_capacity.unwrap_or(1024 * 1024 * 64),
+                        })
+                    }
+                },
+            },
+        })
     }
 
-    pub(crate) fn repo(&self) -> &Repo {
-        &self.repo
+    pub(crate) fn store(&self) -> anyhow::Result<Storage> {
+        Ok(match self.store {
+            Store::Filesystem => Storage::Filesystem(RequiredFilesystemStorage {
+                path: self
+                    .filesystem_storage
+                    .as_ref()
+                    .and_then(|f| f.filesystem_storage_path.clone())
+                    .unwrap_or_else(|| {
+                        let mut path = self.path.clone();
+                        path.push("files");
+                        path
+                    }),
+            }),
+            Store::ObjectStorage => Storage::ObjectStorage(
+                self.object_storage
+                    .as_ref()
+                    .cloned()
+                    .unwrap_or_default()
+                    .required()?,
+            ),
+        })
     }
 
-    pub(crate) fn object_storage(&self) -> Result<RequiredObjectStorage, RequiredError> {
-        self.object_storage.required()
-    }
+    pub(crate) fn repo(&self) -> Repository {
+        match self.repo {
+            Repo::Sled => {
+                let sled = self.sled.as_ref().cloned().unwrap_or_default();
 
-    pub(crate) fn filesystem_storage_path(&self) -> Option<&PathBuf> {
-        self.filesystem_storage.filesystem_storage_path.as_ref()
+                Repository::Sled(RequiredSledRepo {
+                    path: sled.sled_path.unwrap_or_else(|| {
+                        let mut path = self.path.clone();
+                        path.push("sled-repo");
+                        path
+                    }),
+                    cache_capacity: sled.sled_cache_capacity.unwrap_or(1024 * 1024 * 64),
+                })
+            }
+        }
     }
 
     pub(crate) fn bind_address(&self) -> SocketAddr {
@@ -372,10 +464,6 @@ impl Config {
         self.path.clone()
     }
 
-    pub(crate) fn sled_cache_capacity(&self) -> u64 {
-        self.sled_cache_capacity
-    }
-
     pub(crate) fn console_buffer_capacity(&self) -> Option<usize> {
         self.console_buffer_capacity
     }
@@ -430,10 +518,10 @@ pub(crate) struct StoreError(String);
 pub(crate) struct RepoError(String);
 
 #[derive(Debug, thiserror::Error)]
-#[error("Missing required fields")]
-pub(crate) struct RequiredError;
+#[error("Missing required {0} field")]
+pub(crate) struct RequiredError(&'static str);
 
-#[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
+#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
 #[serde(rename_all = "snake_case")]
 pub(crate) enum Format {
     Jpeg,
diff --git a/src/error.rs b/src/error.rs
index deec103..7e02205 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -69,7 +69,6 @@ pub(crate) enum UploadError {
     #[error(transparent)]
     FileStore(#[from] crate::store::file_store::FileError),
 
-    #[cfg(feature = "object-storage")]
     #[error(transparent)]
     ObjectStore(#[from] crate::store::object_store::ObjectError),
 
diff --git a/src/main.rs b/src/main.rs
index 6013799..305df02 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -9,7 +9,7 @@ use futures_util::{
     stream::{empty, once},
     Stream,
 };
-use once_cell::sync::{Lazy, OnceCell};
+use once_cell::sync::Lazy;
 use std::{
     collections::HashSet,
     future::ready,
@@ -47,18 +47,17 @@ mod tmp_file;
 mod upload_manager;
 mod validate;
 
-use crate::{magick::details_hint, store::file_store::FileStore};
-
 use self::{
     concurrent_processor::CancelSafeProcessor,
-    config::{Config, Format, Migrate},
+    config::{CommandConfig, Config, Format, RequiredFilesystemStorage, RequiredObjectStorage},
     details::Details,
     either::Either,
     error::{Error, UploadError},
     init_tracing::init_tracing,
+    magick::details_hint,
     middleware::{Deadline, Internal},
     migrate::LatestDb,
-    store::Store,
+    store::{file_store::FileStore, object_store::ObjectStore, Store},
     upload_manager::{UploadManager, UploadManagerSession},
 };
 
@@ -67,7 +66,6 @@ const MINUTES: u32 = 60;
 const HOURS: u32 = 60 * MINUTES;
 const DAYS: u32 = 24 * HOURS;
 
-static MIGRATE: OnceCell<Migrate> = OnceCell::new();
 static CONFIG: Lazy<Config> = Lazy::new(|| Config::build().unwrap());
 static PROCESS_SEMAPHORE: Lazy<Semaphore> =
     Lazy::new(|| Semaphore::new(num_cpus::get().saturating_sub(1).max(1)));
@@ -694,7 +692,6 @@ fn build_client() -> awc::Client {
         .finish()
 }
 
-#[cfg(feature = "object-storage")]
 fn build_reqwest_client() -> reqwest::Result<reqwest::Client> {
     reqwest::Client::builder()
         .user_agent("pict-rs v0.3.0-main")
@@ -839,35 +836,30 @@ async fn migrate_inner<S1>(
     manager: &UploadManager,
     db: &sled::Db,
     from: S1,
-    to: &config::Store,
+    to: &config::Storage,
 ) -> anyhow::Result<()>
 where
     S1: Store,
     Error: From<S1::Error>,
 {
     match to {
-        config::Store::FileStore { path } => {
-            let path = path.to_owned().unwrap_or_else(|| CONFIG.data_dir());
-
-            let to = FileStore::build(path, db)?;
+        config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
+            let to = FileStore::build(path.clone(), db)?;
             manager.restructure(&to).await?;
 
             manager.migrate_store::<S1, FileStore>(from, to).await?;
         }
-        #[cfg(feature = "object-storage")]
-        config::Store::S3Store {
+        config::Storage::ObjectStorage(RequiredObjectStorage {
             bucket_name,
             region,
             access_key,
             secret_key,
             security_token,
             session_token,
-        } => {
-            use store::object_store::ObjectStore;
-
+        }) => {
             let to = ObjectStore::build(
                 bucket_name,
-                (**region).clone(),
+                region.clone(),
                 access_key.clone(),
                 secret_key.clone(),
                 security_token.clone(),
@@ -891,75 +883,78 @@ async fn main() -> anyhow::Result<()> {
         CONFIG.console_buffer_capacity(),
     )?;
 
-    let db = LatestDb::exists(CONFIG.data_dir(), CONFIG.sled_cache_capacity()).migrate()?;
+    let db = LatestDb::exists(CONFIG.data_dir()).migrate()?;
+
+    let repo = self::repo::Repo::open(CONFIG.repo())?;
+
+    repo.from_db(db).await?;
 
     let manager = UploadManager::new(db.clone(), CONFIG.format()).await?;
 
-    if let Some(m) = MIGRATE.get() {
-        let from = m.from();
-        let to = m.to();
-
-        match from {
-            config::Store::FileStore { path } => {
-                let path = path.to_owned().unwrap_or_else(|| CONFIG.data_dir());
-
-                let from = FileStore::build(path, &db)?;
-                manager.restructure(&from).await?;
-
-                migrate_inner(&manager, &db, from, to).await?;
-            }
-            #[cfg(feature = "object-storage")]
-            config::Store::S3Store {
-                bucket_name,
-                region,
-                access_key,
-                secret_key,
-                security_token,
-                session_token,
-            } => {
-                let from = crate::store::object_store::ObjectStore::build(
-                    bucket_name,
-                    (**region).clone(),
-                    access_key.clone(),
-                    secret_key.clone(),
-                    security_token.clone(),
-                    session_token.clone(),
-                    &db,
-                    build_reqwest_client()?,
-                )?;
-
-                migrate_inner(&manager, &db, from, to).await?;
-            }
+    match CONFIG.command()? {
+        CommandConfig::Run => (),
+        CommandConfig::MigrateRepo { to: _ } => {
+            unimplemented!("Repo migrations are currently unsupported")
         }
+        CommandConfig::MigrateStore { to } => {
+            let from = CONFIG.store()?;
 
-        return Ok(());
+            match from {
+                config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
+                    let from = FileStore::build(path.clone(), &db)?;
+                    manager.restructure(&from).await?;
+
+                    migrate_inner(&manager, &db, from, &to).await?;
+                }
+                config::Storage::ObjectStorage(RequiredObjectStorage {
+                    bucket_name,
+                    region,
+                    access_key,
+                    secret_key,
+                    security_token,
+                    session_token,
+                }) => {
+                    let from = ObjectStore::build(
+                        &bucket_name,
+                        region,
+                        access_key,
+                        secret_key,
+                        security_token,
+                        session_token,
+                        &db,
+                        build_reqwest_client()?,
+                    )?;
+
+                    migrate_inner(&manager, &db, from, &to).await?;
+                }
+            }
+
+            return Ok(());
+        }
     }
 
-    match CONFIG.store() {
-        config::Store::FileStore { path } => {
-            let path = path.to_owned().unwrap_or_else(|| CONFIG.data_dir());
-
-            let store = FileStore::build(path.clone(), &db)?;
+    match CONFIG.store()? {
+        config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
+            let store = FileStore::build(path, &db)?;
             manager.restructure(&store).await?;
 
             launch(manager, store).await
         }
-        #[cfg(feature = "object-storage")]
-        config::Store::S3Store {
+        config::Storage::ObjectStorage(RequiredObjectStorage {
             bucket_name,
             region,
             access_key,
             secret_key,
             security_token,
             session_token,
-        } => {
-            let store = crate::store::object_store::ObjectStore::build(
-                bucket_name,
-                (**region).clone(),
-                access_key.clone(),
-                secret_key.clone(),
-                security_token.clone(),
-                session_token.clone(),
+        }) => {
+            let store = ObjectStore::build(
+                &bucket_name,
+                region,
+                access_key,
+                secret_key,
+                security_token,
+                session_token,
                 &db,
                 build_reqwest_client()?,
             )?;
diff --git a/src/migrate.rs b/src/migrate.rs
index 90a391d..8ac65d6 100644
--- a/src/migrate.rs
+++ b/src/migrate.rs
@@ -51,30 +51,21 @@ trait SledTree {
 pub(crate) struct LatestDb {
     root_dir: PathBuf,
     version: DbVersion,
-    cache_capacity: u64,
 }
 
 impl LatestDb {
-    pub(crate) fn exists(root_dir: PathBuf, cache_capacity: u64) -> Self {
-        let version = DbVersion::exists(root_dir.clone(), cache_capacity);
+    pub(crate) fn exists(root_dir: PathBuf) -> Self {
+        let version = DbVersion::exists(root_dir.clone());
 
-        LatestDb {
-            root_dir,
-            version,
-            cache_capacity,
-        }
+        LatestDb { root_dir, version }
     }
 
     pub(crate) fn migrate(self) -> Result<sled::Db, UploadError> {
-        let LatestDb {
-            root_dir,
-            version,
-            cache_capacity,
-        } = self;
+        let LatestDb { root_dir, version } = self;
 
         loop {
             let root_dir2 = root_dir.clone();
-            let res = std::panic::catch_unwind(move || version.migrate(root_dir2, cache_capacity));
+            let res = std::panic::catch_unwind(move || version.migrate(root_dir2));
 
             if let Ok(res) = res {
                 return res;
@@ -90,17 +81,17 @@ enum DbVersion {
 }
 
 impl DbVersion {
-    fn exists(root: PathBuf, cache_capacity: u64) -> Self {
-        if s034::exists(root.clone()) && !s034::migrating(root, cache_capacity) {
+    fn exists(root: PathBuf) -> Self {
+        if s034::exists(root.clone()) && !s034::migrating(root) {
             return DbVersion::Sled034;
         }
 
         DbVersion::Fresh
     }
 
-    fn migrate(self, root: PathBuf, cache_capacity: u64) -> Result<sled::Db, UploadError> {
+    fn migrate(self, root: PathBuf) -> Result<sled::Db, UploadError> {
         match self {
-            DbVersion::Sled034 | DbVersion::Fresh => s034::open(root, cache_capacity),
+            DbVersion::Sled034 | DbVersion::Fresh => s034::open(root),
         }
     }
 }
diff --git a/src/migrate/repo.rs b/src/migrate/repo.rs
deleted file mode 100644
index e69de29..0000000
diff --git a/src/migrate/s034.rs b/src/migrate/s034.rs
index 5638b46..5bcad24 100644
--- a/src/migrate/s034.rs
+++ b/src/migrate/s034.rs
@@ -14,8 +14,8 @@ pub(crate) fn exists(mut base: PathBuf) -> bool {
     std::fs::metadata(base).is_ok()
 }
 
-pub(crate) fn migrating(base: PathBuf, cache_capacity: u64) -> bool {
-    if let Ok(db) = open(base, cache_capacity) {
+pub(crate) fn migrating(base: PathBuf) -> bool {
+    if let Ok(db) = open(base) {
         if let Ok(tree) = db.open_tree("migrate") {
             if let Ok(Some(_)) = tree.get("done") {
                 return false;
@@ -26,12 +26,12 @@ pub(crate) fn migrating(base: PathBuf, cache_capacity: u64) -> bool {
     true
 }
 
-pub(crate) fn open(mut base: PathBuf, cache_capacity: u64) -> Result<sled034::Db, UploadError> {
+pub(crate) fn open(mut base: PathBuf) -> Result<sled034::Db, UploadError> {
     base.push("sled");
     base.push(SLED_034);
 
     let db = sled034::Config::default()
-        .cache_capacity(cache_capacity)
+        .cache_capacity(1024 * 1024 * 64)
         .path(base)
         .open()?;
 
diff --git a/src/repo.rs b/src/repo.rs
index 87d751a..13d1b78 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -1,9 +1,17 @@
-use crate::{details::Details, store::Identifier};
+use crate::config::RequiredSledRepo;
+use crate::{config::Repository, details::Details, store::Identifier};
 use futures_util::Stream;
 use uuid::Uuid;
 
+mod old;
 pub(crate) mod sled;
 
+#[derive(Debug)]
+pub(crate) enum Repo {
+    Sled(self::sled::SledRepo),
+}
+
+#[derive(Clone, Debug)]
 pub(crate) struct Alias {
     id: Uuid,
     extension: String,
@@ -15,6 +23,201 @@ pub(crate) struct DeleteToken {
 
 pub(crate) struct AlreadyExists;
 
+#[async_trait::async_trait]
+pub(crate) trait SettingsRepo {
+    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
+    type Error: std::error::Error;
+
+    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Self::Error>;
+    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Self::Error>;
+    async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error>;
+}
+
+#[async_trait::async_trait]
+pub(crate) trait IdentifierRepo<I: Identifier> {
+    type Error: std::error::Error;
+
+    async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error>;
+    async fn details(&self, identifier: I) -> Result<Option<Details>, Self::Error>;
+
+    async fn cleanup(&self, identifier: I) -> Result<(), Self::Error>;
+}
+
+#[async_trait::async_trait]
+pub(crate) trait HashRepo<I: Identifier> {
+    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
+    type Error: std::error::Error;
+    type Stream: Stream<Item = Result<Self::Bytes, Self::Error>>;
+
+    async fn hashes(&self) -> Self::Stream;
+
+    async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Self::Error>;
+
+    async fn relate_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
+    async fn remove_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
+    async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Self::Error>;
+
+    async fn relate_identifier(&self, hash: Self::Bytes, identifier: I) -> Result<(), Self::Error>;
+    async fn identifier(&self, hash: Self::Bytes) -> Result<I, Self::Error>;
+
+    async fn relate_variant_identifier(
+        &self,
+        hash: Self::Bytes,
+        variant: String,
+        identifier: I,
+    ) -> Result<(), Self::Error>;
+    async fn variant_identifier(
+        &self,
+        hash: Self::Bytes,
+        variant: String,
+    ) -> Result<Option<I>, Self::Error>;
+
+    async fn relate_motion_identifier(
+        &self,
+        hash: Self::Bytes,
+        identifier: I,
+    ) -> Result<(), Self::Error>;
+    async fn motion_identifier(&self, hash: Self::Bytes) -> Result<Option<I>, Self::Error>;
+
+    async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error>;
+}
+
+#[async_trait::async_trait]
+pub(crate) trait AliasRepo {
+    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
+    type Error: std::error::Error;
+
+    async fn create(&self, alias: Alias) -> Result<Result<(), AlreadyExists>, Self::Error>;
+
+    async fn relate_delete_token(
+        &self,
+        alias: Alias,
+        delete_token: DeleteToken,
+    ) -> Result<Result<(), AlreadyExists>, Self::Error>;
+    async fn delete_token(&self, alias: Alias) -> Result<DeleteToken, Self::Error>;
+
+    async fn relate_hash(&self, alias: Alias, hash: Self::Bytes) -> Result<(), Self::Error>;
+    async fn hash(&self, alias: Alias) -> Result<Self::Bytes, Self::Error>;
+
+    async fn cleanup(&self, alias: Alias) -> Result<(), Self::Error>;
+}
+
+impl Repo {
+    pub(crate) fn open(config: Repository) -> anyhow::Result<Self> {
+        match config {
+            Repository::Sled(RequiredSledRepo {
+                mut path,
+                cache_capacity,
+            }) => {
+                path.push("v0.4.0-alpha.1");
+
+                let db = ::sled::Config::new()
+                    .cache_capacity(cache_capacity)
+                    .path(path)
+                    .open()?;
+
+                Ok(Self::Sled(self::sled::SledRepo::new(db)?))
+            }
+        }
+    }
+
+    #[tracing::instrument(skip_all)]
+    pub(crate) async fn from_db(&self, db: ::sled::Db) -> anyhow::Result<()> {
+        if self.has_migrated().await? {
+            return Ok(());
+        }
+
+        let old = self::old::Old::open(db)?;
+
+        for hash in old.hashes() {
+            match self {
+                Self::Sled(repo) => {
+                    if let Err(e) = migrate_hash(repo, &old, hash).await {
+                        tracing::error!("Failed to migrate hash: {}", e);
+                    }
+                }
+            }
+        }
+
+        self.mark_migrated().await?;
+
+        Ok(())
+    }
+
+    async fn has_migrated(&self) -> anyhow::Result<bool> {
+        match self {
+            Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_O1).await?.is_some()),
+        }
+    }
+
+    async fn mark_migrated(&self) -> anyhow::Result<()> {
+        match self {
+            Self::Sled(repo) => {
+                repo.set(REPO_MIGRATION_O1, b"1".to_vec().into()).await?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+const REPO_MIGRATION_O1: &[u8] = b"repo-migration-01";
+const STORE_MIGRATION_PROGRESS: &[u8] = b"store-migration-progress";
+const GENERATOR_KEY: &[u8] = b"last-path";
+
+async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> anyhow::Result<()>
+where
+    T: IdentifierRepo<::sled::IVec>,
+    <T as IdentifierRepo<::sled::IVec>>::Error: Send + Sync + 'static,
+    T: HashRepo<::sled::IVec>,
+    <T as HashRepo<::sled::IVec>>::Error: Send + Sync + 'static,
+    T: AliasRepo,
+    <T as AliasRepo>::Error: Send + Sync + 'static,
+    T: SettingsRepo,
+    <T as SettingsRepo>::Error: Send + Sync + 'static,
+{
+    HashRepo::create(repo, hash.to_vec().into()).await?;
+
+    let main_ident = old.main_identifier(&hash)?;
+
+    HashRepo::relate_identifier(repo, hash.to_vec().into(), main_ident.clone()).await?;
+
+    for alias in old.aliases(&hash) {
+        if let Ok(Ok(())) = AliasRepo::create(repo, alias.clone()).await {
+            let _ = HashRepo::relate_alias(repo, hash.to_vec().into(), alias.clone()).await;
+            let _ = AliasRepo::relate_hash(repo, alias.clone(), hash.to_vec().into()).await;
+
+            if let Ok(Some(delete_token)) = old.delete_token(&alias) {
+                let _ = AliasRepo::relate_delete_token(repo, alias, delete_token).await;
+            }
+        }
+    }
+
+    if let Ok(Some(identifier)) = old.motion_identifier(&hash) {
+        HashRepo::relate_motion_identifier(repo, hash.to_vec().into(), identifier).await;
+    }
+
+    for (variant, identifier) in old.variants(&hash)? {
+        let _ =
+            HashRepo::relate_variant_identifier(repo, hash.to_vec().into(), variant, identifier)
+                .await;
+    }
+
+    for (identifier, details) in old.details(&hash)? {
+        let _ = IdentifierRepo::relate_details(repo, identifier, details).await;
+    }
+
+    if let Ok(Some(value)) = old.setting(STORE_MIGRATION_PROGRESS) {
+        SettingsRepo::set(repo, STORE_MIGRATION_PROGRESS, value.to_vec().into()).await?;
+    }
+
+    if let Ok(Some(value)) = old.setting(GENERATOR_KEY) {
+        SettingsRepo::set(repo, GENERATOR_KEY, value.to_vec().into()).await?;
+    }
+
+    Ok(())
+}
+
 impl Alias {
     fn to_bytes(&self) -> Vec<u8> {
         let mut v = self.id.as_bytes().to_vec();
@@ -48,66 +251,38 @@ impl DeleteToken {
     }
 }
 
-#[async_trait::async_trait]
-pub(crate) trait SettingsRepo {
-    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error;
-
-    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Self::Error>;
-    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Self::Error>;
-    async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error>;
+impl std::fmt::Display for Alias {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}{}", self.id, self.extension)
+    }
 }
 
-#[async_trait::async_trait]
-pub(crate) trait IdentifierRepo<I: Identifier> {
-    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error;
+impl Identifier for Vec<u8> {
+    type Error = std::convert::Infallible;
 
-    async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error>;
-    async fn details(&self, identifier: I) -> Result<Option<Details>, Self::Error>;
+    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error>
+    where
+        Self: Sized,
+    {
+        Ok(bytes)
+    }
 
-    async fn relate_hash(&self, identifier: I, hash: Self::Bytes) -> Result<(), Self::Error>;
-    async fn hash(&self, identifier: I) -> Result<Self::Bytes, Self::Error>;
-
-    async fn cleanup(&self, identifier: I) -> Result<(), Self::Error>;
+    fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> {
+        Ok(self.clone())
+    }
 }
 
-#[async_trait::async_trait]
-pub(crate) trait HashRepo<I: Identifier> {
-    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error;
-    type Stream: Stream<Item = Result<Self::Bytes, Self::Error>>;
+impl Identifier for ::sled::IVec {
+    type Error = std::convert::Infallible;
 
-    async fn hashes(&self) -> Self::Stream;
+    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error>
+    where
+        Self: Sized,
+    {
+        Ok(bytes.into())
+    }
 
-    async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Self::Error>;
-
-    async fn relate_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
-    async fn remove_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
-    async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Self::Error>;
-
-    async fn relate_identifier(&self, hash: Self::Bytes, identifier: I) -> Result<(), Self::Error>;
-    async fn identifier(&self, hash: Self::Bytes) -> Result<I, Self::Error>;
-
-    async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error>;
-}
-
-#[async_trait::async_trait]
-pub(crate) trait AliasRepo {
-    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error;
-
-    async fn create(&self, alias: Alias) -> Result<Result<(), AlreadyExists>, Self::Error>;
-
-    async fn relate_delete_token(
-        &self,
-        alias: Alias,
-        delete_token: DeleteToken,
-    ) -> Result<Result<(), AlreadyExists>, Self::Error>;
-    async fn delete_token(&self, alias: Alias) -> Result<DeleteToken, Self::Error>;
-
-    async fn relate_hash(&self, alias: Alias, hash: Self::Bytes) -> Result<(), Self::Error>;
-    async fn hash(&self, alias: Alias) -> Result<Self::Bytes, Self::Error>;
-
-    async fn cleanup(&self, alias: Alias) -> Result<(), Self::Error>;
+    fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> {
+        Ok(self.to_vec())
+    }
 }
diff --git a/src/repo/old.rs b/src/repo/old.rs
new file mode 100644
index 0000000..a61b402
--- /dev/null
+++ b/src/repo/old.rs
@@ -0,0 +1,171 @@
+// TREE STRUCTURE
+// - Alias Tree
+//   - alias -> hash
+//   - alias / id -> u64(id)
+//   - alias / delete -> delete token
+// - Main Tree
+//   - hash -> filename
+//   - hash 0 u64(id) -> alias
+// - Filename Tree
+//   - filename -> hash
+// - Details Tree
+//   - filename / S::Identifier -> details
+// - Identifier Tree
+//   - filename -> S::Identifier
+//   - filename / variant path -> S::Identifier
+//   - filename / motion -> S::Identifier
+// - Settings Tree
+//   - store-migration-progress -> Path Tree Key
+
+use super::{Alias, DeleteToken, Details};
+use uuid::Uuid;
+
+pub(super) struct Old {
+    alias_tree: ::sled::Tree,
+    filename_tree: ::sled::Tree,
+    main_tree: ::sled::Tree,
+    details_tree: ::sled::Tree,
+    settings_tree: ::sled::Tree,
+    identifier_tree: ::sled::Tree,
+    _db: ::sled::Db,
+}
+
+impl Old {
+    pub(super) fn open(db: sled::Db) -> anyhow::Result<Self> {
+        Ok(Self {
+            alias_tree: db.open_tree("alias")?,
+            filename_tree: db.open_tree("filename")?,
+            main_tree: db.open_tree("main")?,
+            details_tree: db.open_tree("details")?,
+            settings_tree: db.open_tree("settings")?,
+            identifier_tree: db.open_tree("path")?,
+            _db: db,
+        })
+    }
+
+    pub(super) fn setting(&self, key: &[u8]) -> anyhow::Result<Option<sled::IVec>> {
+        Ok(self.settings_tree.get(key)?)
+    }
+
+    pub(super) fn hashes(&self) -> impl std::iter::Iterator<Item = sled::IVec> {
+        self.filename_tree
+            .iter()
+            .values()
+            .filter_map(|res| res.ok())
+    }
+
+    pub(super) fn details(&self, hash: &sled::IVec) -> anyhow::Result<Vec<(sled::IVec, Details)>> {
+        let filename = self
+            .main_tree
+            .get(hash)?
+            .ok_or_else(|| anyhow::anyhow!("missing filename"))?;
+
+        let filename = String::from_utf8_lossy(&filename);
+
+        Ok(self
+            .identifier_tree
+            .scan_prefix(filename.as_bytes())
+            .values()
+            .filter_map(Result::ok)
+            .filter_map(|identifier| {
+                let mut key = filename.as_bytes().to_vec();
+                key.push(b'/');
+                key.extend_from_slice(&identifier);
+
+                let details = self.details_tree.get(key).ok()??;
+                let details = serde_json::from_slice(&details).ok()?;
+
+                Some((identifier, details))
+            })
+            .collect())
+    }
+
+    pub(super) fn main_identifier(&self, hash: &sled::IVec) -> anyhow::Result<sled::IVec> {
+        let filename = self
+            .main_tree
+            .get(hash)?
+            .ok_or_else(|| anyhow::anyhow!("Missing filename"))?;
+
+        self.filename_tree
+            .get(filename)?
+            .ok_or_else(|| anyhow::anyhow!("Missing identifier"))
+    }
+
+    pub(super) fn variants(&self, hash: &sled::IVec) -> anyhow::Result<Vec<(String, sled::IVec)>> {
+        let filename = self
+            .main_tree
+            .get(hash)?
+            .ok_or_else(|| anyhow::anyhow!("Missing filename"))?;
+
+        let filename_string = String::from_utf8_lossy(&filename);
+
+        let variant_prefix = format!("{}/", filename_string);
+
+        Ok(self
+            .identifier_tree
+            .scan_prefix(&variant_prefix)
+            .filter_map(|res| res.ok())
+            .filter_map(|(key, value)| {
+                let key_str = String::from_utf8_lossy(&key);
+                let variant_path = key_str.trim_start_matches(&variant_prefix);
+                if variant_path == "motion" {
+                    return None;
+                }
+
+                Some((variant_path.to_string(), value))
+            })
+            .collect())
+    }
+
+    pub(super) fn motion_identifier(
+        &self,
+        hash: &sled::IVec,
+    ) -> anyhow::Result<Option<sled::IVec>> {
+        let filename = self
+            .main_tree
+            .get(hash)?
+            .ok_or_else(|| anyhow::anyhow!("Missing filename"))?;
+
+        let filename_string = String::from_utf8_lossy(&filename);
+
+        let motion_key = format!("{}/motion", filename_string);
+
+        Ok(self.filename_tree.get(motion_key)?)
+    }
+
+    pub(super) fn aliases(&self, hash: &sled::IVec) -> Vec<Alias> {
+        let mut key = hash.to_vec();
+        key.push(0);
+
+        self.main_tree
+            .scan_prefix(key)
+            .values()
+            .filter_map(|res| res.ok())
+            .filter_map(|alias| {
+                let alias_str = String::from_utf8_lossy(&alias);
+
+                let (uuid, ext) = alias_str.split_once('.')?;
+
+                let uuid = uuid.parse::<Uuid>().ok()?;
+
+                Some(Alias {
+                    id: uuid,
+                    extension: ext.to_string(),
+                })
+            })
+            .collect()
+    }
+
+    pub(super) fn delete_token(&self, alias: &Alias) -> anyhow::Result<Option<DeleteToken>> {
+        let key = format!("{}{}/delete", alias.id, alias.extension);
+
+        if let Some(ivec) = self.alias_tree.get(key)? {
+            let token_str = String::from_utf8_lossy(&ivec);
+            if let Ok(uuid) = token_str.parse::<Uuid>() {
+                return Ok(Some(DeleteToken { id: uuid }));
+            }
+        }
+
+        Ok(None)
+    }
+}
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index b543202..b35500b 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -18,7 +18,7 @@ pub(crate) enum Error {
     Sled(#[from] sled::Error),
 
     #[error("Invalid identifier")]
-    Identifier(#[source] Box<dyn std::error::Error + Send>),
+    Identifier(#[source] Box<dyn std::error::Error + Send + Sync>),
 
     #[error("Invalid details json")]
     Details(#[from] serde_json::Error),
@@ -32,11 +32,12 @@ pub(crate) enum Error {
 
 pub(crate) struct SledRepo {
     settings: Tree,
-    identifier_hashes: Tree,
     identifier_details: Tree,
     hashes: Tree,
     hash_aliases: Tree,
     hash_identifiers: Tree,
+    hash_variant_identifiers: Tree,
+    hash_motion_identifiers: Tree,
     aliases: Tree,
     alias_hashes: Tree,
     alias_delete_tokens: Tree,
@@ -47,11 +48,12 @@ impl SledRepo {
     pub(crate) fn new(db: Db) -> Result<Self, Error> {
         Ok(SledRepo {
             settings: db.open_tree("pict-rs-settings-tree")?,
-            identifier_hashes: db.open_tree("pict-rs-identifier-hashes-tree")?,
             identifier_details: db.open_tree("pict-rs-identifier-details-tree")?,
             hashes: db.open_tree("pict-rs-hashes-tree")?,
             hash_aliases: db.open_tree("pict-rs-hash-aliases-tree")?,
             hash_identifiers: db.open_tree("pict-rs-hash-identifiers-tree")?,
+            hash_variant_identifiers: db.open_tree("pict-rs-hash-variant-identifiers-tree")?,
+            hash_motion_identifiers: db.open_tree("pict-rs-hash-motion-identifiers-tree")?,
             aliases: db.open_tree("pict-rs-aliases-tree")?,
             alias_hashes: db.open_tree("pict-rs-alias-hashes-tree")?,
             alias_delete_tokens: db.open_tree("pict-rs-alias-delete-tokens-tree")?,
@@ -87,20 +89,26 @@ impl SettingsRepo for SledRepo {
 fn identifier_bytes<I>(identifier: &I) -> Result<Vec<u8>, Error>
 where
     I: Identifier,
-    I::Error: Send + 'static,
+    I::Error: Send + Sync + 'static,
 {
     identifier
         .to_bytes()
         .map_err(|e| Error::Identifier(Box::new(e)))
 }
 
+fn variant_key(hash: &[u8], variant: &str) -> Result<Vec<u8>, Error> {
+    let mut bytes = hash.to_vec();
+    bytes.push(b'/');
+    bytes.extend_from_slice(variant.as_bytes());
+    Ok(bytes)
+}
+
 #[async_trait::async_trait]
 impl<I> IdentifierRepo<I> for SledRepo
 where
     I: Identifier + 'static,
-    I::Error: Send + 'static,
+    I::Error: Send + Sync + 'static,
 {
-    type Bytes = IVec;
     type Error = Error;
 
     async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error> {
@@ -128,27 +136,9 @@ where
         }
     }
 
-    async fn relate_hash(&self, identifier: I, hash: Self::Bytes) -> Result<(), Self::Error> {
-        let key = identifier_bytes(&identifier)?;
-
-        b!(self.identifier_hashes, identifier_hashes.insert(key, hash));
-
-        Ok(())
-    }
-
-    async fn hash(&self, identifier: I) -> Result<Self::Bytes, Self::Error> {
-        let key = identifier_bytes(&identifier)?;
-
-        let opt = b!(self.identifier_hashes, identifier_hashes.get(key));
-
-        opt.ok_or(Error::Missing)
-    }
-
     async fn cleanup(&self, identifier: I) -> Result<(), Self::Error> {
         let key = identifier_bytes(&identifier)?;
 
-        let key2 = key.clone();
-        b!(self.identifier_hashes, identifier_hashes.remove(key2));
         b!(self.identifier_details, identifier_details.remove(key));
 
         Ok(())
@@ -218,7 +208,7 @@ fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> {
 impl<I> HashRepo<I> for SledRepo
 where
     I: Identifier + 'static,
-    I::Error: Send + 'static,
+    I::Error: Send + Sync + 'static,
 {
     type Bytes = IVec;
     type Error = Error;
@@ -290,6 +280,74 @@ where
         })
     }
 
+    async fn relate_variant_identifier(
+        &self,
+        hash: Self::Bytes,
+        variant: String,
+        identifier: I,
+    ) -> Result<(), Self::Error> {
+        let key = variant_key(&hash, &variant)?;
+        let value = identifier_bytes(&identifier)?;
+
+        b!(
+            self.hash_variant_identifiers,
+            hash_variant_identifiers.insert(key, value)
+        );
+
+        Ok(())
+    }
+
+    async fn variant_identifier(
+        &self,
+        hash: Self::Bytes,
+        variant: String,
+    ) -> Result<Option<I>, Self::Error> {
+        let key = variant_key(&hash, &variant)?;
+
+        let opt = b!(
+            self.hash_variant_identifiers,
+            hash_variant_identifiers.get(key)
+        );
+
+        if let Some(ivec) = opt {
+            Ok(Some(
+                I::from_bytes(ivec.to_vec()).map_err(|e| Error::Identifier(Box::new(e)))?,
+            ))
+        } else {
+            Ok(None)
+        }
+    }
+
+    async fn relate_motion_identifier(
+        &self,
+        hash: Self::Bytes,
+        identifier: I,
+    ) -> Result<(), Self::Error> {
+        let bytes = identifier_bytes(&identifier)?;
+
+        b!(
+            self.hash_motion_identifiers,
+            hash_motion_identifiers.insert(hash, bytes)
+        );
+
+        Ok(())
+    }
+
+    async fn motion_identifier(&self, hash: Self::Bytes) -> Result<Option<I>, Self::Error> {
+        let opt = b!(
+            self.hash_motion_identifiers,
+            hash_motion_identifiers.get(hash)
+        );
+
+        if let Some(ivec) = opt {
+            Ok(Some(
+                I::from_bytes(ivec.to_vec()).map_err(|e| Error::Identifier(Box::new(e)))?,
+            ))
+        } else {
+            Ok(None)
+        }
+    }
+
     async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error> {
         let hash2 = hash.clone();
         b!(self.hashes, hashes.remove(hash2));
@@ -297,17 +355,39 @@ where
         let hash2 = hash.clone();
         b!(self.hash_identifiers, hash_identifiers.remove(hash2));
 
-        let aliases = HashRepo::<I>::aliases(self, hash.clone()).await?;
+        let hash2 = hash.clone();
+        b!(
+            self.hash_motion_identifiers,
+            hash_motion_identifiers.remove(hash2)
+        );
 
+        let aliases = HashRepo::<I>::aliases(self, hash.clone()).await?;
+        let hash2 = hash.clone();
         b!(self.hash_aliases, {
             for alias in aliases {
-                let key = hash_alias_key(&hash, &alias);
+                let key = hash_alias_key(&hash2, &alias);
 
                 let _ = hash_aliases.remove(key);
             }
             Ok(()) as Result<(), Error>
         });
 
+        let variant_keys = b!(self.hash_variant_identifiers, {
+            let v = hash_variant_identifiers
+                .scan_prefix(hash)
+                .keys()
+                .filter_map(Result::ok)
+                .collect::<Vec<_>>();
+
+            Ok(v) as Result<Vec<_>, Error>
+        });
+        b!(self.hash_variant_identifiers, {
+            for key in variant_keys {
+                let _ = hash_variant_identifiers.remove(key);
+            }
+            Ok(()) as Result<(), Error>
+        });
+
         Ok(())
     }
 }
diff --git a/src/store.rs b/src/store.rs
index d9af237..46fad1e 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -5,7 +5,6 @@ use futures_util::stream::Stream;
 use tokio::io::{AsyncRead, AsyncWrite};
 
 pub(crate) mod file_store;
-#[cfg(feature = "object-storage")]
 pub(crate) mod object_store;
 
 pub(crate) trait Identifier: Send + Sync + Clone + Debug {
diff --git a/src/store/file_store.rs b/src/store/file_store.rs
index 1b885df..84b2f0e 100644
--- a/src/store/file_store.rs
+++ b/src/store/file_store.rs
@@ -159,7 +159,7 @@ impl FileStore {
         self.settings_tree
             .insert(GENERATOR_KEY, path.to_be_bytes())?;
 
-        let mut target_path = self.root_dir.join("files");
+        let mut target_path = self.root_dir.clone();
         for dir in path.to_strings() {
             target_path.push(dir)
         }

From 15b52ba6ecdba35e4ff2a4c7f5de99c75153c568 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sat, 26 Mar 2022 16:49:23 -0500
Subject: [PATCH 05/37] Get it compiling again

---
 Cargo.lock                          |   2 +-
 Cargo.toml                          |   2 +-
 src/error.rs                        |  39 +-
 src/magick.rs                       |   6 +-
 src/main.rs                         | 207 +++-----
 src/migrate.rs                      |  23 -
 src/processor.rs                    |   7 +-
 src/repo.rs                         | 488 +++++++++++++++---
 src/repo/old.rs                     |  36 +-
 src/repo/sled.rs                    | 149 +++---
 src/store.rs                        |  11 +-
 src/store/file_store.rs             |  71 +--
 src/store/file_store/restructure.rs | 118 -----
 src/store/object_store.rs           |  72 +--
 src/upload_manager.rs               | 764 ++++++++++------------------
 src/upload_manager/session.rs       | 300 +++--------
 16 files changed, 1032 insertions(+), 1263 deletions(-)
 delete mode 100644 src/store/file_store/restructure.rs

diff --git a/Cargo.lock b/Cargo.lock
index 76fc925..33fed7a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1610,7 +1610,7 @@ dependencies = [
 
 [[package]]
 name = "pict-rs"
-version = "0.3.0-rc.7"
+version = "0.4.0-alpha.1"
 dependencies = [
  "actix-form-data",
  "actix-rt",
diff --git a/Cargo.toml b/Cargo.toml
index 478f4d1..4857e63 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pict-rs"
 description = "A simple image hosting service"
-version = "0.3.0-rc.7"
+version = "0.4.0-alpha.1"
 authors = ["asonix <asonix@asonix.dog>"]
 license = "AGPL-3.0"
 readme = "README.md"
diff --git a/src/error.rs b/src/error.rs
index 7e02205..faca216 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -37,39 +37,33 @@ where
     }
 }
 
-impl From<sled::transaction::TransactionError<Error>> for Error {
-    fn from(e: sled::transaction::TransactionError<Error>) -> Self {
-        match e {
-            sled::transaction::TransactionError::Abort(t) => t,
-            sled::transaction::TransactionError::Storage(e) => e.into(),
-        }
-    }
-}
-
 #[derive(Debug, thiserror::Error)]
 pub(crate) enum UploadError {
-    #[error("Couln't upload file, {0}")]
+    #[error("Couln't upload file")]
     Upload(#[from] actix_form_data::Error),
 
-    #[error("Error in DB, {0}")]
-    Db(#[from] sled::Error),
+    #[error("Error in DB")]
+    Sled(#[from] crate::repo::sled::Error),
 
-    #[error("Error parsing string, {0}")]
+    #[error("Error in old sled DB")]
+    OldSled(#[from] ::sled::Error),
+
+    #[error("Error parsing string")]
     ParseString(#[from] std::string::FromUtf8Error),
 
-    #[error("Error interacting with filesystem, {0}")]
+    #[error("Error interacting with filesystem")]
     Io(#[from] std::io::Error),
 
-    #[error(transparent)]
+    #[error("Error generating path")]
     PathGenerator(#[from] storage_path_generator::PathError),
 
-    #[error(transparent)]
+    #[error("Error stripping prefix")]
     StripPrefix(#[from] std::path::StripPrefixError),
 
-    #[error(transparent)]
+    #[error("Error storing file")]
     FileStore(#[from] crate::store::file_store::FileError),
 
-    #[error(transparent)]
+    #[error("Error storing object")]
     ObjectStore(#[from] crate::store::object_store::ObjectError),
 
     #[error("Provided process path is invalid")]
@@ -87,9 +81,6 @@ pub(crate) enum UploadError {
     #[error("Requested a file that doesn't exist")]
     MissingAlias,
 
-    #[error("Alias directed to missing file")]
-    MissingFile,
-
     #[error("Provided token did not match expected token")]
     InvalidToken,
 
@@ -102,7 +93,7 @@ pub(crate) enum UploadError {
     #[error("Unable to download image, bad response {0}")]
     Download(actix_web::http::StatusCode),
 
-    #[error("Unable to download image, {0}")]
+    #[error("Unable to download image")]
     Payload(#[from] awc::error::PayloadError),
 
     #[error("Unable to send request, {0}")]
@@ -117,13 +108,13 @@ pub(crate) enum UploadError {
     #[error("Tried to save an image with an already-taken name")]
     DuplicateAlias,
 
-    #[error("{0}")]
+    #[error("Error in json")]
     Json(#[from] serde_json::Error),
 
     #[error("Range header not satisfiable")]
     Range,
 
-    #[error(transparent)]
+    #[error("Hit limit")]
     Limit(#[from] super::LimitError),
 }
 
diff --git a/src/magick.rs b/src/magick.rs
index 327052e..772b333 100644
--- a/src/magick.rs
+++ b/src/magick.rs
@@ -2,6 +2,7 @@ use crate::{
     config::Format,
     error::{Error, UploadError},
     process::Process,
+    repo::Alias,
     store::Store,
 };
 use actix_web::web::Bytes;
@@ -11,8 +12,9 @@ use tokio::{
 };
 use tracing::instrument;
 
-pub(crate) fn details_hint(filename: &str) -> Option<ValidInputType> {
-    if filename.ends_with(".mp4") {
+pub(crate) fn details_hint(alias: &Alias) -> Option<ValidInputType> {
+    let ext = alias.extension()?;
+    if ext.ends_with(".mp4") {
         Some(ValidInputType::Mp4)
     } else {
         None
diff --git a/src/main.rs b/src/main.rs
index 305df02..d7acc11 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -57,6 +57,7 @@ use self::{
     magick::details_hint,
     middleware::{Deadline, Internal},
     migrate::LatestDb,
+    repo::{Alias, DeleteToken, Repo},
     store::{file_store::FileStore, object_store::ObjectStore, Store},
     upload_manager::{UploadManager, UploadManagerSession},
 };
@@ -96,30 +97,26 @@ where
             info!("Uploaded {} as {:?}", image.filename, alias);
             let delete_token = image.result.delete_token().await?;
 
-            let name = manager.from_alias(alias.to_owned()).await?;
-            let identifier = manager.identifier_from_filename::<S>(name.clone()).await?;
-
-            let details = manager.variant_details(&identifier, name.clone()).await?;
+            let identifier = manager.identifier_from_alias::<S>(alias).await?;
+            let details = manager.details(&identifier).await?;
 
             let details = if let Some(details) = details {
                 debug!("details exist");
                 details
             } else {
                 debug!("generating new details from {:?}", identifier);
-                let hint = details_hint(&name);
+                let hint = details_hint(alias);
                 let new_details =
                     Details::from_store((**store).clone(), identifier.clone(), hint).await?;
-                debug!("storing details for {:?} {}", identifier, name);
-                manager
-                    .store_variant_details(&identifier, name, &new_details)
-                    .await?;
+                debug!("storing details for {:?}", identifier);
+                manager.store_details(&identifier, &new_details).await?;
                 debug!("stored");
                 new_details
             };
 
             files.push(serde_json::json!({
-                "file": alias,
-                "delete_token": delete_token,
+                "file": alias.to_string(),
+                "delete_token": delete_token.to_string(),
                 "details": details,
             }));
         }
@@ -222,19 +219,16 @@ where
     drop(permit);
     let delete_token = session.delete_token().await?;
 
-    let name = manager.from_alias(alias.to_owned()).await?;
-    let identifier = manager.identifier_from_filename::<S>(name.clone()).await?;
+    let identifier = manager.identifier_from_alias::<S>(&alias).await?;
 
-    let details = manager.variant_details(&identifier, name.clone()).await?;
+    let details = manager.details(&identifier).await?;
 
     let details = if let Some(details) = details {
         details
     } else {
-        let hint = details_hint(&name);
+        let hint = details_hint(&alias);
         let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
-        manager
-            .store_variant_details(&identifier, name, &new_details)
-            .await?;
+        manager.store_details(&identifier, &new_details).await?;
         new_details
     };
 
@@ -242,8 +236,8 @@ where
     Ok(HttpResponse::Created().json(&serde_json::json!({
         "msg": "ok",
         "files": [{
-            "file": alias,
-            "delete_token": delete_token,
+            "file": alias.to_string(),
+            "delete_token": delete_token.to_string(),
             "details": details,
         }]
     })))
@@ -261,19 +255,21 @@ where
 {
     let (alias, token) = path_entries.into_inner();
 
-    manager.delete((**store).clone(), token, alias).await?;
+    let alias = Alias::from_existing(&alias);
+    let token = DeleteToken::from_existing(&token);
+
+    manager.delete((**store).clone(), alias, token).await?;
 
     Ok(HttpResponse::NoContent().finish())
 }
 
 type ProcessQuery = Vec<(String, String)>;
 
-async fn prepare_process(
+fn prepare_process(
     query: web::Query<ProcessQuery>,
     ext: &str,
-    manager: &UploadManager,
     filters: &Option<HashSet<String>>,
-) -> Result<(Format, String, PathBuf, Vec<String>), Error> {
+) -> Result<(Format, Alias, PathBuf, Vec<String>), Error> {
     let (alias, operations) =
         query
             .into_inner()
@@ -291,7 +287,7 @@ async fn prepare_process(
         return Err(UploadError::MissingFilename.into());
     }
 
-    let name = manager.from_alias(alias).await?;
+    let alias = Alias::from_existing(&alias);
 
     let operations = if let Some(filters) = filters.as_ref() {
         operations
@@ -305,12 +301,10 @@ async fn prepare_process(
     let format = ext
         .parse::<Format>()
         .map_err(|_| UploadError::UnsupportedFormat)?;
-    let processed_name = format!("{}.{}", name, ext);
 
-    let (thumbnail_path, thumbnail_args) =
-        self::processor::build_chain(&operations, processed_name)?;
+    let (thumbnail_path, thumbnail_args) = self::processor::build_chain(&operations)?;
 
-    Ok((format, name, thumbnail_path, thumbnail_args))
+    Ok((format, alias, thumbnail_path, thumbnail_args))
 }
 
 #[instrument(name = "Fetching derived details", skip(manager, filters))]
@@ -324,15 +318,14 @@ async fn process_details<S: Store>(
 where
     Error: From<S::Error>,
 {
-    let (_, name, thumbnail_path, _) =
-        prepare_process(query, ext.as_str(), &manager, &filters).await?;
+    let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
 
     let identifier = manager
-        .variant_identifier::<S>(&thumbnail_path, &name)
+        .variant_identifier::<S>(&alias, &thumbnail_path)
         .await?
         .ok_or(UploadError::MissingAlias)?;
 
-    let details = manager.variant_details(&identifier, name).await?;
+    let details = manager.details(&identifier).await?;
 
     let details = details.ok_or(UploadError::NoFiles)?;
 
@@ -352,24 +345,22 @@ async fn process<S: Store + 'static>(
 where
     Error: From<S::Error>,
 {
-    let (format, name, thumbnail_path, thumbnail_args) =
-        prepare_process(query, ext.as_str(), &manager, &filters).await?;
+    let (format, alias, thumbnail_path, thumbnail_args) =
+        prepare_process(query, ext.as_str(), &filters)?;
 
     let identifier_opt = manager
-        .variant_identifier::<S>(&thumbnail_path, &name)
+        .variant_identifier::<S>(&alias, &thumbnail_path)
         .await?;
 
     if let Some(identifier) = identifier_opt {
-        let details_opt = manager.variant_details(&identifier, name.clone()).await?;
+        let details_opt = manager.details(&identifier).await?;
 
         let details = if let Some(details) = details_opt {
             details
         } else {
-            let hint = details_hint(&name);
+            let hint = details_hint(&alias);
             let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
-            manager
-                .store_variant_details(&identifier, name, &details)
-                .await?;
+            manager.store_details(&identifier, &details).await?;
             details
         };
 
@@ -377,7 +368,7 @@ where
     }
 
     let identifier = manager
-        .still_identifier_from_filename((**store).clone(), name.clone())
+        .still_identifier_from_alias((**store).clone(), &alias)
         .await?;
 
     let thumbnail_path2 = thumbnail_path.clone();
@@ -405,29 +396,27 @@ where
             parent: None,
             "Saving variant information",
             path = tracing::field::debug(&thumbnail_path),
-            name = tracing::field::display(&name),
+            name = tracing::field::display(&alias),
         );
         save_span.follows_from(Span::current());
         let details2 = details.clone();
         let bytes2 = bytes.clone();
+        let alias2 = alias.clone();
         actix_rt::spawn(
             async move {
-                let identifier = match store.save_bytes(bytes2, &name).await {
+                let identifier = match store.save_bytes(bytes2).await {
                     Ok(identifier) => identifier,
                     Err(e) => {
                         tracing::warn!("Failed to generate directory path: {}", e);
                         return;
                     }
                 };
-                if let Err(e) = manager
-                    .store_variant_details(&identifier, name.clone(), &details2)
-                    .await
-                {
+                if let Err(e) = manager.store_details(&identifier, &details2).await {
                     tracing::warn!("Error saving variant details: {}", e);
                     return;
                 }
                 if let Err(e) = manager
-                    .store_variant(Some(&thumbnail_path), &identifier, &name)
+                    .store_variant(&alias2, &thumbnail_path, &identifier)
                     .await
                 {
                     tracing::warn!("Error saving variant info: {}", e);
@@ -483,19 +472,19 @@ async fn details<S: Store>(
 where
     Error: From<S::Error>,
 {
-    let name = manager.from_alias(alias.into_inner()).await?;
-    let identifier = manager.identifier_from_filename::<S>(name.clone()).await?;
+    let alias = alias.into_inner();
+    let alias = Alias::from_existing(&alias);
 
-    let details = manager.variant_details(&identifier, name.clone()).await?;
+    let identifier = manager.identifier_from_alias::<S>(&alias).await?;
+
+    let details = manager.details(&identifier).await?;
 
     let details = if let Some(details) = details {
         details
     } else {
-        let hint = details_hint(&name);
+        let hint = details_hint(&alias);
         let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
-        manager
-            .store_variant_details(&identifier, name, &new_details)
-            .await?;
+        manager.store_details(&identifier, &new_details).await?;
         new_details
     };
 
@@ -513,19 +502,18 @@ async fn serve<S: Store>(
 where
     Error: From<S::Error>,
 {
-    let name = manager.from_alias(alias.into_inner()).await?;
-    let identifier = manager.identifier_from_filename::<S>(name.clone()).await?;
+    let alias = alias.into_inner();
+    let alias = Alias::from_existing(&alias);
+    let identifier = manager.identifier_from_alias::<S>(&alias).await?;
 
-    let details = manager.variant_details(&identifier, name.clone()).await?;
+    let details = manager.details(&identifier).await?;
 
     let details = if let Some(details) = details {
         details
     } else {
-        let hint = details_hint(&name);
+        let hint = details_hint(&alias);
         let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
-        manager
-            .store_variant_details(&identifier, name, &details)
-            .await?;
+        manager.store_details(&identifier, &details).await?;
         details
     };
 
@@ -605,25 +593,21 @@ where
 }
 
 #[derive(Debug, serde::Deserialize)]
-#[serde(untagged)]
-enum FileOrAlias {
-    File { file: String },
-    Alias { alias: String },
+struct AliasQuery {
+    alias: String,
 }
 
 #[instrument(name = "Purging file", skip(upload_manager))]
 async fn purge<S: Store>(
-    query: web::Query<FileOrAlias>,
+    query: web::Query<AliasQuery>,
     upload_manager: web::Data<UploadManager>,
     store: web::Data<S>,
 ) -> Result<HttpResponse, Error>
 where
     Error: From<S::Error>,
 {
-    let aliases = match query.into_inner() {
-        FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
-        FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
-    };
+    let alias = Alias::from_existing(&query.alias);
+    let aliases = upload_manager.aliases_by_alias(&alias).await?;
 
     for alias in aliases.iter() {
         upload_manager
@@ -633,49 +617,25 @@ where
 
     Ok(HttpResponse::Ok().json(&serde_json::json!({
         "msg": "ok",
-        "aliases": aliases
+        "aliases": aliases.iter().map(|a| a.to_string()).collect::<Vec<_>>()
     })))
 }
 
 #[instrument(name = "Fetching aliases", skip(upload_manager))]
 async fn aliases<S: Store>(
-    query: web::Query<FileOrAlias>,
+    query: web::Query<AliasQuery>,
     upload_manager: web::Data<UploadManager>,
     store: web::Data<S>,
 ) -> Result<HttpResponse, Error>
 where
     Error: From<S::Error>,
 {
-    let aliases = match query.into_inner() {
-        FileOrAlias::File { file } => upload_manager.aliases_by_filename(file).await?,
-        FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
-    };
+    let alias = Alias::from_existing(&query.alias);
+    let aliases = upload_manager.aliases_by_alias(&alias).await?;
 
     Ok(HttpResponse::Ok().json(&serde_json::json!({
         "msg": "ok",
-        "aliases": aliases,
-    })))
-}
-
-#[derive(Debug, serde::Deserialize)]
-struct ByAlias {
-    alias: String,
-}
-
-#[instrument(name = "Fetching filename", skip(upload_manager))]
-async fn filename_by_alias<S: Store>(
-    query: web::Query<ByAlias>,
-    upload_manager: web::Data<UploadManager>,
-    store: web::Data<S>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
-    let filename = upload_manager.from_alias(query.into_inner().alias).await?;
-
-    Ok(HttpResponse::Ok().json(&serde_json::json!({
-        "msg": "ok",
-        "filename": filename,
+        "aliases": aliases.iter().map(|a| a.to_string()).collect::<Vec<_>>()
     })))
 }
 
@@ -817,10 +777,7 @@ where
                             .route(web::post().to(upload::<S>)),
                     )
                     .service(web::resource("/purge").route(web::post().to(purge::<S>)))
-                    .service(web::resource("/aliases").route(web::get().to(aliases::<S>)))
-                    .service(
-                        web::resource("/filename").route(web::get().to(filename_by_alias::<S>)),
-                    ),
+                    .service(web::resource("/aliases").route(web::get().to(aliases::<S>))),
             )
     })
     .bind(CONFIG.bind_address())?
@@ -834,7 +791,7 @@ where
 
 async fn migrate_inner<S1>(
     manager: &UploadManager,
-    db: &sled::Db,
+    repo: &Repo,
     from: S1,
     to: &config::Storage,
 ) -> anyhow::Result<()>
@@ -844,9 +801,7 @@ where
 {
     match to {
         config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
-            let to = FileStore::build(path.clone(), db)?;
-            manager.restructure(&to).await?;
-
+            let to = FileStore::build(path.clone(), repo.clone()).await?;
             manager.migrate_store::<S1, FileStore>(from, to).await?;
         }
         config::Storage::ObjectStorage(RequiredObjectStorage {
@@ -864,9 +819,10 @@ where
                 secret_key.clone(),
                 security_token.clone(),
                 session_token.clone(),
-                db,
+                repo.clone(),
                 build_reqwest_client()?,
-            )?;
+            )
+            .await?;
 
             manager.migrate_store::<S1, ObjectStore>(from, to).await?;
         }
@@ -883,13 +839,12 @@ async fn main() -> anyhow::Result<()> {
         CONFIG.console_buffer_capacity(),
     )?;
 
+    let repo = Repo::open(CONFIG.repo())?;
+
     let db = LatestDb::exists(CONFIG.data_dir()).migrate()?;
-
-    let repo = self::repo::Repo::open(CONFIG.repo())?;
-
     repo.from_db(db).await?;
 
-    let manager = UploadManager::new(db.clone(), CONFIG.format()).await?;
+    let manager = UploadManager::new(repo.clone(), CONFIG.format()).await?;
 
     match CONFIG.command()? {
         CommandConfig::Run => (),
@@ -901,10 +856,8 @@ async fn main() -> anyhow::Result<()> {
 
             match from {
                 config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
-                    let from = FileStore::build(path.clone(), &db)?;
-                    manager.restructure(&from).await?;
-
-                    migrate_inner(&manager, &db, from, &to).await?;
+                    let from = FileStore::build(path.clone(), repo.clone()).await?;
+                    migrate_inner(&manager, &repo, from, &to).await?;
                 }
                 config::Storage::ObjectStorage(RequiredObjectStorage {
                     bucket_name,
@@ -921,11 +874,12 @@ async fn main() -> anyhow::Result<()> {
                         secret_key,
                         security_token,
                         session_token,
-                        &db,
+                        repo.clone(),
                         build_reqwest_client()?,
-                    )?;
+                    )
+                    .await?;
 
-                    migrate_inner(&manager, &db, from, &to).await?;
+                    migrate_inner(&manager, &repo, from, &to).await?;
                 }
             }
 
@@ -935,9 +889,7 @@ async fn main() -> anyhow::Result<()> {
 
     match CONFIG.store()? {
         config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
-            let store = FileStore::build(path, &db)?;
-            manager.restructure(&store).await?;
-
+            let store = FileStore::build(path, repo).await?;
             launch(manager, store).await
         }
         config::Storage::ObjectStorage(RequiredObjectStorage {
@@ -955,9 +907,10 @@ async fn main() -> anyhow::Result<()> {
                 secret_key,
                 security_token,
                 session_token,
-                &db,
+                repo,
                 build_reqwest_client()?,
-            )?;
+            )
+            .await?;
 
             launch(manager, store).await
         }
diff --git a/src/migrate.rs b/src/migrate.rs
index 8ac65d6..f33ffb5 100644
--- a/src/migrate.rs
+++ b/src/migrate.rs
@@ -95,26 +95,3 @@ impl DbVersion {
         }
     }
 }
-
-pub(crate) fn alias_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
-    let mut start = hash.to_vec();
-    start.extend(&[0]);
-
-    let mut end = hash.to_vec();
-    end.extend(&[1]);
-
-    (start, end)
-}
-
-pub(crate) fn alias_id_key(alias: &str) -> String {
-    format!("{}/id", alias)
-}
-
-pub(crate) fn alias_key(hash: &[u8], id: &str) -> Vec<u8> {
-    let mut key = hash.to_vec();
-    // add a separator to the key between the hash and the ID
-    key.extend(&[0]);
-    key.extend(id.as_bytes());
-
-    key
-}
diff --git a/src/processor.rs b/src/processor.rs
index feb3c09..0e01a5d 100644
--- a/src/processor.rs
+++ b/src/processor.rs
@@ -20,10 +20,7 @@ pub(crate) struct Crop(usize, usize);
 pub(crate) struct Blur(f64);
 
 #[instrument]
-pub(crate) fn build_chain(
-    args: &[(String, String)],
-    filename: String,
-) -> Result<(PathBuf, Vec<String>), Error> {
+pub(crate) fn build_chain(args: &[(String, String)]) -> Result<(PathBuf, Vec<String>), Error> {
     fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, UploadError> {
         if key == P::NAME {
             return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?));
@@ -56,7 +53,7 @@ pub(crate) fn build_chain(
                 }
             })?;
 
-    Ok((path.join(filename), args))
+    Ok((path, args))
 }
 
 impl Processor for Identity {
diff --git a/src/repo.rs b/src/repo.rs
index 13d1b78..b7fcf37 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -1,105 +1,131 @@
 use crate::config::RequiredSledRepo;
 use crate::{config::Repository, details::Details, store::Identifier};
 use futures_util::Stream;
+use tracing::debug;
 use uuid::Uuid;
 
 mod old;
 pub(crate) mod sled;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub(crate) enum Repo {
     Sled(self::sled::SledRepo),
 }
 
-#[derive(Clone, Debug)]
-pub(crate) struct Alias {
-    id: Uuid,
-    extension: String,
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+enum MaybeUuid {
+    Uuid(Uuid),
+    Name(String),
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub(crate) struct Alias {
+    id: MaybeUuid,
+    extension: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub(crate) struct DeleteToken {
-    id: Uuid,
+    id: MaybeUuid,
 }
 
 pub(crate) struct AlreadyExists;
 
-#[async_trait::async_trait]
+#[async_trait::async_trait(?Send)]
 pub(crate) trait SettingsRepo {
     type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error;
+    type Error: std::error::Error + Send + Sync + 'static;
 
     async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Self::Error>;
     async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Self::Error>;
     async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error>;
 }
 
-#[async_trait::async_trait]
-pub(crate) trait IdentifierRepo<I: Identifier> {
-    type Error: std::error::Error;
+#[async_trait::async_trait(?Send)]
+pub(crate) trait IdentifierRepo {
+    type Error: std::error::Error + Send + Sync + 'static;
 
-    async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error>;
-    async fn details(&self, identifier: I) -> Result<Option<Details>, Self::Error>;
+    async fn relate_details<I: Identifier>(
+        &self,
+        identifier: &I,
+        details: &Details,
+    ) -> Result<(), Self::Error>;
+    async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Self::Error>;
 
-    async fn cleanup(&self, identifier: I) -> Result<(), Self::Error>;
+    async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Self::Error>;
 }
 
-#[async_trait::async_trait]
-pub(crate) trait HashRepo<I: Identifier> {
+#[async_trait::async_trait(?Send)]
+pub(crate) trait HashRepo {
     type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error;
+    type Error: std::error::Error + Send + Sync + 'static;
     type Stream: Stream<Item = Result<Self::Bytes, Self::Error>>;
 
     async fn hashes(&self) -> Self::Stream;
 
     async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Self::Error>;
 
-    async fn relate_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
-    async fn remove_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error>;
+    async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error>;
+    async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error>;
     async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Self::Error>;
 
-    async fn relate_identifier(&self, hash: Self::Bytes, identifier: I) -> Result<(), Self::Error>;
-    async fn identifier(&self, hash: Self::Bytes) -> Result<I, Self::Error>;
+    async fn relate_identifier<I: Identifier>(
+        &self,
+        hash: Self::Bytes,
+        identifier: &I,
+    ) -> Result<(), Self::Error>;
+    async fn identifier<I: Identifier + 'static>(
+        &self,
+        hash: Self::Bytes,
+    ) -> Result<I, Self::Error>;
 
-    async fn relate_variant_identifier(
+    async fn relate_variant_identifier<I: Identifier>(
         &self,
         hash: Self::Bytes,
         variant: String,
-        identifier: I,
+        identifier: &I,
     ) -> Result<(), Self::Error>;
-    async fn variant_identifier(
+    async fn variant_identifier<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
         variant: String,
     ) -> Result<Option<I>, Self::Error>;
-
-    async fn relate_motion_identifier(
+    async fn variants<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
-        identifier: I,
+    ) -> Result<Vec<(String, I)>, Self::Error>;
+
+    async fn relate_motion_identifier<I: Identifier>(
+        &self,
+        hash: Self::Bytes,
+        identifier: &I,
     ) -> Result<(), Self::Error>;
-    async fn motion_identifier(&self, hash: Self::Bytes) -> Result<Option<I>, Self::Error>;
+    async fn motion_identifier<I: Identifier + 'static>(
+        &self,
+        hash: Self::Bytes,
+    ) -> Result<Option<I>, Self::Error>;
 
     async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error>;
 }
 
-#[async_trait::async_trait]
+#[async_trait::async_trait(?Send)]
 pub(crate) trait AliasRepo {
     type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error;
+    type Error: std::error::Error + Send + Sync + 'static;
 
-    async fn create(&self, alias: Alias) -> Result<Result<(), AlreadyExists>, Self::Error>;
+    async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Self::Error>;
 
     async fn relate_delete_token(
         &self,
-        alias: Alias,
-        delete_token: DeleteToken,
+        alias: &Alias,
+        delete_token: &DeleteToken,
     ) -> Result<Result<(), AlreadyExists>, Self::Error>;
-    async fn delete_token(&self, alias: Alias) -> Result<DeleteToken, Self::Error>;
+    async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Self::Error>;
 
-    async fn relate_hash(&self, alias: Alias, hash: Self::Bytes) -> Result<(), Self::Error>;
-    async fn hash(&self, alias: Alias) -> Result<Self::Bytes, Self::Error>;
+    async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Self::Error>;
+    async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Self::Error>;
 
-    async fn cleanup(&self, alias: Alias) -> Result<(), Self::Error>;
+    async fn cleanup(&self, alias: &Alias) -> Result<(), Self::Error>;
 }
 
 impl Repo {
@@ -167,72 +193,134 @@ const GENERATOR_KEY: &[u8] = b"last-path";
 
 async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> anyhow::Result<()>
 where
-    T: IdentifierRepo<::sled::IVec>,
-    <T as IdentifierRepo<::sled::IVec>>::Error: Send + Sync + 'static,
-    T: HashRepo<::sled::IVec>,
-    <T as HashRepo<::sled::IVec>>::Error: Send + Sync + 'static,
-    T: AliasRepo,
-    <T as AliasRepo>::Error: Send + Sync + 'static,
-    T: SettingsRepo,
-    <T as SettingsRepo>::Error: Send + Sync + 'static,
+    T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo,
 {
-    HashRepo::create(repo, hash.to_vec().into()).await?;
+    if HashRepo::create(repo, hash.to_vec().into()).await?.is_err() {
+        debug!("Duplicate hash detected");
+        return Ok(());
+    }
 
-    let main_ident = old.main_identifier(&hash)?;
+    let main_ident = old.main_identifier(&hash)?.to_vec();
 
-    HashRepo::relate_identifier(repo, hash.to_vec().into(), main_ident.clone()).await?;
+    repo.relate_identifier(hash.to_vec().into(), &main_ident)
+        .await?;
 
     for alias in old.aliases(&hash) {
-        if let Ok(Ok(())) = AliasRepo::create(repo, alias.clone()).await {
-            let _ = HashRepo::relate_alias(repo, hash.to_vec().into(), alias.clone()).await;
-            let _ = AliasRepo::relate_hash(repo, alias.clone(), hash.to_vec().into()).await;
+        if let Ok(Ok(())) = AliasRepo::create(repo, &alias).await {
+            let _ = repo.relate_alias(hash.to_vec().into(), &alias).await;
+            let _ = repo.relate_hash(&alias, hash.to_vec().into()).await;
 
             if let Ok(Some(delete_token)) = old.delete_token(&alias) {
-                let _ = AliasRepo::relate_delete_token(repo, alias, delete_token).await;
+                let _ = repo.relate_delete_token(&alias, &delete_token).await;
             }
         }
     }
 
     if let Ok(Some(identifier)) = old.motion_identifier(&hash) {
-        HashRepo::relate_motion_identifier(repo, hash.to_vec().into(), identifier).await;
+        let _ = repo
+            .relate_motion_identifier(hash.to_vec().into(), &identifier.to_vec())
+            .await;
     }
 
-    for (variant, identifier) in old.variants(&hash)? {
-        let _ =
-            HashRepo::relate_variant_identifier(repo, hash.to_vec().into(), variant, identifier)
-                .await;
+    for (variant_path, identifier) in old.variants(&hash)? {
+        let variant = variant_path.to_string_lossy().to_string();
+
+        let _ = repo
+            .relate_variant_identifier(hash.to_vec().into(), variant, &identifier.to_vec())
+            .await;
     }
 
     for (identifier, details) in old.details(&hash)? {
-        let _ = IdentifierRepo::relate_details(repo, identifier, details).await;
+        let _ = repo.relate_details(&identifier.to_vec(), &details).await;
     }
 
     if let Ok(Some(value)) = old.setting(STORE_MIGRATION_PROGRESS) {
-        SettingsRepo::set(repo, STORE_MIGRATION_PROGRESS, value.to_vec().into()).await?;
+        repo.set(STORE_MIGRATION_PROGRESS, value.to_vec().into())
+            .await?;
     }
 
     if let Ok(Some(value)) = old.setting(GENERATOR_KEY) {
-        SettingsRepo::set(repo, GENERATOR_KEY, value.to_vec().into()).await?;
+        repo.set(GENERATOR_KEY, value.to_vec().into()).await?;
     }
 
     Ok(())
 }
 
+impl MaybeUuid {
+    fn from_str(s: &str) -> Self {
+        if let Ok(uuid) = Uuid::parse_str(s) {
+            MaybeUuid::Uuid(uuid)
+        } else {
+            MaybeUuid::Name(s.into())
+        }
+    }
+
+    fn as_bytes(&self) -> &[u8] {
+        match self {
+            Self::Uuid(uuid) => &uuid.as_bytes()[..],
+            Self::Name(name) => name.as_bytes(),
+        }
+    }
+}
+
+fn split_at_dot(s: &str) -> Option<(&str, &str)> {
+    let index = s.find('.')?;
+
+    Some(s.split_at(index))
+}
+
 impl Alias {
+    pub(crate) fn generate(extension: String) -> Self {
+        Alias {
+            id: MaybeUuid::Uuid(Uuid::new_v4()),
+            extension: Some(extension),
+        }
+    }
+
+    pub(crate) fn from_existing(alias: &str) -> Self {
+        if let Some((start, end)) = split_at_dot(alias) {
+            Alias {
+                id: MaybeUuid::from_str(start),
+                extension: Some(end.into()),
+            }
+        } else {
+            Alias {
+                id: MaybeUuid::from_str(alias),
+                extension: None,
+            }
+        }
+    }
+
+    pub(crate) fn extension(&self) -> Option<&str> {
+        self.extension.as_deref()
+    }
+
     fn to_bytes(&self) -> Vec<u8> {
         let mut v = self.id.as_bytes().to_vec();
 
-        v.extend_from_slice(self.extension.as_bytes());
+        if let Some(ext) = self.extension() {
+            v.extend_from_slice(ext.as_bytes());
+        }
 
         v
     }
 
     fn from_slice(bytes: &[u8]) -> Option<Self> {
-        if bytes.len() > 16 {
+        if let Ok(s) = std::str::from_utf8(bytes) {
+            Some(Self::from_existing(s))
+        } else if bytes.len() >= 16 {
             let id = Uuid::from_slice(&bytes[0..16]).expect("Already checked length");
-            let extension = String::from_utf8_lossy(&bytes[16..]).to_string();
 
-            Some(Self { id, extension })
+            let extension = if bytes.len() > 16 {
+                Some(String::from_utf8_lossy(&bytes[16..]).to_string())
+            } else {
+                None
+            };
+
+            Some(Self {
+                id: MaybeUuid::Uuid(id),
+                extension,
+            })
         } else {
             None
         }
@@ -240,20 +328,63 @@ impl Alias {
 }
 
 impl DeleteToken {
+    pub(crate) fn from_existing(existing: &str) -> Self {
+        if let Ok(uuid) = Uuid::parse_str(existing) {
+            DeleteToken {
+                id: MaybeUuid::Uuid(uuid),
+            }
+        } else {
+            DeleteToken {
+                id: MaybeUuid::Name(existing.into()),
+            }
+        }
+    }
+
+    pub(crate) fn generate() -> Self {
+        Self {
+            id: MaybeUuid::Uuid(Uuid::new_v4()),
+        }
+    }
+
     fn to_bytes(&self) -> Vec<u8> {
         self.id.as_bytes().to_vec()
     }
 
     fn from_slice(bytes: &[u8]) -> Option<Self> {
-        Some(DeleteToken {
-            id: Uuid::from_slice(bytes).ok()?,
-        })
+        if let Ok(s) = std::str::from_utf8(bytes) {
+            Some(DeleteToken::from_existing(s))
+        } else if bytes.len() == 16 {
+            Some(DeleteToken {
+                id: MaybeUuid::Uuid(Uuid::from_slice(bytes).ok()?),
+            })
+        } else {
+            None
+        }
+    }
+}
+
+impl std::fmt::Display for MaybeUuid {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Uuid(id) => write!(f, "{}", id),
+            Self::Name(name) => write!(f, "{}", name),
+        }
+    }
+}
+
+impl std::fmt::Display for DeleteToken {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.id)
     }
 }
 
 impl std::fmt::Display for Alias {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}{}", self.id, self.extension)
+        if let Some(ext) = self.extension() {
+            write!(f, "{}{}", self.id, ext)
+        } else {
+            write!(f, "{}", self.id)
+        }
     }
 }
 
@@ -272,17 +403,220 @@ impl Identifier for Vec<u8> {
     }
 }
 
-impl Identifier for ::sled::IVec {
-    type Error = std::convert::Infallible;
+#[cfg(test)]
+mod tests {
+    use super::{Alias, DeleteToken, MaybeUuid, Uuid};
 
-    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error>
-    where
-        Self: Sized,
-    {
-        Ok(bytes.into())
+    #[test]
+    fn string_delete_token() {
+        let delete_token = DeleteToken::from_existing("blah");
+
+        assert_eq!(
+            delete_token,
+            DeleteToken {
+                id: MaybeUuid::Name(String::from("blah"))
+            }
+        )
     }
 
-    fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> {
-        Ok(self.to_vec())
+    #[test]
+    fn uuid_string_delete_token() {
+        let uuid = Uuid::new_v4();
+
+        let delete_token = DeleteToken::from_existing(&uuid.to_string());
+
+        assert_eq!(
+            delete_token,
+            DeleteToken {
+                id: MaybeUuid::Uuid(uuid),
+            }
+        )
+    }
+
+    #[test]
+    fn bytes_delete_token() {
+        let delete_token = DeleteToken::from_slice(b"blah").unwrap();
+
+        assert_eq!(
+            delete_token,
+            DeleteToken {
+                id: MaybeUuid::Name(String::from("blah"))
+            }
+        )
+    }
+
+    #[test]
+    fn uuid_bytes_delete_token() {
+        let uuid = Uuid::new_v4();
+
+        let delete_token = DeleteToken::from_slice(&uuid.as_bytes()[..]).unwrap();
+
+        assert_eq!(
+            delete_token,
+            DeleteToken {
+                id: MaybeUuid::Uuid(uuid),
+            }
+        )
+    }
+
+    #[test]
+    fn uuid_bytes_string_delete_token() {
+        let uuid = Uuid::new_v4();
+
+        let delete_token = DeleteToken::from_slice(uuid.to_string().as_bytes()).unwrap();
+
+        assert_eq!(
+            delete_token,
+            DeleteToken {
+                id: MaybeUuid::Uuid(uuid),
+            }
+        )
+    }
+
+    #[test]
+    fn string_alias() {
+        let alias = Alias::from_existing("blah");
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Name(String::from("blah")),
+                extension: None
+            }
+        );
+    }
+
+    #[test]
+    fn string_alias_ext() {
+        let alias = Alias::from_existing("blah.mp4");
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Name(String::from("blah")),
+                extension: Some(String::from(".mp4")),
+            }
+        );
+    }
+
+    #[test]
+    fn uuid_string_alias() {
+        let uuid = Uuid::new_v4();
+
+        let alias = Alias::from_existing(&uuid.to_string());
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Uuid(uuid),
+                extension: None,
+            }
+        )
+    }
+
+    #[test]
+    fn uuid_string_alias_ext() {
+        let uuid = Uuid::new_v4();
+
+        let alias_str = format!("{}.mp4", uuid);
+        let alias = Alias::from_existing(&alias_str);
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Uuid(uuid),
+                extension: Some(String::from(".mp4")),
+            }
+        )
+    }
+
+    #[test]
+    fn bytes_alias() {
+        let alias = Alias::from_slice(b"blah").unwrap();
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Name(String::from("blah")),
+                extension: None
+            }
+        );
+    }
+
+    #[test]
+    fn bytes_alias_ext() {
+        let alias = Alias::from_slice(b"blah.mp4").unwrap();
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Name(String::from("blah")),
+                extension: Some(String::from(".mp4")),
+            }
+        );
+    }
+
+    #[test]
+    fn uuid_bytes_alias() {
+        let uuid = Uuid::new_v4();
+
+        let alias = Alias::from_slice(&uuid.as_bytes()[..]).unwrap();
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Uuid(uuid),
+                extension: None,
+            }
+        )
+    }
+
+    #[test]
+    fn uuid_bytes_string_alias() {
+        let uuid = Uuid::new_v4();
+
+        let alias = Alias::from_slice(uuid.to_string().as_bytes()).unwrap();
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Uuid(uuid),
+                extension: None,
+            }
+        )
+    }
+
+    #[test]
+    fn uuid_bytes_alias_ext() {
+        let uuid = Uuid::new_v4();
+
+        let mut alias_bytes = uuid.as_bytes().to_vec();
+        alias_bytes.extend_from_slice(b".mp4");
+
+        let alias = Alias::from_slice(&alias_bytes).unwrap();
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Uuid(uuid),
+                extension: Some(String::from(".mp4")),
+            }
+        )
+    }
+
+    #[test]
+    fn uuid_bytes_string_alias_ext() {
+        let uuid = Uuid::new_v4();
+
+        let alias_str = format!("{}.mp4", uuid);
+        let alias = Alias::from_slice(alias_str.as_bytes()).unwrap();
+
+        assert_eq!(
+            alias,
+            Alias {
+                id: MaybeUuid::Uuid(uuid),
+                extension: Some(String::from(".mp4")),
+            }
+        )
     }
 }
diff --git a/src/repo/old.rs b/src/repo/old.rs
index a61b402..bf4ae65 100644
--- a/src/repo/old.rs
+++ b/src/repo/old.rs
@@ -17,8 +17,9 @@
 // - Settings Tree
 //   - store-migration-progress -> Path Tree Key
 
+use std::path::PathBuf;
+
 use super::{Alias, DeleteToken, Details};
-use uuid::Uuid;
 
 pub(super) struct Old {
     alias_tree: ::sled::Tree,
@@ -91,7 +92,7 @@ impl Old {
             .ok_or_else(|| anyhow::anyhow!("Missing identifier"))
     }
 
-    pub(super) fn variants(&self, hash: &sled::IVec) -> anyhow::Result<Vec<(String, sled::IVec)>> {
+    pub(super) fn variants(&self, hash: &sled::IVec) -> anyhow::Result<Vec<(PathBuf, sled::IVec)>> {
         let filename = self
             .main_tree
             .get(hash)?
@@ -106,13 +107,16 @@ impl Old {
             .scan_prefix(&variant_prefix)
             .filter_map(|res| res.ok())
             .filter_map(|(key, value)| {
-                let key_str = String::from_utf8_lossy(&key);
-                let variant_path = key_str.trim_start_matches(&variant_prefix);
-                if variant_path == "motion" {
+                let variant_path_bytes = &key[variant_prefix.as_bytes().len()..];
+                if variant_path_bytes == b"motion" {
                     return None;
                 }
 
-                Some((variant_path.to_string(), value))
+                let path = String::from_utf8(variant_path_bytes.to_vec()).ok()?;
+                let mut path = PathBuf::from(path);
+                path.pop();
+
+                Some((path, value))
             })
             .collect())
     }
@@ -141,29 +145,15 @@ impl Old {
             .scan_prefix(key)
             .values()
             .filter_map(|res| res.ok())
-            .filter_map(|alias| {
-                let alias_str = String::from_utf8_lossy(&alias);
-
-                let (uuid, ext) = alias_str.split_once('.')?;
-
-                let uuid = uuid.parse::<Uuid>().ok()?;
-
-                Some(Alias {
-                    id: uuid,
-                    extension: ext.to_string(),
-                })
-            })
+            .filter_map(|alias| Alias::from_slice(&alias))
             .collect()
     }
 
     pub(super) fn delete_token(&self, alias: &Alias) -> anyhow::Result<Option<DeleteToken>> {
-        let key = format!("{}{}/delete", alias.id, alias.extension);
+        let key = format!("{}/delete", alias);
 
         if let Some(ivec) = self.alias_tree.get(key)? {
-            let token_str = String::from_utf8_lossy(&ivec);
-            if let Ok(uuid) = token_str.parse::<Uuid>() {
-                return Ok(Some(DeleteToken { id: uuid }));
-            }
+            return Ok(DeleteToken::from_slice(&ivec));
         }
 
         Ok(None)
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index b35500b..d7a3f2f 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -18,7 +18,7 @@ pub(crate) enum Error {
     Sled(#[from] sled::Error),
 
     #[error("Invalid identifier")]
-    Identifier(#[source] Box<dyn std::error::Error + Send + Sync>),
+    Identifier(#[source] Box<dyn std::error::Error + Sync + Send>),
 
     #[error("Invalid details json")]
     Details(#[from] serde_json::Error),
@@ -30,6 +30,7 @@ pub(crate) enum Error {
     Panic,
 }
 
+#[derive(Clone)]
 pub(crate) struct SledRepo {
     settings: Tree,
     identifier_details: Tree,
@@ -62,7 +63,7 @@ impl SledRepo {
     }
 }
 
-#[async_trait::async_trait]
+#[async_trait::async_trait(?Send)]
 impl SettingsRepo for SledRepo {
     type Bytes = IVec;
     type Error = Error;
@@ -89,31 +90,35 @@ impl SettingsRepo for SledRepo {
 fn identifier_bytes<I>(identifier: &I) -> Result<Vec<u8>, Error>
 where
     I: Identifier,
-    I::Error: Send + Sync + 'static,
 {
     identifier
         .to_bytes()
         .map_err(|e| Error::Identifier(Box::new(e)))
 }
 
-fn variant_key(hash: &[u8], variant: &str) -> Result<Vec<u8>, Error> {
+fn variant_key(hash: &[u8], variant: &str) -> Vec<u8> {
     let mut bytes = hash.to_vec();
     bytes.push(b'/');
     bytes.extend_from_slice(variant.as_bytes());
-    Ok(bytes)
+    bytes
 }
 
-#[async_trait::async_trait]
-impl<I> IdentifierRepo<I> for SledRepo
-where
-    I: Identifier + 'static,
-    I::Error: Send + Sync + 'static,
-{
+fn variant_from_key(hash: &[u8], key: &[u8]) -> Option<String> {
+    let prefix_len = hash.len() + 1;
+    let variant_bytes = key.get(prefix_len..)?.to_vec();
+    String::from_utf8(variant_bytes).ok()
+}
+
+#[async_trait::async_trait(?Send)]
+impl IdentifierRepo for SledRepo {
     type Error = Error;
 
-    async fn relate_details(&self, identifier: I, details: Details) -> Result<(), Self::Error> {
-        let key = identifier_bytes(&identifier)?;
-
+    async fn relate_details<I: Identifier>(
+        &self,
+        identifier: &I,
+        details: &Details,
+    ) -> Result<(), Self::Error> {
+        let key = identifier_bytes(identifier)?;
         let details = serde_json::to_vec(&details)?;
 
         b!(
@@ -124,8 +129,8 @@ where
         Ok(())
     }
 
-    async fn details(&self, identifier: I) -> Result<Option<Details>, Self::Error> {
-        let key = identifier_bytes(&identifier)?;
+    async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Self::Error> {
+        let key = identifier_bytes(identifier)?;
 
         let opt = b!(self.identifier_details, identifier_details.get(key));
 
@@ -136,8 +141,8 @@ where
         }
     }
 
-    async fn cleanup(&self, identifier: I) -> Result<(), Self::Error> {
-        let key = identifier_bytes(&identifier)?;
+    async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Self::Error> {
+        let key = identifier_bytes(identifier)?;
 
         b!(self.identifier_details, identifier_details.remove(key));
 
@@ -182,11 +187,12 @@ impl futures_util::Stream for HashStream {
         } else if let Some(mut iter) = this.hashes.take() {
             let fut = Box::pin(async move {
                 actix_rt::task::spawn_blocking(move || {
-                    let opt = iter.next().map(|res| res.map_err(Error::from));
+                    let opt = iter.next();
 
                     (iter, opt)
                 })
                 .await
+                .map(|(iter, opt)| (iter, opt.map(|res| res.map_err(Error::from))))
                 .map_err(Error::from)
             });
 
@@ -204,12 +210,8 @@ fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> {
     v
 }
 
-#[async_trait::async_trait]
-impl<I> HashRepo<I> for SledRepo
-where
-    I: Identifier + 'static,
-    I::Error: Send + Sync + 'static,
-{
+#[async_trait::async_trait(?Send)]
+impl HashRepo for SledRepo {
     type Bytes = IVec;
     type Error = Error;
     type Stream = HashStream;
@@ -232,19 +234,17 @@ where
         Ok(res.map_err(|_| AlreadyExists))
     }
 
-    async fn relate_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error> {
-        let key = hash_alias_key(&hash, &alias);
+    async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error> {
+        let key = hash_alias_key(&hash, alias);
+        let value = alias.to_bytes();
 
-        b!(
-            self.hash_aliases,
-            hash_aliases.insert(key, alias.to_bytes())
-        );
+        b!(self.hash_aliases, hash_aliases.insert(key, value));
 
         Ok(())
     }
 
-    async fn remove_alias(&self, hash: Self::Bytes, alias: Alias) -> Result<(), Self::Error> {
-        let key = hash_alias_key(&hash, &alias);
+    async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error> {
+        let key = hash_alias_key(&hash, alias);
 
         b!(self.hash_aliases, hash_aliases.remove(key));
 
@@ -258,21 +258,28 @@ where
                 .values()
                 .filter_map(Result::ok)
                 .filter_map(|ivec| Alias::from_slice(&ivec))
-                .collect::<Vec<_>>()) as Result<_, Error>
+                .collect::<Vec<_>>()) as Result<_, sled::Error>
         });
 
         Ok(v)
     }
 
-    async fn relate_identifier(&self, hash: Self::Bytes, identifier: I) -> Result<(), Self::Error> {
-        let bytes = identifier_bytes(&identifier)?;
+    async fn relate_identifier<I: Identifier>(
+        &self,
+        hash: Self::Bytes,
+        identifier: &I,
+    ) -> Result<(), Self::Error> {
+        let bytes = identifier_bytes(identifier)?;
 
         b!(self.hash_identifiers, hash_identifiers.insert(hash, bytes));
 
         Ok(())
     }
 
-    async fn identifier(&self, hash: Self::Bytes) -> Result<I, Self::Error> {
+    async fn identifier<I: Identifier + 'static>(
+        &self,
+        hash: Self::Bytes,
+    ) -> Result<I, Self::Error> {
         let opt = b!(self.hash_identifiers, hash_identifiers.get(hash));
 
         opt.ok_or(Error::Missing).and_then(|ivec| {
@@ -280,14 +287,14 @@ where
         })
     }
 
-    async fn relate_variant_identifier(
+    async fn relate_variant_identifier<I: Identifier>(
         &self,
         hash: Self::Bytes,
         variant: String,
-        identifier: I,
+        identifier: &I,
     ) -> Result<(), Self::Error> {
-        let key = variant_key(&hash, &variant)?;
-        let value = identifier_bytes(&identifier)?;
+        let key = variant_key(&hash, &variant);
+        let value = identifier_bytes(identifier)?;
 
         b!(
             self.hash_variant_identifiers,
@@ -297,12 +304,12 @@ where
         Ok(())
     }
 
-    async fn variant_identifier(
+    async fn variant_identifier<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
         variant: String,
     ) -> Result<Option<I>, Self::Error> {
-        let key = variant_key(&hash, &variant)?;
+        let key = variant_key(&hash, &variant);
 
         let opt = b!(
             self.hash_variant_identifiers,
@@ -318,12 +325,33 @@ where
         }
     }
 
-    async fn relate_motion_identifier(
+    async fn variants<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
-        identifier: I,
+    ) -> Result<Vec<(String, I)>, Self::Error> {
+        let vec = b!(
+            self.hash_variant_identifiers,
+            Ok(hash_variant_identifiers
+                .scan_prefix(&hash)
+                .filter_map(|res| res.ok())
+                .filter_map(|(key, ivec)| {
+                    let identifier = I::from_bytes(ivec.to_vec()).ok()?;
+                    let variant = variant_from_key(&hash, &key)?;
+
+                    Some((variant, identifier))
+                })
+                .collect::<Vec<_>>()) as Result<Vec<_>, sled::Error>
+        );
+
+        Ok(vec)
+    }
+
+    async fn relate_motion_identifier<I: Identifier>(
+        &self,
+        hash: Self::Bytes,
+        identifier: &I,
     ) -> Result<(), Self::Error> {
-        let bytes = identifier_bytes(&identifier)?;
+        let bytes = identifier_bytes(identifier)?;
 
         b!(
             self.hash_motion_identifiers,
@@ -333,7 +361,10 @@ where
         Ok(())
     }
 
-    async fn motion_identifier(&self, hash: Self::Bytes) -> Result<Option<I>, Self::Error> {
+    async fn motion_identifier<I: Identifier + 'static>(
+        &self,
+        hash: Self::Bytes,
+    ) -> Result<Option<I>, Self::Error> {
         let opt = b!(
             self.hash_motion_identifiers,
             hash_motion_identifiers.get(hash)
@@ -361,7 +392,7 @@ where
             hash_motion_identifiers.remove(hash2)
         );
 
-        let aliases = HashRepo::<I>::aliases(self, hash.clone()).await?;
+        let aliases = self.aliases(hash.clone()).await?;
         let hash2 = hash.clone();
         b!(self.hash_aliases, {
             for alias in aliases {
@@ -369,7 +400,7 @@ where
 
                 let _ = hash_aliases.remove(key);
             }
-            Ok(()) as Result<(), Error>
+            Ok(()) as Result<(), sled::Error>
         });
 
         let variant_keys = b!(self.hash_variant_identifiers, {
@@ -379,25 +410,25 @@ where
                 .filter_map(Result::ok)
                 .collect::<Vec<_>>();
 
-            Ok(v) as Result<Vec<_>, Error>
+            Ok(v) as Result<Vec<_>, sled::Error>
         });
         b!(self.hash_variant_identifiers, {
             for key in variant_keys {
                 let _ = hash_variant_identifiers.remove(key);
             }
-            Ok(()) as Result<(), Error>
+            Ok(()) as Result<(), sled::Error>
         });
 
         Ok(())
     }
 }
 
-#[async_trait::async_trait]
+#[async_trait::async_trait(?Send)]
 impl AliasRepo for SledRepo {
     type Bytes = sled::IVec;
     type Error = Error;
 
-    async fn create(&self, alias: Alias) -> Result<Result<(), AlreadyExists>, Self::Error> {
+    async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Self::Error> {
         let bytes = alias.to_bytes();
         let bytes2 = bytes.clone();
 
@@ -411,8 +442,8 @@ impl AliasRepo for SledRepo {
 
     async fn relate_delete_token(
         &self,
-        alias: Alias,
-        delete_token: DeleteToken,
+        alias: &Alias,
+        delete_token: &DeleteToken,
     ) -> Result<Result<(), AlreadyExists>, Self::Error> {
         let key = alias.to_bytes();
         let token = delete_token.to_bytes();
@@ -425,7 +456,7 @@ impl AliasRepo for SledRepo {
         Ok(res.map_err(|_| AlreadyExists))
     }
 
-    async fn delete_token(&self, alias: Alias) -> Result<DeleteToken, Self::Error> {
+    async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Self::Error> {
         let key = alias.to_bytes();
 
         let opt = b!(self.alias_delete_tokens, alias_delete_tokens.get(key));
@@ -434,7 +465,7 @@ impl AliasRepo for SledRepo {
             .ok_or(Error::Missing)
     }
 
-    async fn relate_hash(&self, alias: Alias, hash: Self::Bytes) -> Result<(), Self::Error> {
+    async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Self::Error> {
         let key = alias.to_bytes();
 
         b!(self.alias_hashes, alias_hashes.insert(key, hash));
@@ -442,7 +473,7 @@ impl AliasRepo for SledRepo {
         Ok(())
     }
 
-    async fn hash(&self, alias: Alias) -> Result<Self::Bytes, Self::Error> {
+    async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Self::Error> {
         let key = alias.to_bytes();
 
         let opt = b!(self.alias_hashes, alias_hashes.get(key));
@@ -450,7 +481,7 @@ impl AliasRepo for SledRepo {
         opt.ok_or(Error::Missing)
     }
 
-    async fn cleanup(&self, alias: Alias) -> Result<(), Self::Error> {
+    async fn cleanup(&self, alias: &Alias) -> Result<(), Self::Error> {
         let key = alias.to_bytes();
 
         let key2 = key.clone();
diff --git a/src/store.rs b/src/store.rs
index 46fad1e..668950b 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -8,7 +8,7 @@ pub(crate) mod file_store;
 pub(crate) mod object_store;
 
 pub(crate) trait Identifier: Send + Sync + Clone + Debug {
-    type Error: std::error::Error;
+    type Error: std::error::Error + Send + Sync + 'static;
 
     fn to_bytes(&self) -> Result<Vec<u8>, Self::Error>;
 
@@ -19,23 +19,18 @@ pub(crate) trait Identifier: Send + Sync + Clone + Debug {
 
 #[async_trait::async_trait(?Send)]
 pub(crate) trait Store: Send + Sync + Clone + Debug + 'static {
-    type Error: std::error::Error;
+    type Error: std::error::Error + Send + Sync + 'static;
     type Identifier: Identifier<Error = Self::Error>;
     type Stream: Stream<Item = std::io::Result<Bytes>>;
 
     async fn save_async_read<Reader>(
         &self,
         reader: &mut Reader,
-        filename: &str,
     ) -> Result<Self::Identifier, Self::Error>
     where
         Reader: AsyncRead + Unpin;
 
-    async fn save_bytes(
-        &self,
-        bytes: Bytes,
-        filename: &str,
-    ) -> Result<Self::Identifier, Self::Error>;
+    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error>;
 
     async fn to_stream(
         &self,
diff --git a/src/store/file_store.rs b/src/store/file_store.rs
index 84b2f0e..c923731 100644
--- a/src/store/file_store.rs
+++ b/src/store/file_store.rs
@@ -1,4 +1,8 @@
-use crate::{file::File, store::Store};
+use crate::{
+    file::File,
+    repo::{Repo, SettingsRepo},
+    store::Store,
+};
 use actix_web::web::Bytes;
 use futures_util::stream::Stream;
 use std::{
@@ -10,24 +14,22 @@ use tokio::io::{AsyncRead, AsyncWrite};
 use tracing::{debug, error, instrument};
 
 mod file_id;
-mod restructure;
 pub(crate) use file_id::FileId;
 
 // - Settings Tree
 //   - last-path -> last generated path
-//   - fs-restructure-01-complete -> bool
 
 const GENERATOR_KEY: &[u8] = b"last-path";
 
 #[derive(Debug, thiserror::Error)]
 pub(crate) enum FileError {
-    #[error(transparent)]
-    Sled(#[from] sled::Error),
+    #[error("Failed to interact with sled db")]
+    Sled(#[from] crate::repo::sled::Error),
 
-    #[error(transparent)]
+    #[error("Failed to read or write file")]
     Io(#[from] std::io::Error),
 
-    #[error(transparent)]
+    #[error("Failed to generate path")]
     PathGenerator(#[from] storage_path_generator::PathError),
 
     #[error("Error formatting file store identifier")]
@@ -44,7 +46,7 @@ pub(crate) enum FileError {
 pub(crate) struct FileStore {
     path_gen: Generator,
     root_dir: PathBuf,
-    settings_tree: sled::Tree,
+    repo: Repo,
 }
 
 #[async_trait::async_trait(?Send)]
@@ -57,12 +59,11 @@ impl Store for FileStore {
     async fn save_async_read<Reader>(
         &self,
         reader: &mut Reader,
-        filename: &str,
     ) -> Result<Self::Identifier, Self::Error>
     where
         Reader: AsyncRead + Unpin,
     {
-        let path = self.next_file(filename)?;
+        let path = self.next_file().await?;
 
         if let Err(e) = self.safe_save_reader(&path, reader).await {
             self.safe_remove_file(&path).await?;
@@ -73,12 +74,8 @@ impl Store for FileStore {
     }
 
     #[tracing::instrument(skip(bytes))]
-    async fn save_bytes(
-        &self,
-        bytes: Bytes,
-        filename: &str,
-    ) -> Result<Self::Identifier, Self::Error> {
-        let path = self.next_file(filename)?;
+    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
+        let path = self.next_file().await?;
 
         if let Err(e) = self.safe_save_bytes(&path, bytes).await {
             self.safe_remove_file(&path).await?;
@@ -141,23 +138,26 @@ impl Store for FileStore {
 }
 
 impl FileStore {
-    pub fn build(root_dir: PathBuf, db: &sled::Db) -> Result<Self, FileError> {
-        let settings_tree = db.open_tree("settings")?;
-
-        let path_gen = init_generator(&settings_tree)?;
+    pub(crate) async fn build(root_dir: PathBuf, repo: Repo) -> Result<Self, FileError> {
+        let path_gen = init_generator(&repo).await?;
 
         Ok(FileStore {
             root_dir,
             path_gen,
-            settings_tree,
+            repo,
         })
     }
 
-    fn next_directory(&self) -> Result<PathBuf, FileError> {
+    async fn next_directory(&self) -> Result<PathBuf, FileError> {
         let path = self.path_gen.next();
 
-        self.settings_tree
-            .insert(GENERATOR_KEY, path.to_be_bytes())?;
+        match self.repo {
+            Repo::Sled(ref sled_repo) => {
+                sled_repo
+                    .set(GENERATOR_KEY, path.to_be_bytes().into())
+                    .await?;
+            }
+        }
 
         let mut target_path = self.root_dir.clone();
         for dir in path.to_strings() {
@@ -167,8 +167,9 @@ impl FileStore {
         Ok(target_path)
     }
 
-    fn next_file(&self, filename: &str) -> Result<PathBuf, FileError> {
-        let target_path = self.next_directory()?;
+    async fn next_file(&self) -> Result<PathBuf, FileError> {
+        let target_path = self.next_directory().await?;
+        let filename = uuid::Uuid::new_v4().to_string();
 
         Ok(target_path.join(filename))
     }
@@ -289,13 +290,17 @@ pub(crate) async fn safe_create_parent<P: AsRef<Path>>(path: P) -> Result<(), Fi
     Ok(())
 }
 
-fn init_generator(settings: &sled::Tree) -> Result<Generator, FileError> {
-    if let Some(ivec) = settings.get(GENERATOR_KEY)? {
-        Ok(Generator::from_existing(
-            storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
-        ))
-    } else {
-        Ok(Generator::new())
+async fn init_generator(repo: &Repo) -> Result<Generator, FileError> {
+    match repo {
+        Repo::Sled(sled_repo) => {
+            if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
+                Ok(Generator::from_existing(
+                    storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
+                ))
+            } else {
+                Ok(Generator::new())
+            }
+        }
     }
 }
 
diff --git a/src/store/file_store/restructure.rs b/src/store/file_store/restructure.rs
deleted file mode 100644
index 81e990c..0000000
--- a/src/store/file_store/restructure.rs
+++ /dev/null
@@ -1,118 +0,0 @@
-use crate::{
-    error::{Error, UploadError},
-    store::file_store::FileStore,
-    upload_manager::UploadManager,
-};
-use std::path::{Path, PathBuf};
-
-const RESTRUCTURE_COMPLETE: &[u8] = b"fs-restructure-01-complete";
-const DETAILS: &[u8] = b"details";
-
-impl UploadManager {
-    #[tracing::instrument(skip(self))]
-    pub(crate) async fn restructure(&self, store: &FileStore) -> Result<(), Error> {
-        if self.restructure_complete(store)? {
-            return Ok(());
-        }
-
-        for res in self.inner().filename_tree.iter() {
-            let (filename, hash) = res?;
-            let filename = String::from_utf8(filename.to_vec())?;
-            tracing::info!("Migrating {}", filename);
-
-            let file_path = store.root_dir.join("files").join(&filename);
-
-            if tokio::fs::metadata(&file_path).await.is_ok() {
-                let target_path = store.next_directory()?.join(&filename);
-
-                let target_path_bytes = self
-                    .generalize_path(store, &target_path)?
-                    .to_str()
-                    .ok_or(UploadError::Path)?
-                    .as_bytes()
-                    .to_vec();
-
-                self.inner()
-                    .identifier_tree
-                    .insert(filename.as_bytes(), target_path_bytes)?;
-
-                store.safe_move_file(file_path, target_path).await?;
-            }
-
-            let (start, end) = variant_key_bounds(&hash);
-
-            for res in self.inner().main_tree.range(start..end) {
-                let (hash_variant_key, variant_path_or_details) = res?;
-
-                if !hash_variant_key.ends_with(DETAILS) {
-                    let variant_path =
-                        PathBuf::from(String::from_utf8(variant_path_or_details.to_vec())?);
-                    if tokio::fs::metadata(&variant_path).await.is_ok() {
-                        let target_path = store.next_directory()?.join(&filename);
-
-                        let relative_target_path_bytes = self
-                            .generalize_path(store, &target_path)?
-                            .to_str()
-                            .ok_or(UploadError::Path)?
-                            .as_bytes()
-                            .to_vec();
-
-                        let variant_key =
-                            self.migrate_variant_key(store, &variant_path, &filename)?;
-
-                        self.inner()
-                            .identifier_tree
-                            .insert(variant_key, relative_target_path_bytes)?;
-
-                        store
-                            .safe_move_file(variant_path.clone(), target_path)
-                            .await?;
-                        store.try_remove_parents(&variant_path).await;
-                    }
-                }
-
-                self.inner().main_tree.remove(hash_variant_key)?;
-            }
-        }
-
-        self.mark_restructure_complete(store)?;
-        Ok(())
-    }
-
-    fn restructure_complete(&self, store: &FileStore) -> Result<bool, Error> {
-        Ok(store.settings_tree.get(RESTRUCTURE_COMPLETE)?.is_some())
-    }
-
-    fn mark_restructure_complete(&self, store: &FileStore) -> Result<(), Error> {
-        store.settings_tree.insert(RESTRUCTURE_COMPLETE, b"true")?;
-
-        Ok(())
-    }
-
-    fn generalize_path<'a>(&self, store: &FileStore, path: &'a Path) -> Result<&'a Path, Error> {
-        Ok(path.strip_prefix(&store.root_dir)?)
-    }
-
-    fn migrate_variant_key(
-        &self,
-        store: &FileStore,
-        variant_process_path: &Path,
-        filename: &str,
-    ) -> Result<Vec<u8>, Error> {
-        let path = self
-            .generalize_path(store, variant_process_path)?
-            .strip_prefix("files")?;
-
-        self.variant_key(path, filename)
-    }
-}
-
-pub(crate) fn variant_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
-    let mut start = hash.to_vec();
-    start.extend(&[2]);
-
-    let mut end = hash.to_vec();
-    end.extend(&[3]);
-
-    (start, end)
-}
diff --git a/src/store/object_store.rs b/src/store/object_store.rs
index d07f29a..8fea267 100644
--- a/src/store/object_store.rs
+++ b/src/store/object_store.rs
@@ -1,4 +1,7 @@
-use crate::store::Store;
+use crate::{
+    repo::{Repo, SettingsRepo},
+    store::Store,
+};
 use actix_web::web::Bytes;
 use futures_util::stream::Stream;
 use s3::{
@@ -22,26 +25,26 @@ const GENERATOR_KEY: &[u8] = b"last-path";
 
 #[derive(Debug, thiserror::Error)]
 pub(crate) enum ObjectError {
-    #[error(transparent)]
+    #[error("Failed to generate path")]
     PathGenerator(#[from] storage_path_generator::PathError),
 
-    #[error(transparent)]
-    Sled(#[from] sled::Error),
+    #[error("Failed to interact with sled repo")]
+    Sled(#[from] crate::repo::sled::Error),
 
-    #[error(transparent)]
+    #[error("Failed to parse string")]
     Utf8(#[from] FromUtf8Error),
 
     #[error("Invalid length")]
     Length,
 
-    #[error("Storage error: {0}")]
+    #[error("Storage error")]
     Anyhow(#[from] anyhow::Error),
 }
 
 #[derive(Clone)]
 pub(crate) struct ObjectStore {
     path_gen: Generator,
-    settings_tree: sled::Tree,
+    repo: Repo,
     bucket: Bucket,
     client: reqwest::Client,
 }
@@ -63,12 +66,11 @@ impl Store for ObjectStore {
     async fn save_async_read<Reader>(
         &self,
         reader: &mut Reader,
-        filename: &str,
     ) -> Result<Self::Identifier, Self::Error>
     where
         Reader: AsyncRead + Unpin,
     {
-        let path = self.next_file(filename)?;
+        let path = self.next_file().await?;
 
         self.bucket
             .put_object_stream(&self.client, reader, &path)
@@ -78,12 +80,8 @@ impl Store for ObjectStore {
     }
 
     #[tracing::instrument(skip(bytes))]
-    async fn save_bytes(
-        &self,
-        bytes: Bytes,
-        filename: &str,
-    ) -> Result<Self::Identifier, Self::Error> {
-        let path = self.next_file(filename)?;
+    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
+        let path = self.next_file().await?;
 
         self.bucket.put_object(&self.client, &path, &bytes).await?;
 
@@ -154,23 +152,21 @@ impl Store for ObjectStore {
 
 impl ObjectStore {
     #[allow(clippy::too_many_arguments)]
-    pub(crate) fn build(
+    pub(crate) async fn build(
         bucket_name: &str,
         region: Region,
         access_key: Option<String>,
         secret_key: Option<String>,
         security_token: Option<String>,
         session_token: Option<String>,
-        db: &sled::Db,
+        repo: Repo,
         client: reqwest::Client,
     ) -> Result<ObjectStore, ObjectError> {
-        let settings_tree = db.open_tree("settings")?;
-
-        let path_gen = init_generator(&settings_tree)?;
+        let path_gen = init_generator(&repo).await?;
 
         Ok(ObjectStore {
             path_gen,
-            settings_tree,
+            repo,
             bucket: Bucket::new_with_path_style(
                 bucket_name,
                 match region {
@@ -191,29 +187,39 @@ impl ObjectStore {
         })
     }
 
-    fn next_directory(&self) -> Result<Path, ObjectError> {
+    async fn next_directory(&self) -> Result<Path, ObjectError> {
         let path = self.path_gen.next();
 
-        self.settings_tree
-            .insert(GENERATOR_KEY, path.to_be_bytes())?;
+        match self.repo {
+            Repo::Sled(ref sled_repo) => {
+                sled_repo
+                    .set(GENERATOR_KEY, path.to_be_bytes().into())
+                    .await?;
+            }
+        }
 
         Ok(path)
     }
 
-    fn next_file(&self, filename: &str) -> Result<String, ObjectError> {
-        let path = self.next_directory()?.to_strings().join("/");
+    async fn next_file(&self) -> Result<String, ObjectError> {
+        let path = self.next_directory().await?.to_strings().join("/");
+        let filename = uuid::Uuid::new_v4().to_string();
 
         Ok(format!("{}/{}", path, filename))
     }
 }
 
-fn init_generator(settings: &sled::Tree) -> Result<Generator, ObjectError> {
-    if let Some(ivec) = settings.get(GENERATOR_KEY)? {
-        Ok(Generator::from_existing(
-            storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
-        ))
-    } else {
-        Ok(Generator::new())
+async fn init_generator(repo: &Repo) -> Result<Generator, ObjectError> {
+    match repo {
+        Repo::Sled(sled_repo) => {
+            if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
+                Ok(Generator::from_existing(
+                    storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
+                ))
+            } else {
+                Ok(Generator::new())
+            }
+        }
     }
 }
 
diff --git a/src/upload_manager.rs b/src/upload_manager.rs
index 7e1b7cf..3052414 100644
--- a/src/upload_manager.rs
+++ b/src/upload_manager.rs
@@ -4,13 +4,15 @@ use crate::{
     error::{Error, UploadError},
     ffmpeg::{InputFormat, ThumbnailFormat},
     magick::details_hint,
-    migrate::{alias_id_key, alias_key, alias_key_bounds},
+    repo::{
+        sled::SledRepo, Alias, AliasRepo, DeleteToken, HashRepo, IdentifierRepo, Repo, SettingsRepo,
+    },
     store::{Identifier, Store},
 };
-use actix_web::web;
+use futures_util::StreamExt;
 use sha2::Digest;
-use std::{string::FromUtf8Error, sync::Arc};
-use tracing::{debug, error, info, instrument, warn, Span};
+use std::sync::Arc;
+use tracing::{debug, error, instrument, Span};
 use tracing_futures::Instrument;
 
 mod hasher;
@@ -18,28 +20,6 @@ mod session;
 
 pub(super) use session::UploadManagerSession;
 
-// TREE STRUCTURE
-// - Alias Tree
-//   - alias -> hash
-//   - alias / id -> u64(id)
-//   - alias / delete -> delete token
-// - Main Tree
-//   - hash -> filename
-//   - hash 0 u64(id) -> alias
-//   - DEPRECATED:
-//     - hash 2 variant path -> variant path
-//     - hash 2 vairant path details -> details
-// - Filename Tree
-//   - filename -> hash
-// - Details Tree
-//   - filename / S::Identifier -> details
-// - Identifier Tree
-//   - filename -> S::Identifier
-//   - filename / variant path -> S::Identifier
-//   - filename / motion -> S::Identifier
-// - Settings Tree
-//   - store-migration-progress -> Path Tree Key
-
 const STORE_MIGRATION_PROGRESS: &[u8] = b"store-migration-progress";
 
 #[derive(Clone)]
@@ -50,33 +30,17 @@ pub(crate) struct UploadManager {
 pub(crate) struct UploadManagerInner {
     format: Option<Format>,
     hasher: sha2::Sha256,
-    pub(crate) alias_tree: sled::Tree,
-    pub(crate) filename_tree: sled::Tree,
-    pub(crate) main_tree: sled::Tree,
-    details_tree: sled::Tree,
-    settings_tree: sled::Tree,
-    pub(crate) identifier_tree: sled::Tree,
-    db: sled::Db,
-}
-
-struct FilenameIVec {
-    inner: sled::IVec,
+    repo: Repo,
 }
 
 impl UploadManager {
     /// Create a new UploadManager
-    pub(crate) async fn new(db: sled::Db, format: Option<Format>) -> Result<Self, Error> {
+    pub(crate) async fn new(repo: Repo, format: Option<Format>) -> Result<Self, Error> {
         let manager = UploadManager {
             inner: Arc::new(UploadManagerInner {
                 format,
                 hasher: sha2::Sha256::new(),
-                alias_tree: db.open_tree("alias")?,
-                filename_tree: db.open_tree("filename")?,
-                main_tree: db.open_tree("main")?,
-                details_tree: db.open_tree("details")?,
-                settings_tree: db.open_tree("settings")?,
-                identifier_tree: db.open_tree("path")?,
-                db,
+                repo,
             }),
         };
 
@@ -89,92 +53,32 @@ impl UploadManager {
         S2: Store,
         Error: From<S1::Error> + From<S2::Error>,
     {
-        let iter =
-            if let Some(starting_line) = self.inner.settings_tree.get(STORE_MIGRATION_PROGRESS)? {
-                self.inner.identifier_tree.range(starting_line..)
-            } else {
-                self.inner.identifier_tree.iter()
-            };
-
-        for res in iter {
-            let (key, identifier) = res?;
-
-            let identifier = S1::Identifier::from_bytes(identifier.to_vec())?;
-
-            let filename =
-                if let Some((filename, _)) = String::from_utf8_lossy(&key).split_once('/') {
-                    filename.to_string()
-                } else {
-                    String::from_utf8_lossy(&key).to_string()
-                };
-
-            let stream = from.to_stream(&identifier, None, None).await?;
-            futures_util::pin_mut!(stream);
-            let mut reader = tokio_util::io::StreamReader::new(stream);
-
-            let new_identifier = to.save_async_read(&mut reader, &filename).await?;
-
-            let details_key = self.details_key(&identifier, &filename)?;
-
-            if let Some(details) = self.inner.details_tree.get(details_key.clone())? {
-                let new_details_key = self.details_key(&new_identifier, &filename)?;
-
-                self.inner.details_tree.insert(new_details_key, details)?;
-            }
-
-            self.inner
-                .identifier_tree
-                .insert(key.clone(), new_identifier.to_bytes()?)?;
-            self.inner.details_tree.remove(details_key)?;
-            self.inner
-                .settings_tree
-                .insert(STORE_MIGRATION_PROGRESS, key)?;
-
-            let (ident, detail, settings) = futures_util::future::join3(
-                self.inner.identifier_tree.flush_async(),
-                self.inner.details_tree.flush_async(),
-                self.inner.settings_tree.flush_async(),
-            )
-            .await;
-
-            ident?;
-            detail?;
-            settings?;
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => do_migrate_store(sled_repo, from, to).await,
         }
-
-        // clean up the migration key to avoid interfering with future migrations
-        self.inner.settings_tree.remove(STORE_MIGRATION_PROGRESS)?;
-        self.inner.settings_tree.flush_async().await?;
-
-        Ok(())
     }
 
-    pub(crate) fn inner(&self) -> &UploadManagerInner {
-        &self.inner
-    }
-
-    pub(crate) async fn still_identifier_from_filename<S: Store + Clone>(
+    pub(crate) async fn still_identifier_from_alias<S: Store + Clone>(
         &self,
         store: S,
-        filename: String,
+        alias: &Alias,
     ) -> Result<S::Identifier, Error>
     where
         Error: From<S::Error>,
     {
-        let identifier = self.identifier_from_filename::<S>(filename.clone()).await?;
-        let details =
-            if let Some(details) = self.variant_details(&identifier, filename.clone()).await? {
-                details
-            } else {
-                let hint = details_hint(&filename);
-                Details::from_store(store.clone(), identifier.clone(), hint).await?
-            };
+        let identifier = self.identifier_from_alias::<S>(alias).await?;
+        let details = if let Some(details) = self.details(&identifier).await? {
+            details
+        } else {
+            let hint = details_hint(alias);
+            Details::from_store(store.clone(), identifier.clone(), hint).await?
+        };
 
         if !details.is_motion() {
             return Ok(identifier);
         }
 
-        if let Some(motion_identifier) = self.motion_identifier::<S>(&filename).await? {
+        if let Some(motion_identifier) = self.motion_identifier::<S>(alias).await? {
             return Ok(motion_identifier);
         }
 
@@ -186,101 +90,93 @@ impl UploadManager {
             ThumbnailFormat::Jpeg,
         )
         .await?;
-        let motion_identifier = store.save_async_read(&mut reader, &filename).await?;
+        let motion_identifier = store.save_async_read(&mut reader).await?;
         drop(permit);
 
-        self.store_motion_path(&filename, &motion_identifier)
+        self.store_motion_identifier(alias, &motion_identifier)
             .await?;
         Ok(motion_identifier)
     }
 
     async fn motion_identifier<S: Store>(
         &self,
-        filename: &str,
-    ) -> Result<Option<S::Identifier>, Error>
-    where
-        Error: From<S::Error>,
-    {
-        let identifier_tree = self.inner.identifier_tree.clone();
-        let motion_key = format!("{}/motion", filename);
-
-        let opt = web::block(move || identifier_tree.get(motion_key.as_bytes())).await??;
-
-        if let Some(ivec) = opt {
-            return Ok(Some(S::Identifier::from_bytes(ivec.to_vec())?));
+        alias: &Alias,
+    ) -> Result<Option<S::Identifier>, Error> {
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let hash = sled_repo.hash(alias).await?;
+                Ok(sled_repo.motion_identifier(hash).await?)
+            }
         }
-
-        Ok(None)
     }
 
-    async fn store_motion_path<I: Identifier>(
+    async fn store_motion_identifier<I: Identifier + 'static>(
         &self,
-        filename: &str,
+        alias: &Alias,
         identifier: &I,
-    ) -> Result<(), Error>
-    where
-        Error: From<I::Error>,
-    {
-        let identifier_bytes = identifier.to_bytes()?;
-        let motion_key = format!("{}/motion", filename);
-        let identifier_tree = self.inner.identifier_tree.clone();
-
-        web::block(move || identifier_tree.insert(motion_key.as_bytes(), identifier_bytes))
-            .await??;
-        Ok(())
+    ) -> Result<(), Error> {
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let hash = sled_repo.hash(alias).await?;
+                Ok(sled_repo.relate_motion_identifier(hash, identifier).await?)
+            }
+        }
     }
 
     #[instrument(skip(self))]
-    pub(crate) async fn identifier_from_filename<S: Store>(
+    pub(crate) async fn identifier_from_alias<S: Store>(
         &self,
-        filename: String,
-    ) -> Result<S::Identifier, Error>
-    where
-        Error: From<S::Error>,
-    {
-        let identifier_tree = self.inner.identifier_tree.clone();
-        let path_ivec = web::block(move || identifier_tree.get(filename.as_bytes()))
-            .await??
-            .ok_or(UploadError::MissingFile)?;
-
-        let identifier = S::Identifier::from_bytes(path_ivec.to_vec())?;
-
-        Ok(identifier)
+        alias: &Alias,
+    ) -> Result<S::Identifier, Error> {
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let hash = sled_repo.hash(alias).await?;
+                Ok(sled_repo.identifier(hash).await?)
+            }
+        }
     }
 
     #[instrument(skip(self))]
     async fn store_identifier<I: Identifier>(
         &self,
-        filename: String,
+        hash: Vec<u8>,
         identifier: &I,
-    ) -> Result<(), Error>
-    where
-        Error: From<I::Error>,
-    {
-        let identifier_bytes = identifier.to_bytes()?;
-        let identifier_tree = self.inner.identifier_tree.clone();
-        web::block(move || identifier_tree.insert(filename.as_bytes(), identifier_bytes)).await??;
-        Ok(())
+    ) -> Result<(), Error> {
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                Ok(sled_repo.relate_identifier(hash.into(), identifier).await?)
+            }
+        }
     }
 
     #[instrument(skip(self))]
     pub(crate) async fn variant_identifier<S: Store>(
         &self,
+        alias: &Alias,
         process_path: &std::path::Path,
-        filename: &str,
-    ) -> Result<Option<S::Identifier>, Error>
-    where
-        Error: From<S::Error>,
-    {
-        let key = self.variant_key(process_path, filename)?;
-        let identifier_tree = self.inner.identifier_tree.clone();
-        let path_opt = web::block(move || identifier_tree.get(key)).await??;
+    ) -> Result<Option<S::Identifier>, Error> {
+        let variant = process_path.to_string_lossy().to_string();
 
-        if let Some(ivec) = path_opt {
-            let identifier = S::Identifier::from_bytes(ivec.to_vec())?;
-            Ok(Some(identifier))
-        } else {
-            Ok(None)
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let hash = sled_repo.hash(alias).await?;
+                Ok(sled_repo.variant_identifier(hash, variant).await?)
+            }
+        }
+    }
+
+    /// Store the path to a generated image variant so we can easily clean it up later
+    #[instrument(skip(self))]
+    pub(crate) async fn store_full_res<I: Identifier>(
+        &self,
+        alias: &Alias,
+        identifier: &I,
+    ) -> Result<(), Error> {
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let hash = sled_repo.hash(alias).await?;
+                Ok(sled_repo.relate_identifier(hash, identifier).await?)
+            }
         }
     }
 
@@ -288,139 +184,71 @@ impl UploadManager {
     #[instrument(skip(self))]
     pub(crate) async fn store_variant<I: Identifier>(
         &self,
-        variant_process_path: Option<&std::path::Path>,
+        alias: &Alias,
+        variant_process_path: &std::path::Path,
         identifier: &I,
-        filename: &str,
-    ) -> Result<(), Error>
-    where
-        Error: From<I::Error>,
-    {
-        let key = if let Some(path) = variant_process_path {
-            self.variant_key(path, filename)?
-        } else {
-            let mut vec = filename.as_bytes().to_vec();
-            vec.extend(b"/");
-            vec.extend(&identifier.to_bytes()?);
-            vec
-        };
-        let identifier_tree = self.inner.identifier_tree.clone();
-        let identifier_bytes = identifier.to_bytes()?;
+    ) -> Result<(), Error> {
+        let variant = variant_process_path.to_string_lossy().to_string();
 
-        debug!("Storing variant");
-        web::block(move || identifier_tree.insert(key, identifier_bytes)).await??;
-        debug!("Stored variant");
-
-        Ok(())
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let hash = sled_repo.hash(alias).await?;
+                Ok(sled_repo
+                    .relate_variant_identifier(hash, variant, identifier)
+                    .await?)
+            }
+        }
     }
 
     /// Get the image details for a given variant
     #[instrument(skip(self))]
-    pub(crate) async fn variant_details<I: Identifier>(
+    pub(crate) async fn details<I: Identifier>(
         &self,
         identifier: &I,
-        filename: String,
     ) -> Result<Option<Details>, Error>
     where
         Error: From<I::Error>,
     {
-        let key = self.details_key(identifier, &filename)?;
-        let details_tree = self.inner.details_tree.clone();
-
-        debug!("Getting details");
-        let opt = match web::block(move || details_tree.get(key)).await?? {
-            Some(ivec) => match serde_json::from_slice(&ivec) {
-                Ok(details) => Some(details),
-                Err(_) => None,
-            },
-            None => None,
-        };
-        debug!("Got details");
-
-        Ok(opt)
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => Ok(sled_repo.details(identifier).await?),
+        }
     }
 
     #[instrument(skip(self))]
-    pub(crate) async fn store_variant_details<I: Identifier>(
+    pub(crate) async fn store_details<I: Identifier>(
         &self,
         identifier: &I,
-        filename: String,
         details: &Details,
-    ) -> Result<(), Error>
-    where
-        Error: From<I::Error>,
-    {
-        let key = self.details_key(identifier, &filename)?;
-        let details_tree = self.inner.details_tree.clone();
-        let details_value = serde_json::to_vec(details)?;
-
-        debug!("Storing details");
-        web::block(move || details_tree.insert(key, details_value)).await??;
-        debug!("Stored details");
-
-        Ok(())
-    }
-
-    /// Get a list of aliases for a given file
-    pub(crate) async fn aliases_by_filename(&self, filename: String) -> Result<Vec<String>, Error> {
-        let fname_tree = self.inner.filename_tree.clone();
-        let hash = web::block(move || fname_tree.get(filename.as_bytes()))
-            .await??
-            .ok_or(UploadError::MissingAlias)?;
-
-        self.aliases_by_hash(&hash).await
+    ) -> Result<(), Error> {
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => Ok(sled_repo.relate_details(identifier, details).await?),
+        }
     }
 
     /// Get a list of aliases for a given alias
-    pub(crate) async fn aliases_by_alias(&self, alias: String) -> Result<Vec<String>, Error> {
-        let alias_tree = self.inner.alias_tree.clone();
-        let hash = web::block(move || alias_tree.get(alias.as_bytes()))
-            .await??
-            .ok_or(UploadError::MissingFilename)?;
-
-        self.aliases_by_hash(&hash).await
-    }
-
-    async fn aliases_by_hash(&self, hash: &sled::IVec) -> Result<Vec<String>, Error> {
-        let (start, end) = alias_key_bounds(hash);
-        let main_tree = self.inner.main_tree.clone();
-        let aliases = web::block(move || {
-            main_tree
-                .range(start..end)
-                .values()
-                .collect::<Result<Vec<_>, _>>()
-        })
-        .await??;
-
-        debug!("Got {} aliases for hash", aliases.len());
-        let aliases = aliases
-            .into_iter()
-            .filter_map(|s| String::from_utf8(s.to_vec()).ok())
-            .collect::<Vec<_>>();
-
-        for alias in aliases.iter() {
-            debug!("{}", alias);
+    pub(crate) async fn aliases_by_alias(&self, alias: &Alias) -> Result<Vec<Alias>, Error> {
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let hash = sled_repo.hash(alias).await?;
+                Ok(sled_repo.aliases(hash).await?)
+            }
         }
-
-        Ok(aliases)
     }
 
     /// Delete an alias without a delete token
     pub(crate) async fn delete_without_token<S: Store + 'static>(
         &self,
         store: S,
-        alias: String,
+        alias: Alias,
     ) -> Result<(), Error>
     where
         Error: From<S::Error>,
     {
-        let token_key = delete_key(&alias);
-        let alias_tree = self.inner.alias_tree.clone();
-        let token = web::block(move || alias_tree.get(token_key.as_bytes()))
-            .await??
-            .ok_or(UploadError::MissingAlias)?;
+        let token = match self.inner.repo {
+            Repo::Sled(ref sled_repo) => sled_repo.delete_token(&alias).await?,
+        };
 
-        self.delete(store, alias, String::from_utf8(token.to_vec())?)
-            .await
+        self.delete(store, alias, token).await
     }
 
     /// Delete the alias, and the file & variants if no more aliases exist
@@ -428,57 +256,24 @@ impl UploadManager {
     pub(crate) async fn delete<S: Store + 'static>(
         &self,
         store: S,
-        alias: String,
-        token: String,
+        alias: Alias,
+        token: DeleteToken,
     ) -> Result<(), Error>
     where
         Error: From<S::Error>,
     {
-        use sled::Transactional;
-        let main_tree = self.inner.main_tree.clone();
-        let alias_tree = self.inner.alias_tree.clone();
-
-        let span = Span::current();
-        let alias2 = alias.clone();
-        let hash = web::block(move || {
-            [&main_tree, &alias_tree].transaction(|v| {
-                let entered = span.enter();
-                let main_tree = &v[0];
-                let alias_tree = &v[1];
-
-                // -- GET TOKEN --
-                debug!("Deleting alias -> delete-token mapping");
-                let existing_token = alias_tree
-                    .remove(delete_key(&alias2).as_bytes())?
-                    .ok_or_else(|| trans_upload_error(UploadError::MissingAlias))?;
-
-                // Bail if invalid token
-                if existing_token != token {
-                    warn!("Invalid delete token");
-                    return Err(trans_upload_error(UploadError::InvalidToken));
+        let hash = match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let saved_delete_token = sled_repo.delete_token(&alias).await?;
+                if saved_delete_token != token {
+                    return Err(UploadError::InvalidToken.into());
                 }
-
-                // -- GET ID FOR HASH TREE CLEANUP --
-                debug!("Deleting alias -> id mapping");
-                let id = alias_tree
-                    .remove(alias_id_key(&alias2).as_bytes())?
-                    .ok_or_else(|| trans_upload_error(UploadError::MissingAlias))?;
-                let id = String::from_utf8(id.to_vec()).map_err(trans_utf8_error)?;
-
-                // -- GET HASH FOR HASH TREE CLEANUP --
-                debug!("Deleting alias -> hash mapping");
-                let hash = alias_tree
-                    .remove(alias2.as_bytes())?
-                    .ok_or_else(|| trans_upload_error(UploadError::MissingAlias))?;
-
-                // -- REMOVE HASH TREE ELEMENT --
-                debug!("Deleting hash -> alias mapping");
-                main_tree.remove(alias_key(&hash, &id))?;
-                drop(entered);
-                Ok(hash)
-            })
-        })
-        .await??;
+                let hash = sled_repo.hash(&alias).await?;
+                AliasRepo::cleanup(sled_repo, &alias).await?;
+                sled_repo.remove_alias(hash.clone(), &alias).await?;
+                hash.to_vec()
+            }
+        };
 
         self.check_delete_files(store, hash).await
     }
@@ -486,206 +281,163 @@ impl UploadManager {
     async fn check_delete_files<S: Store + 'static>(
         &self,
         store: S,
-        hash: sled::IVec,
+        hash: Vec<u8>,
     ) -> Result<(), Error>
     where
         Error: From<S::Error>,
     {
-        // -- CHECK IF ANY OTHER ALIASES EXIST --
-        let main_tree = self.inner.main_tree.clone();
-        let (start, end) = alias_key_bounds(&hash);
-        debug!("Checking for additional aliases referencing hash");
-        let any_aliases = web::block(move || {
-            Ok(main_tree.range(start..end).next().is_some()) as Result<bool, Error>
-        })
-        .await??;
+        match self.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let hash: <SledRepo as HashRepo>::Bytes = hash.into();
 
-        // Bail if there are existing aliases
-        if any_aliases {
-            debug!("Other aliases reference file, not removing from disk");
-            return Ok(());
-        }
+                let aliases = sled_repo.aliases(hash.clone()).await?;
 
-        // -- DELETE HASH ENTRY --
-        let main_tree = self.inner.main_tree.clone();
-        let hash2 = hash.clone();
-        debug!("Deleting hash -> filename mapping");
-        let filename = web::block(move || main_tree.remove(&hash2))
-            .await??
-            .ok_or(UploadError::MissingFile)?;
-
-        // -- DELETE FILES --
-        let this = self.clone();
-        let cleanup_span = tracing::info_span!(
-            parent: None,
-            "Cleanup",
-            filename = &tracing::field::display(String::from_utf8_lossy(&filename)),
-        );
-        cleanup_span.follows_from(Span::current());
-        debug!("Spawning cleanup task");
-        actix_rt::spawn(
-            async move {
-                if let Err(e) = this
-                    .cleanup_files(store, FilenameIVec::new(filename.clone()))
-                    .await
-                {
-                    error!("Error removing files from fs, {}", e);
+                if !aliases.is_empty() {
+                    return Ok(());
                 }
-                info!(
-                    "Files deleted for {:?}",
-                    String::from_utf8(filename.to_vec())
+
+                let variant_idents = sled_repo
+                    .variants::<S::Identifier>(hash.clone())
+                    .await?
+                    .into_iter()
+                    .map(|(_, v)| v)
+                    .collect::<Vec<_>>();
+                let main_ident = sled_repo.identifier(hash.clone()).await?;
+                let motion_ident = sled_repo.motion_identifier(hash.clone()).await?;
+
+                let repo = sled_repo.clone();
+
+                HashRepo::cleanup(sled_repo, hash).await?;
+
+                let cleanup_span = tracing::info_span!("Cleaning files");
+                cleanup_span.follows_from(Span::current());
+
+                actix_rt::spawn(
+                    async move {
+                        let mut errors = Vec::new();
+
+                        for identifier in variant_idents
+                            .iter()
+                            .chain(&[main_ident])
+                            .chain(motion_ident.iter())
+                        {
+                            debug!("Deleting {:?}", identifier);
+                            if let Err(e) = store.remove(identifier).await {
+                                let e: Error = e.into();
+                                errors.push(e);
+                            }
+
+                            if let Err(e) = IdentifierRepo::cleanup(&repo, identifier).await {
+                                let e: Error = e.into();
+                                errors.push(e);
+                            }
+                        }
+
+                        if !errors.is_empty() {
+                            let span = tracing::error_span!("Error deleting files");
+                            span.in_scope(|| {
+                                for error in errors {
+                                    error!("{}", error);
+                                }
+                            });
+                        }
+                    }
+                    .instrument(cleanup_span),
                 );
             }
-            .instrument(cleanup_span),
-        );
+        }
 
         Ok(())
     }
 
-    /// Fetch the real on-disk filename given an alias
-    #[instrument(skip(self))]
-    pub(crate) async fn from_alias(&self, alias: String) -> Result<String, Error> {
-        let tree = self.inner.alias_tree.clone();
-        debug!("Getting hash from alias");
-        let hash = web::block(move || tree.get(alias.as_bytes()))
-            .await??
-            .ok_or(UploadError::MissingAlias)?;
-
-        let main_tree = self.inner.main_tree.clone();
-        debug!("Getting filename from hash");
-        let filename = web::block(move || main_tree.get(hash))
-            .await??
-            .ok_or(UploadError::MissingFile)?;
-
-        let filename = String::from_utf8(filename.to_vec())?;
-
-        Ok(filename)
-    }
-
     pub(crate) fn session<S: Store + Clone + 'static>(&self, store: S) -> UploadManagerSession<S>
     where
         Error: From<S::Error>,
     {
         UploadManagerSession::new(self.clone(), store)
     }
-
-    // Find image variants and remove them from the DB and the disk
-    #[instrument(skip(self))]
-    async fn cleanup_files<S: Store>(&self, store: S, filename: FilenameIVec) -> Result<(), Error>
-    where
-        Error: From<S::Error>,
-    {
-        let filename = filename.inner;
-
-        let filename2 = filename.clone();
-        let identifier_tree = self.inner.identifier_tree.clone();
-        let identifier = web::block(move || identifier_tree.remove(filename2)).await??;
-
-        let mut errors = Vec::new();
-        if let Some(identifier) = identifier {
-            let identifier = S::Identifier::from_bytes(identifier.to_vec())?;
-            debug!("Deleting {:?}", identifier);
-            if let Err(e) = store.remove(&identifier).await {
-                errors.push(e);
-            }
-        }
-
-        let filename2 = filename.clone();
-        let fname_tree = self.inner.filename_tree.clone();
-        debug!("Deleting filename -> hash mapping");
-        web::block(move || fname_tree.remove(filename2)).await??;
-
-        let path_prefix = filename.clone();
-        let identifier_tree = self.inner.identifier_tree.clone();
-        debug!("Fetching file variants");
-        let identifiers = web::block(move || {
-            identifier_tree
-                .scan_prefix(path_prefix)
-                .values()
-                .collect::<Result<Vec<sled::IVec>, sled::Error>>()
-        })
-        .await??;
-
-        debug!("{} files prepared for deletion", identifiers.len());
-
-        for id in identifiers {
-            let identifier = S::Identifier::from_bytes(id.to_vec())?;
-
-            debug!("Deleting {:?}", identifier);
-            if let Err(e) = store.remove(&identifier).await {
-                errors.push(e);
-            }
-        }
-
-        let path_prefix = filename.clone();
-        let identifier_tree = self.inner.identifier_tree.clone();
-        debug!("Deleting path info");
-        web::block(move || {
-            for res in identifier_tree.scan_prefix(path_prefix).keys() {
-                let key = res?;
-                identifier_tree.remove(key)?;
-            }
-            Ok(()) as Result<(), Error>
-        })
-        .await??;
-
-        for error in errors {
-            error!("Error deleting files, {}", error);
-        }
-        Ok(())
-    }
-
-    pub(crate) fn variant_key(
-        &self,
-        variant_process_path: &std::path::Path,
-        filename: &str,
-    ) -> Result<Vec<u8>, Error> {
-        let path_string = variant_process_path
-            .to_str()
-            .ok_or(UploadError::Path)?
-            .to_string();
-
-        let vec = format!("{}/{}", filename, path_string).as_bytes().to_vec();
-        Ok(vec)
-    }
-
-    fn details_key<I: Identifier>(&self, identifier: &I, filename: &str) -> Result<Vec<u8>, Error>
-    where
-        Error: From<I::Error>,
-    {
-        let mut vec = filename.as_bytes().to_vec();
-        vec.extend(b"/");
-        vec.extend(&identifier.to_bytes()?);
-
-        Ok(vec)
-    }
 }
 
-impl FilenameIVec {
-    fn new(inner: sled::IVec) -> Self {
-        FilenameIVec { inner }
-    }
-}
-
-fn trans_upload_error(
-    upload_error: UploadError,
-) -> sled::transaction::ConflictableTransactionError<Error> {
-    trans_err(upload_error)
-}
-
-fn trans_utf8_error(e: FromUtf8Error) -> sled::transaction::ConflictableTransactionError<Error> {
-    trans_err(e)
-}
-
-fn trans_err<E>(e: E) -> sled::transaction::ConflictableTransactionError<Error>
+async fn migrate_file<S1, S2>(
+    from: &S1,
+    to: &S2,
+    identifier: &S1::Identifier,
+) -> Result<S2::Identifier, Error>
 where
-    Error: From<E>,
+    S1: Store,
+    S2: Store,
+    Error: From<S1::Error> + From<S2::Error>,
 {
-    sled::transaction::ConflictableTransactionError::Abort(e.into())
+    let stream = from.to_stream(identifier, None, None).await?;
+    futures_util::pin_mut!(stream);
+    let mut reader = tokio_util::io::StreamReader::new(stream);
+
+    let new_identifier = to.save_async_read(&mut reader).await?;
+
+    Ok(new_identifier)
 }
 
-fn delete_key(alias: &str) -> String {
-    format!("{}/delete", alias)
+async fn migrate_details<R, I1, I2>(repo: &R, from: I1, to: &I2) -> Result<(), Error>
+where
+    R: IdentifierRepo,
+    I1: Identifier,
+    I2: Identifier,
+    Error: From<<R as IdentifierRepo>::Error>,
+{
+    if let Some(details) = repo.details(&from).await? {
+        repo.relate_details(to, &details).await?;
+        repo.cleanup(&from).await?;
+    }
+
+    Ok(())
+}
+
+async fn do_migrate_store<R, S1, S2>(repo: &R, from: S1, to: S2) -> Result<(), Error>
+where
+    S1: Store,
+    S2: Store,
+    Error: From<S1::Error> + From<S2::Error>,
+    R: IdentifierRepo + HashRepo + SettingsRepo,
+    Error: From<<R as IdentifierRepo>::Error>,
+    Error: From<<R as HashRepo>::Error>,
+    Error: From<<R as SettingsRepo>::Error>,
+{
+    let stream = repo.hashes().await;
+    let mut stream = Box::pin(stream);
+
+    while let Some(hash) = stream.next().await {
+        let hash = hash?;
+        if let Some(identifier) = repo
+            .motion_identifier(hash.as_ref().to_vec().into())
+            .await?
+        {
+            let new_identifier = migrate_file(&from, &to, &identifier).await?;
+            migrate_details(repo, identifier, &new_identifier).await?;
+            repo.relate_motion_identifier(hash.as_ref().to_vec().into(), &new_identifier)
+                .await?;
+        }
+
+        for (variant, identifier) in repo.variants(hash.as_ref().to_vec().into()).await? {
+            let new_identifier = migrate_file(&from, &to, &identifier).await?;
+            migrate_details(repo, identifier, &new_identifier).await?;
+            repo.relate_variant_identifier(hash.as_ref().to_vec().into(), variant, &new_identifier)
+                .await?;
+        }
+
+        let identifier = repo.identifier(hash.as_ref().to_vec().into()).await?;
+        let new_identifier = migrate_file(&from, &to, &identifier).await?;
+        migrate_details(repo, identifier, &new_identifier).await?;
+        repo.relate_identifier(hash.as_ref().to_vec().into(), &new_identifier)
+            .await?;
+
+        repo.set(STORE_MIGRATION_PROGRESS, hash.as_ref().to_vec().into())
+            .await?;
+    }
+
+    // clean up the migration key to avoid interfering with future migrations
+    repo.remove(STORE_MIGRATION_PROGRESS).await?;
+
+    Ok(())
 }
 
 impl std::fmt::Debug for UploadManager {
@@ -693,9 +445,3 @@ impl std::fmt::Debug for UploadManager {
         f.debug_struct("UploadManager").finish()
     }
 }
-
-impl std::fmt::Debug for FilenameIVec {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "{:?}", String::from_utf8(self.inner.to_vec()))
-    }
-}
diff --git a/src/upload_manager/session.rs b/src/upload_manager/session.rs
index 9274697..dca15a1 100644
--- a/src/upload_manager/session.rs
+++ b/src/upload_manager/session.rs
@@ -1,19 +1,17 @@
 use crate::{
     error::{Error, UploadError},
     magick::ValidInputType,
-    migrate::{alias_id_key, alias_key},
+    repo::{Alias, AliasRepo, AlreadyExists, DeleteToken, HashRepo, IdentifierRepo, Repo},
     store::Store,
     upload_manager::{
-        delete_key,
         hasher::{Hash, Hasher},
         UploadManager,
     },
 };
 use actix_web::web;
 use futures_util::stream::{Stream, StreamExt};
-use tracing::{debug, instrument, warn, Span};
+use tracing::{debug, instrument, Span};
 use tracing_futures::Instrument;
-use uuid::Uuid;
 
 pub(crate) struct UploadManagerSession<S: Store + Clone + 'static>
 where
@@ -21,7 +19,7 @@ where
 {
     store: S,
     manager: UploadManager,
-    alias: Option<String>,
+    alias: Option<Alias>,
     finished: bool,
 }
 
@@ -42,19 +40,8 @@ where
         self.finished = true;
     }
 
-    pub(crate) fn alias(&self) -> Option<&str> {
-        self.alias.as_deref()
-    }
-}
-
-enum Dup {
-    Exists,
-    New,
-}
-
-impl Dup {
-    fn exists(&self) -> bool {
-        matches!(self, Dup::Exists)
+    pub(crate) fn alias(&self) -> Option<&Alias> {
+        self.alias.as_ref()
     }
 }
 
@@ -79,20 +66,23 @@ where
             actix_rt::spawn(
                 async move {
                     // undo alias -> hash mapping
-                    debug!("Remove alias -> hash mapping");
-                    if let Ok(Some(hash)) = manager.inner.alias_tree.remove(&alias) {
-                        // undo alias -> id mapping
-                        debug!("Remove alias -> id mapping");
-                        let key = alias_id_key(&alias);
-                        if let Ok(Some(id)) = manager.inner.alias_tree.remove(&key) {
-                            // undo hash/id -> alias mapping
-                            debug!("Remove hash/id -> alias mapping");
-                            let id = String::from_utf8_lossy(&id);
-                            let key = alias_key(&hash, &id);
-                            let _ = manager.inner.main_tree.remove(&key);
-                        }
+                    match manager.inner.repo {
+                        Repo::Sled(ref sled_repo) => {
+                            if let Ok(hash) = sled_repo.hash(&alias).await {
+                                debug!("Clean alias repo");
+                                let _ = AliasRepo::cleanup(sled_repo, &alias).await;
 
-                        let _ = manager.check_delete_files(store, hash).await;
+                                if let Ok(identifier) = sled_repo.identifier(hash.clone()).await {
+                                    debug!("Clean identifier repo");
+                                    let _ = IdentifierRepo::cleanup(sled_repo, &identifier).await;
+
+                                    debug!("Remove stored files");
+                                    let _ = store.remove(&identifier).await;
+                                }
+                                debug!("Clean hash repo");
+                                let _ = HashRepo::cleanup(sled_repo, hash).await;
+                            }
+                        }
                     }
                 }
                 .instrument(cleanup_span),
@@ -107,42 +97,30 @@ where
 {
     /// Generate a delete token for an alias
     #[instrument(skip(self))]
-    pub(crate) async fn delete_token(&self) -> Result<String, Error> {
+    pub(crate) async fn delete_token(&self) -> Result<DeleteToken, Error> {
         let alias = self.alias.clone().ok_or(UploadError::MissingAlias)?;
 
         debug!("Generating delete token");
-        let s: String = Uuid::new_v4().to_string();
-        let delete_token = s.clone();
+        let delete_token = DeleteToken::generate();
 
         debug!("Saving delete token");
-        let alias_tree = self.manager.inner.alias_tree.clone();
-        let key = delete_key(&alias);
-        let res = web::block(move || {
-            alias_tree.compare_and_swap(
-                key.as_bytes(),
-                None as Option<sled::IVec>,
-                Some(s.as_bytes()),
-            )
-        })
-        .await??;
+        match self.manager.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                let res = sled_repo.relate_delete_token(&alias, &delete_token).await?;
 
-        if let Err(sled::CompareAndSwapError {
-            current: Some(ivec),
-            ..
-        }) = res
-        {
-            let s = String::from_utf8(ivec.to_vec())?;
-
-            debug!("Returning existing delete token, {}", s);
-            return Ok(s);
+                Ok(if res.is_err() {
+                    let delete_token = sled_repo.delete_token(&alias).await?;
+                    debug!("Returning existing delete token, {:?}", delete_token);
+                    delete_token
+                } else {
+                    debug!("Returning new delete token, {:?}", delete_token);
+                    delete_token
+                })
+            }
         }
-
-        debug!("Returning new delete token, {}", delete_token);
-        Ok(delete_token)
     }
 
-    /// Upload the file while preserving the filename, optionally validating the uploaded image
-    #[instrument(skip(self, stream))]
+    /// Import the file, discarding bytes if it's already present, or saving if it's new
     pub(crate) async fn import(
         mut self,
         alias: String,
@@ -158,7 +136,7 @@ where
         }
 
         debug!("Validating bytes");
-        let (content_type, validated_reader) = crate::validate::validate_image_bytes(
+        let (_, validated_reader) = crate::validate::validate_image_bytes(
             bytes_mut.freeze(),
             self.manager.inner.format,
             validate,
@@ -167,20 +145,14 @@ where
 
         let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
 
-        let filename = self.next_file(content_type).await?;
-
-        let identifier = self
-            .store
-            .save_async_read(&mut hasher_reader, &filename)
-            .await?;
+        let identifier = self.store.save_async_read(&mut hasher_reader).await?;
         let hash = hasher_reader.finalize_reset().await?;
 
-        debug!("Storing alias");
-        self.alias = Some(alias.clone());
-        self.add_existing_alias(&hash, &alias).await?;
+        debug!("Adding alias");
+        self.add_existing_alias(&hash, alias).await?;
 
         debug!("Saving file");
-        self.save_upload(&identifier, hash, filename).await?;
+        self.save_upload(&identifier, hash).await?;
 
         // Return alias to file
         Ok(self)
@@ -210,106 +182,65 @@ where
 
         let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
 
-        let filename = self.next_file(input_type).await?;
-
-        let identifier = self
-            .store
-            .save_async_read(&mut hasher_reader, &filename)
-            .await?;
+        let identifier = self.store.save_async_read(&mut hasher_reader).await?;
         let hash = hasher_reader.finalize_reset().await?;
 
         debug!("Adding alias");
         self.add_alias(&hash, input_type).await?;
 
         debug!("Saving file");
-        self.save_upload(&identifier, hash, filename).await?;
+        self.save_upload(&identifier, hash).await?;
 
         // Return alias to file
         Ok(self)
     }
 
     // check duplicates & store image if new
-    async fn save_upload(
-        &self,
-        identifier: &S::Identifier,
-        hash: Hash,
-        filename: String,
-    ) -> Result<(), Error> {
-        let dup = self.check_duplicate(hash, filename.clone()).await?;
+    #[instrument(skip(self, hash))]
+    async fn save_upload(&self, identifier: &S::Identifier, hash: Hash) -> Result<(), Error> {
+        let res = self.check_duplicate(&hash).await?;
 
         // bail early with alias to existing file if this is a duplicate
-        if dup.exists() {
+        if res.is_err() {
             debug!("Duplicate exists, removing file");
 
             self.store.remove(identifier).await?;
             return Ok(());
         }
 
-        self.manager.store_identifier(filename, identifier).await?;
+        self.manager
+            .store_identifier(hash.into_inner(), identifier)
+            .await?;
 
         Ok(())
     }
 
     // check for an already-uploaded image with this hash, returning the path to the target file
     #[instrument(skip(self, hash))]
-    async fn check_duplicate(&self, hash: Hash, filename: String) -> Result<Dup, Error> {
-        let main_tree = self.manager.inner.main_tree.clone();
+    async fn check_duplicate(&self, hash: &Hash) -> Result<Result<(), AlreadyExists>, Error> {
+        let hash = hash.as_slice().to_vec();
 
-        let filename2 = filename.clone();
-        let hash2 = hash.as_slice().to_vec();
-        debug!("Inserting filename for hash");
-        let res = web::block(move || {
-            main_tree.compare_and_swap(
-                hash2,
-                None as Option<sled::IVec>,
-                Some(filename2.as_bytes()),
-            )
-        })
-        .await??;
-
-        if let Err(sled::CompareAndSwapError {
-            current: Some(ivec),
-            ..
-        }) = res
-        {
-            let name = String::from_utf8(ivec.to_vec())?;
-            debug!("Filename exists for hash, {}", name);
-            return Ok(Dup::Exists);
+        match self.manager.inner.repo {
+            Repo::Sled(ref sled_repo) => Ok(HashRepo::create(sled_repo, hash.into()).await?),
         }
-
-        let fname_tree = self.manager.inner.filename_tree.clone();
-        debug!("Saving filename -> hash relation");
-        web::block(move || fname_tree.insert(filename, hash.into_inner())).await??;
-
-        Ok(Dup::New)
     }
 
-    // generate a short filename that isn't already in-use
-    #[instrument(skip(self, input_type))]
-    async fn next_file(&self, input_type: ValidInputType) -> Result<String, Error> {
-        loop {
-            debug!("Filename generation loop");
-            let filename = file_name(Uuid::new_v4(), input_type);
+    // Add an alias from an existing filename
+    async fn add_existing_alias(&mut self, hash: &Hash, filename: String) -> Result<(), Error> {
+        let alias = Alias::from_existing(&filename);
 
-            let identifier_tree = self.manager.inner.identifier_tree.clone();
-            let filename2 = filename.clone();
-            let filename_exists = web::block(move || identifier_tree.get(filename2.as_bytes()))
-                .await??
-                .is_some();
+        match self.manager.inner.repo {
+            Repo::Sled(ref sled_repo) => {
+                AliasRepo::create(sled_repo, &alias)
+                    .await?
+                    .map_err(|_| UploadError::DuplicateAlias)?;
+                self.alias = Some(alias.clone());
 
-            if !filename_exists {
-                return Ok(filename);
+                let hash = hash.as_slice().to_vec();
+                sled_repo.relate_hash(&alias, hash.clone().into()).await?;
+                sled_repo.relate_alias(hash.into(), &alias).await?;
             }
-
-            debug!("Filename exists, trying again");
         }
-    }
-
-    #[instrument(skip(self, hash, alias))]
-    async fn add_existing_alias(&self, hash: &Hash, alias: &str) -> Result<(), Error> {
-        self.save_alias_hash_mapping(hash, alias).await??;
-
-        self.store_hash_id_alias_mapping(hash, alias).await?;
 
         Ok(())
     }
@@ -319,96 +250,25 @@ where
     // This will help if multiple 'users' upload the same file, and one of them wants to delete it
     #[instrument(skip(self, hash, input_type))]
     async fn add_alias(&mut self, hash: &Hash, input_type: ValidInputType) -> Result<(), Error> {
-        let alias = self.next_alias(hash, input_type).await?;
-
-        self.store_hash_id_alias_mapping(hash, &alias).await?;
-
-        Ok(())
-    }
-
-    // Add a pre-defined alias to an existin file
-    //
-    // DANGER: this can cause BAD BAD BAD conflicts if the same alias is used for multiple files
-    #[instrument(skip(self, hash))]
-    async fn store_hash_id_alias_mapping(&self, hash: &Hash, alias: &str) -> Result<(), Error> {
-        let alias = alias.to_string();
-        loop {
-            debug!("hash -> alias save loop");
-            let db = self.manager.inner.db.clone();
-            let id = web::block(move || db.generate_id()).await??.to_string();
-
-            let alias_tree = self.manager.inner.alias_tree.clone();
-            let key = alias_id_key(&alias);
-            let id2 = id.clone();
-            debug!("Saving alias -> id mapping");
-            web::block(move || alias_tree.insert(key.as_bytes(), id2.as_bytes())).await??;
-
-            let key = alias_key(hash.as_slice(), &id);
-            let main_tree = self.manager.inner.main_tree.clone();
-            let alias2 = alias.clone();
-            debug!("Saving hash/id -> alias mapping");
-            let res = web::block(move || {
-                main_tree.compare_and_swap(key, None as Option<sled::IVec>, Some(alias2.as_bytes()))
-            })
-            .await??;
-
-            if res.is_ok() {
-                break;
-            }
-
-            debug!("Id exists, trying again");
-        }
-
-        Ok(())
-    }
-
-    // Generate an alias to the file
-    #[instrument(skip(self, hash, input_type))]
-    async fn next_alias(
-        &mut self,
-        hash: &Hash,
-        input_type: ValidInputType,
-    ) -> Result<String, Error> {
         loop {
             debug!("Alias gen loop");
-            let alias = file_name(Uuid::new_v4(), input_type);
-            self.alias = Some(alias.clone());
+            let alias = Alias::generate(input_type.as_ext().to_string());
 
-            let res = self.save_alias_hash_mapping(hash, &alias).await?;
+            match self.manager.inner.repo {
+                Repo::Sled(ref sled_repo) => {
+                    let res = AliasRepo::create(sled_repo, &alias).await?;
+
+                    if res.is_ok() {
+                        self.alias = Some(alias.clone());
+                        let hash = hash.as_slice().to_vec();
+                        sled_repo.relate_hash(&alias, hash.clone().into()).await?;
+                        sled_repo.relate_alias(hash.into(), &alias).await?;
+                        return Ok(());
+                    }
+                }
+            };
 
-            if res.is_ok() {
-                return Ok(alias);
-            }
             debug!("Alias exists, regenning");
         }
     }
-
-    // Save an alias to the database
-    #[instrument(skip(self, hash))]
-    async fn save_alias_hash_mapping(
-        &self,
-        hash: &Hash,
-        alias: &str,
-    ) -> Result<Result<(), Error>, Error> {
-        let tree = self.manager.inner.alias_tree.clone();
-        let vec = hash.as_slice().to_vec();
-        let alias = alias.to_string();
-
-        debug!("Saving alias -> hash mapping");
-        let res = web::block(move || {
-            tree.compare_and_swap(alias.as_bytes(), None as Option<sled::IVec>, Some(vec))
-        })
-        .await??;
-
-        if res.is_err() {
-            warn!("Duplicate alias");
-            return Ok(Err(UploadError::DuplicateAlias.into()));
-        }
-
-        Ok(Ok(()))
-    }
-}
-
-fn file_name(name: Uuid, input_type: ValidInputType) -> String {
-    format!("{}{}", name, input_type.as_ext())
 }

From 37e6b21b55470c65a232a97aaa2f7e004e916427 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sat, 26 Mar 2022 20:45:12 -0500
Subject: [PATCH 06/37] Fix Range, consolidate errors, test object storage

---
 Cargo.lock                                    |   1 +
 Cargo.toml                                    |   1 +
 README.md                                     | 166 ++++++++++++------
 docker/object-storage/Dockerfile              |  18 +-
 docker/object-storage/pict-rs.toml            |  22 ++-
 .../config-Q16HDRI/policy.xml                 |   6 +-
 .../config-Q16HDRI/policy.xml                 |   6 +-
 pict-rs.toml                                  | 116 ++++++------
 src/config.rs                                 |  17 +-
 src/details.rs                                |   5 +-
 src/error.rs                                  |  16 +-
 src/ffmpeg.rs                                 |   7 +-
 src/magick.rs                                 |   7 +-
 src/main.rs                                   |  69 +++-----
 src/range.rs                                  |  15 +-
 src/repo.rs                                   |  72 ++++----
 src/repo/sled.rs                              | 144 ++++++++-------
 src/store.rs                                  |  26 ++-
 src/store/file_store.rs                       |  34 ++--
 src/store/file_store/file_id.rs               |  15 +-
 src/store/object_store.rs                     |  55 +++---
 src/store/object_store/object_id.rs           |  15 +-
 src/upload_manager.rs                         |  41 +----
 src/upload_manager/session.rs                 |  20 +--
 24 files changed, 458 insertions(+), 436 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 33fed7a..3a45ff7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1643,6 +1643,7 @@ dependencies = [
  "tokio",
  "tokio-uring",
  "tokio-util 0.7.0",
+ "toml",
  "tracing",
  "tracing-actix-web",
  "tracing-awc",
diff --git a/Cargo.toml b/Cargo.toml
index 4857e63..0ed1dec 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -56,6 +56,7 @@ time = { version = "0.3.0", features = ["serde"] }
 tokio = { version = "1", features = ["full", "tracing"] }
 tokio-uring = { version = "0.3", optional = true, features = ["bytes"] }
 tokio-util = { version = "0.7", default-features = false, features = ["codec"] }
+toml = "0.5.8"
 tracing = "0.1.15"
 tracing-error = "0.2.0"
 tracing-futures = "0.2.4"
diff --git a/README.md b/README.md
index dfb7266..c45163a 100644
--- a/README.md
+++ b/README.md
@@ -9,83 +9,145 @@ _a simple image hosting service_
 ## Usage
 ### Running
 ```
-pict-rs 0.3.0-rc.7
+pict-rs 
 
 USAGE:
-    pict-rs [FLAGS] [OPTIONS] [SUBCOMMAND]
-
-FLAGS:
-    -h, --help                     Prints help information
-    -s, --skip-validate-imports    Whether to skip validating images uploaded via the internal import API
-    -V, --version                  Prints version information
+    pict-rs [OPTIONS] <SUBCOMMAND>
 
 OPTIONS:
-    -a, --addr <addr>                                          The address and port the server binds to.
-        --api-key <api-key>
+    -a, --addr <ADDR>
+            The address and port the server binds to.
+
+        --api-key <API_KEY>
             An optional string to be checked on requests to privileged endpoints
 
-    -c, --config-file <config-file>                            Path to the pict-rs configuration file
-        --console-buffer-capacity <console-buffer-capacity>
+    -c, --config-file <CONFIG_FILE>
+            Path to the pict-rs configuration file
+
+        --console-buffer-capacity <CONSOLE_BUFFER_CAPACITY>
             Specify the number of events the console subscriber is allowed to buffer
 
-    -f, --filters <filters>...
-            An optional list of filters to permit, supports 'identity', 'thumbnail', 'resize', 'crop', and 'blur'
+    -f, --filters <FILTERS>
+            An optional list of filters to permit, supports 'identity', 'thumbnail', 'resize',
+            'crop', and 'blur'
 
-    -i, --image-format <image-format>
-            An optional image format to convert all uploaded files into, supports 'jpg', 'png', and 'webp'
+        --filesystem-storage-path <FILESYSTEM_STORAGE_PATH>
+            Path in which pict-rs will create it's 'files' directory
 
-    -m, --max-file-size <max-file-size>
+    -h, --help
+            Print help information
+
+    -i, --image-format <IMAGE_FORMAT>
+            An optional image format to convert all uploaded files into, supports 'jpg', 'png', and
+            'webp'
+
+    -m, --max-file-size <MAX_FILE_SIZE>
             Specify the maximum allowed uploaded file size (in Megabytes)
 
-        --max-image-area <max-image-area>                      Specify the maximum area in pixels allowed in an image
-        --max-image-height <max-image-height>                  Specify the maximum width in pixels allowed on an image
-        --max-image-width <max-image-width>                    Specify the maximum width in pixels allowed on an image
-        --migrate-file <migrate-file>                          Path to a file defining a store migration
-    -o, --opentelemetry-url <opentelemetry-url>
+        --max-image-area <MAX_IMAGE_AREA>
+            Specify the maximum area in pixels allowed in an image
+
+        --max-image-height <MAX_IMAGE_HEIGHT>
+            Specify the maximum width in pixels allowed on an image
+
+        --max-image-width <MAX_IMAGE_WIDTH>
+            Specify the maximum width in pixels allowed on an image
+
+    -o, --opentelemetry-url <OPENTELEMETRY_URL>
             Enable OpenTelemetry Tracing exports to the given OpenTelemetry collector
 
-    -p, --path <path>                                          The path to the data directory, e.g. data/
-        --sled-cache-capacity <sled-cache-capacity>
-            Specify the number of bytes sled is allowed to use for it's cache
+        --object-store-access-key <OBJECT_STORE_ACCESS_KEY>
+            
 
+        --object-store-bucket-name <OBJECT_STORE_BUCKET_NAME>
+            Name of the bucket in which pict-rs will store images
+
+        --object-store-region <OBJECT_STORE_REGION>
+            Region in which the bucket exists, can be an http endpoint
+
+        --object-store-secret-key <OBJECT_STORE_SECRET_KEY>
+            
+
+        --object-store-security-token <OBJECT_STORE_SECURITY_TOKEN>
+            
+
+        --object-store-session-token <OBJECT_STORE_SESSION_TOKEN>
+            
+
+    -p, --path <PATH>
+            The path to the data directory, e.g. data/
+
+    -R, --repo <REPO>
+            Set the database implementation. Available options are 'sled'. Default is 'sled'
+
+    -s, --skip-validate-imports
+            Whether to skip validating images uploaded via the internal import API
+
+    -S, --store <STORE>
+            Set the image store. Available options are 'object-storage' or 'filesystem'. Default is
+            'filesystem'
+
+        --sled-cache-capacity <SLED_CACHE_CAPACITY>
+            The number of bytes sled is allowed to use for it's in-memory cache
+
+        --sled-path <SLED_PATH>
+            Path in which pict-rs will create it's 'repo' directory
 
 SUBCOMMANDS:
-    file-store    
-    help          Prints this message or the help of the given subcommand(s)
-    s3-store
+    dump             
+    help             Print this message or the help of the given subcommand(s)
+    migrate-repo     
+    migrate-store    
+    run
 ```
 
 ```
-pict-rs-file-store 0.3.0-rc.1
+pict-rs-dump 
 
 USAGE:
-    pict-rs file-store [OPTIONS]
+    pict-rs dump <PATH>
 
-FLAGS:
-    -h, --help       Prints help information
-    -V, --version    Prints version information
+ARGS:
+    <PATH>    
 
 OPTIONS:
-        --path <path>    Path in which pict-rs will create it's 'files' directory
+    -h, --help    Print help information
 ```
 
 ```
-pict-rs-s3-store 0.3.0-rc.1
+pict-rs-migrate-repo 
 
 USAGE:
-    pict-rs s3-store [OPTIONS] --bucket-name <bucket-name> --region <region>
+    pict-rs migrate-repo <TO>
 
-FLAGS:
-    -h, --help       Prints help information
-    -V, --version    Prints version information
+ARGS:
+    <TO>    
 
 OPTIONS:
-        --access-key <access-key>            
-        --bucket-name <bucket-name>          Name of the bucket in which pict-rs will store images
-        --region <region>                    Region in which the bucket exists, can be an http endpoint
-        --secret-key <secret-key>            
-        --security-token <security-token>    
-        --session-token <session-token>   
+    -h, --help    Print help information
+```
+
+```
+pict-rs-migrate-store 
+
+USAGE:
+    pict-rs migrate-store <TO>
+
+ARGS:
+    <TO>    
+
+OPTIONS:
+    -h, --help    Print help information
+```
+
+```
+pict-rs-run 
+
+USAGE:
+    pict-rs run
+
+OPTIONS:
+    -h, --help    Print help information
 ```
 
 See [`pict-rs.toml`](https://git.asonix.dog/asonix/pict-rs/src/branch/main/pict-rs.toml) and 
@@ -95,23 +157,27 @@ configuration
 #### Example:
 Running on all interfaces, port 8080, storing data in /opt/data
 ```
-$ ./pict-rs -a 0.0.0.0:8080 -p /opt/data
+$ ./pict-rs -a 0.0.0.0:8080 -p /opt/data run
 ```
 Running locally, port 9000, storing data in data/, and converting all uploads to PNG
 ```
-$ ./pict-rs -a 127.0.0.1:9000 -p data/ -f png
+$ ./pict-rs -a 127.0.0.1:9000 -p data/ -f png run
 ```
 Running locally, port 8080, storing data in data/, and only allowing the `thumbnail` and `identity` filters
 ```
-$ ./pict-rs -a 127.0.0.1:8080 -p data/ -w thumbnail identity
+$ ./pict-rs -a 127.0.0.1:8080 -p data/ -w thumbnail identity run
 ```
 Running from a configuration file
 ```
-$ ./pict-rs -c ./pict-rs.toml
+$ ./pict-rs -c ./pict-rs.toml run
 ```
-Migrating between storage backends
+Migrating to object storage from filesystem storage (both storages must be configured in pict-rs.toml)
 ```
-$ ./pict-rs -p ./data --migrate-file ./migrate.toml
+$ ./pict-rs -c ./pict-rs.toml --store filesystem migrate-store object-storage
+```
+Dumping commandline flags to a toml file
+```
+$ ./pict-rs -p data/ --store object-storage --object-storage-bucket-name pict-rs --object-storage-region us-east-1 dump pict-rs.toml
 ```
 
 #### Docker
diff --git a/docker/object-storage/Dockerfile b/docker/object-storage/Dockerfile
index 9b07035..fcb932e 100644
--- a/docker/object-storage/Dockerfile
+++ b/docker/object-storage/Dockerfile
@@ -1,25 +1,17 @@
-FROM archlinux:latest
+FROM alpine:edge
 
 ARG UID=1000
 ARG GID=1000
 
 RUN \
- pacman -Syu --noconfirm \
-        perl-image-exiftool \
-        imagemagick \
-        ffmpeg && \
- groupadd -g 1000 app && \
- useradd -m \
-        -d /opt/app \
-        -u $UID \
-        -g $GID \
-        app
+  apk add exiftool imagemagick ffmpeg && \
+  addgroup -g $GID app && \
+  adduser -h /opt/app -g "" -G app -u $UID -D app && \
+  chown -R app:app /mnt
 
 COPY root/ /
 
 COPY ./pict-rs.toml /etc/pict-rs.toml
 
-ENV PATH=$PATH:/usr/bin/vendor_perl
-
 WORKDIR /opt/app
 USER app
diff --git a/docker/object-storage/pict-rs.toml b/docker/object-storage/pict-rs.toml
index 29c3c9d..04fc42a 100644
--- a/docker/object-storage/pict-rs.toml
+++ b/docker/object-storage/pict-rs.toml
@@ -1,9 +1,17 @@
-path = '/mnt'
+path = 'data/'
 addr = '0.0.0.0:8080'
 
-[store]
-type = 's3_store'
-bucket_name = 'pict-rs'
-region = 'http://minio:9000'
-access_key = '09ODZ3BGBISV4U92JLIM'
-secret_key = 'j35YE9RrxhBP0dpiD5mmdXRXvPkEJR4k6zK12q3o'
+repo = 'sled'
+store = 'object_storage'
+
+[sled]
+sled_cache_capacity = 67108864
+
+[filesystem_storage]
+filesystem_storage_path = '/mnt/files'
+
+[object_storage]
+object_store_bucket_name = 'pict-rs'
+object_store_region = 'http://minio:9000'
+object_store_access_key = 'XZEZ5B8Y3UCINU1KCVF6'
+object_store_secret_key = 'cWbE5LcCK9YH8j1NvhOZocl+vH+b6T5Zvy3z+BZu'
diff --git a/docker/object-storage/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml b/docker/object-storage/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml
index 06166a9..9586799 100644
--- a/docker/object-storage/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml
+++ b/docker/object-storage/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml
@@ -8,12 +8,14 @@
     <policy domain="resource" name="disk" value="1GiB" />
     <policy domain="resource" name="file" value="768" />
     <policy domain="resource" name="thread" value="2" />
+    <policy domain="path" rights="none" pattern="@*" />
     <policy domain="coder" rights="none" pattern="*" />
     <policy domain="coder" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,MP4,TMP,PAM}" />
+    <policy domain="delegate" rights="none" pattern="*" />
+    <policy domain="delegate" rights="execute" pattern="ffmpeg" />
     <policy domain="filter" rights="none" pattern="*" />
-    <policy domain="path" rights="none" pattern="@*" />
     <policy domain="module" rights="none" pattern="*" />
-    <policy domain="module" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,MP4,TMP,PAM}" />
+    <policy domain="module" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,TMP,PAM,VIDEO}" />
         <!-- indirect reads not permitted -->
     <policy domain="system" name="precision" value="6" />
 </policymap>
diff --git a/docker/prod/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml b/docker/prod/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml
index 06166a9..9586799 100644
--- a/docker/prod/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml
+++ b/docker/prod/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml
@@ -8,12 +8,14 @@
     <policy domain="resource" name="disk" value="1GiB" />
     <policy domain="resource" name="file" value="768" />
     <policy domain="resource" name="thread" value="2" />
+    <policy domain="path" rights="none" pattern="@*" />
     <policy domain="coder" rights="none" pattern="*" />
     <policy domain="coder" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,MP4,TMP,PAM}" />
+    <policy domain="delegate" rights="none" pattern="*" />
+    <policy domain="delegate" rights="execute" pattern="ffmpeg" />
     <policy domain="filter" rights="none" pattern="*" />
-    <policy domain="path" rights="none" pattern="@*" />
     <policy domain="module" rights="none" pattern="*" />
-    <policy domain="module" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,MP4,TMP,PAM}" />
+    <policy domain="module" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,TMP,PAM,VIDEO}" />
         <!-- indirect reads not permitted -->
     <policy domain="system" name="precision" value="6" />
 </policymap>
diff --git a/pict-rs.toml b/pict-rs.toml
index 645d002..16c3182 100644
--- a/pict-rs.toml
+++ b/pict-rs.toml
@@ -43,13 +43,6 @@ max_image_area = 40_000_000 # in Pixels
 # default: false
 skip_validate_imports = false
 
-## Optional: set sled's cache capacity to a given number of bytes
-# environment variable: PICTRS_SLED_CACHE_CAPACITY
-# default: 67_108_864 (1024 * 1024 * 64) e.g. 64MB
-#
-# Increasing this value can improve performance by keeping more of the database in RAM
-sled_cache_capacity = 67_108_864 # in bytes
-
 ## Optional: enable tokio-console and set the event buffer size
 # environment variable: PICTRS_CONSOLE_BUFFER_CAPACITY
 # default: empty
@@ -95,58 +88,65 @@ api_key = 'API_KEY'
 # Not specifying opentelemetry_url means no traces will be exported
 opentelemetry_url = 'http://localhost:4317/'
 
-## Optional: store definition
-# default store: file_store
-#
-# Not specifying a store means a file_store will be used with the top-level pict-rs' path
-[store]
-type = "file_store"
+## Optional: the data repository to use
+# environment variable: PICTRS_REPO
+# default: 'sled'
+# available options: 'sled'
+repo = 'sled'
 
-## Example file store
-# [store]
-#
-# # environment variable: PICTRS_STORE__TYPE
-# type = 'file_store'
-#
-# # Optional: file path
-# # environment variable: PICTRS_STORE__PATH
-# # default: empty
-# #
-# # Not specifying path means pict-rs' top-level `path` config is used
-# path = './data'
+## Optional: the file storage to use
+# environment variable: PICTRS_STORE
+# default: 'filesystem'
+# available options: 'filesystem', 'object_storage'
+store = 'filesystem'
 
-## Example s3 store
-# [store]
+
+## Optional: Sled store configration definition
+[sled]
+## Optional: set sled's cache capacity to a given number of bytes
+# environment variable: PICTRS_SLED__SLED_CACHE_CAPACITY
+# default: 67_108_864 (1024 * 1024 * 64) e.g. 64MB
 #
-# # environment variable: PICTRS_STORE__TYPE
-# type = 's3_store'
+# Increasing this value can improve performance by keeping more of the database in RAM
+sled_cache_capacity = 67_108_864 # in bytes
+
+
+## Optional: Filesystem storage configuration
+[filesystem_storage]
+## Optional: set the path for pict-rs filesystem file storage
+# environment variable: PICTRS_FILESYSTEM_STORAGE__FILESYSTEM_STORAGE_PATH
+# default '${path}/files'
+filesystem_storage_path = 'data/files'
+
+
+## Optional: Object Storage configuration
+[object_storage]
+## Required: bucket name
+# environment variable: PICTRS_OBJECT_STORAGE__OBJECT_STORE_BUCKET_NAME
+object_store_bucket_name = 'pict-rs'
+
+## Required: bucket region
+# environment variable: PICTRS_OBJECT_STORAGE__OBJECT_STORE_REGION
 #
-# # Required: bucket name
-# # environment variable: PICTRS_STORE__BUCKET_NAME
-# bucket_name = 'rust_s3'
-#
-# # Required: bucket region
-# # environment variable: PICTRS_STORE__REGION
-# #
-# # can also be endpoint of local s3 store, e.g. 'http://minio:9000'
-# region = 'eu-central-1'
-#
-# # Optional: bucket access key
-# # environment variable: PICTRS_STORE__ACCESS_KEY
-# # default: empty
-# access_key = 'ACCESS_KEY'
-#
-# # Optional: bucket secret key
-# # environment variable: PICTRS_STORE__SECRET_KEY
-# # default: empty
-# secret_key = 'SECRET_KEY'
-#
-# # Optional: bucket security token
-# # environment variable: PICTRS_STORE__SECURITY_TOKEN
-# # default: empty
-# security_token = 'SECURITY_TOKEN'
-#
-# # Optional: bucket session token
-# # environment variable: PICTRS_STORE__SESSION_TOKEN
-# # default: empty
-# session_token = 'SESSION_TOKEN'
+# can also be endpoint of local s3 store, e.g. 'http://minio:9000'
+object_store_region = 'eu-central-1'
+
+## Optional: bucket access key
+# environment variable: PICTRS_OBJECT_STORAGE__OBJECT_STORE_ACCESS_KEY
+# default: empty
+object_store_access_key = '09ODZ3BGBISV4U92JLIM'
+
+## Optional: bucket secret key
+# environment variable: PICTRS_OBJECT_STORAGE__OBJECT_STORE_SECRET_KEY
+# default: empty
+object_store_secret_key = 'j35YE9RrxhBP0dpiD5mmdXRXvPkEJR4k6zK12q3o'
+
+## Optional: bucket security token
+# environment variable: PICTRS_OBJECT_STORAGE__OBJECT_STORE_SECURITY_TOKEN
+# default: empty
+object_store_security_token = 'SECURITY_TOKEN'
+
+## Optional: bucket session token
+# environment variable: PICTRS_OBJECT_STORAGE__OBJECT_STORE_SESSION_TOKEN
+# default: empty
+object_store_session_token = 'SESSION_TOKEN'
diff --git a/src/config.rs b/src/config.rs
index 1c68b96..c7cb2be 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -170,12 +170,16 @@ impl Overrides {
 #[serde(tag = "type")]
 pub(crate) enum Command {
     Run,
+    Dump { path: PathBuf },
     MigrateStore { to: Store },
     MigrateRepo { to: Repo },
 }
 
 pub(crate) enum CommandConfig {
     Run,
+    Dump {
+        path: PathBuf,
+    },
     MigrateStore {
         to: Storage,
     },
@@ -287,7 +291,6 @@ pub(crate) enum Repository {
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
 #[serde(rename_all = "snake_case")]
 pub(crate) struct Config {
-    command: Command,
     skip_validate_imports: bool,
     addr: SocketAddr,
     path: PathBuf,
@@ -301,8 +304,9 @@ pub(crate) struct Config {
     api_key: Option<String>,
     opentelemetry_url: Option<Url>,
     repo: Repo,
-    sled: Option<Sled>,
     store: Store,
+    command: Command,
+    sled: Option<Sled>,
     filesystem_storage: Option<FilesystemStorage>,
     object_storage: Option<ObjectStorage>,
 }
@@ -329,9 +333,9 @@ struct SledDefaults {
 }
 
 impl Defaults {
-    fn new() -> Self {
+    fn new(command: Command) -> Self {
         Defaults {
-            command: Command::Run,
+            command,
             skip_validate_imports: false,
             addr: ([0, 0, 0, 0], 8080).into(),
             max_file_size: 40,
@@ -351,8 +355,8 @@ impl Config {
     pub(crate) fn build() -> anyhow::Result<Self> {
         let args = Args::parse();
 
-        let mut base_config =
-            config::Config::builder().add_source(config::Config::try_from(&Defaults::new())?);
+        let mut base_config = config::Config::builder()
+            .add_source(config::Config::try_from(&Defaults::new(args.command))?);
 
         if let Some(path) = args.config_file {
             base_config = base_config.add_source(config::File::from(path));
@@ -375,6 +379,7 @@ impl Config {
     pub(crate) fn command(&self) -> anyhow::Result<CommandConfig> {
         Ok(match &self.command {
             Command::Run => CommandConfig::Run,
+            Command::Dump { path } => CommandConfig::Dump { path: path.clone() },
             Command::MigrateStore { to } => CommandConfig::MigrateStore {
                 to: match to {
                     Store::ObjectStorage => Storage::ObjectStorage(
diff --git a/src/details.rs b/src/details.rs
index 0730689..1a9e9b8 100644
--- a/src/details.rs
+++ b/src/details.rs
@@ -34,10 +34,7 @@ impl Details {
         store: S,
         identifier: S::Identifier,
         expected_format: Option<ValidInputType>,
-    ) -> Result<Self, Error>
-    where
-        Error: From<S::Error>,
-    {
+    ) -> Result<Self, Error> {
         let details = crate::magick::details_store(store, identifier, expected_format).await?;
 
         Ok(Details::now(
diff --git a/src/error.rs b/src/error.rs
index faca216..bfb336b 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -15,6 +15,20 @@ impl std::fmt::Debug for Error {
 impl std::fmt::Display for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(f, "{}", self.kind)?;
+        writeln!(f)?;
+        let mut count = 0;
+        let mut source = std::error::Error::source(self);
+        if source.is_some() {
+            writeln!(f, "Chain:")?;
+        }
+        while let Some(err) = source {
+            write!(f, "{}. ", count)?;
+            writeln!(f, "{}", err)?;
+
+            count += 1;
+            source = std::error::Error::source(err);
+        }
+
         std::fmt::Display::fmt(&self.context, f)
     }
 }
@@ -43,7 +57,7 @@ pub(crate) enum UploadError {
     Upload(#[from] actix_form_data::Error),
 
     #[error("Error in DB")]
-    Sled(#[from] crate::repo::sled::Error),
+    Sled(#[from] crate::repo::sled::SledError),
 
     #[error("Error in old sled DB")]
     OldSled(#[from] ::sled::Error),
diff --git a/src/ffmpeg.rs b/src/ffmpeg.rs
index 1ec9c6a..3dd9124 100644
--- a/src/ffmpeg.rs
+++ b/src/ffmpeg.rs
@@ -44,7 +44,7 @@ impl ThumbnailFormat {
 
     fn as_format(&self) -> &'static str {
         match self {
-            ThumbnailFormat::Jpeg => "singlejpeg",
+            ThumbnailFormat::Jpeg => "image2",
             // ThumbnailFormat::Webp => "webp",
         }
     }
@@ -101,10 +101,7 @@ pub(crate) async fn thumbnail<S: Store>(
     from: S::Identifier,
     input_format: InputFormat,
     format: ThumbnailFormat,
-) -> Result<impl AsyncRead + Unpin, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<impl AsyncRead + Unpin, Error> {
     let input_file = crate::tmp_file::tmp_file(Some(input_format.to_ext()));
     let input_file_str = input_file.to_str().ok_or(UploadError::Path)?;
     crate::store::file_store::safe_create_parent(&input_file).await?;
diff --git a/src/magick.rs b/src/magick.rs
index 772b333..1d9f3c6 100644
--- a/src/magick.rs
+++ b/src/magick.rs
@@ -139,14 +139,12 @@ pub(crate) async fn details_bytes(
     parse_details(s)
 }
 
+#[tracing::instrument(skip(store))]
 pub(crate) async fn details_store<S: Store>(
     store: S,
     identifier: S::Identifier,
     hint: Option<ValidInputType>,
-) -> Result<Details, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<Details, Error> {
     if hint.as_ref().map(|h| h.is_mp4()).unwrap_or(false) {
         let input_file = crate::tmp_file::tmp_file(Some(".mp4"));
         let input_file_str = input_file.to_str().ok_or(UploadError::Path)?;
@@ -182,6 +180,7 @@ where
     parse_details(s)
 }
 
+#[tracing::instrument]
 pub(crate) async fn details_file(path_str: &str) -> Result<Details, Error> {
     let process = Process::run(
         "magick",
diff --git a/src/main.rs b/src/main.rs
index d7acc11..88daa92 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -77,10 +77,7 @@ async fn upload<S: Store>(
     value: Value<UploadManagerSession<S>>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<HttpResponse, Error> {
     let images = value
         .map()
         .and_then(|mut m| m.remove("images"))
@@ -196,10 +193,7 @@ async fn download<S: Store>(
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
     query: web::Query<UrlQuery>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<HttpResponse, Error> {
     let res = client.get(&query.url).send().await?;
 
     if !res.status().is_success() {
@@ -249,14 +243,11 @@ async fn delete<S: Store>(
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
     path_entries: web::Path<(String, String)>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
-    let (alias, token) = path_entries.into_inner();
+) -> Result<HttpResponse, Error> {
+    let (token, alias) = path_entries.into_inner();
 
-    let alias = Alias::from_existing(&alias);
     let token = DeleteToken::from_existing(&token);
+    let alias = Alias::from_existing(&alias);
 
     manager.delete((**store).clone(), alias, token).await?;
 
@@ -314,10 +305,7 @@ async fn process_details<S: Store>(
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
     filters: web::Data<Option<HashSet<String>>>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<HttpResponse, Error> {
     let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
 
     let identifier = manager
@@ -341,10 +329,7 @@ async fn process<S: Store + 'static>(
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
     filters: web::Data<Option<HashSet<String>>>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<HttpResponse, Error> {
     let (format, alias, thumbnail_path, thumbnail_args) =
         prepare_process(query, ext.as_str(), &filters)?;
 
@@ -468,10 +453,7 @@ async fn details<S: Store>(
     alias: web::Path<String>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<HttpResponse, Error> {
     let alias = alias.into_inner();
     let alias = Alias::from_existing(&alias);
 
@@ -498,10 +480,7 @@ async fn serve<S: Store>(
     alias: web::Path<String>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<HttpResponse, Error> {
     let alias = alias.into_inner();
     let alias = Alias::from_existing(&alias);
     let identifier = manager.identifier_from_alias::<S>(&alias).await?;
@@ -525,10 +504,7 @@ async fn ranged_file_resp<S: Store>(
     identifier: S::Identifier,
     range: Option<web::Header<Range>>,
     details: Details,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<HttpResponse, Error> {
     let (builder, stream) = if let Some(web::Header(range_header)) = range {
         //Range header exists - return as ranged
         if let Some(range) = range::single_bytes_range(&range_header) {
@@ -602,10 +578,7 @@ async fn purge<S: Store>(
     query: web::Query<AliasQuery>,
     upload_manager: web::Data<UploadManager>,
     store: web::Data<S>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<HttpResponse, Error> {
     let alias = Alias::from_existing(&query.alias);
     let aliases = upload_manager.aliases_by_alias(&alias).await?;
 
@@ -626,10 +599,7 @@ async fn aliases<S: Store>(
     query: web::Query<AliasQuery>,
     upload_manager: web::Data<UploadManager>,
     store: web::Data<S>,
-) -> Result<HttpResponse, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<HttpResponse, Error> {
     let alias = Alias::from_existing(&query.alias);
     let aliases = upload_manager.aliases_by_alias(&alias).await?;
 
@@ -658,11 +628,10 @@ fn build_reqwest_client() -> reqwest::Result<reqwest::Client> {
         .build()
 }
 
-async fn launch<S: Store + Clone + 'static>(manager: UploadManager, store: S) -> anyhow::Result<()>
-where
-    S::Error: Unpin,
-    Error: From<S::Error>,
-{
+async fn launch<S: Store + Clone + 'static>(
+    manager: UploadManager,
+    store: S,
+) -> anyhow::Result<()> {
     // Create a new Multipart Form validator
     //
     // This form is expecting a single array field, 'images' with at most 10 files in it
@@ -797,7 +766,6 @@ async fn migrate_inner<S1>(
 ) -> anyhow::Result<()>
 where
     S1: Store,
-    Error: From<S1::Error>,
 {
     match to {
         config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
@@ -848,6 +816,11 @@ async fn main() -> anyhow::Result<()> {
 
     match CONFIG.command()? {
         CommandConfig::Run => (),
+        CommandConfig::Dump { path } => {
+            let configuration = toml::to_string_pretty(&*CONFIG)?;
+            tokio::fs::write(path, configuration).await?;
+            return Ok(());
+        }
         CommandConfig::MigrateRepo { to: _ } => {
             unimplemented!("Repo migrations are currently unsupported")
         }
diff --git a/src/range.rs b/src/range.rs
index e5d9efa..51066e9 100644
--- a/src/range.rs
+++ b/src/range.rs
@@ -15,7 +15,9 @@ pub(crate) fn chop_bytes(
     length: u64,
 ) -> Result<impl Stream<Item = Result<Bytes, Error>>, Error> {
     if let Some((start, end)) = byte_range.to_satisfiable_range(length) {
-        return Ok(once(ready(Ok(bytes.slice(start as usize..end as usize)))));
+        // END IS INCLUSIVE
+        let end = end as usize + 1;
+        return Ok(once(ready(Ok(bytes.slice(start as usize..end)))));
     }
 
     Err(UploadError::Range.into())
@@ -26,14 +28,13 @@ pub(crate) async fn chop_store<S: Store>(
     store: &S,
     identifier: &S::Identifier,
     length: u64,
-) -> Result<impl Stream<Item = std::io::Result<Bytes>>, Error>
-where
-    Error: From<S::Error>,
-{
+) -> Result<impl Stream<Item = std::io::Result<Bytes>>, Error> {
     if let Some((start, end)) = byte_range.to_satisfiable_range(length) {
-        return Ok(store
+        // END IS INCLUSIVE
+        let end = end + 1;
+        return store
             .to_stream(identifier, Some(start), Some(end.saturating_sub(start)))
-            .await?);
+            .await;
     }
 
     Err(UploadError::Range.into())
diff --git a/src/repo.rs b/src/repo.rs
index b7fcf37..c1af53d 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -1,5 +1,9 @@
-use crate::config::RequiredSledRepo;
-use crate::{config::Repository, details::Details, store::Identifier};
+use crate::{
+    config::{Repository, RequiredSledRepo},
+    details::Details,
+    error::Error,
+    store::Identifier,
+};
 use futures_util::Stream;
 use tracing::debug;
 use uuid::Uuid;
@@ -34,98 +38,90 @@ pub(crate) struct AlreadyExists;
 #[async_trait::async_trait(?Send)]
 pub(crate) trait SettingsRepo {
     type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error + Send + Sync + 'static;
 
-    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Self::Error>;
-    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Self::Error>;
-    async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error>;
+    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Error>;
+    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Error>;
+    async fn remove(&self, key: &'static [u8]) -> Result<(), Error>;
 }
 
 #[async_trait::async_trait(?Send)]
 pub(crate) trait IdentifierRepo {
-    type Error: std::error::Error + Send + Sync + 'static;
-
     async fn relate_details<I: Identifier>(
         &self,
         identifier: &I,
         details: &Details,
-    ) -> Result<(), Self::Error>;
-    async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Self::Error>;
+    ) -> Result<(), Error>;
+    async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Error>;
 
-    async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Self::Error>;
+    async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Error>;
 }
 
 #[async_trait::async_trait(?Send)]
 pub(crate) trait HashRepo {
     type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error + Send + Sync + 'static;
-    type Stream: Stream<Item = Result<Self::Bytes, Self::Error>>;
+    type Stream: Stream<Item = Result<Self::Bytes, Error>>;
 
     async fn hashes(&self) -> Self::Stream;
 
-    async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Self::Error>;
+    async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Error>;
 
-    async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error>;
-    async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error>;
-    async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Self::Error>;
+    async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error>;
+    async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error>;
+    async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Error>;
 
     async fn relate_identifier<I: Identifier>(
         &self,
         hash: Self::Bytes,
         identifier: &I,
-    ) -> Result<(), Self::Error>;
-    async fn identifier<I: Identifier + 'static>(
-        &self,
-        hash: Self::Bytes,
-    ) -> Result<I, Self::Error>;
+    ) -> Result<(), Error>;
+    async fn identifier<I: Identifier + 'static>(&self, hash: Self::Bytes) -> Result<I, Error>;
 
     async fn relate_variant_identifier<I: Identifier>(
         &self,
         hash: Self::Bytes,
         variant: String,
         identifier: &I,
-    ) -> Result<(), Self::Error>;
+    ) -> Result<(), Error>;
     async fn variant_identifier<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
         variant: String,
-    ) -> Result<Option<I>, Self::Error>;
+    ) -> Result<Option<I>, Error>;
     async fn variants<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
-    ) -> Result<Vec<(String, I)>, Self::Error>;
+    ) -> Result<Vec<(String, I)>, Error>;
 
     async fn relate_motion_identifier<I: Identifier>(
         &self,
         hash: Self::Bytes,
         identifier: &I,
-    ) -> Result<(), Self::Error>;
+    ) -> Result<(), Error>;
     async fn motion_identifier<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
-    ) -> Result<Option<I>, Self::Error>;
+    ) -> Result<Option<I>, Error>;
 
-    async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error>;
+    async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Error>;
 }
 
 #[async_trait::async_trait(?Send)]
 pub(crate) trait AliasRepo {
     type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-    type Error: std::error::Error + Send + Sync + 'static;
 
-    async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Self::Error>;
+    async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Error>;
 
     async fn relate_delete_token(
         &self,
         alias: &Alias,
         delete_token: &DeleteToken,
-    ) -> Result<Result<(), AlreadyExists>, Self::Error>;
-    async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Self::Error>;
+    ) -> Result<Result<(), AlreadyExists>, Error>;
+    async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Error>;
 
-    async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Self::Error>;
-    async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Self::Error>;
+    async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Error>;
+    async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Error>;
 
-    async fn cleanup(&self, alias: &Alias) -> Result<(), Self::Error>;
+    async fn cleanup(&self, alias: &Alias) -> Result<(), Error>;
 }
 
 impl Repo {
@@ -389,16 +385,14 @@ impl std::fmt::Display for Alias {
 }
 
 impl Identifier for Vec<u8> {
-    type Error = std::convert::Infallible;
-
-    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error>
+    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error>
     where
         Self: Sized,
     {
         Ok(bytes)
     }
 
-    fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> {
+    fn to_bytes(&self) -> Result<Vec<u8>, Error> {
         Ok(self.clone())
     }
 }
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index d7a3f2f..c1f323f 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -2,24 +2,24 @@ use super::{
     Alias, AliasRepo, AlreadyExists, DeleteToken, Details, HashRepo, Identifier, IdentifierRepo,
     SettingsRepo,
 };
+use crate::error::Error;
 use sled::{Db, IVec, Tree};
 
 macro_rules! b {
     ($self:ident.$ident:ident, $expr:expr) => {{
         let $ident = $self.$ident.clone();
 
-        actix_rt::task::spawn_blocking(move || $expr).await??
+        actix_rt::task::spawn_blocking(move || $expr)
+            .await
+            .map_err(SledError::from)??
     }};
 }
 
 #[derive(Debug, thiserror::Error)]
-pub(crate) enum Error {
+pub(crate) enum SledError {
     #[error("Error in database")]
     Sled(#[from] sled::Error),
 
-    #[error("Invalid identifier")]
-    Identifier(#[source] Box<dyn std::error::Error + Sync + Send>),
-
     #[error("Invalid details json")]
     Details(#[from] serde_json::Error),
 
@@ -46,7 +46,7 @@ pub(crate) struct SledRepo {
 }
 
 impl SledRepo {
-    pub(crate) fn new(db: Db) -> Result<Self, Error> {
+    pub(crate) fn new(db: Db) -> Result<Self, SledError> {
         Ok(SledRepo {
             settings: db.open_tree("pict-rs-settings-tree")?,
             identifier_details: db.open_tree("pict-rs-identifier-details-tree")?,
@@ -66,36 +66,29 @@ impl SledRepo {
 #[async_trait::async_trait(?Send)]
 impl SettingsRepo for SledRepo {
     type Bytes = IVec;
-    type Error = Error;
 
-    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Self::Error> {
+    #[tracing::instrument(skip(value))]
+    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Error> {
         b!(self.settings, settings.insert(key, value));
 
         Ok(())
     }
 
-    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Self::Error> {
+    #[tracing::instrument]
+    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Error> {
         let opt = b!(self.settings, settings.get(key));
 
         Ok(opt)
     }
 
-    async fn remove(&self, key: &'static [u8]) -> Result<(), Self::Error> {
+    #[tracing::instrument]
+    async fn remove(&self, key: &'static [u8]) -> Result<(), Error> {
         b!(self.settings, settings.remove(key));
 
         Ok(())
     }
 }
 
-fn identifier_bytes<I>(identifier: &I) -> Result<Vec<u8>, Error>
-where
-    I: Identifier,
-{
-    identifier
-        .to_bytes()
-        .map_err(|e| Error::Identifier(Box::new(e)))
-}
-
 fn variant_key(hash: &[u8], variant: &str) -> Vec<u8> {
     let mut bytes = hash.to_vec();
     bytes.push(b'/');
@@ -111,14 +104,13 @@ fn variant_from_key(hash: &[u8], key: &[u8]) -> Option<String> {
 
 #[async_trait::async_trait(?Send)]
 impl IdentifierRepo for SledRepo {
-    type Error = Error;
-
+    #[tracing::instrument]
     async fn relate_details<I: Identifier>(
         &self,
         identifier: &I,
         details: &Details,
-    ) -> Result<(), Self::Error> {
-        let key = identifier_bytes(identifier)?;
+    ) -> Result<(), Error> {
+        let key = identifier.to_bytes()?;
         let details = serde_json::to_vec(&details)?;
 
         b!(
@@ -129,8 +121,9 @@ impl IdentifierRepo for SledRepo {
         Ok(())
     }
 
-    async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Self::Error> {
-        let key = identifier_bytes(identifier)?;
+    #[tracing::instrument]
+    async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, Error> {
+        let key = identifier.to_bytes()?;
 
         let opt = b!(self.identifier_details, identifier_details.get(key));
 
@@ -141,8 +134,9 @@ impl IdentifierRepo for SledRepo {
         }
     }
 
-    async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Self::Error> {
-        let key = identifier_bytes(identifier)?;
+    #[tracing::instrument]
+    async fn cleanup<I: Identifier>(&self, identifier: &I) -> Result<(), Error> {
+        let key = identifier.to_bytes()?;
 
         b!(self.identifier_details, identifier_details.remove(key));
 
@@ -192,7 +186,13 @@ impl futures_util::Stream for HashStream {
                     (iter, opt)
                 })
                 .await
-                .map(|(iter, opt)| (iter, opt.map(|res| res.map_err(Error::from))))
+                .map(|(iter, opt)| {
+                    (
+                        iter,
+                        opt.map(|res| res.map_err(SledError::from).map_err(Error::from)),
+                    )
+                })
+                .map_err(SledError::from)
                 .map_err(Error::from)
             });
 
@@ -213,7 +213,6 @@ fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> {
 #[async_trait::async_trait(?Send)]
 impl HashRepo for SledRepo {
     type Bytes = IVec;
-    type Error = Error;
     type Stream = HashStream;
 
     async fn hashes(&self) -> Self::Stream {
@@ -225,7 +224,8 @@ impl HashRepo for SledRepo {
         }
     }
 
-    async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Self::Error> {
+    #[tracing::instrument]
+    async fn create(&self, hash: Self::Bytes) -> Result<Result<(), AlreadyExists>, Error> {
         let res = b!(self.hashes, {
             let hash2 = hash.clone();
             hashes.compare_and_swap(hash, None as Option<Self::Bytes>, Some(hash2))
@@ -234,7 +234,8 @@ impl HashRepo for SledRepo {
         Ok(res.map_err(|_| AlreadyExists))
     }
 
-    async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error> {
+    #[tracing::instrument]
+    async fn relate_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error> {
         let key = hash_alias_key(&hash, alias);
         let value = alias.to_bytes();
 
@@ -243,7 +244,8 @@ impl HashRepo for SledRepo {
         Ok(())
     }
 
-    async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Self::Error> {
+    #[tracing::instrument]
+    async fn remove_alias(&self, hash: Self::Bytes, alias: &Alias) -> Result<(), Error> {
         let key = hash_alias_key(&hash, alias);
 
         b!(self.hash_aliases, hash_aliases.remove(key));
@@ -251,7 +253,8 @@ impl HashRepo for SledRepo {
         Ok(())
     }
 
-    async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Self::Error> {
+    #[tracing::instrument]
+    async fn aliases(&self, hash: Self::Bytes) -> Result<Vec<Alias>, Error> {
         let v = b!(self.hash_aliases, {
             Ok(hash_aliases
                 .scan_prefix(hash)
@@ -264,37 +267,37 @@ impl HashRepo for SledRepo {
         Ok(v)
     }
 
+    #[tracing::instrument]
     async fn relate_identifier<I: Identifier>(
         &self,
         hash: Self::Bytes,
         identifier: &I,
-    ) -> Result<(), Self::Error> {
-        let bytes = identifier_bytes(identifier)?;
+    ) -> Result<(), Error> {
+        let bytes = identifier.to_bytes()?;
 
         b!(self.hash_identifiers, hash_identifiers.insert(hash, bytes));
 
         Ok(())
     }
 
-    async fn identifier<I: Identifier + 'static>(
-        &self,
-        hash: Self::Bytes,
-    ) -> Result<I, Self::Error> {
+    #[tracing::instrument]
+    async fn identifier<I: Identifier + 'static>(&self, hash: Self::Bytes) -> Result<I, Error> {
         let opt = b!(self.hash_identifiers, hash_identifiers.get(hash));
 
-        opt.ok_or(Error::Missing).and_then(|ivec| {
-            I::from_bytes(ivec.to_vec()).map_err(|e| Error::Identifier(Box::new(e)))
-        })
+        opt.ok_or(SledError::Missing)
+            .map_err(Error::from)
+            .and_then(|ivec| I::from_bytes(ivec.to_vec()))
     }
 
+    #[tracing::instrument]
     async fn relate_variant_identifier<I: Identifier>(
         &self,
         hash: Self::Bytes,
         variant: String,
         identifier: &I,
-    ) -> Result<(), Self::Error> {
+    ) -> Result<(), Error> {
         let key = variant_key(&hash, &variant);
-        let value = identifier_bytes(identifier)?;
+        let value = identifier.to_bytes()?;
 
         b!(
             self.hash_variant_identifiers,
@@ -304,11 +307,12 @@ impl HashRepo for SledRepo {
         Ok(())
     }
 
+    #[tracing::instrument]
     async fn variant_identifier<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
         variant: String,
-    ) -> Result<Option<I>, Self::Error> {
+    ) -> Result<Option<I>, Error> {
         let key = variant_key(&hash, &variant);
 
         let opt = b!(
@@ -317,18 +321,17 @@ impl HashRepo for SledRepo {
         );
 
         if let Some(ivec) = opt {
-            Ok(Some(
-                I::from_bytes(ivec.to_vec()).map_err(|e| Error::Identifier(Box::new(e)))?,
-            ))
+            Ok(Some(I::from_bytes(ivec.to_vec())?))
         } else {
             Ok(None)
         }
     }
 
+    #[tracing::instrument]
     async fn variants<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
-    ) -> Result<Vec<(String, I)>, Self::Error> {
+    ) -> Result<Vec<(String, I)>, Error> {
         let vec = b!(
             self.hash_variant_identifiers,
             Ok(hash_variant_identifiers
@@ -346,12 +349,13 @@ impl HashRepo for SledRepo {
         Ok(vec)
     }
 
+    #[tracing::instrument]
     async fn relate_motion_identifier<I: Identifier>(
         &self,
         hash: Self::Bytes,
         identifier: &I,
-    ) -> Result<(), Self::Error> {
-        let bytes = identifier_bytes(identifier)?;
+    ) -> Result<(), Error> {
+        let bytes = identifier.to_bytes()?;
 
         b!(
             self.hash_motion_identifiers,
@@ -361,25 +365,25 @@ impl HashRepo for SledRepo {
         Ok(())
     }
 
+    #[tracing::instrument]
     async fn motion_identifier<I: Identifier + 'static>(
         &self,
         hash: Self::Bytes,
-    ) -> Result<Option<I>, Self::Error> {
+    ) -> Result<Option<I>, Error> {
         let opt = b!(
             self.hash_motion_identifiers,
             hash_motion_identifiers.get(hash)
         );
 
         if let Some(ivec) = opt {
-            Ok(Some(
-                I::from_bytes(ivec.to_vec()).map_err(|e| Error::Identifier(Box::new(e)))?,
-            ))
+            Ok(Some(I::from_bytes(ivec.to_vec())?))
         } else {
             Ok(None)
         }
     }
 
-    async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Self::Error> {
+    #[tracing::instrument]
+    async fn cleanup(&self, hash: Self::Bytes) -> Result<(), Error> {
         let hash2 = hash.clone();
         b!(self.hashes, hashes.remove(hash2));
 
@@ -426,9 +430,9 @@ impl HashRepo for SledRepo {
 #[async_trait::async_trait(?Send)]
 impl AliasRepo for SledRepo {
     type Bytes = sled::IVec;
-    type Error = Error;
 
-    async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Self::Error> {
+    #[tracing::instrument]
+    async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Error> {
         let bytes = alias.to_bytes();
         let bytes2 = bytes.clone();
 
@@ -440,11 +444,12 @@ impl AliasRepo for SledRepo {
         Ok(res.map_err(|_| AlreadyExists))
     }
 
+    #[tracing::instrument]
     async fn relate_delete_token(
         &self,
         alias: &Alias,
         delete_token: &DeleteToken,
-    ) -> Result<Result<(), AlreadyExists>, Self::Error> {
+    ) -> Result<Result<(), AlreadyExists>, Error> {
         let key = alias.to_bytes();
         let token = delete_token.to_bytes();
 
@@ -456,16 +461,19 @@ impl AliasRepo for SledRepo {
         Ok(res.map_err(|_| AlreadyExists))
     }
 
-    async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Self::Error> {
+    #[tracing::instrument]
+    async fn delete_token(&self, alias: &Alias) -> Result<DeleteToken, Error> {
         let key = alias.to_bytes();
 
         let opt = b!(self.alias_delete_tokens, alias_delete_tokens.get(key));
 
         opt.and_then(|ivec| DeleteToken::from_slice(&ivec))
-            .ok_or(Error::Missing)
+            .ok_or(SledError::Missing)
+            .map_err(Error::from)
     }
 
-    async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Self::Error> {
+    #[tracing::instrument]
+    async fn relate_hash(&self, alias: &Alias, hash: Self::Bytes) -> Result<(), Error> {
         let key = alias.to_bytes();
 
         b!(self.alias_hashes, alias_hashes.insert(key, hash));
@@ -473,15 +481,17 @@ impl AliasRepo for SledRepo {
         Ok(())
     }
 
-    async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Self::Error> {
+    #[tracing::instrument]
+    async fn hash(&self, alias: &Alias) -> Result<Self::Bytes, Error> {
         let key = alias.to_bytes();
 
         let opt = b!(self.alias_hashes, alias_hashes.get(key));
 
-        opt.ok_or(Error::Missing)
+        opt.ok_or(SledError::Missing).map_err(Error::from)
     }
 
-    async fn cleanup(&self, alias: &Alias) -> Result<(), Self::Error> {
+    #[tracing::instrument]
+    async fn cleanup(&self, alias: &Alias) -> Result<(), Error> {
         let key = alias.to_bytes();
 
         let key2 = key.clone();
@@ -502,8 +512,8 @@ impl std::fmt::Debug for SledRepo {
     }
 }
 
-impl From<actix_rt::task::JoinError> for Error {
+impl From<actix_rt::task::JoinError> for SledError {
     fn from(_: actix_rt::task::JoinError) -> Self {
-        Error::Panic
+        SledError::Panic
     }
 }
diff --git a/src/store.rs b/src/store.rs
index 668950b..fea8374 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -1,43 +1,37 @@
-use std::fmt::Debug;
-
+use crate::error::Error;
 use actix_web::web::Bytes;
 use futures_util::stream::Stream;
+use std::fmt::Debug;
 use tokio::io::{AsyncRead, AsyncWrite};
 
 pub(crate) mod file_store;
 pub(crate) mod object_store;
 
 pub(crate) trait Identifier: Send + Sync + Clone + Debug {
-    type Error: std::error::Error + Send + Sync + 'static;
+    fn to_bytes(&self) -> Result<Vec<u8>, Error>;
 
-    fn to_bytes(&self) -> Result<Vec<u8>, Self::Error>;
-
-    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error>
+    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error>
     where
         Self: Sized;
 }
 
 #[async_trait::async_trait(?Send)]
 pub(crate) trait Store: Send + Sync + Clone + Debug + 'static {
-    type Error: std::error::Error + Send + Sync + 'static;
-    type Identifier: Identifier<Error = Self::Error>;
+    type Identifier: Identifier;
     type Stream: Stream<Item = std::io::Result<Bytes>>;
 
-    async fn save_async_read<Reader>(
-        &self,
-        reader: &mut Reader,
-    ) -> Result<Self::Identifier, Self::Error>
+    async fn save_async_read<Reader>(&self, reader: &mut Reader) -> Result<Self::Identifier, Error>
     where
         Reader: AsyncRead + Unpin;
 
-    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error>;
+    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Error>;
 
     async fn to_stream(
         &self,
         identifier: &Self::Identifier,
         from_start: Option<u64>,
         len: Option<u64>,
-    ) -> Result<Self::Stream, Self::Error>;
+    ) -> Result<Self::Stream, Error>;
 
     async fn read_into<Writer>(
         &self,
@@ -47,7 +41,7 @@ pub(crate) trait Store: Send + Sync + Clone + Debug + 'static {
     where
         Writer: AsyncWrite + Send + Unpin;
 
-    async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Self::Error>;
+    async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Error>;
 
-    async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error>;
+    async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error>;
 }
diff --git a/src/store/file_store.rs b/src/store/file_store.rs
index c923731..4145d98 100644
--- a/src/store/file_store.rs
+++ b/src/store/file_store.rs
@@ -1,4 +1,5 @@
 use crate::{
+    error::Error,
     file::File,
     repo::{Repo, SettingsRepo},
     store::Store,
@@ -23,9 +24,6 @@ const GENERATOR_KEY: &[u8] = b"last-path";
 
 #[derive(Debug, thiserror::Error)]
 pub(crate) enum FileError {
-    #[error("Failed to interact with sled db")]
-    Sled(#[from] crate::repo::sled::Error),
-
     #[error("Failed to read or write file")]
     Io(#[from] std::io::Error),
 
@@ -51,15 +49,11 @@ pub(crate) struct FileStore {
 
 #[async_trait::async_trait(?Send)]
 impl Store for FileStore {
-    type Error = FileError;
     type Identifier = FileId;
     type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
 
     #[tracing::instrument(skip(reader))]
-    async fn save_async_read<Reader>(
-        &self,
-        reader: &mut Reader,
-    ) -> Result<Self::Identifier, Self::Error>
+    async fn save_async_read<Reader>(&self, reader: &mut Reader) -> Result<Self::Identifier, Error>
     where
         Reader: AsyncRead + Unpin,
     {
@@ -67,22 +61,22 @@ impl Store for FileStore {
 
         if let Err(e) = self.safe_save_reader(&path, reader).await {
             self.safe_remove_file(&path).await?;
-            return Err(e);
+            return Err(e.into());
         }
 
-        self.file_id_from_path(path)
+        Ok(self.file_id_from_path(path)?)
     }
 
     #[tracing::instrument(skip(bytes))]
-    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
+    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Error> {
         let path = self.next_file().await?;
 
         if let Err(e) = self.safe_save_bytes(&path, bytes).await {
             self.safe_remove_file(&path).await?;
-            return Err(e);
+            return Err(e.into());
         }
 
-        self.file_id_from_path(path)
+        Ok(self.file_id_from_path(path)?)
     }
 
     #[tracing::instrument]
@@ -91,7 +85,7 @@ impl Store for FileStore {
         identifier: &Self::Identifier,
         from_start: Option<u64>,
         len: Option<u64>,
-    ) -> Result<Self::Stream, Self::Error> {
+    ) -> Result<Self::Stream, Error> {
         let path = self.path_from_file_id(identifier);
 
         let stream = File::open(path)
@@ -119,7 +113,7 @@ impl Store for FileStore {
     }
 
     #[tracing::instrument]
-    async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Self::Error> {
+    async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Error> {
         let path = self.path_from_file_id(identifier);
 
         let len = tokio::fs::metadata(path).await?.len();
@@ -128,7 +122,7 @@ impl Store for FileStore {
     }
 
     #[tracing::instrument]
-    async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error> {
+    async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error> {
         let path = self.path_from_file_id(identifier);
 
         self.safe_remove_file(path).await?;
@@ -138,7 +132,7 @@ impl Store for FileStore {
 }
 
 impl FileStore {
-    pub(crate) async fn build(root_dir: PathBuf, repo: Repo) -> Result<Self, FileError> {
+    pub(crate) async fn build(root_dir: PathBuf, repo: Repo) -> Result<Self, Error> {
         let path_gen = init_generator(&repo).await?;
 
         Ok(FileStore {
@@ -148,7 +142,7 @@ impl FileStore {
         })
     }
 
-    async fn next_directory(&self) -> Result<PathBuf, FileError> {
+    async fn next_directory(&self) -> Result<PathBuf, Error> {
         let path = self.path_gen.next();
 
         match self.repo {
@@ -167,7 +161,7 @@ impl FileStore {
         Ok(target_path)
     }
 
-    async fn next_file(&self) -> Result<PathBuf, FileError> {
+    async fn next_file(&self) -> Result<PathBuf, Error> {
         let target_path = self.next_directory().await?;
         let filename = uuid::Uuid::new_v4().to_string();
 
@@ -290,7 +284,7 @@ pub(crate) async fn safe_create_parent<P: AsRef<Path>>(path: P) -> Result<(), Fi
     Ok(())
 }
 
-async fn init_generator(repo: &Repo) -> Result<Generator, FileError> {
+async fn init_generator(repo: &Repo) -> Result<Generator, Error> {
     match repo {
         Repo::Sled(sled_repo) => {
             if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
diff --git a/src/store/file_store/file_id.rs b/src/store/file_store/file_id.rs
index e811466..bb7a3ec 100644
--- a/src/store/file_store/file_id.rs
+++ b/src/store/file_store/file_id.rs
@@ -1,6 +1,9 @@
-use crate::store::{
-    file_store::{FileError, FileStore},
-    Identifier,
+use crate::{
+    error::Error,
+    store::{
+        file_store::{FileError, FileStore},
+        Identifier,
+    },
 };
 use std::path::PathBuf;
 
@@ -8,9 +11,7 @@ use std::path::PathBuf;
 pub(crate) struct FileId(PathBuf);
 
 impl Identifier for FileId {
-    type Error = FileError;
-
-    fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> {
+    fn to_bytes(&self) -> Result<Vec<u8>, Error> {
         let vec = self
             .0
             .to_str()
@@ -21,7 +22,7 @@ impl Identifier for FileId {
         Ok(vec)
     }
 
-    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error>
+    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error>
     where
         Self: Sized,
     {
diff --git a/src/store/object_store.rs b/src/store/object_store.rs
index 8fea267..4a30165 100644
--- a/src/store/object_store.rs
+++ b/src/store/object_store.rs
@@ -1,4 +1,5 @@
 use crate::{
+    error::Error,
     repo::{Repo, SettingsRepo},
     store::Store,
 };
@@ -28,9 +29,6 @@ pub(crate) enum ObjectError {
     #[error("Failed to generate path")]
     PathGenerator(#[from] storage_path_generator::PathError),
 
-    #[error("Failed to interact with sled repo")]
-    Sled(#[from] crate::repo::sled::Error),
-
     #[error("Failed to parse string")]
     Utf8(#[from] FromUtf8Error),
 
@@ -58,15 +56,11 @@ pin_project_lite::pin_project! {
 
 #[async_trait::async_trait(?Send)]
 impl Store for ObjectStore {
-    type Error = ObjectError;
     type Identifier = ObjectId;
     type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
 
     #[tracing::instrument(skip(reader))]
-    async fn save_async_read<Reader>(
-        &self,
-        reader: &mut Reader,
-    ) -> Result<Self::Identifier, Self::Error>
+    async fn save_async_read<Reader>(&self, reader: &mut Reader) -> Result<Self::Identifier, Error>
     where
         Reader: AsyncRead + Unpin,
     {
@@ -74,16 +68,20 @@ impl Store for ObjectStore {
 
         self.bucket
             .put_object_stream(&self.client, reader, &path)
-            .await?;
+            .await
+            .map_err(ObjectError::from)?;
 
         Ok(ObjectId::from_string(path))
     }
 
     #[tracing::instrument(skip(bytes))]
-    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
+    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Error> {
         let path = self.next_file().await?;
 
-        self.bucket.put_object(&self.client, &path, &bytes).await?;
+        self.bucket
+            .put_object(&self.client, &path, &bytes)
+            .await
+            .map_err(ObjectError::from)?;
 
         Ok(ObjectId::from_string(path))
     }
@@ -94,7 +92,7 @@ impl Store for ObjectStore {
         identifier: &Self::Identifier,
         from_start: Option<u64>,
         len: Option<u64>,
-    ) -> Result<Self::Stream, Self::Error> {
+    ) -> Result<Self::Stream, Error> {
         let path = identifier.as_str();
 
         let start = from_start.unwrap_or(0);
@@ -107,7 +105,7 @@ impl Store for ObjectStore {
             Command::GetObjectRange { start, end },
         );
 
-        let response = request.response().await?;
+        let response = request.response().await.map_err(ObjectError::from)?;
 
         Ok(Box::pin(io_error(response.bytes_stream())))
     }
@@ -126,26 +124,34 @@ impl Store for ObjectStore {
         self.bucket
             .get_object_stream(&self.client, path, writer)
             .await
-            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, Self::Error::from(e)))?;
+            .map_err(ObjectError::from)
+            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, Error::from(e)))?;
 
         Ok(())
     }
 
     #[tracing::instrument]
-    async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Self::Error> {
+    async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Error> {
         let path = identifier.as_str();
 
-        let (head, _) = self.bucket.head_object(&self.client, path).await?;
+        let (head, _) = self
+            .bucket
+            .head_object(&self.client, path)
+            .await
+            .map_err(ObjectError::from)?;
         let length = head.content_length.ok_or(ObjectError::Length)?;
 
         Ok(length as u64)
     }
 
     #[tracing::instrument]
-    async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error> {
+    async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error> {
         let path = identifier.as_str();
 
-        self.bucket.delete_object(&self.client, path).await?;
+        self.bucket
+            .delete_object(&self.client, path)
+            .await
+            .map_err(ObjectError::from)?;
         Ok(())
     }
 }
@@ -161,7 +167,7 @@ impl ObjectStore {
         session_token: Option<String>,
         repo: Repo,
         client: reqwest::Client,
-    ) -> Result<ObjectStore, ObjectError> {
+    ) -> Result<ObjectStore, Error> {
         let path_gen = init_generator(&repo).await?;
 
         Ok(ObjectStore {
@@ -182,12 +188,13 @@ impl ObjectStore {
                     security_token,
                     session_token,
                 },
-            )?,
+            )
+            .map_err(ObjectError::from)?,
             client,
         })
     }
 
-    async fn next_directory(&self) -> Result<Path, ObjectError> {
+    async fn next_directory(&self) -> Result<Path, Error> {
         let path = self.path_gen.next();
 
         match self.repo {
@@ -201,7 +208,7 @@ impl ObjectStore {
         Ok(path)
     }
 
-    async fn next_file(&self) -> Result<String, ObjectError> {
+    async fn next_file(&self) -> Result<String, Error> {
         let path = self.next_directory().await?.to_strings().join("/");
         let filename = uuid::Uuid::new_v4().to_string();
 
@@ -209,7 +216,7 @@ impl ObjectStore {
     }
 }
 
-async fn init_generator(repo: &Repo) -> Result<Generator, ObjectError> {
+async fn init_generator(repo: &Repo) -> Result<Generator, Error> {
     match repo {
         Repo::Sled(sled_repo) => {
             if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
@@ -250,7 +257,7 @@ where
 impl std::fmt::Debug for ObjectStore {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("ObjectStore")
-            .field("path_gen", &self.path_gen)
+            .field("path_gen", &"generator")
             .field("bucket", &self.bucket.name)
             .field("region", &self.bucket.region)
             .finish()
diff --git a/src/store/object_store/object_id.rs b/src/store/object_store/object_id.rs
index 6b3bb32..d9c8a4e 100644
--- a/src/store/object_store/object_id.rs
+++ b/src/store/object_store/object_id.rs
@@ -1,17 +1,20 @@
-use crate::store::{object_store::ObjectError, Identifier};
+use crate::{
+    error::Error,
+    store::{object_store::ObjectError, Identifier},
+};
 
 #[derive(Debug, Clone)]
 pub(crate) struct ObjectId(String);
 
 impl Identifier for ObjectId {
-    type Error = ObjectError;
-
-    fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> {
+    fn to_bytes(&self) -> Result<Vec<u8>, Error> {
         Ok(self.0.as_bytes().to_vec())
     }
 
-    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error> {
-        Ok(ObjectId(String::from_utf8(bytes)?))
+    fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> {
+        Ok(ObjectId(
+            String::from_utf8(bytes).map_err(ObjectError::from)?,
+        ))
     }
 }
 
diff --git a/src/upload_manager.rs b/src/upload_manager.rs
index 3052414..5261227 100644
--- a/src/upload_manager.rs
+++ b/src/upload_manager.rs
@@ -51,7 +51,6 @@ impl UploadManager {
     where
         S1: Store,
         S2: Store,
-        Error: From<S1::Error> + From<S2::Error>,
     {
         match self.inner.repo {
             Repo::Sled(ref sled_repo) => do_migrate_store(sled_repo, from, to).await,
@@ -62,10 +61,7 @@ impl UploadManager {
         &self,
         store: S,
         alias: &Alias,
-    ) -> Result<S::Identifier, Error>
-    where
-        Error: From<S::Error>,
-    {
+    ) -> Result<S::Identifier, Error> {
         let identifier = self.identifier_from_alias::<S>(alias).await?;
         let details = if let Some(details) = self.details(&identifier).await? {
             details
@@ -205,10 +201,7 @@ impl UploadManager {
     pub(crate) async fn details<I: Identifier>(
         &self,
         identifier: &I,
-    ) -> Result<Option<Details>, Error>
-    where
-        Error: From<I::Error>,
-    {
+    ) -> Result<Option<Details>, Error> {
         match self.inner.repo {
             Repo::Sled(ref sled_repo) => Ok(sled_repo.details(identifier).await?),
         }
@@ -240,10 +233,7 @@ impl UploadManager {
         &self,
         store: S,
         alias: Alias,
-    ) -> Result<(), Error>
-    where
-        Error: From<S::Error>,
-    {
+    ) -> Result<(), Error> {
         let token = match self.inner.repo {
             Repo::Sled(ref sled_repo) => sled_repo.delete_token(&alias).await?,
         };
@@ -258,10 +248,7 @@ impl UploadManager {
         store: S,
         alias: Alias,
         token: DeleteToken,
-    ) -> Result<(), Error>
-    where
-        Error: From<S::Error>,
-    {
+    ) -> Result<(), Error> {
         let hash = match self.inner.repo {
             Repo::Sled(ref sled_repo) => {
                 let saved_delete_token = sled_repo.delete_token(&alias).await?;
@@ -282,10 +269,7 @@ impl UploadManager {
         &self,
         store: S,
         hash: Vec<u8>,
-    ) -> Result<(), Error>
-    where
-        Error: From<S::Error>,
-    {
+    ) -> Result<(), Error> {
         match self.inner.repo {
             Repo::Sled(ref sled_repo) => {
                 let hash: <SledRepo as HashRepo>::Bytes = hash.into();
@@ -309,7 +293,7 @@ impl UploadManager {
 
                 HashRepo::cleanup(sled_repo, hash).await?;
 
-                let cleanup_span = tracing::info_span!("Cleaning files");
+                let cleanup_span = tracing::info_span!(parent: None, "Cleaning files");
                 cleanup_span.follows_from(Span::current());
 
                 actix_rt::spawn(
@@ -323,12 +307,10 @@ impl UploadManager {
                         {
                             debug!("Deleting {:?}", identifier);
                             if let Err(e) = store.remove(identifier).await {
-                                let e: Error = e.into();
                                 errors.push(e);
                             }
 
                             if let Err(e) = IdentifierRepo::cleanup(&repo, identifier).await {
-                                let e: Error = e.into();
                                 errors.push(e);
                             }
                         }
@@ -350,10 +332,7 @@ impl UploadManager {
         Ok(())
     }
 
-    pub(crate) fn session<S: Store + Clone + 'static>(&self, store: S) -> UploadManagerSession<S>
-    where
-        Error: From<S::Error>,
-    {
+    pub(crate) fn session<S: Store + Clone + 'static>(&self, store: S) -> UploadManagerSession<S> {
         UploadManagerSession::new(self.clone(), store)
     }
 }
@@ -366,7 +345,6 @@ async fn migrate_file<S1, S2>(
 where
     S1: Store,
     S2: Store,
-    Error: From<S1::Error> + From<S2::Error>,
 {
     let stream = from.to_stream(identifier, None, None).await?;
     futures_util::pin_mut!(stream);
@@ -382,7 +360,6 @@ where
     R: IdentifierRepo,
     I1: Identifier,
     I2: Identifier,
-    Error: From<<R as IdentifierRepo>::Error>,
 {
     if let Some(details) = repo.details(&from).await? {
         repo.relate_details(to, &details).await?;
@@ -396,11 +373,7 @@ async fn do_migrate_store<R, S1, S2>(repo: &R, from: S1, to: S2) -> Result<(), E
 where
     S1: Store,
     S2: Store,
-    Error: From<S1::Error> + From<S2::Error>,
     R: IdentifierRepo + HashRepo + SettingsRepo,
-    Error: From<<R as IdentifierRepo>::Error>,
-    Error: From<<R as HashRepo>::Error>,
-    Error: From<<R as SettingsRepo>::Error>,
 {
     let stream = repo.hashes().await;
     let mut stream = Box::pin(stream);
diff --git a/src/upload_manager/session.rs b/src/upload_manager/session.rs
index dca15a1..726ac77 100644
--- a/src/upload_manager/session.rs
+++ b/src/upload_manager/session.rs
@@ -13,20 +13,14 @@ use futures_util::stream::{Stream, StreamExt};
 use tracing::{debug, instrument, Span};
 use tracing_futures::Instrument;
 
-pub(crate) struct UploadManagerSession<S: Store + Clone + 'static>
-where
-    Error: From<S::Error>,
-{
+pub(crate) struct UploadManagerSession<S: Store + Clone + 'static> {
     store: S,
     manager: UploadManager,
     alias: Option<Alias>,
     finished: bool,
 }
 
-impl<S: Store + Clone + 'static> UploadManagerSession<S>
-where
-    Error: From<S::Error>,
-{
+impl<S: Store + Clone + 'static> UploadManagerSession<S> {
     pub(super) fn new(manager: UploadManager, store: S) -> Self {
         UploadManagerSession {
             store,
@@ -45,10 +39,7 @@ where
     }
 }
 
-impl<S: Store + Clone + 'static> Drop for UploadManagerSession<S>
-where
-    Error: From<S::Error>,
-{
+impl<S: Store + Clone + 'static> Drop for UploadManagerSession<S> {
     fn drop(&mut self) {
         if self.finished {
             return;
@@ -91,10 +82,7 @@ where
     }
 }
 
-impl<S: Store> UploadManagerSession<S>
-where
-    Error: From<S::Error>,
-{
+impl<S: Store> UploadManagerSession<S> {
     /// Generate a delete token for an alias
     #[instrument(skip(self))]
     pub(crate) async fn delete_token(&self) -> Result<DeleteToken, Error> {

From d3d0817cb088720964d9170d5023facc56b071f2 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sat, 26 Mar 2022 20:49:15 -0500
Subject: [PATCH 07/37] Remove object-storage drone flag

---
 .drone.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index d929325..2541bef 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -24,8 +24,6 @@ steps:
   - rustup component add clippy
   - cargo clippy --no-default-features -- -D warnings
   - cargo clippy --no-default-features --features io-uring -- -D warnings
-  - cargo clippy --no-default-features --features object-storage -- -D warnings
-  - cargo clippy --no-default-features --features object-storage,io-uring -- -D warnings
 
 trigger:
   event:

From ca28f68ef5c88c26bf4416d41be43c75a52eaf4f Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sun, 27 Mar 2022 19:10:06 -0500
Subject: [PATCH 08/37] Start redoing config again

---
 src/config.rs             | 594 +-------------------------------------
 src/config/commandline.rs | 216 ++++++++++++++
 src/config/defaults.rs    | 181 ++++++++++++
 src/config/file.rs        | 131 +++++++++
 src/config/primitives.rs  | 111 +++++++
 5 files changed, 646 insertions(+), 587 deletions(-)
 create mode 100644 src/config/commandline.rs
 create mode 100644 src/config/defaults.rs
 create mode 100644 src/config/file.rs
 create mode 100644 src/config/primitives.rs

diff --git a/src/config.rs b/src/config.rs
index c7cb2be..737a61c 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,592 +1,12 @@
-use crate::serde_str::Serde;
-use clap::{ArgEnum, Parser, Subcommand};
-use std::{collections::HashSet, net::SocketAddr, path::PathBuf};
-use url::Url;
+mod commandline;
+mod defaults;
+mod file;
+mod primitives;
 
 use crate::magick::ValidInputType;
 
-#[derive(Clone, Debug, Parser)]
-pub(crate) struct Args {
-    #[clap(short, long, help = "Path to the pict-rs configuration file")]
-    config_file: Option<PathBuf>,
+pub(crate) use file::ConfigFile as Configuration;
 
-    #[clap(subcommand)]
-    command: Command,
-
-    #[clap(flatten)]
-    overrides: Overrides,
-}
-
-fn is_false(b: &bool) -> bool {
-    !b
-}
-
-#[derive(Clone, Debug, serde::Serialize, Parser)]
-#[serde(rename_all = "snake_case")]
-pub(crate) struct Overrides {
-    #[clap(
-        short,
-        long,
-        help = "Whether to skip validating images uploaded via the internal import API"
-    )]
-    #[serde(skip_serializing_if = "is_false")]
-    skip_validate_imports: bool,
-
-    #[clap(short, long, help = "The address and port the server binds to.")]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    addr: Option<SocketAddr>,
-
-    #[clap(short, long, help = "The path to the data directory, e.g. data/")]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    path: Option<PathBuf>,
-
-    #[clap(
-        short,
-        long,
-        help = "An optional image format to convert all uploaded files into, supports 'jpg', 'png', and 'webp'"
-    )]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    image_format: Option<Format>,
-
-    #[clap(
-        short,
-        long,
-        help = "An optional list of filters to permit, supports 'identity', 'thumbnail', 'resize', 'crop', and 'blur'"
-    )]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    filters: Option<Vec<String>>,
-
-    #[clap(
-        short,
-        long,
-        help = "Specify the maximum allowed uploaded file size (in Megabytes)"
-    )]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    max_file_size: Option<usize>,
-
-    #[clap(long, help = "Specify the maximum width in pixels allowed on an image")]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    max_image_width: Option<usize>,
-
-    #[clap(long, help = "Specify the maximum width in pixels allowed on an image")]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    max_image_height: Option<usize>,
-
-    #[clap(long, help = "Specify the maximum area in pixels allowed in an image")]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    max_image_area: Option<usize>,
-
-    #[clap(
-        long,
-        help = "Specify the number of events the console subscriber is allowed to buffer"
-    )]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    console_buffer_capacity: Option<usize>,
-
-    #[clap(
-        long,
-        help = "An optional string to be checked on requests to privileged endpoints"
-    )]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    api_key: Option<String>,
-
-    #[clap(
-        short,
-        long,
-        help = "Enable OpenTelemetry Tracing exports to the given OpenTelemetry collector"
-    )]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    opentelemetry_url: Option<Url>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[clap(
-        short = 'R',
-        long,
-        help = "Set the database implementation. Available options are 'sled'. Default is 'sled'"
-    )]
-    repo: Option<Repo>,
-
-    #[clap(flatten)]
-    sled: Sled,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[clap(
-        short = 'S',
-        long,
-        help = "Set the image store. Available options are 'object-storage' or 'filesystem'. Default is 'filesystem'"
-    )]
-    store: Option<Store>,
-
-    #[clap(flatten)]
-    filesystem_storage: FilesystemStorage,
-
-    #[clap(flatten)]
-    object_storage: ObjectStorage,
-}
-
-impl ObjectStorage {
-    pub(crate) fn required(&self) -> Result<RequiredObjectStorage, RequiredError> {
-        Ok(RequiredObjectStorage {
-            bucket_name: self
-                .object_store_bucket_name
-                .as_ref()
-                .cloned()
-                .ok_or(RequiredError("object-store-bucket-name"))?,
-            region: self
-                .object_store_region
-                .as_ref()
-                .cloned()
-                .map(Serde::into_inner)
-                .ok_or(RequiredError("object-store-region"))?,
-            access_key: self.object_store_access_key.as_ref().cloned(),
-            secret_key: self.object_store_secret_key.as_ref().cloned(),
-            security_token: self.object_store_security_token.as_ref().cloned(),
-            session_token: self.object_store_session_token.as_ref().cloned(),
-        })
-    }
-}
-
-impl Overrides {
-    fn is_default(&self) -> bool {
-        !self.skip_validate_imports
-            && self.addr.is_none()
-            && self.path.is_none()
-            && self.image_format.is_none()
-            && self.filters.is_none()
-            && self.max_file_size.is_none()
-            && self.max_image_width.is_none()
-            && self.max_image_height.is_none()
-            && self.max_image_area.is_none()
-            && self.console_buffer_capacity.is_none()
-            && self.api_key.is_none()
-            && self.opentelemetry_url.is_none()
-            && self.repo.is_none()
-            && self.store.is_none()
-    }
-}
-
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Subcommand)]
-#[serde(rename_all = "snake_case")]
-#[serde(tag = "type")]
-pub(crate) enum Command {
-    Run,
-    Dump { path: PathBuf },
-    MigrateStore { to: Store },
-    MigrateRepo { to: Repo },
-}
-
-pub(crate) enum CommandConfig {
-    Run,
-    Dump {
-        path: PathBuf,
-    },
-    MigrateStore {
-        to: Storage,
-    },
-    MigrateRepo {
-        #[allow(dead_code)]
-        to: Repository,
-    },
-}
-
-#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
-#[serde(rename_all = "snake_case")]
-pub(crate) enum Repo {
-    Sled,
-}
-
-#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize, Parser)]
-#[serde(rename_all = "snake_case")]
-pub(crate) struct Sled {
-    // defaults to {config.path}
-    #[clap(long, help = "Path in which pict-rs will create it's 'repo' directory")]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) sled_path: Option<PathBuf>,
-
-    #[clap(
-        long,
-        help = "The number of bytes sled is allowed to use for it's in-memory cache"
-    )]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) sled_cache_capacity: Option<u64>,
-}
-
-#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
-#[serde(rename_all = "snake_case")]
-pub(crate) enum Store {
-    Filesystem,
-    ObjectStorage,
-}
-
-#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize, Parser)]
-#[serde(rename_all = "snake_case")]
-pub(crate) struct FilesystemStorage {
-    // defaults to {config.path}
-    #[clap(
-        long,
-        help = "Path in which pict-rs will create it's 'files' directory"
-    )]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) filesystem_storage_path: Option<PathBuf>,
-}
-
-#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize, Parser)]
-#[serde(rename_all = "snake_case")]
-pub(crate) struct ObjectStorage {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[clap(long, help = "Name of the bucket in which pict-rs will store images")]
-    object_store_bucket_name: Option<String>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[clap(
-        long,
-        help = "Region in which the bucket exists, can be an http endpoint"
-    )]
-    object_store_region: Option<Serde<s3::Region>>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[clap(long)]
-    object_store_access_key: Option<String>,
-
-    #[clap(long)]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    object_store_secret_key: Option<String>,
-
-    #[clap(long)]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    object_store_security_token: Option<String>,
-
-    #[clap(long)]
-    #[serde(skip_serializing_if = "Option::is_none")]
-    object_store_session_token: Option<String>,
-}
-
-pub(crate) struct RequiredSledRepo {
-    pub(crate) path: PathBuf,
-    pub(crate) cache_capacity: u64,
-}
-
-pub(crate) struct RequiredObjectStorage {
-    pub(crate) bucket_name: String,
-    pub(crate) region: s3::Region,
-    pub(crate) access_key: Option<String>,
-    pub(crate) secret_key: Option<String>,
-    pub(crate) security_token: Option<String>,
-    pub(crate) session_token: Option<String>,
-}
-
-pub(crate) struct RequiredFilesystemStorage {
-    pub(crate) path: PathBuf,
-}
-
-pub(crate) enum Storage {
-    ObjectStorage(RequiredObjectStorage),
-    Filesystem(RequiredFilesystemStorage),
-}
-
-pub(crate) enum Repository {
-    Sled(RequiredSledRepo),
-}
-
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
-#[serde(rename_all = "snake_case")]
-pub(crate) struct Config {
-    skip_validate_imports: bool,
-    addr: SocketAddr,
-    path: PathBuf,
-    image_format: Option<Format>,
-    filters: Option<Vec<String>>,
-    max_file_size: usize,
-    max_image_width: usize,
-    max_image_height: usize,
-    max_image_area: usize,
-    console_buffer_capacity: Option<usize>,
-    api_key: Option<String>,
-    opentelemetry_url: Option<Url>,
-    repo: Repo,
-    store: Store,
-    command: Command,
-    sled: Option<Sled>,
-    filesystem_storage: Option<FilesystemStorage>,
-    object_storage: Option<ObjectStorage>,
-}
-
-#[derive(serde::Serialize)]
-#[serde(rename_all = "snake_case")]
-pub(crate) struct Defaults {
-    command: Command,
-    skip_validate_imports: bool,
-    addr: SocketAddr,
-    max_file_size: usize,
-    max_image_width: usize,
-    max_image_height: usize,
-    max_image_area: usize,
-    repo: Repo,
-    sled: SledDefaults,
-    store: Store,
-}
-
-#[derive(serde::Serialize)]
-#[serde(rename_all = "snake_case")]
-struct SledDefaults {
-    sled_cache_capacity: usize,
-}
-
-impl Defaults {
-    fn new(command: Command) -> Self {
-        Defaults {
-            command,
-            skip_validate_imports: false,
-            addr: ([0, 0, 0, 0], 8080).into(),
-            max_file_size: 40,
-            max_image_width: 10_000,
-            max_image_height: 10_000,
-            max_image_area: 40_000_000,
-            repo: Repo::Sled,
-            sled: SledDefaults {
-                sled_cache_capacity: 1024 * 1024 * 64,
-            },
-            store: Store::Filesystem,
-        }
-    }
-}
-
-impl Config {
-    pub(crate) fn build() -> anyhow::Result<Self> {
-        let args = Args::parse();
-
-        let mut base_config = config::Config::builder()
-            .add_source(config::Config::try_from(&Defaults::new(args.command))?);
-
-        if let Some(path) = args.config_file {
-            base_config = base_config.add_source(config::File::from(path));
-        };
-
-        if !args.overrides.is_default() {
-            let merging = config::Config::try_from(&args.overrides)?;
-
-            base_config = base_config.add_source(merging);
-        }
-
-        let config: Self = base_config
-            .add_source(config::Environment::with_prefix("PICTRS").separator("__"))
-            .build()?
-            .try_deserialize()?;
-
-        Ok(config)
-    }
-
-    pub(crate) fn command(&self) -> anyhow::Result<CommandConfig> {
-        Ok(match &self.command {
-            Command::Run => CommandConfig::Run,
-            Command::Dump { path } => CommandConfig::Dump { path: path.clone() },
-            Command::MigrateStore { to } => CommandConfig::MigrateStore {
-                to: match to {
-                    Store::ObjectStorage => Storage::ObjectStorage(
-                        self.object_storage
-                            .as_ref()
-                            .cloned()
-                            .unwrap_or_default()
-                            .required()?,
-                    ),
-                    Store::Filesystem => Storage::Filesystem(RequiredFilesystemStorage {
-                        path: self
-                            .filesystem_storage
-                            .as_ref()
-                            .and_then(|f| f.filesystem_storage_path.clone())
-                            .unwrap_or_else(|| {
-                                let mut path = self.path.clone();
-                                path.push("files");
-                                path
-                            }),
-                    }),
-                },
-            },
-            Command::MigrateRepo { to } => CommandConfig::MigrateRepo {
-                to: match to {
-                    Repo::Sled => {
-                        let sled = self.sled.as_ref().cloned().unwrap_or_default();
-
-                        Repository::Sled(RequiredSledRepo {
-                            path: sled.sled_path.unwrap_or_else(|| {
-                                let mut path = self.path.clone();
-                                path.push("sled-repo");
-                                path
-                            }),
-                            cache_capacity: sled.sled_cache_capacity.unwrap_or(1024 * 1024 * 64),
-                        })
-                    }
-                },
-            },
-        })
-    }
-
-    pub(crate) fn store(&self) -> anyhow::Result<Storage> {
-        Ok(match self.store {
-            Store::Filesystem => Storage::Filesystem(RequiredFilesystemStorage {
-                path: self
-                    .filesystem_storage
-                    .as_ref()
-                    .and_then(|f| f.filesystem_storage_path.clone())
-                    .unwrap_or_else(|| {
-                        let mut path = self.path.clone();
-                        path.push("files");
-                        path
-                    }),
-            }),
-            Store::ObjectStorage => Storage::ObjectStorage(
-                self.object_storage
-                    .as_ref()
-                    .cloned()
-                    .unwrap_or_default()
-                    .required()?,
-            ),
-        })
-    }
-
-    pub(crate) fn repo(&self) -> Repository {
-        match self.repo {
-            Repo::Sled => {
-                let sled = self.sled.as_ref().cloned().unwrap_or_default();
-
-                Repository::Sled(RequiredSledRepo {
-                    path: sled.sled_path.unwrap_or_else(|| {
-                        let mut path = self.path.clone();
-                        path.push("sled-repo");
-                        path
-                    }),
-                    cache_capacity: sled.sled_cache_capacity.unwrap_or(1024 * 1024 * 64),
-                })
-            }
-        }
-    }
-
-    pub(crate) fn bind_address(&self) -> SocketAddr {
-        self.addr
-    }
-
-    pub(crate) fn data_dir(&self) -> PathBuf {
-        self.path.clone()
-    }
-
-    pub(crate) fn console_buffer_capacity(&self) -> Option<usize> {
-        self.console_buffer_capacity
-    }
-
-    pub(crate) fn format(&self) -> Option<Format> {
-        self.image_format
-    }
-
-    pub(crate) fn allowed_filters(&self) -> Option<HashSet<String>> {
-        self.filters.as_ref().map(|wl| wl.iter().cloned().collect())
-    }
-
-    pub(crate) fn validate_imports(&self) -> bool {
-        !self.skip_validate_imports
-    }
-
-    pub(crate) fn max_file_size(&self) -> usize {
-        self.max_file_size
-    }
-
-    pub(crate) fn max_width(&self) -> usize {
-        self.max_image_width
-    }
-
-    pub(crate) fn max_height(&self) -> usize {
-        self.max_image_height
-    }
-
-    pub(crate) fn max_area(&self) -> usize {
-        self.max_image_area
-    }
-
-    pub(crate) fn api_key(&self) -> Option<&str> {
-        self.api_key.as_deref()
-    }
-
-    pub(crate) fn opentelemetry_url(&self) -> Option<&Url> {
-        self.opentelemetry_url.as_ref()
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-#[error("Invalid format supplied, {0}")]
-pub(crate) struct FormatError(String);
-
-#[derive(Debug, thiserror::Error)]
-#[error("Invalid store supplied, {0}")]
-pub(crate) struct StoreError(String);
-
-#[derive(Debug, thiserror::Error)]
-#[error("Invalid repo supplied, {0}")]
-pub(crate) struct RepoError(String);
-
-#[derive(Debug, thiserror::Error)]
-#[error("Missing required {0} field")]
-pub(crate) struct RequiredError(&'static str);
-
-#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
-#[serde(rename_all = "snake_case")]
-pub(crate) enum Format {
-    Jpeg,
-    Png,
-    Webp,
-}
-
-impl Format {
-    pub(crate) fn as_magick_format(&self) -> &'static str {
-        match self {
-            Format::Jpeg => "JPEG",
-            Format::Png => "PNG",
-            Format::Webp => "WEBP",
-        }
-    }
-
-    pub(crate) fn as_hint(&self) -> Option<ValidInputType> {
-        match self {
-            Format::Jpeg => Some(ValidInputType::Jpeg),
-            Format::Png => Some(ValidInputType::Png),
-            Format::Webp => Some(ValidInputType::Webp),
-        }
-    }
-}
-
-impl std::str::FromStr for Format {
-    type Err = FormatError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        for variant in Self::value_variants() {
-            if variant.to_possible_value().unwrap().matches(s, false) {
-                return Ok(*variant);
-            }
-        }
-        Err(FormatError(s.into()))
-    }
-}
-
-impl std::str::FromStr for Store {
-    type Err = StoreError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        for variant in Self::value_variants() {
-            if variant.to_possible_value().unwrap().matches(s, false) {
-                return Ok(*variant);
-            }
-        }
-        Err(StoreError(s.into()))
-    }
-}
-
-impl std::str::FromStr for Repo {
-    type Err = RepoError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        for variant in Self::value_variants() {
-            if variant.to_possible_value().unwrap().matches(s, false) {
-                return Ok(*variant);
-            }
-        }
-        Err(RepoError(s.into()))
-    }
+pub(crate) fn configure() -> anyhow::Result<Configuration> {
+    unimplemented!()
 }
diff --git a/src/config/commandline.rs b/src/config/commandline.rs
new file mode 100644
index 0000000..89dd726
--- /dev/null
+++ b/src/config/commandline.rs
@@ -0,0 +1,216 @@
+use crate::config::primitives::{ImageFormat, LogFormat, Targets};
+use clap::{Parser, Subcommand};
+use std::{net::SocketAddr, path::PathBuf};
+use url::Url;
+
+/// Run the pict-rs application
+#[derive(Debug, Parser)]
+#[clap(author, version, about, long_about = None)]
+pub(crate) struct Args {
+    /// Path to the pict-rs configuration file
+    #[clap(short, long)]
+    pub(crate) config_file: Option<PathBuf>,
+
+    /// Format of logs printed to stdout
+    #[clap(long)]
+    pub(crate) log_format: Option<LogFormat>,
+    /// Log levels to print to stdout, respects RUST_LOG formatting
+    #[clap(long)]
+    pub(crate) log_targets: Option<Targets>,
+
+    /// Address and port to expose tokio-console metrics
+    #[clap(long)]
+    pub(crate) console_address: Option<SocketAddr>,
+    /// Capacity of the console-subscriber Event Buffer
+    #[clap(long)]
+    pub(crate) console_buffer_capacity: Option<usize>,
+
+    /// URL to send OpenTelemetry metrics
+    #[clap(long)]
+    pub(crate) opentelemetry_url: Option<Url>,
+    /// Service Name to use for OpenTelemetry
+    #[clap(long)]
+    pub(crate) opentelemetry_service_name: Option<String>,
+    /// Log levels to use for OpenTelemetry, respects RUST_LOG formatting
+    #[clap(long)]
+    pub(crate) opentelemetry_targets: Option<Targets>,
+
+    /// File to save the current configuration for reproducible runs
+    #[clap(long)]
+    pub(crate) save_to: Option<PathBuf>,
+
+    #[clap(subcommand)]
+    pub(crate) command: Command,
+}
+
+#[derive(Debug, Subcommand)]
+pub(crate) enum Command {
+    /// Runs the pict-rs web server
+    Run(Run),
+
+    /// Migrates from one provided media store to another
+    #[clap(flatten)]
+    MigrateStore(MigrateStore),
+}
+
+#[derive(Debug, Parser)]
+pub(crate) struct Run {
+    /// The address and port to bind the pict-rs web server
+    #[clap(short, long)]
+    pub(crate) address: SocketAddr,
+
+    /// The API KEY required to access restricted routes
+    #[clap(long)]
+    pub(crate) api_key: Option<String>,
+
+    /// Whether to validate media on the "import" endpoint
+    #[clap(long)]
+    pub(crate) media_skip_validate_imports: Option<bool>,
+    /// The maximum width, in pixels, for uploaded media
+    #[clap(long)]
+    pub(crate) media_max_width: Option<usize>,
+    /// The maximum height, in pixels, for uploaded media
+    #[clap(long)]
+    pub(crate) media_max_height: Option<usize>,
+    /// The maximum area, in pixels, for uploaded media
+    #[clap(long)]
+    pub(crate) media_max_area: Option<usize>,
+    /// The maximum size, in megabytes, for uploaded media
+    #[clap(long)]
+    pub(crate) media_max_file_size: Option<usize>,
+    /// Whether to enable GIF and silent MP4 uploads. Full videos are unsupported
+    #[clap(long)]
+    pub(crate) media_enable_silent_video: Option<bool>,
+    /// Which media filters should be enabled on the `process` endpoint
+    #[clap(long)]
+    pub(crate) media_filters: Option<Vec<String>>,
+    /// Enforce uploaded media is transcoded to the provided format
+    #[clap(long)]
+    pub(crate) media_format: Option<ImageFormat>,
+
+    #[clap(subcommand)]
+    pub(crate) store: Option<RunStore>,
+}
+
+/// Configure the provided storage
+#[derive(Debug, Subcommand)]
+pub(crate) enum Store {
+    /// configure filesystem storage
+    Filesystem(Filesystem),
+
+    /// configure object storage
+    ObjectStorage(ObjectStorage),
+}
+
+/// Run pict-rs with the provided storage
+#[derive(Debug, Subcommand)]
+pub(crate) enum RunStore {
+    /// Run pict-rs with filesystem storage
+    Filesystem(RunFilesystem),
+
+    /// Run pict-rs with object storage
+    ObjectStorage(RunObjectStorage),
+}
+
+/// Configure the pict-rs storage migration
+#[derive(Debug, Subcommand)]
+pub(crate) enum MigrateStore {
+    /// Migrate from the provided filesystem storage
+    Filesystem(MigrateFilesystem),
+
+    /// Migrate from the provided object storage
+    ObjectStorage(MigrateObjectStorage),
+}
+
+/// Migrate pict-rs' storage from the provided filesystem storage
+#[derive(Debug, Parser)]
+pub(crate) struct MigrateFilesystem {
+    #[clap(flatten)]
+    pub(crate) from: Filesystem,
+
+    #[clap(subcommand)]
+    pub(crate) to: RunStore,
+}
+
+/// Migrate pict-rs' storage from the provided object storage
+#[derive(Debug, Parser)]
+pub(crate) struct MigrateObjectStorage {
+    #[clap(flatten)]
+    pub(crate) from: ObjectStorage,
+
+    #[clap(subcommand)]
+    pub(crate) to: RunStore,
+}
+
+/// Run pict-rs with the provided filesystem storage
+#[derive(Debug, Parser)]
+pub(crate) struct RunFilesystem {
+    #[clap(flatten)]
+    pub(crate) system: Filesystem,
+
+    #[clap(subcommand)]
+    pub(crate) repo: Repo,
+}
+
+/// Run pict-rs with the provided object storage
+#[derive(Debug, Parser)]
+pub(crate) struct RunObjectStorage {
+    #[clap(flatten)]
+    pub(crate) storage: ObjectStorage,
+
+    #[clap(subcommand)]
+    pub(crate) repo: Repo,
+}
+
+/// Configuration for data repositories
+#[derive(Debug, Subcommand)]
+pub(crate) enum Repo {
+    /// Run pict-rs with the provided sled-backed data repository
+    Sled(Sled),
+}
+
+/// Configuration for filesystem media storage
+#[derive(Debug, Parser)]
+pub(crate) struct Filesystem {
+    /// The path to store uploaded media
+    #[clap(short, long)]
+    pub(crate) path: Option<PathBuf>,
+}
+
+/// Configuration for Object Storage
+#[derive(Debug, Parser)]
+pub(crate) struct ObjectStorage {
+    /// The bucket in which to store media
+    #[clap(short, long)]
+    pub(crate) bucket_name: Option<String>,
+
+    /// The region the bucket is located in
+    #[clap(short, long)]
+    pub(crate) region: Option<s3::Region>,
+
+    /// The Access Key for the user accessing the bucket
+    #[clap(short, long)]
+    pub(crate) access_key: Option<String>,
+
+    /// The secret key for the user accessing the bucket
+    #[clap(short, long)]
+    pub(crate) secret_key: Option<String>,
+
+    /// The security token for accessing the bucket
+    #[clap(long)]
+    pub(crate) security_token: Option<String>,
+
+    /// The session token for accessing the bucket
+    #[clap(long)]
+    pub(crate) session_token: Option<String>,
+}
+
+/// Configuration for the sled-backed data repository
+#[derive(Debug, Parser)]
+pub(crate) struct Sled {
+    /// The path to store the sled database
+    pub(crate) path: Option<PathBuf>,
+
+    /// The cache capacity, in bytes, allowed to sled for in-memory operations
+    pub(crate) cache_capacity: Option<u64>,
+}
diff --git a/src/config/defaults.rs b/src/config/defaults.rs
new file mode 100644
index 0000000..04ad61f
--- /dev/null
+++ b/src/config/defaults.rs
@@ -0,0 +1,181 @@
+use crate::{
+    config::primitives::{LogFormat, Targets},
+    serde_str::Serde,
+};
+use std::{net::SocketAddr, path::PathBuf};
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Defaults {
+    server: ServerDefaults,
+    tracing: TracingDefaults,
+    old_db: OldDbDefaults,
+    media: MediaDefaults,
+    repo: RepoDefaults,
+    store: StoreDefaults,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+struct ServerDefaults {
+    address: SocketAddr,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+struct TracingDefaults {
+    logging: LoggingDefaults,
+
+    console: ConsoleDefaults,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+struct LoggingDefaults {
+    format: LogFormat,
+    targets: Serde<Targets>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+struct ConsoleDefaults {
+    buffer_capacity: usize,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+struct OldDbDefaults {
+    path: PathBuf,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+struct MediaDefaults {
+    max_width: usize,
+    max_height: usize,
+    max_area: usize,
+    max_file_size: usize,
+    enable_silent_video: bool,
+    filters: Vec<String>,
+    skip_validate_imports: bool,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(tag = "type")]
+enum RepoDefaults {
+    Sled(SledDefaults),
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+struct SledDefaults {
+    path: PathBuf,
+    cache_capacity: u64,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(tag = "type")]
+enum StoreDefaults {
+    Filesystem(FilesystemDefaults),
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+struct FilesystemDefaults {
+    path: PathBuf,
+}
+
+impl Default for Defaults {
+    fn default() -> Self {
+        Defaults {
+            server: ServerDefaults::default(),
+            tracing: TracingDefaults::default(),
+            old_db: OldDbDefaults::default(),
+            media: MediaDefaults::default(),
+            repo: RepoDefaults::default(),
+            store: StoreDefaults::default(),
+        }
+    }
+}
+
+impl Default for ServerDefaults {
+    fn default() -> Self {
+        ServerDefaults {
+            address: "0.0.0.0:8080".parse().expect("Valid address string"),
+        }
+    }
+}
+
+impl Default for TracingDefaults {
+    fn default() -> TracingDefaults {
+        TracingDefaults {
+            logging: LoggingDefaults::default(),
+            console: ConsoleDefaults::default(),
+        }
+    }
+}
+
+impl Default for LoggingDefaults {
+    fn default() -> Self {
+        LoggingDefaults {
+            format: LogFormat::Normal,
+            targets: "info".parse().expect("Valid targets string"),
+        }
+    }
+}
+
+impl Default for ConsoleDefaults {
+    fn default() -> Self {
+        ConsoleDefaults {
+            buffer_capacity: 1024 * 100,
+        }
+    }
+}
+
+impl Default for OldDbDefaults {
+    fn default() -> Self {
+        OldDbDefaults {
+            path: PathBuf::from(String::from("/mnt")),
+        }
+    }
+}
+
+impl Default for MediaDefaults {
+    fn default() -> Self {
+        MediaDefaults {
+            max_width: 10_000,
+            max_height: 10_000,
+            max_area: 40_000_000,
+            max_file_size: 40,
+            enable_silent_video: true,
+            filters: vec![
+                "identity".into(),
+                "thumbnail".into(),
+                "resize".into(),
+                "crop".into(),
+                "blur".into(),
+            ],
+            skip_validate_imports: false,
+        }
+    }
+}
+
+impl Default for RepoDefaults {
+    fn default() -> Self {
+        Self::Sled(SledDefaults::default())
+    }
+}
+
+impl Default for SledDefaults {
+    fn default() -> Self {
+        SledDefaults {
+            path: PathBuf::from(String::from("/mnt/sled-repo")),
+            cache_capacity: 1024 * 1024 * 64,
+        }
+    }
+}
+
+impl Default for StoreDefaults {
+    fn default() -> Self {
+        Self::Filesystem(FilesystemDefaults::default())
+    }
+}
+
+impl Default for FilesystemDefaults {
+    fn default() -> Self {
+        Self {
+            path: PathBuf::from(String::from("/mnt/files")),
+        }
+    }
+}
diff --git a/src/config/file.rs b/src/config/file.rs
new file mode 100644
index 0000000..26e298b
--- /dev/null
+++ b/src/config/file.rs
@@ -0,0 +1,131 @@
+use crate::{
+    config::primitives::{ImageFormat, LogFormat, Targets},
+    serde_str::Serde,
+};
+use std::{net::SocketAddr, path::PathBuf};
+use url::Url;
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct ConfigFile {
+    pub(crate) server: Server,
+
+    pub(crate) tracing: Tracing,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) old_db: Option<OldDb>,
+
+    pub(crate) media: Media,
+
+    pub(crate) repo: Repo,
+
+    pub(crate) store: Store,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(tag = "type")]
+pub(crate) enum Repo {
+    Sled(Sled),
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(tag = "type")]
+pub(crate) enum Store {
+    Filesystem(Filesystem),
+
+    ObjectStorage(ObjectStorage),
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Server {
+    pub(crate) address: SocketAddr,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) api_key: Option<String>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Tracing {
+    logging: Logging,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    console: Option<Console>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    opentelemetry: Option<OpenTelemetry>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Logging {
+    pub(crate) format: LogFormat,
+
+    pub(crate) targets: Serde<Targets>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct OpenTelemetry {
+    pub(crate) url: Url,
+
+    pub(crate) service_name: String,
+
+    pub(crate) targets: Serde<Targets>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Console {
+    pub(crate) address: SocketAddr,
+    pub(crate) buffer_capacity: usize,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct OldDb {
+    pub(crate) path: PathBuf,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Media {
+    pub(crate) max_width: usize,
+
+    pub(crate) max_height: usize,
+
+    pub(crate) max_area: usize,
+
+    pub(crate) max_file_size: usize,
+
+    pub(crate) enable_silent_video: bool,
+
+    pub(crate) filters: Vec<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) format: Option<ImageFormat>,
+
+    pub(crate) skip_validate_imports: bool,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Filesystem {
+    pub(crate) path: PathBuf,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct ObjectStorage {
+    pub(crate) bucket_name: String,
+
+    pub(crate) region: Serde<s3::Region>,
+
+    pub(crate) access_key: String,
+
+    pub(crate) secret_key: String,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) security_token: Option<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) session_token: Option<String>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Sled {
+    pub(crate) path: PathBuf,
+
+    pub(crate) cache_capacity: u64,
+}
diff --git a/src/config/primitives.rs b/src/config/primitives.rs
new file mode 100644
index 0000000..44d1ccd
--- /dev/null
+++ b/src/config/primitives.rs
@@ -0,0 +1,111 @@
+use clap::ArgEnum;
+use std::{fmt::Display, str::FromStr};
+
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    serde::Deserialize,
+    serde::Serialize,
+    ArgEnum,
+)]
+pub(crate) enum LogFormat {
+    Compact,
+    Json,
+    Normal,
+    Pretty,
+}
+
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    serde::Deserialize,
+    serde::Serialize,
+    ArgEnum,
+)]
+pub(crate) enum ImageFormat {
+    Jpeg,
+    Webp,
+    Png,
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct Targets {
+    pub(crate) targets: tracing_subscriber::filter::Targets,
+}
+
+impl FromStr for Targets {
+    type Err = <tracing_subscriber::filter::Targets as FromStr>::Err;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Targets {
+            targets: s.parse()?,
+        })
+    }
+}
+
+impl Display for Targets {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let targets = self
+            .targets
+            .iter()
+            .map(|(path, level)| format!("{}={}", path, level))
+            .collect::<Vec<_>>()
+            .join(",");
+
+        write!(f, "{}", targets)
+    }
+}
+
+impl FromStr for ImageFormat {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        for variant in Self::value_variants() {
+            if variant.to_possible_value().unwrap().matches(s, false) {
+                return Ok(*variant);
+            }
+        }
+        Err(format!("Invalid variant: {}", s))
+    }
+}
+
+impl FromStr for LogFormat {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        for variant in Self::value_variants() {
+            if variant.to_possible_value().unwrap().matches(s, false) {
+                return Ok(*variant);
+            }
+        }
+        Err(format!("Invalid variant: {}", s))
+    }
+}
+
+impl Display for ImageFormat {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.to_possible_value()
+            .expect("no values are skipped")
+            .get_name()
+            .fmt(f)
+    }
+}
+
+impl Display for LogFormat {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.to_possible_value()
+            .expect("no values are skipped")
+            .get_name()
+            .fmt(f)
+    }
+}

From 3792a8923ae91a58b27af2bfe8a94218bd757d49 Mon Sep 17 00:00:00 2001
From: "Aode (Lion)" <asonix@asonix.dog>
Date: Sun, 27 Mar 2022 23:27:07 -0500
Subject: [PATCH 09/37] Finish redoing config probably

---
 Cargo.lock                |  13 ++
 Cargo.toml                |   3 +
 defaults.toml             |  39 ++++
 dev.toml                  |  39 ++++
 src/config.rs             |  41 +++-
 src/config/commandline.rs | 466 +++++++++++++++++++++++++++++++++-----
 src/config/defaults.rs    |  73 +++---
 src/config/file.rs        |  64 ++----
 src/config/primitives.rs  | 144 +++++++++++-
 src/init_tracing.rs       |  78 +++----
 src/magick.rs             |  20 +-
 src/main.rs               |  90 ++++----
 src/repo.rs               |  11 +-
 src/serde_str.rs          |  12 +
 src/upload_manager.rs     |   6 +-
 src/validate.rs           |  10 +-
 16 files changed, 856 insertions(+), 253 deletions(-)
 create mode 100644 defaults.toml
 create mode 100644 dev.toml

diff --git a/Cargo.lock b/Cargo.lock
index 3a45ff7..9bb78d4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2651,6 +2651,16 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "tracing-serde"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.9"
@@ -2661,12 +2671,15 @@ dependencies = [
  "lazy_static",
  "matchers",
  "regex",
+ "serde",
+ "serde_json",
  "sharded-slab",
  "smallvec",
  "thread_local",
  "tracing",
  "tracing-core",
  "tracing-log",
+ "tracing-serde",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 0ed1dec..aea8f57 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -63,8 +63,11 @@ tracing-futures = "0.2.4"
 tracing-log = "0.1.2"
 tracing-opentelemetry = "0.17"
 tracing-subscriber = { version = "0.3.0", features = [
+  "ansi",
   "env-filter",
   "fmt",
+  "json",
+  "registry",
   "tracing-log",
 ] }
 url = { version = "2.2", features = ["serde"] }
diff --git a/defaults.toml b/defaults.toml
new file mode 100644
index 0000000..01834cc
--- /dev/null
+++ b/defaults.toml
@@ -0,0 +1,39 @@
+[server]
+address = '0.0.0.0:8080'
+[tracing.logging]
+format = 'normal'
+targets = 'info'
+
+[tracing.console]
+buffer_capacity = 102400
+
+[tracing.opentelemetry]
+service_name = 'pict-rs'
+targets = 'info'
+
+[old_db]
+path = '/mnt'
+
+[media]
+max_width = 10000
+max_height = 10000
+max_area = 40000000
+max_file_size = 40
+enable_silent_video = true
+filters = [
+    'crop',
+    'blur',
+    'resize',
+    'identity',
+    'thumbnail',
+]
+skip_validate_imports = false
+
+[repo]
+type = 'sled'
+path = '/mnt/sled-repo'
+cache_capacity = 67108864
+
+[store]
+type = 'filesystem'
+path = '/mnt/files'
diff --git a/dev.toml b/dev.toml
new file mode 100644
index 0000000..91c5b00
--- /dev/null
+++ b/dev.toml
@@ -0,0 +1,39 @@
+[server]
+address = '0.0.0.0:8080'
+[tracing.logging]
+format = 'normal'
+targets = 'info'
+
+[tracing.console]
+buffer_capacity = 102400
+
+[tracing.opentelemetry]
+service_name = 'pict-rs'
+targets = 'info'
+
+[old_db]
+path = 'data/'
+
+[media]
+max_width = 10000
+max_height = 10000
+max_area = 40000000
+max_file_size = 40
+enable_silent_video = true
+filters = [
+    'identity',
+    'resize',
+    'crop',
+    'thumbnail',
+    'blur',
+]
+skip_validate_imports = false
+
+[repo]
+type = 'sled'
+path = 'data/sled-repo'
+cache_capacity = 67108864
+
+[store]
+type = 'filesystem'
+path = 'data/files'
diff --git a/src/config.rs b/src/config.rs
index 737a61c..eb4901f 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,12 +1,45 @@
+use clap::Parser;
+
 mod commandline;
 mod defaults;
 mod file;
 mod primitives;
 
-use crate::magick::ValidInputType;
+use commandline::{Args, Output};
+use config::Config;
+use defaults::Defaults;
 
-pub(crate) use file::ConfigFile as Configuration;
+pub(crate) use commandline::Operation;
+pub(crate) use file::{ConfigFile as Configuration, OpenTelemetry, Repo, Sled, Tracing};
+pub(crate) use primitives::{Filesystem, ImageFormat, LogFormat, ObjectStorage, Store};
 
-pub(crate) fn configure() -> anyhow::Result<Configuration> {
-    unimplemented!()
+pub(crate) fn configure() -> anyhow::Result<(Configuration, Operation)> {
+    let Output {
+        config_format,
+        operation,
+        save_to,
+        config_file,
+    } = Args::parse().into_output();
+
+    let config = Config::builder().add_source(config::Config::try_from(&Defaults::default())?);
+
+    let config = if let Some(config_file) = config_file {
+        config.add_source(config::File::from(config_file))
+    } else {
+        config
+    };
+
+    let built = config
+        .add_source(config::Environment::with_prefix("PICTRS").separator("__"))
+        .add_source(config::Config::try_from(&config_format)?)
+        .build()?;
+
+    let config: Configuration = built.try_deserialize()?;
+
+    if let Some(save_to) = save_to {
+        let output = toml::to_string_pretty(&config)?;
+        std::fs::write(save_to, output)?;
+    }
+
+    Ok((config, operation))
 }
diff --git a/src/config/commandline.rs b/src/config/commandline.rs
index 89dd726..08900d3 100644
--- a/src/config/commandline.rs
+++ b/src/config/commandline.rs
@@ -1,50 +1,359 @@
-use crate::config::primitives::{ImageFormat, LogFormat, Targets};
+use crate::{
+    config::primitives::{ImageFormat, LogFormat, Targets},
+    serde_str::Serde,
+};
 use clap::{Parser, Subcommand};
 use std::{net::SocketAddr, path::PathBuf};
 use url::Url;
 
+impl Args {
+    pub(super) fn into_output(self) -> Output {
+        let Args {
+            config_file,
+            old_db_path,
+            log_format,
+            log_targets,
+            console_address,
+            console_buffer_capacity,
+            opentelemetry_url,
+            opentelemetry_service_name,
+            opentelemetry_targets,
+            save_to,
+            command,
+        } = self;
+
+        let old_db = OldDb { path: old_db_path };
+
+        let tracing = Tracing {
+            logging: Logging {
+                format: log_format,
+                targets: log_targets.map(Serde::new),
+            },
+            console: Console {
+                address: console_address,
+                buffer_capacity: console_buffer_capacity,
+            },
+            opentelemetry: OpenTelemetry {
+                url: opentelemetry_url,
+                service_name: opentelemetry_service_name,
+                targets: opentelemetry_targets.map(Serde::new),
+            },
+        };
+
+        match command {
+            Command::Run(Run {
+                address,
+                api_key,
+                media_skip_validate_imports,
+                media_max_width,
+                media_max_height,
+                media_max_area,
+                media_max_file_size,
+                media_enable_silent_video,
+                media_filters,
+                media_format,
+                store,
+            }) => {
+                let server = Server { address, api_key };
+                let media = Media {
+                    skip_validate_imports: media_skip_validate_imports,
+                    max_width: media_max_width,
+                    max_height: media_max_height,
+                    max_area: media_max_area,
+                    max_file_size: media_max_file_size,
+                    enable_silent_video: media_enable_silent_video,
+                    filters: media_filters,
+                    format: media_format,
+                };
+                let operation = Operation::Run;
+
+                match store {
+                    Some(RunStore::Filesystem(RunFilesystem { system, repo })) => {
+                        let store = Some(Store::Filesystem(system));
+                        Output {
+                            config_format: ConfigFormat {
+                                server,
+                                old_db,
+                                tracing,
+                                media,
+                                store,
+                                repo,
+                            },
+                            operation,
+                            config_file,
+                            save_to,
+                        }
+                    }
+                    Some(RunStore::ObjectStorage(RunObjectStorage { storage, repo })) => {
+                        let store = Some(Store::ObjectStorage(storage));
+                        Output {
+                            config_format: ConfigFormat {
+                                server,
+                                old_db,
+                                tracing,
+                                media,
+                                store,
+                                repo,
+                            },
+                            operation,
+                            config_file,
+                            save_to,
+                        }
+                    }
+                    None => Output {
+                        config_format: ConfigFormat {
+                            server,
+                            old_db,
+                            tracing,
+                            media,
+                            store: None,
+                            repo: None,
+                        },
+                        operation,
+                        config_file,
+                        save_to,
+                    },
+                }
+            }
+            Command::MigrateStore(migrate_store) => {
+                let server = Server::default();
+                let media = Media::default();
+
+                match migrate_store {
+                    MigrateStore::Filesystem(MigrateFilesystem { from, to }) => match to {
+                        MigrateStoreInner::Filesystem(MigrateFilesystemInner { to, repo }) => {
+                            Output {
+                                config_format: ConfigFormat {
+                                    server,
+                                    old_db,
+                                    tracing,
+                                    media,
+                                    store: None,
+                                    repo,
+                                },
+                                operation: Operation::MigrateStore {
+                                    from: from.into(),
+                                    to: to.into(),
+                                },
+                                config_file,
+                                save_to,
+                            }
+                        }
+                        MigrateStoreInner::ObjectStorage(MigrateObjectStorageInner {
+                            to,
+                            repo,
+                        }) => Output {
+                            config_format: ConfigFormat {
+                                server,
+                                old_db,
+                                tracing,
+                                media,
+                                store: None,
+                                repo,
+                            },
+                            operation: Operation::MigrateStore {
+                                from: from.into(),
+                                to: to.into(),
+                            },
+                            config_file,
+                            save_to,
+                        },
+                    },
+                    MigrateStore::ObjectStorage(MigrateObjectStorage { from, to }) => match to {
+                        MigrateStoreInner::Filesystem(MigrateFilesystemInner { to, repo }) => {
+                            Output {
+                                config_format: ConfigFormat {
+                                    server,
+                                    old_db,
+                                    tracing,
+                                    media,
+                                    store: None,
+                                    repo,
+                                },
+                                operation: Operation::MigrateStore {
+                                    from: from.into(),
+                                    to: to.into(),
+                                },
+                                config_file,
+                                save_to,
+                            }
+                        }
+                        MigrateStoreInner::ObjectStorage(MigrateObjectStorageInner {
+                            to,
+                            repo,
+                        }) => Output {
+                            config_format: ConfigFormat {
+                                server,
+                                old_db,
+                                tracing,
+                                media,
+                                store: None,
+                                repo,
+                            },
+                            operation: Operation::MigrateStore {
+                                from: from.into(),
+                                to: to.into(),
+                            },
+                            config_file,
+                            save_to,
+                        },
+                    },
+                }
+            }
+        }
+    }
+}
+
+pub(super) struct Output {
+    pub(super) config_format: ConfigFormat,
+    pub(super) operation: Operation,
+    pub(super) save_to: Option<PathBuf>,
+    pub(super) config_file: Option<PathBuf>,
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone)]
+pub(crate) enum Operation {
+    Run,
+    MigrateStore {
+        from: crate::config::primitives::Store,
+        to: crate::config::primitives::Store,
+    },
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(super) struct ConfigFormat {
+    server: Server,
+    old_db: OldDb,
+    tracing: Tracing,
+    media: Media,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    repo: Option<Repo>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    store: Option<Store>,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct Server {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    address: Option<SocketAddr>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    api_key: Option<String>,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct Tracing {
+    logging: Logging,
+    console: Console,
+    opentelemetry: OpenTelemetry,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct Logging {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    format: Option<LogFormat>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    targets: Option<Serde<Targets>>,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct Console {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    address: Option<SocketAddr>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    buffer_capacity: Option<usize>,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct OpenTelemetry {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    url: Option<Url>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    service_name: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    targets: Option<Serde<Targets>>,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct OldDb {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    path: Option<PathBuf>,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct Media {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    max_width: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    max_height: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    max_area: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    max_file_size: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    enable_silent_video: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    filters: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    format: Option<ImageFormat>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    skip_validate_imports: Option<bool>,
+}
+
 /// Run the pict-rs application
 #[derive(Debug, Parser)]
 #[clap(author, version, about, long_about = None)]
-pub(crate) struct Args {
+pub(super) struct Args {
     /// Path to the pict-rs configuration file
     #[clap(short, long)]
-    pub(crate) config_file: Option<PathBuf>,
+    config_file: Option<PathBuf>,
+
+    /// Path to the old pict-rs sled database
+    #[clap(long)]
+    old_db_path: Option<PathBuf>,
 
     /// Format of logs printed to stdout
     #[clap(long)]
-    pub(crate) log_format: Option<LogFormat>,
+    log_format: Option<LogFormat>,
     /// Log levels to print to stdout, respects RUST_LOG formatting
     #[clap(long)]
-    pub(crate) log_targets: Option<Targets>,
+    log_targets: Option<Targets>,
 
     /// Address and port to expose tokio-console metrics
     #[clap(long)]
-    pub(crate) console_address: Option<SocketAddr>,
+    console_address: Option<SocketAddr>,
     /// Capacity of the console-subscriber Event Buffer
     #[clap(long)]
-    pub(crate) console_buffer_capacity: Option<usize>,
+    console_buffer_capacity: Option<usize>,
 
     /// URL to send OpenTelemetry metrics
     #[clap(long)]
-    pub(crate) opentelemetry_url: Option<Url>,
+    opentelemetry_url: Option<Url>,
     /// Service Name to use for OpenTelemetry
     #[clap(long)]
-    pub(crate) opentelemetry_service_name: Option<String>,
+    opentelemetry_service_name: Option<String>,
     /// Log levels to use for OpenTelemetry, respects RUST_LOG formatting
     #[clap(long)]
-    pub(crate) opentelemetry_targets: Option<Targets>,
+    opentelemetry_targets: Option<Targets>,
 
     /// File to save the current configuration for reproducible runs
     #[clap(long)]
-    pub(crate) save_to: Option<PathBuf>,
+    save_to: Option<PathBuf>,
 
     #[clap(subcommand)]
-    pub(crate) command: Command,
+    command: Command,
 }
 
 #[derive(Debug, Subcommand)]
-pub(crate) enum Command {
+enum Command {
     /// Runs the pict-rs web server
     Run(Run),
 
@@ -54,47 +363,49 @@ pub(crate) enum Command {
 }
 
 #[derive(Debug, Parser)]
-pub(crate) struct Run {
+struct Run {
     /// The address and port to bind the pict-rs web server
     #[clap(short, long)]
-    pub(crate) address: SocketAddr,
+    address: Option<SocketAddr>,
 
     /// The API KEY required to access restricted routes
     #[clap(long)]
-    pub(crate) api_key: Option<String>,
+    api_key: Option<String>,
 
     /// Whether to validate media on the "import" endpoint
     #[clap(long)]
-    pub(crate) media_skip_validate_imports: Option<bool>,
+    media_skip_validate_imports: Option<bool>,
     /// The maximum width, in pixels, for uploaded media
     #[clap(long)]
-    pub(crate) media_max_width: Option<usize>,
+    media_max_width: Option<usize>,
     /// The maximum height, in pixels, for uploaded media
     #[clap(long)]
-    pub(crate) media_max_height: Option<usize>,
+    media_max_height: Option<usize>,
     /// The maximum area, in pixels, for uploaded media
     #[clap(long)]
-    pub(crate) media_max_area: Option<usize>,
+    media_max_area: Option<usize>,
     /// The maximum size, in megabytes, for uploaded media
     #[clap(long)]
-    pub(crate) media_max_file_size: Option<usize>,
+    media_max_file_size: Option<usize>,
     /// Whether to enable GIF and silent MP4 uploads. Full videos are unsupported
     #[clap(long)]
-    pub(crate) media_enable_silent_video: Option<bool>,
+    media_enable_silent_video: Option<bool>,
     /// Which media filters should be enabled on the `process` endpoint
     #[clap(long)]
-    pub(crate) media_filters: Option<Vec<String>>,
+    media_filters: Option<Vec<String>>,
     /// Enforce uploaded media is transcoded to the provided format
     #[clap(long)]
-    pub(crate) media_format: Option<ImageFormat>,
+    media_format: Option<ImageFormat>,
 
     #[clap(subcommand)]
-    pub(crate) store: Option<RunStore>,
+    store: Option<RunStore>,
 }
 
 /// Configure the provided storage
-#[derive(Debug, Subcommand)]
-pub(crate) enum Store {
+#[derive(Clone, Debug, Subcommand, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(tag = "type")]
+enum Store {
     /// configure filesystem storage
     Filesystem(Filesystem),
 
@@ -104,7 +415,7 @@ pub(crate) enum Store {
 
 /// Run pict-rs with the provided storage
 #[derive(Debug, Subcommand)]
-pub(crate) enum RunStore {
+enum RunStore {
     /// Run pict-rs with filesystem storage
     Filesystem(RunFilesystem),
 
@@ -114,7 +425,7 @@ pub(crate) enum RunStore {
 
 /// Configure the pict-rs storage migration
 #[derive(Debug, Subcommand)]
-pub(crate) enum MigrateStore {
+enum MigrateStore {
     /// Migrate from the provided filesystem storage
     Filesystem(MigrateFilesystem),
 
@@ -122,95 +433,134 @@ pub(crate) enum MigrateStore {
     ObjectStorage(MigrateObjectStorage),
 }
 
+/// Configure the destination storage for pict-rs storage migration
+#[derive(Debug, Subcommand)]
+enum MigrateStoreInner {
+    /// Migrate to the provided filesystem storage
+    Filesystem(MigrateFilesystemInner),
+
+    /// Migrate to the provided object storage
+    ObjectStorage(MigrateObjectStorageInner),
+}
+
 /// Migrate pict-rs' storage from the provided filesystem storage
 #[derive(Debug, Parser)]
-pub(crate) struct MigrateFilesystem {
+struct MigrateFilesystem {
     #[clap(flatten)]
-    pub(crate) from: Filesystem,
+    from: crate::config::primitives::Filesystem,
 
     #[clap(subcommand)]
-    pub(crate) to: RunStore,
+    to: MigrateStoreInner,
+}
+
+/// Migrate pict-rs' storage to the provided filesystem storage
+#[derive(Debug, Parser)]
+struct MigrateFilesystemInner {
+    #[clap(flatten)]
+    to: crate::config::primitives::Filesystem,
+
+    #[clap(subcommand)]
+    repo: Option<Repo>,
 }
 
 /// Migrate pict-rs' storage from the provided object storage
 #[derive(Debug, Parser)]
-pub(crate) struct MigrateObjectStorage {
+struct MigrateObjectStorage {
     #[clap(flatten)]
-    pub(crate) from: ObjectStorage,
+    from: crate::config::primitives::ObjectStorage,
 
     #[clap(subcommand)]
-    pub(crate) to: RunStore,
+    to: MigrateStoreInner,
+}
+
+/// Migrate pict-rs' storage to the provided object storage
+#[derive(Debug, Parser)]
+struct MigrateObjectStorageInner {
+    #[clap(flatten)]
+    to: crate::config::primitives::ObjectStorage,
+
+    #[clap(subcommand)]
+    repo: Option<Repo>,
 }
 
 /// Run pict-rs with the provided filesystem storage
 #[derive(Debug, Parser)]
-pub(crate) struct RunFilesystem {
+struct RunFilesystem {
     #[clap(flatten)]
-    pub(crate) system: Filesystem,
+    system: Filesystem,
 
     #[clap(subcommand)]
-    pub(crate) repo: Repo,
+    repo: Option<Repo>,
 }
 
 /// Run pict-rs with the provided object storage
 #[derive(Debug, Parser)]
-pub(crate) struct RunObjectStorage {
+struct RunObjectStorage {
     #[clap(flatten)]
-    pub(crate) storage: ObjectStorage,
+    storage: ObjectStorage,
 
     #[clap(subcommand)]
-    pub(crate) repo: Repo,
+    repo: Option<Repo>,
 }
 
 /// Configuration for data repositories
-#[derive(Debug, Subcommand)]
-pub(crate) enum Repo {
+#[derive(Debug, Subcommand, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(tag = "type")]
+enum Repo {
     /// Run pict-rs with the provided sled-backed data repository
     Sled(Sled),
 }
 
 /// Configuration for filesystem media storage
-#[derive(Debug, Parser)]
-pub(crate) struct Filesystem {
+#[derive(Clone, Debug, Parser, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct Filesystem {
     /// The path to store uploaded media
     #[clap(short, long)]
-    pub(crate) path: Option<PathBuf>,
+    path: Option<PathBuf>,
 }
 
 /// Configuration for Object Storage
-#[derive(Debug, Parser)]
-pub(crate) struct ObjectStorage {
+#[derive(Clone, Debug, Parser, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct ObjectStorage {
     /// The bucket in which to store media
     #[clap(short, long)]
-    pub(crate) bucket_name: Option<String>,
+    bucket_name: Option<String>,
 
     /// The region the bucket is located in
     #[clap(short, long)]
-    pub(crate) region: Option<s3::Region>,
+    region: Option<Serde<s3::Region>>,
 
     /// The Access Key for the user accessing the bucket
     #[clap(short, long)]
-    pub(crate) access_key: Option<String>,
+    access_key: Option<String>,
 
     /// The secret key for the user accessing the bucket
     #[clap(short, long)]
-    pub(crate) secret_key: Option<String>,
+    secret_key: Option<String>,
 
     /// The security token for accessing the bucket
     #[clap(long)]
-    pub(crate) security_token: Option<String>,
+    security_token: Option<String>,
 
     /// The session token for accessing the bucket
     #[clap(long)]
-    pub(crate) session_token: Option<String>,
+    session_token: Option<String>,
 }
 
 /// Configuration for the sled-backed data repository
-#[derive(Debug, Parser)]
-pub(crate) struct Sled {
+#[derive(Debug, Parser, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct Sled {
     /// The path to store the sled database
-    pub(crate) path: Option<PathBuf>,
+    #[clap(short, long)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    path: Option<PathBuf>,
 
     /// The cache capacity, in bytes, allowed to sled for in-memory operations
-    pub(crate) cache_capacity: Option<u64>,
+    #[clap(short, long)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    cache_capacity: Option<u64>,
 }
diff --git a/src/config/defaults.rs b/src/config/defaults.rs
index 04ad61f..e45066f 100644
--- a/src/config/defaults.rs
+++ b/src/config/defaults.rs
@@ -4,7 +4,8 @@ use crate::{
 };
 use std::{net::SocketAddr, path::PathBuf};
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct Defaults {
     server: ServerDefaults,
     tracing: TracingDefaults,
@@ -14,35 +15,50 @@ pub(crate) struct Defaults {
     store: StoreDefaults,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 struct ServerDefaults {
     address: SocketAddr,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 struct TracingDefaults {
     logging: LoggingDefaults,
 
     console: ConsoleDefaults,
+
+    opentelemetry: OpenTelemetryDefaults,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 struct LoggingDefaults {
     format: LogFormat,
     targets: Serde<Targets>,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 struct ConsoleDefaults {
     buffer_capacity: usize,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct OpenTelemetryDefaults {
+    service_name: String,
+    targets: Serde<Targets>,
+}
+
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 struct OldDbDefaults {
     path: PathBuf,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 struct MediaDefaults {
     max_width: usize,
     max_height: usize,
@@ -53,42 +69,33 @@ struct MediaDefaults {
     skip_validate_imports: bool,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 #[serde(tag = "type")]
 enum RepoDefaults {
     Sled(SledDefaults),
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 struct SledDefaults {
     path: PathBuf,
     cache_capacity: u64,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 #[serde(tag = "type")]
 enum StoreDefaults {
     Filesystem(FilesystemDefaults),
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 struct FilesystemDefaults {
     path: PathBuf,
 }
 
-impl Default for Defaults {
-    fn default() -> Self {
-        Defaults {
-            server: ServerDefaults::default(),
-            tracing: TracingDefaults::default(),
-            old_db: OldDbDefaults::default(),
-            media: MediaDefaults::default(),
-            repo: RepoDefaults::default(),
-            store: StoreDefaults::default(),
-        }
-    }
-}
-
 impl Default for ServerDefaults {
     fn default() -> Self {
         ServerDefaults {
@@ -97,15 +104,6 @@ impl Default for ServerDefaults {
     }
 }
 
-impl Default for TracingDefaults {
-    fn default() -> TracingDefaults {
-        TracingDefaults {
-            logging: LoggingDefaults::default(),
-            console: ConsoleDefaults::default(),
-        }
-    }
-}
-
 impl Default for LoggingDefaults {
     fn default() -> Self {
         LoggingDefaults {
@@ -123,6 +121,15 @@ impl Default for ConsoleDefaults {
     }
 }
 
+impl Default for OpenTelemetryDefaults {
+    fn default() -> Self {
+        OpenTelemetryDefaults {
+            service_name: String::from("pict-rs"),
+            targets: "info".parse().expect("Valid targets string"),
+        }
+    }
+}
+
 impl Default for OldDbDefaults {
     fn default() -> Self {
         OldDbDefaults {
diff --git a/src/config/file.rs b/src/config/file.rs
index 26e298b..f8cded8 100644
--- a/src/config/file.rs
+++ b/src/config/file.rs
@@ -1,18 +1,18 @@
 use crate::{
-    config::primitives::{ImageFormat, LogFormat, Targets},
+    config::primitives::{ImageFormat, LogFormat, Store, Targets},
     serde_str::Serde,
 };
-use std::{net::SocketAddr, path::PathBuf};
+use std::{collections::HashSet, net::SocketAddr, path::PathBuf};
 use url::Url;
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct ConfigFile {
     pub(crate) server: Server,
 
     pub(crate) tracing: Tracing,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) old_db: Option<OldDb>,
+    pub(crate) old_db: OldDb,
 
     pub(crate) media: Media,
 
@@ -22,20 +22,14 @@ pub(crate) struct ConfigFile {
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 #[serde(tag = "type")]
 pub(crate) enum Repo {
     Sled(Sled),
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
-#[serde(tag = "type")]
-pub(crate) enum Store {
-    Filesystem(Filesystem),
-
-    ObjectStorage(ObjectStorage),
-}
-
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct Server {
     pub(crate) address: SocketAddr,
 
@@ -44,17 +38,17 @@ pub(crate) struct Server {
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct Tracing {
-    logging: Logging,
+    pub(crate) logging: Logging,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    console: Option<Console>,
+    pub(crate) console: Console,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    opentelemetry: Option<OpenTelemetry>,
+    pub(crate) opentelemetry: OpenTelemetry,
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct Logging {
     pub(crate) format: LogFormat,
 
@@ -62,8 +56,10 @@ pub(crate) struct Logging {
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct OpenTelemetry {
-    pub(crate) url: Url,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) url: Option<Url>,
 
     pub(crate) service_name: String,
 
@@ -71,17 +67,22 @@ pub(crate) struct OpenTelemetry {
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct Console {
-    pub(crate) address: SocketAddr,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) address: Option<SocketAddr>,
+
     pub(crate) buffer_capacity: usize,
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct OldDb {
     pub(crate) path: PathBuf,
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct Media {
     pub(crate) max_width: usize,
 
@@ -93,7 +94,7 @@ pub(crate) struct Media {
 
     pub(crate) enable_silent_video: bool,
 
-    pub(crate) filters: Vec<String>,
+    pub(crate) filters: HashSet<String>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) format: Option<ImageFormat>,
@@ -102,28 +103,7 @@ pub(crate) struct Media {
 }
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
-pub(crate) struct Filesystem {
-    pub(crate) path: PathBuf,
-}
-
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
-pub(crate) struct ObjectStorage {
-    pub(crate) bucket_name: String,
-
-    pub(crate) region: Serde<s3::Region>,
-
-    pub(crate) access_key: String,
-
-    pub(crate) secret_key: String,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) security_token: Option<String>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) session_token: Option<String>,
-}
-
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
 pub(crate) struct Sled {
     pub(crate) path: PathBuf,
 
diff --git a/src/config/primitives.rs b/src/config/primitives.rs
index 44d1ccd..fd94797 100644
--- a/src/config/primitives.rs
+++ b/src/config/primitives.rs
@@ -1,8 +1,12 @@
+use crate::magick::ValidInputType;
+use crate::serde_str::Serde;
 use clap::ArgEnum;
-use std::{fmt::Display, str::FromStr};
+use std::{fmt::Display, path::PathBuf, str::FromStr};
+use tracing::Level;
 
 #[derive(
     Clone,
+    Copy,
     Debug,
     PartialEq,
     Eq,
@@ -13,6 +17,7 @@ use std::{fmt::Display, str::FromStr};
     serde::Serialize,
     ArgEnum,
 )]
+#[serde(rename_all = "snake_case")]
 pub(crate) enum LogFormat {
     Compact,
     Json,
@@ -22,6 +27,7 @@ pub(crate) enum LogFormat {
 
 #[derive(
     Clone,
+    Copy,
     Debug,
     PartialEq,
     Eq,
@@ -32,6 +38,7 @@ pub(crate) enum LogFormat {
     serde::Serialize,
     ArgEnum,
 )]
+#[serde(rename_all = "snake_case")]
 pub(crate) enum ImageFormat {
     Jpeg,
     Webp,
@@ -43,6 +50,81 @@ pub(crate) struct Targets {
     pub(crate) targets: tracing_subscriber::filter::Targets,
 }
 
+/// Configuration for filesystem media storage
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, clap::Parser)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct Filesystem {
+    /// Path to store media
+    #[clap(short, long)]
+    pub(crate) path: PathBuf,
+}
+
+/// Configuration for object media storage
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, clap::Parser)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct ObjectStorage {
+    /// The bucket in which to store media
+    #[clap(short, long)]
+    pub(crate) bucket_name: String,
+
+    /// The region the bucket is located in
+    #[clap(short, long)]
+    pub(crate) region: Serde<s3::Region>,
+
+    /// The Access Key for the user accessing the bucket
+    #[clap(short, long)]
+    pub(crate) access_key: String,
+
+    /// The secret key for the user accessing the bucket
+    #[clap(short, long)]
+    pub(crate) secret_key: String,
+
+    /// The security token for accessing the bucket
+    #[clap(long)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) security_token: Option<String>,
+
+    /// The session token for accessing the bucket
+    #[clap(long)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) session_token: Option<String>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(tag = "type")]
+pub(crate) enum Store {
+    Filesystem(Filesystem),
+
+    ObjectStorage(ObjectStorage),
+}
+
+impl ImageFormat {
+    pub(crate) fn as_hint(self) -> Option<ValidInputType> {
+        Some(ValidInputType::from_format(self))
+    }
+
+    pub(crate) fn as_magick_format(self) -> &'static str {
+        match self {
+            Self::Jpeg => "JPEG",
+            Self::Png => "PNG",
+            Self::Webp => "WEBP",
+        }
+    }
+}
+
+impl From<Filesystem> for Store {
+    fn from(f: Filesystem) -> Self {
+        Self::Filesystem(f)
+    }
+}
+
+impl From<ObjectStorage> for Store {
+    fn from(o: ObjectStorage) -> Self {
+        Self::ObjectStorage(o)
+    }
+}
+
 impl FromStr for Targets {
     type Err = <tracing_subscriber::filter::Targets as FromStr>::Err;
 
@@ -62,7 +144,37 @@ impl Display for Targets {
             .collect::<Vec<_>>()
             .join(",");
 
-        write!(f, "{}", targets)
+        let max_level = [
+            Level::TRACE,
+            Level::DEBUG,
+            Level::INFO,
+            Level::WARN,
+            Level::ERROR,
+        ]
+        .iter()
+        .fold(None, |found, level| {
+            if found.is_none()
+                && self
+                    .targets
+                    .would_enable("not_a_real_target_so_nothing_can_conflict", level)
+            {
+                Some(level.to_string().to_lowercase())
+            } else {
+                found
+            }
+        });
+
+        if let Some(level) = max_level {
+            if !targets.is_empty() {
+                write!(f, "{},{}", level, targets)
+            } else {
+                write!(f, "{}", level)
+            }
+        } else if !targets.is_empty() {
+            write!(f, "{}", targets)
+        } else {
+            Ok(())
+        }
     }
 }
 
@@ -109,3 +221,31 @@ impl Display for LogFormat {
             .fmt(f)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{Serde, Targets};
+
+    #[test]
+    fn builds_info_targets() {
+        let t: Serde<Targets> = "info".parse().unwrap();
+
+        println!("{:?}", t);
+
+        assert_eq!(t.to_string(), "info");
+    }
+
+    #[test]
+    fn builds_specific_targets() {
+        let t: Serde<Targets> = "pict_rs=info".parse().unwrap();
+
+        assert_eq!(t.to_string(), "pict_rs=info");
+    }
+
+    #[test]
+    fn builds_warn_and_specific_targets() {
+        let t: Serde<Targets> = "warn,pict_rs=info".parse().unwrap();
+
+        assert_eq!(t.to_string(), "warn,pict_rs=info");
+    }
+}
diff --git a/src/init_tracing.rs b/src/init_tracing.rs
index 65a5d51..9840391 100644
--- a/src/init_tracing.rs
+++ b/src/init_tracing.rs
@@ -1,3 +1,4 @@
+use crate::config::{LogFormat, OpenTelemetry, Tracing};
 use console_subscriber::ConsoleLayer;
 use opentelemetry::{
     sdk::{propagation::TraceContextPropagator, Resource},
@@ -8,74 +9,73 @@ use tracing::subscriber::set_global_default;
 use tracing_error::ErrorLayer;
 use tracing_log::LogTracer;
 use tracing_subscriber::{
-    filter::Targets, fmt::format::FmtSpan, layer::SubscriberExt, registry::LookupSpan, Layer,
-    Registry,
+    fmt::format::FmtSpan, layer::SubscriberExt, registry::LookupSpan, Layer, Registry,
 };
-use url::Url;
 
-pub(super) fn init_tracing(
-    servic_name: &'static str,
-    opentelemetry_url: Option<&Url>,
-    buffer_capacity: Option<usize>,
-) -> anyhow::Result<()> {
+pub(super) fn init_tracing(tracing: &Tracing) -> anyhow::Result<()> {
     LogTracer::init()?;
 
     opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
 
-    let targets = std::env::var("RUST_LOG")
-        .unwrap_or_else(|_| "info".into())
-        .parse::<Targets>()?;
+    let format_layer =
+        tracing_subscriber::fmt::layer().with_span_events(FmtSpan::NEW | FmtSpan::CLOSE);
 
-    let format_layer = tracing_subscriber::fmt::layer()
-        .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
-        .with_filter(targets.clone());
+    match tracing.logging.format {
+        LogFormat::Compact => with_format(format_layer.compact(), tracing),
+        LogFormat::Json => with_format(format_layer.json(), tracing),
+        LogFormat::Normal => with_format(format_layer, tracing),
+        LogFormat::Pretty => with_format(format_layer.pretty(), tracing),
+    }
+}
+
+fn with_format<F>(format_layer: F, tracing: &Tracing) -> anyhow::Result<()>
+where
+    F: Layer<Registry> + Send + Sync,
+{
+    let format_layer = format_layer.with_filter(tracing.logging.targets.targets.clone());
 
     let subscriber = Registry::default()
         .with(format_layer)
         .with(ErrorLayer::default());
 
-    if let Some(buffer_capacity) = buffer_capacity {
+    if let Some(address) = tracing.console.address {
         let console_layer = ConsoleLayer::builder()
             .with_default_env()
-            .event_buffer_capacity(buffer_capacity)
-            .server_addr(([0, 0, 0, 0], 6669))
+            .event_buffer_capacity(tracing.console.buffer_capacity)
+            .server_addr(address)
             .spawn();
 
         let subscriber = subscriber.with(console_layer);
 
-        with_otel(subscriber, targets, servic_name, opentelemetry_url)
+        with_subscriber(subscriber, &tracing.opentelemetry)
     } else {
-        with_otel(subscriber, targets, servic_name, opentelemetry_url)
+        with_subscriber(subscriber, &tracing.opentelemetry)
     }
 }
 
-fn with_otel<S>(
-    subscriber: S,
-    targets: Targets,
-    servic_name: &'static str,
-    opentelemetry_url: Option<&Url>,
-) -> anyhow::Result<()>
+fn with_subscriber<S>(subscriber: S, otel: &OpenTelemetry) -> anyhow::Result<()>
 where
     S: SubscriberExt + Send + Sync,
     for<'a> S: LookupSpan<'a>,
 {
-    if let Some(url) = opentelemetry_url {
-        let tracer =
-            opentelemetry_otlp::new_pipeline()
-                .tracing()
-                .with_trace_config(opentelemetry::sdk::trace::config().with_resource(
-                    Resource::new(vec![KeyValue::new("service.name", servic_name)]),
-                ))
-                .with_exporter(
-                    opentelemetry_otlp::new_exporter()
-                        .tonic()
-                        .with_endpoint(url.as_str()),
-                )
-                .install_batch(opentelemetry::runtime::Tokio)?;
+    if let Some(url) = otel.url.as_ref() {
+        let tracer = opentelemetry_otlp::new_pipeline()
+            .tracing()
+            .with_trace_config(
+                opentelemetry::sdk::trace::config().with_resource(Resource::new(vec![
+                    KeyValue::new("service.name", otel.service_name.clone()),
+                ])),
+            )
+            .with_exporter(
+                opentelemetry_otlp::new_exporter()
+                    .tonic()
+                    .with_endpoint(url.as_str()),
+            )
+            .install_batch(opentelemetry::runtime::Tokio)?;
 
         let otel_layer = tracing_opentelemetry::layer()
             .with_tracer(tracer)
-            .with_filter(targets);
+            .with_filter(otel.targets.as_ref().targets.clone());
 
         let subscriber = subscriber.with(otel_layer);
 
diff --git a/src/magick.rs b/src/magick.rs
index 1d9f3c6..8efbf23 100644
--- a/src/magick.rs
+++ b/src/magick.rs
@@ -1,5 +1,5 @@
 use crate::{
-    config::Format,
+    config::ImageFormat,
     error::{Error, UploadError},
     process::Process,
     repo::Alias,
@@ -63,11 +63,11 @@ impl ValidInputType {
         matches!(self, Self::Mp4)
     }
 
-    pub(crate) fn from_format(format: Format) -> Self {
+    pub(crate) fn from_format(format: ImageFormat) -> Self {
         match format {
-            Format::Jpeg => ValidInputType::Jpeg,
-            Format::Png => ValidInputType::Png,
-            Format::Webp => ValidInputType::Webp,
+            ImageFormat::Jpeg => ValidInputType::Jpeg,
+            ImageFormat::Png => ValidInputType::Png,
+            ImageFormat::Webp => ValidInputType::Webp,
         }
     }
 }
@@ -87,7 +87,7 @@ pub(crate) fn clear_metadata_bytes_read(input: Bytes) -> std::io::Result<impl As
 
 pub(crate) fn convert_bytes_read(
     input: Bytes,
-    format: Format,
+    format: ImageFormat,
 ) -> std::io::Result<impl AsyncRead + Unpin> {
     let process = Process::run(
         "magick",
@@ -259,7 +259,7 @@ pub(crate) fn process_image_store_read<S: Store>(
     store: S,
     identifier: S::Identifier,
     args: Vec<String>,
-    format: Format,
+    format: ImageFormat,
 ) -> std::io::Result<impl AsyncRead + Unpin> {
     let command = "magick";
     let convert_args = ["convert", "-"];
@@ -278,9 +278,9 @@ pub(crate) fn process_image_store_read<S: Store>(
 impl Details {
     #[instrument(name = "Validating input type")]
     fn validate_input(&self) -> Result<ValidInputType, Error> {
-        if self.width > crate::CONFIG.max_width()
-            || self.height > crate::CONFIG.max_height()
-            || self.width * self.height > crate::CONFIG.max_area()
+        if self.width > crate::CONFIG.media.max_width
+            || self.height > crate::CONFIG.media.max_height
+            || self.width * self.height > crate::CONFIG.media.max_area
         {
             return Err(UploadError::Dimensions.into());
         }
diff --git a/src/main.rs b/src/main.rs
index 88daa92..51b1e87 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -49,7 +49,7 @@ mod validate;
 
 use self::{
     concurrent_processor::CancelSafeProcessor,
-    config::{CommandConfig, Config, Format, RequiredFilesystemStorage, RequiredObjectStorage},
+    config::{Configuration, ImageFormat, Operation},
     details::Details,
     either::Either,
     error::{Error, UploadError},
@@ -58,6 +58,7 @@ use self::{
     middleware::{Deadline, Internal},
     migrate::LatestDb,
     repo::{Alias, DeleteToken, Repo},
+    serde_str::Serde,
     store::{file_store::FileStore, object_store::ObjectStore, Store},
     upload_manager::{UploadManager, UploadManagerSession},
 };
@@ -67,7 +68,10 @@ const MINUTES: u32 = 60;
 const HOURS: u32 = 60 * MINUTES;
 const DAYS: u32 = 24 * HOURS;
 
-static CONFIG: Lazy<Config> = Lazy::new(|| Config::build().unwrap());
+static DO_CONFIG: Lazy<(Configuration, Operation)> =
+    Lazy::new(|| config::configure().expect("Failed to configure"));
+static CONFIG: Lazy<Configuration> = Lazy::new(|| DO_CONFIG.0.clone());
+static OPERATION: Lazy<Operation> = Lazy::new(|| DO_CONFIG.1.clone());
 static PROCESS_SEMAPHORE: Lazy<Semaphore> =
     Lazy::new(|| Semaphore::new(num_cpus::get().saturating_sub(1).max(1)));
 
@@ -202,7 +206,7 @@ async fn download<S: Store>(
 
     let stream = Limit::new(
         map_error::map_crate_error(res),
-        (CONFIG.max_file_size() * MEGABYTES) as u64,
+        (CONFIG.media.max_file_size * MEGABYTES) as u64,
     );
 
     futures_util::pin_mut!(stream);
@@ -260,7 +264,7 @@ fn prepare_process(
     query: web::Query<ProcessQuery>,
     ext: &str,
     filters: &Option<HashSet<String>>,
-) -> Result<(Format, Alias, PathBuf, Vec<String>), Error> {
+) -> Result<(ImageFormat, Alias, PathBuf, Vec<String>), Error> {
     let (alias, operations) =
         query
             .into_inner()
@@ -290,7 +294,7 @@ fn prepare_process(
     };
 
     let format = ext
-        .parse::<Format>()
+        .parse::<ImageFormat>()
         .map_err(|_| UploadError::UnsupportedFormat)?;
 
     let (thumbnail_path, thumbnail_args) = self::processor::build_chain(&operations)?;
@@ -639,7 +643,7 @@ async fn launch<S: Store + Clone + 'static>(
     let store2 = store.clone();
     let form = Form::new()
         .max_files(10)
-        .max_file_size(CONFIG.max_file_size() * MEGABYTES)
+        .max_file_size(CONFIG.media.max_file_size * MEGABYTES)
         .transform_error(transform_error)
         .field(
             "images",
@@ -667,12 +671,12 @@ async fn launch<S: Store + Clone + 'static>(
     // Create a new Multipart Form validator for internal imports
     //
     // This form is expecting a single array field, 'images' with at most 10 files in it
-    let validate_imports = CONFIG.validate_imports();
+    let validate_imports = !CONFIG.media.skip_validate_imports;
     let manager2 = manager.clone();
     let store2 = store.clone();
     let import_form = Form::new()
         .max_files(10)
-        .max_file_size(CONFIG.max_file_size() * MEGABYTES)
+        .max_file_size(CONFIG.media.max_file_size * MEGABYTES)
         .transform_error(transform_error)
         .field(
             "images",
@@ -708,7 +712,7 @@ async fn launch<S: Store + Clone + 'static>(
             .app_data(web::Data::new(store.clone()))
             .app_data(web::Data::new(manager.clone()))
             .app_data(web::Data::new(build_client()))
-            .app_data(web::Data::new(CONFIG.allowed_filters()))
+            .app_data(web::Data::new(CONFIG.media.filters.clone()))
             .service(
                 web::scope("/image")
                     .service(
@@ -739,7 +743,9 @@ async fn launch<S: Store + Clone + 'static>(
             )
             .service(
                 web::scope("/internal")
-                    .wrap(Internal(CONFIG.api_key().map(|s| s.to_owned())))
+                    .wrap(Internal(
+                        CONFIG.server.api_key.as_ref().map(|s| s.to_owned()),
+                    ))
                     .service(
                         web::resource("/import")
                             .wrap(import_form.clone())
@@ -749,7 +755,7 @@ async fn launch<S: Store + Clone + 'static>(
                     .service(web::resource("/aliases").route(web::get().to(aliases::<S>))),
             )
     })
-    .bind(CONFIG.bind_address())?
+    .bind(CONFIG.server.address)?
     .run()
     .await?;
 
@@ -762,17 +768,17 @@ async fn migrate_inner<S1>(
     manager: &UploadManager,
     repo: &Repo,
     from: S1,
-    to: &config::Storage,
+    to: &config::Store,
 ) -> anyhow::Result<()>
 where
     S1: Store,
 {
     match to {
-        config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
+        config::Store::Filesystem(config::Filesystem { path }) => {
             let to = FileStore::build(path.clone(), repo.clone()).await?;
             manager.migrate_store::<S1, FileStore>(from, to).await?;
         }
-        config::Storage::ObjectStorage(RequiredObjectStorage {
+        config::Store::ObjectStorage(config::ObjectStorage {
             bucket_name,
             region,
             access_key,
@@ -782,9 +788,9 @@ where
         }) => {
             let to = ObjectStore::build(
                 bucket_name,
-                region.clone(),
-                access_key.clone(),
-                secret_key.clone(),
+                region.as_ref().clone(),
+                Some(access_key.clone()),
+                Some(secret_key.clone()),
                 security_token.clone(),
                 session_token.clone(),
                 repo.clone(),
@@ -801,38 +807,24 @@ where
 
 #[actix_rt::main]
 async fn main() -> anyhow::Result<()> {
-    init_tracing(
-        "pict-rs",
-        CONFIG.opentelemetry_url(),
-        CONFIG.console_buffer_capacity(),
-    )?;
+    init_tracing(&CONFIG.tracing)?;
 
-    let repo = Repo::open(CONFIG.repo())?;
+    let repo = Repo::open(CONFIG.repo.clone())?;
 
-    let db = LatestDb::exists(CONFIG.data_dir()).migrate()?;
+    let db = LatestDb::exists(CONFIG.old_db.path.clone()).migrate()?;
     repo.from_db(db).await?;
 
-    let manager = UploadManager::new(repo.clone(), CONFIG.format()).await?;
-
-    match CONFIG.command()? {
-        CommandConfig::Run => (),
-        CommandConfig::Dump { path } => {
-            let configuration = toml::to_string_pretty(&*CONFIG)?;
-            tokio::fs::write(path, configuration).await?;
-            return Ok(());
-        }
-        CommandConfig::MigrateRepo { to: _ } => {
-            unimplemented!("Repo migrations are currently unsupported")
-        }
-        CommandConfig::MigrateStore { to } => {
-            let from = CONFIG.store()?;
+    let manager = UploadManager::new(repo.clone(), CONFIG.media.format).await?;
 
+    match (*OPERATION).clone() {
+        Operation::Run => (),
+        Operation::MigrateStore { from, to } => {
             match from {
-                config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
+                config::Store::Filesystem(config::Filesystem { path }) => {
                     let from = FileStore::build(path.clone(), repo.clone()).await?;
                     migrate_inner(&manager, &repo, from, &to).await?;
                 }
-                config::Storage::ObjectStorage(RequiredObjectStorage {
+                config::Store::ObjectStorage(config::ObjectStorage {
                     bucket_name,
                     region,
                     access_key,
@@ -842,9 +834,9 @@ async fn main() -> anyhow::Result<()> {
                 }) => {
                     let from = ObjectStore::build(
                         &bucket_name,
-                        region,
-                        access_key,
-                        secret_key,
+                        Serde::into_inner(region),
+                        Some(access_key),
+                        Some(secret_key),
                         security_token,
                         session_token,
                         repo.clone(),
@@ -860,12 +852,12 @@ async fn main() -> anyhow::Result<()> {
         }
     }
 
-    match CONFIG.store()? {
-        config::Storage::Filesystem(RequiredFilesystemStorage { path }) => {
+    match CONFIG.store.clone() {
+        config::Store::Filesystem(config::Filesystem { path }) => {
             let store = FileStore::build(path, repo).await?;
             launch(manager, store).await
         }
-        config::Storage::ObjectStorage(RequiredObjectStorage {
+        config::Store::ObjectStorage(config::ObjectStorage {
             bucket_name,
             region,
             access_key,
@@ -875,9 +867,9 @@ async fn main() -> anyhow::Result<()> {
         }) => {
             let store = ObjectStore::build(
                 &bucket_name,
-                region,
-                access_key,
-                secret_key,
+                Serde::into_inner(region),
+                Some(access_key),
+                Some(secret_key),
                 security_token,
                 session_token,
                 repo,
diff --git a/src/repo.rs b/src/repo.rs
index c1af53d..802eda6 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -1,9 +1,4 @@
-use crate::{
-    config::{Repository, RequiredSledRepo},
-    details::Details,
-    error::Error,
-    store::Identifier,
-};
+use crate::{config, details::Details, error::Error, store::Identifier};
 use futures_util::Stream;
 use tracing::debug;
 use uuid::Uuid;
@@ -125,9 +120,9 @@ pub(crate) trait AliasRepo {
 }
 
 impl Repo {
-    pub(crate) fn open(config: Repository) -> anyhow::Result<Self> {
+    pub(crate) fn open(config: config::Repo) -> anyhow::Result<Self> {
         match config {
-            Repository::Sled(RequiredSledRepo {
+            config::Repo::Sled(config::Sled {
                 mut path,
                 cache_capacity,
             }) => {
diff --git a/src/serde_str.rs b/src/serde_str.rs
index c255f5b..be311e7 100644
--- a/src/serde_str.rs
+++ b/src/serde_str.rs
@@ -18,6 +18,18 @@ impl<T> Serde<T> {
     }
 }
 
+impl<T> AsRef<T> for Serde<T> {
+    fn as_ref(&self) -> &T {
+        &self.inner
+    }
+}
+
+impl<T> AsMut<T> for Serde<T> {
+    fn as_mut(&mut self) -> &mut T {
+        &mut self.inner
+    }
+}
+
 impl<T> Deref for Serde<T> {
     type Target = T;
 
diff --git a/src/upload_manager.rs b/src/upload_manager.rs
index 5261227..42edaad 100644
--- a/src/upload_manager.rs
+++ b/src/upload_manager.rs
@@ -1,5 +1,5 @@
 use crate::{
-    config::Format,
+    config::ImageFormat,
     details::Details,
     error::{Error, UploadError},
     ffmpeg::{InputFormat, ThumbnailFormat},
@@ -28,14 +28,14 @@ pub(crate) struct UploadManager {
 }
 
 pub(crate) struct UploadManagerInner {
-    format: Option<Format>,
+    format: Option<ImageFormat>,
     hasher: sha2::Sha256,
     repo: Repo,
 }
 
 impl UploadManager {
     /// Create a new UploadManager
-    pub(crate) async fn new(repo: Repo, format: Option<Format>) -> Result<Self, Error> {
+    pub(crate) async fn new(repo: Repo, format: Option<ImageFormat>) -> Result<Self, Error> {
         let manager = UploadManager {
             inner: Arc::new(UploadManagerInner {
                 format,
diff --git a/src/validate.rs b/src/validate.rs
index f04c750..d6bfa44 100644
--- a/src/validate.rs
+++ b/src/validate.rs
@@ -1,5 +1,5 @@
 use crate::{
-    config::Format, either::Either, error::Error, ffmpeg::InputFormat, magick::ValidInputType,
+    config::ImageFormat, either::Either, error::Error, ffmpeg::InputFormat, magick::ValidInputType,
 };
 use actix_web::web::Bytes;
 use tokio::io::AsyncRead;
@@ -35,7 +35,7 @@ impl AsyncRead for UnvalidatedBytes {
 #[instrument(name = "Validate image", skip(bytes))]
 pub(crate) async fn validate_image_bytes(
     bytes: Bytes,
-    prescribed_format: Option<Format>,
+    prescribed_format: Option<ImageFormat>,
     validate: bool,
 ) -> Result<(ValidInputType, impl AsyncRead + Unpin), Error> {
     let input_type = crate::magick::input_type_bytes(bytes.clone()).await?;
@@ -57,19 +57,19 @@ pub(crate) async fn validate_image_bytes(
                 crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Mp4).await?,
             )),
         )),
-        (Some(Format::Jpeg) | None, ValidInputType::Jpeg) => Ok((
+        (Some(ImageFormat::Jpeg) | None, ValidInputType::Jpeg) => Ok((
             ValidInputType::Jpeg,
             Either::right(Either::right(Either::left(
                 crate::exiftool::clear_metadata_bytes_read(bytes)?,
             ))),
         )),
-        (Some(Format::Png) | None, ValidInputType::Png) => Ok((
+        (Some(ImageFormat::Png) | None, ValidInputType::Png) => Ok((
             ValidInputType::Png,
             Either::right(Either::right(Either::left(
                 crate::exiftool::clear_metadata_bytes_read(bytes)?,
             ))),
         )),
-        (Some(Format::Webp) | None, ValidInputType::Webp) => Ok((
+        (Some(ImageFormat::Webp) | None, ValidInputType::Webp) => Ok((
             ValidInputType::Webp,
             Either::right(Either::right(Either::right(Either::left(
                 crate::magick::clear_metadata_bytes_read(bytes)?,

From 1662f153ce657ed3427c5f05580754f9d02ef570 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Mon, 28 Mar 2022 15:34:36 -0500
Subject: [PATCH 10/37] Numerous changes:

- Improve error printing (display chain in addition to spantrace)
- Fix migration (read main identifier from identifier tree, not filename tree)
- Ensure uniqueness for processed images in ConcurrentProcessor (use source identifier in addition to thumbnail path, include extension in thumbnail path)
- Update default log levels (make pict-rs quieter)
- Add timeout for serving images from object storage (5 seconds)
---
 docker/object-storage/pict-rs.toml | 51 +++++++++++++-----
 src/concurrent_processor.rs        | 29 ++++++----
 src/config/defaults.rs             |  4 +-
 src/config/primitives.rs           | 10 ++--
 src/error.rs                       | 37 +++++++------
 src/main.rs                        | 30 +++++------
 src/processor.rs                   |  9 +++-
 src/repo/old.rs                    |  4 +-
 src/store/object_store.rs          | 85 +++++++++++++++++++++++++++++-
 9 files changed, 195 insertions(+), 64 deletions(-)

diff --git a/docker/object-storage/pict-rs.toml b/docker/object-storage/pict-rs.toml
index 04fc42a..8056e35 100644
--- a/docker/object-storage/pict-rs.toml
+++ b/docker/object-storage/pict-rs.toml
@@ -1,17 +1,42 @@
-path = 'data/'
-addr = '0.0.0.0:8080'
+[server]
+address = '0.0.0.0:8080'
+[tracing.logging]
+format = 'normal'
+targets = 'info'
 
-repo = 'sled'
-store = 'object_storage'
+[tracing.console]
+buffer_capacity = 102400
 
-[sled]
-sled_cache_capacity = 67108864
+[tracing.opentelemetry]
+service_name = 'pict-rs'
+targets = 'info'
 
-[filesystem_storage]
-filesystem_storage_path = '/mnt/files'
+[old_db]
+path = '/mnt'
 
-[object_storage]
-object_store_bucket_name = 'pict-rs'
-object_store_region = 'http://minio:9000'
-object_store_access_key = 'XZEZ5B8Y3UCINU1KCVF6'
-object_store_secret_key = 'cWbE5LcCK9YH8j1NvhOZocl+vH+b6T5Zvy3z+BZu'
+[media]
+max_width = 10000
+max_height = 10000
+max_area = 40000000
+max_file_size = 40
+enable_silent_video = true
+filters = [
+    'crop',
+    'resize',
+    'thumbnail',
+    'blur',
+    'identity',
+]
+skip_validate_imports = false
+
+[repo]
+type = 'sled'
+path = '/mnt/sled-repo'
+cache_capacity = 67108864
+
+[store]
+type = 'object_storage'
+bucket_name = 'pict-rs'
+region = 'http://minio:9000'
+access_key = 'XZEZ5B8Y3UCINU1KCVF6'
+secret_key = 'cWbE5LcCK9YH8j1NvhOZocl+vH+b6T5Zvy3z+BZu'
diff --git a/src/concurrent_processor.rs b/src/concurrent_processor.rs
index 52eb27e..3db0e6b 100644
--- a/src/concurrent_processor.rs
+++ b/src/concurrent_processor.rs
@@ -1,6 +1,7 @@
 use crate::{
     details::Details,
     error::{Error, UploadError},
+    store::Identifier,
 };
 use actix_web::web;
 use dashmap::{mapref::entry::Entry, DashMap};
@@ -16,13 +17,15 @@ use tracing::Span;
 
 type OutcomeSender = Sender<(Details, web::Bytes)>;
 
-type ProcessMap = DashMap<PathBuf, Vec<OutcomeSender>>;
+type ProcessMapKey = (Vec<u8>, PathBuf);
+
+type ProcessMap = DashMap<ProcessMapKey, Vec<OutcomeSender>>;
 
 static PROCESS_MAP: Lazy<ProcessMap> = Lazy::new(DashMap::new);
 
 struct CancelToken {
     span: Span,
-    path: PathBuf,
+    key: ProcessMapKey,
     receiver: Option<Receiver<(Details, web::Bytes)>>,
 }
 
@@ -39,14 +42,19 @@ impl<F> CancelSafeProcessor<F>
 where
     F: Future<Output = Result<(Details, web::Bytes), Error>>,
 {
-    pub(super) fn new(path: PathBuf, fut: F) -> Self {
-        let entry = PROCESS_MAP.entry(path.clone());
+    pub(super) fn new<I: Identifier>(identifier: I, path: PathBuf, fut: F) -> Result<Self, Error> {
+        let id_bytes = identifier.to_bytes()?;
+
+        let key = (id_bytes, path.clone());
+
+        let entry = PROCESS_MAP.entry(key.clone());
 
         let (receiver, span) = match entry {
             Entry::Vacant(vacant) => {
                 vacant.insert(Vec::new());
                 let span = tracing::info_span!(
                     "Processing image",
+                    identifier = &tracing::field::debug(&identifier),
                     path = &tracing::field::debug(&path),
                     completed = &tracing::field::Empty,
                 );
@@ -57,20 +65,21 @@ where
                 occupied.get_mut().push(tx);
                 let span = tracing::info_span!(
                     "Waiting for processed image",
+                    identifier = &tracing::field::debug(&identifier),
                     path = &tracing::field::debug(&path),
                 );
                 (Some(rx), span)
             }
         };
 
-        CancelSafeProcessor {
+        Ok(CancelSafeProcessor {
             cancel_token: CancelToken {
                 span,
-                path,
+                key,
                 receiver,
             },
             fut,
-        }
+        })
     }
 }
 
@@ -85,7 +94,7 @@ where
 
         let span = &this.cancel_token.span;
         let receiver = &mut this.cancel_token.receiver;
-        let path = &this.cancel_token.path;
+        let key = &this.cancel_token.key;
         let fut = this.fut;
 
         span.in_scope(|| {
@@ -95,7 +104,7 @@ where
                     .map(|res| res.map_err(|_| UploadError::Canceled.into()))
             } else {
                 fut.poll(cx).map(|res| {
-                    let opt = PROCESS_MAP.remove(path);
+                    let opt = PROCESS_MAP.remove(key);
                     res.map(|tup| {
                         if let Some((_, vec)) = opt {
                             for sender in vec {
@@ -113,7 +122,7 @@ where
 impl Drop for CancelToken {
     fn drop(&mut self) {
         if self.receiver.is_none() {
-            let completed = PROCESS_MAP.remove(&self.path).is_none();
+            let completed = PROCESS_MAP.remove(&self.key).is_none();
             self.span.record("completed", &completed);
         }
     }
diff --git a/src/config/defaults.rs b/src/config/defaults.rs
index e45066f..a459e73 100644
--- a/src/config/defaults.rs
+++ b/src/config/defaults.rs
@@ -108,7 +108,9 @@ impl Default for LoggingDefaults {
     fn default() -> Self {
         LoggingDefaults {
             format: LogFormat::Normal,
-            targets: "info".parse().expect("Valid targets string"),
+            targets: "warn,tracing_actix_web=info,actix_web=info,actix_server=info"
+                .parse()
+                .expect("Valid targets string"),
         }
     }
 }
diff --git a/src/config/primitives.rs b/src/config/primitives.rs
index fd94797..f48027d 100644
--- a/src/config/primitives.rs
+++ b/src/config/primitives.rs
@@ -182,12 +182,12 @@ impl FromStr for ImageFormat {
     type Err = String;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        for variant in Self::value_variants() {
-            if variant.to_possible_value().unwrap().matches(s, false) {
-                return Ok(*variant);
-            }
+        match s.to_lowercase().as_str() {
+            "jpeg" | "jpg" => Ok(Self::Jpeg),
+            "png" => Ok(Self::Png),
+            "webp" => Ok(Self::Webp),
+            other => Err(format!("Invalid variant: {}", other)),
         }
-        Err(format!("Invalid variant: {}", s))
     }
 }
 
diff --git a/src/error.rs b/src/error.rs
index bfb336b..34d30db 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -16,23 +16,32 @@ impl std::fmt::Display for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(f, "{}", self.kind)?;
         writeln!(f)?;
-        let mut count = 0;
-        let mut source = std::error::Error::source(self);
-        if source.is_some() {
-            writeln!(f, "Chain:")?;
-        }
-        while let Some(err) = source {
-            write!(f, "{}. ", count)?;
-            writeln!(f, "{}", err)?;
 
-            count += 1;
-            source = std::error::Error::source(err);
-        }
+        writeln!(f, "Chain:")?;
+        fmt_chain(f, &self.kind)?;
 
+        writeln!(f)?;
+        writeln!(f, "Spantrace:")?;
         std::fmt::Display::fmt(&self.context, f)
     }
 }
 
+fn fmt_chain(
+    f: &mut std::fmt::Formatter<'_>,
+    err: &dyn std::error::Error,
+) -> Result<usize, std::fmt::Error> {
+    let count = if let Some(source) = std::error::Error::source(err) {
+        fmt_chain(f, source)?
+    } else {
+        0
+    };
+
+    write!(f, "\t{}. ", count)?;
+    writeln!(f, "{}", err)?;
+
+    Ok(count + 1)
+}
+
 impl std::error::Error for Error {
     fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
         self.kind.source()
@@ -113,9 +122,6 @@ pub(crate) enum UploadError {
     #[error("Unable to send request, {0}")]
     SendRequest(String),
 
-    #[error("No filename provided in request")]
-    MissingFilename,
-
     #[error("Error converting Path to String")]
     Path,
 
@@ -157,7 +163,8 @@ impl ResponseError for Error {
             | UploadError::Limit(_)
             | UploadError::NoFiles
             | UploadError::Upload(_) => StatusCode::BAD_REQUEST,
-            UploadError::MissingAlias | UploadError::MissingFilename => StatusCode::NOT_FOUND,
+            UploadError::Sled(crate::repo::sled::SledError::Missing)
+            | UploadError::MissingAlias => StatusCode::NOT_FOUND,
             UploadError::InvalidToken => StatusCode::FORBIDDEN,
             UploadError::Range => StatusCode::RANGE_NOT_SATISFIABLE,
             _ => StatusCode::INTERNAL_SERVER_ERROR,
diff --git a/src/main.rs b/src/main.rs
index 51b1e87..91f6883 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -263,7 +263,7 @@ type ProcessQuery = Vec<(String, String)>;
 fn prepare_process(
     query: web::Query<ProcessQuery>,
     ext: &str,
-    filters: &Option<HashSet<String>>,
+    filters: &HashSet<String>,
 ) -> Result<(ImageFormat, Alias, PathBuf, Vec<String>), Error> {
     let (alias, operations) =
         query
@@ -279,25 +279,23 @@ fn prepare_process(
             });
 
     if alias.is_empty() {
-        return Err(UploadError::MissingFilename.into());
+        return Err(UploadError::MissingAlias.into());
     }
 
     let alias = Alias::from_existing(&alias);
 
-    let operations = if let Some(filters) = filters.as_ref() {
-        operations
-            .into_iter()
-            .filter(|(k, _)| filters.contains(&k.to_lowercase()))
-            .collect()
-    } else {
-        operations
-    };
+    let operations = operations
+        .into_iter()
+        .filter(|(k, _)| filters.contains(&k.to_lowercase()))
+        .collect::<Vec<_>>();
 
     let format = ext
         .parse::<ImageFormat>()
         .map_err(|_| UploadError::UnsupportedFormat)?;
 
-    let (thumbnail_path, thumbnail_args) = self::processor::build_chain(&operations)?;
+    let ext = format.to_string();
+
+    let (thumbnail_path, thumbnail_args) = self::processor::build_chain(&operations, &ext)?;
 
     Ok((format, alias, thumbnail_path, thumbnail_args))
 }
@@ -308,7 +306,7 @@ async fn process_details<S: Store>(
     ext: web::Path<String>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
-    filters: web::Data<Option<HashSet<String>>>,
+    filters: web::Data<HashSet<String>>,
 ) -> Result<HttpResponse, Error> {
     let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
 
@@ -332,7 +330,7 @@ async fn process<S: Store + 'static>(
     ext: web::Path<String>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
-    filters: web::Data<Option<HashSet<String>>>,
+    filters: web::Data<HashSet<String>>,
 ) -> Result<HttpResponse, Error> {
     let (format, alias, thumbnail_path, thumbnail_args) =
         prepare_process(query, ext.as_str(), &filters)?;
@@ -361,6 +359,7 @@ async fn process<S: Store + 'static>(
         .await?;
 
     let thumbnail_path2 = thumbnail_path.clone();
+    let identifier2 = identifier.clone();
     let process_fut = async {
         let thumbnail_path = thumbnail_path2;
 
@@ -368,7 +367,7 @@ async fn process<S: Store + 'static>(
 
         let mut processed_reader = crate::magick::process_image_store_read(
             (**store).clone(),
-            identifier,
+            identifier2,
             thumbnail_args,
             format,
         )?;
@@ -417,7 +416,8 @@ async fn process<S: Store + 'static>(
         Ok((details, bytes)) as Result<(Details, web::Bytes), Error>
     };
 
-    let (details, bytes) = CancelSafeProcessor::new(thumbnail_path.clone(), process_fut).await?;
+    let (details, bytes) =
+        CancelSafeProcessor::new(identifier, thumbnail_path.clone(), process_fut)?.await?;
 
     let (builder, stream) = if let Some(web::Header(range_header)) = range {
         if let Some(range) = range::single_bytes_range(&range_header) {
diff --git a/src/processor.rs b/src/processor.rs
index 0e01a5d..bf7c749 100644
--- a/src/processor.rs
+++ b/src/processor.rs
@@ -20,7 +20,10 @@ pub(crate) struct Crop(usize, usize);
 pub(crate) struct Blur(f64);
 
 #[instrument]
-pub(crate) fn build_chain(args: &[(String, String)]) -> Result<(PathBuf, Vec<String>), Error> {
+pub(crate) fn build_chain(
+    args: &[(String, String)],
+    ext: &str,
+) -> Result<(PathBuf, Vec<String>), Error> {
     fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, UploadError> {
         if key == P::NAME {
             return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?));
@@ -37,7 +40,7 @@ pub(crate) fn build_chain(args: &[(String, String)]) -> Result<(PathBuf, Vec<Str
         }};
     }
 
-    let (path, args) =
+    let (mut path, args) =
         args.iter()
             .fold(Ok((PathBuf::default(), vec![])), |inner, (name, value)| {
                 if let Ok(inner) = inner {
@@ -53,6 +56,8 @@ pub(crate) fn build_chain(args: &[(String, String)]) -> Result<(PathBuf, Vec<Str
                 }
             })?;
 
+    path.push(ext);
+
     Ok((path, args))
 }
 
diff --git a/src/repo/old.rs b/src/repo/old.rs
index bf4ae65..584db76 100644
--- a/src/repo/old.rs
+++ b/src/repo/old.rs
@@ -87,7 +87,7 @@ impl Old {
             .get(hash)?
             .ok_or_else(|| anyhow::anyhow!("Missing filename"))?;
 
-        self.filename_tree
+        self.identifier_tree
             .get(filename)?
             .ok_or_else(|| anyhow::anyhow!("Missing identifier"))
     }
@@ -114,7 +114,9 @@ impl Old {
 
                 let path = String::from_utf8(variant_path_bytes.to_vec()).ok()?;
                 let mut path = PathBuf::from(path);
+                let extension = path.extension()?.to_str()?.to_string();
                 path.pop();
+                path.push(extension);
 
                 Some((path, value))
             })
diff --git a/src/store/object_store.rs b/src/store/object_store.rs
index 4a30165..442c67d 100644
--- a/src/store/object_store.rs
+++ b/src/store/object_store.rs
@@ -3,15 +3,21 @@ use crate::{
     repo::{Repo, SettingsRepo},
     store::Store,
 };
+use actix_rt::time::Sleep;
 use actix_web::web::Bytes;
 use futures_util::stream::Stream;
 use s3::{
     client::Client, command::Command, creds::Credentials, request_trait::Request, Bucket, Region,
 };
 use std::{
+    future::Future,
     pin::Pin,
     string::FromUtf8Error,
-    task::{Context, Poll},
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    task::{Context, Poll, Wake, Waker},
 };
 use storage_path_generator::{Generator, Path};
 use tokio::io::{AsyncRead, AsyncWrite};
@@ -54,6 +60,17 @@ pin_project_lite::pin_project! {
     }
 }
 
+pin_project_lite::pin_project! {
+    struct Timeout<S> {
+        sleep: Option<Pin<Box<Sleep>>>,
+
+        woken: Arc<AtomicBool>,
+
+        #[pin]
+        inner: S,
+    }
+}
+
 #[async_trait::async_trait(?Send)]
 impl Store for ObjectStore {
     type Identifier = ObjectId;
@@ -107,7 +124,10 @@ impl Store for ObjectStore {
 
         let response = request.response().await.map_err(ObjectError::from)?;
 
-        Ok(Box::pin(io_error(response.bytes_stream())))
+        Ok(Box::pin(timeout(
+            io_error(response.bytes_stream()),
+            std::time::Duration::from_secs(5),
+        )))
     }
 
     #[tracing::instrument(skip(writer))]
@@ -238,6 +258,17 @@ where
     IoError { inner: stream }
 }
 
+fn timeout<S, T>(stream: S, duration: std::time::Duration) -> impl Stream<Item = std::io::Result<T>>
+where
+    S: Stream<Item = std::io::Result<T>>,
+{
+    Timeout {
+        sleep: Some(Box::pin(actix_rt::time::sleep(duration))),
+        woken: Arc::new(AtomicBool::new(true)),
+        inner: stream,
+    }
+}
+
 impl<S, T, E> Stream for IoError<S>
 where
     S: Stream<Item = Result<T, E>>,
@@ -254,6 +285,56 @@ where
     }
 }
 
+struct TimeoutWaker {
+    woken: Arc<AtomicBool>,
+    inner: Waker,
+}
+
+impl Wake for TimeoutWaker {
+    fn wake(self: Arc<Self>) {
+        self.wake_by_ref()
+    }
+
+    fn wake_by_ref(self: &Arc<Self>) {
+        self.woken.store(true, Ordering::Release);
+        self.inner.wake_by_ref();
+    }
+}
+
+impl<S, T> Stream for Timeout<S>
+where
+    S: Stream<Item = std::io::Result<T>>,
+{
+    type Item = std::io::Result<T>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.as_mut().project();
+
+        if this.woken.swap(false, Ordering::Acquire) {
+            if let Some(mut sleep) = this.sleep.take() {
+                let timeout_waker = Arc::new(TimeoutWaker {
+                    woken: Arc::clone(this.woken),
+                    inner: cx.waker().clone(),
+                })
+                .into();
+                let mut timeout_cx = Context::from_waker(&timeout_waker);
+                if let Poll::Ready(()) = sleep.as_mut().poll(&mut timeout_cx) {
+                    return Poll::Ready(Some(Err(std::io::Error::new(
+                        std::io::ErrorKind::Other,
+                        "Stream timeout".to_string(),
+                    ))));
+                } else {
+                    *this.sleep = Some(sleep);
+                }
+            } else {
+                return Poll::Ready(None);
+            }
+        }
+
+        this.inner.poll_next(cx)
+    }
+}
+
 impl std::fmt::Debug for ObjectStore {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("ObjectStore")

From 3f385c106dc574edee357bd974d79eaca0117680 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Mon, 28 Mar 2022 16:45:34 -0500
Subject: [PATCH 11/37] New minio creds

---
 docker/object-storage/pict-rs.toml | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/docker/object-storage/pict-rs.toml b/docker/object-storage/pict-rs.toml
index 8056e35..4ef7d90 100644
--- a/docker/object-storage/pict-rs.toml
+++ b/docker/object-storage/pict-rs.toml
@@ -20,13 +20,7 @@ max_height = 10000
 max_area = 40000000
 max_file_size = 40
 enable_silent_video = true
-filters = [
-    'crop',
-    'resize',
-    'thumbnail',
-    'blur',
-    'identity',
-]
+filters = ['crop', 'resize', 'thumbnail', 'blur', 'identity']
 skip_validate_imports = false
 
 [repo]
@@ -38,5 +32,5 @@ cache_capacity = 67108864
 type = 'object_storage'
 bucket_name = 'pict-rs'
 region = 'http://minio:9000'
-access_key = 'XZEZ5B8Y3UCINU1KCVF6'
-secret_key = 'cWbE5LcCK9YH8j1NvhOZocl+vH+b6T5Zvy3z+BZu'
+access_key = 'Q7Z3AY3JO01N27UNH5IR'
+secret_key = 'bH3Kj6UVJF+btBtWsExVurN3ukEilC3saECsupzi'

From 1291bf8beb0de58b9965bfe6d4d3e5c40d93568e Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Mon, 28 Mar 2022 18:42:22 -0500
Subject: [PATCH 12/37] Divorce reqwest from main application's spans

---
 src/store/object_store.rs | 47 +++++++++++++++++++++++++++------------
 1 file changed, 33 insertions(+), 14 deletions(-)

diff --git a/src/store/object_store.rs b/src/store/object_store.rs
index 442c67d..9734b3a 100644
--- a/src/store/object_store.rs
+++ b/src/store/object_store.rs
@@ -18,9 +18,11 @@ use std::{
         Arc,
     },
     task::{Context, Poll, Wake, Waker},
+    time::{Duration, Instant},
 };
 use storage_path_generator::{Generator, Path};
 use tokio::io::{AsyncRead, AsyncWrite};
+use tracing::Instrument;
 
 mod object_id;
 pub(crate) use object_id::ObjectId;
@@ -35,6 +37,9 @@ pub(crate) enum ObjectError {
     #[error("Failed to generate path")]
     PathGenerator(#[from] storage_path_generator::PathError),
 
+    #[error("Timeout")]
+    Elapsed,
+
     #[error("Failed to parse string")]
     Utf8(#[from] FromUtf8Error),
 
@@ -113,21 +118,35 @@ impl Store for ObjectStore {
         let path = identifier.as_str();
 
         let start = from_start.unwrap_or(0);
-        let end = len.map(|len| start + len);
+        let end = len.map(|len| start + len - 1);
 
-        let request = Client::request(
-            &self.client,
-            &self.bucket,
-            path,
-            Command::GetObjectRange { start, end },
-        );
+        let request_span = tracing::info_span!(parent: None, "Get Object");
 
-        let response = request.response().await.map_err(ObjectError::from)?;
+        let request = request_span.in_scope(|| {
+            Client::request(
+                &self.client,
+                &self.bucket,
+                path,
+                Command::GetObjectRange { start, end },
+            )
+        });
 
-        Ok(Box::pin(timeout(
-            io_error(response.bytes_stream()),
-            std::time::Duration::from_secs(5),
-        )))
+        let now = Instant::now();
+        let allotted = Duration::from_secs(5);
+
+        let response = request_span
+            .in_scope(|| tokio::time::timeout(allotted, request.response()))
+            .instrument(request_span.clone())
+            .await
+            .map_err(|_| ObjectError::Elapsed)?
+            .map_err(ObjectError::from)?;
+
+        let allotted = allotted.saturating_sub(now.elapsed());
+
+        Ok(
+            request_span
+                .in_scope(|| Box::pin(timeout(allotted, io_error(response.bytes_stream())))),
+        )
     }
 
     #[tracing::instrument(skip(writer))]
@@ -258,7 +277,7 @@ where
     IoError { inner: stream }
 }
 
-fn timeout<S, T>(stream: S, duration: std::time::Duration) -> impl Stream<Item = std::io::Result<T>>
+fn timeout<S, T>(duration: Duration, stream: S) -> impl Stream<Item = std::io::Result<T>>
 where
     S: Stream<Item = std::io::Result<T>>,
 {
@@ -321,7 +340,7 @@ where
                 if let Poll::Ready(()) = sleep.as_mut().poll(&mut timeout_cx) {
                     return Poll::Ready(Some(Err(std::io::Error::new(
                         std::io::ErrorKind::Other,
-                        "Stream timeout".to_string(),
+                        Error::from(ObjectError::Elapsed),
                     ))));
                 } else {
                     *this.sleep = Some(sleep);

From eb5e39c6348d47b8bb53388bceb9d6bb55007f80 Mon Sep 17 00:00:00 2001
From: "Aode (Lion)" <asonix@asonix.dog>
Date: Mon, 28 Mar 2022 20:47:46 -0500
Subject: [PATCH 13/37] Use color-eyre

---
 Cargo.lock                | 95 +++++++++++++++++++++++++++++++++++++++
 Cargo.toml                |  5 ++-
 src/config.rs             |  2 +-
 src/config/file.rs        |  4 +-
 src/error.rs              | 87 +++++++++++++++++------------------
 src/init_tracing.rs       |  8 ++--
 src/main.rs               | 14 +++---
 src/migrate.rs            | 18 ++++----
 src/migrate/s034.rs       | 20 ++++-----
 src/processor.rs          |  2 +-
 src/repo.rs               | 10 ++---
 src/repo/old.rs           | 45 +++++++++++++------
 src/store/object_store.rs |  3 ++
 13 files changed, 211 insertions(+), 102 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 9bb78d4..fee71d6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -219,6 +219,15 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "addr2line"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
+dependencies = [
+ "gimli",
+]
+
 [[package]]
 name = "adler"
 version = "1.0.2"
@@ -372,6 +381,21 @@ dependencies = [
  "anyhow",
 ]
 
+[[package]]
+name = "backtrace"
+version = "0.3.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "base64"
 version = "0.13.0"
@@ -498,6 +522,33 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "color-eyre"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ebf286c900a6d5867aeff75cfee3192857bb7f24b547d4f0df2ed6baa812c90"
+dependencies = [
+ "backtrace",
+ "color-spantrace",
+ "eyre",
+ "indenter",
+ "once_cell",
+ "owo-colors",
+ "tracing-error",
+]
+
+[[package]]
+name = "color-spantrace"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce"
+dependencies = [
+ "once_cell",
+ "owo-colors",
+ "tracing-core",
+ "tracing-error",
+]
+
 [[package]]
 name = "config"
 version = "0.12.0"
@@ -734,6 +785,16 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "eyre"
+version = "0.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9289ed2c0440a6536e65119725cf91fc2c6b5e513bfd2e36e1134d7cca6ca12f"
+dependencies = [
+ "indenter",
+ "once_cell",
+]
+
 [[package]]
 name = "fake-simd"
 version = "0.1.2"
@@ -926,6 +987,12 @@ dependencies = [
  "wasi 0.10.2+wasi-snapshot-preview1",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
+
 [[package]]
 name = "h2"
 version = "0.3.12"
@@ -1122,6 +1189,12 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "indenter"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
+
 [[package]]
 name = "indexmap"
 version = "1.8.0"
@@ -1403,6 +1476,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "object"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.10.0"
@@ -1489,6 +1571,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "owo-colors"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e72e30578e0d0993c8ae20823dd9cff2bc5517d2f586a8aef462a581e8a03eb"
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -1621,6 +1709,7 @@ dependencies = [
  "awc",
  "base64",
  "clap",
+ "color-eyre",
  "config",
  "console-subscriber",
  "dashmap",
@@ -1997,6 +2086,12 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.0"
diff --git a/Cargo.toml b/Cargo.toml
index aea8f57..d27e3dd 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -28,6 +28,7 @@ async-trait = "0.1.51"
 awc = { version = "3.0.0", default-features = false, features = ["rustls"] }
 base64 = "0.13.0"
 clap = { version = "3.1.6", features = ["derive"] }
+color-eyre = "0.6"
 config = "0.12.0"
 console-subscriber = "0.1"
 dashmap = "5.1.0"
@@ -76,9 +77,9 @@ uuid = { version = "0.8.2", features = ["v4", "serde"] }
 [dependencies.tracing-actix-web]
 version = "0.5.0"
 default-features = false
-features = ["emit_event_on_error", "opentelemetry_0_17"]
+features = ["opentelemetry_0_17"]
 
 [dependencies.tracing-awc]
 version = "0.1.0"
 default-features = false
-features = ["emit_event_on_error", "opentelemetry_0_17"]
+features = ["opentelemetry_0_17"]
diff --git a/src/config.rs b/src/config.rs
index eb4901f..93e8e6a 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -13,7 +13,7 @@ pub(crate) use commandline::Operation;
 pub(crate) use file::{ConfigFile as Configuration, OpenTelemetry, Repo, Sled, Tracing};
 pub(crate) use primitives::{Filesystem, ImageFormat, LogFormat, ObjectStorage, Store};
 
-pub(crate) fn configure() -> anyhow::Result<(Configuration, Operation)> {
+pub(crate) fn configure() -> color_eyre::Result<(Configuration, Operation)> {
     let Output {
         config_format,
         operation,
diff --git a/src/config/file.rs b/src/config/file.rs
index f8cded8..10cd8dc 100644
--- a/src/config/file.rs
+++ b/src/config/file.rs
@@ -2,7 +2,7 @@ use crate::{
     config::primitives::{ImageFormat, LogFormat, Store, Targets},
     serde_str::Serde,
 };
-use std::{collections::HashSet, net::SocketAddr, path::PathBuf};
+use std::{collections::BTreeSet, net::SocketAddr, path::PathBuf};
 use url::Url;
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
@@ -94,7 +94,7 @@ pub(crate) struct Media {
 
     pub(crate) enable_silent_video: bool,
 
-    pub(crate) filters: HashSet<String>,
+    pub(crate) filters: BTreeSet<String>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) format: Option<ImageFormat>,
diff --git a/src/error.rs b/src/error.rs
index 34d30db..60b33ac 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,50 +1,31 @@
 use actix_web::{http::StatusCode, HttpResponse, ResponseError};
-use tracing_error::SpanTrace;
+use color_eyre::Report;
 
 pub(crate) struct Error {
-    context: SpanTrace,
-    kind: UploadError,
+    inner: color_eyre::Report,
+}
+
+impl Error {
+    fn kind(&self) -> Option<&UploadError> {
+        self.inner.downcast_ref()
+    }
 }
 
 impl std::fmt::Debug for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(f, "{}", self.kind)
+        std::fmt::Debug::fmt(&self.inner, f)
     }
 }
 
 impl std::fmt::Display for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(f, "{}", self.kind)?;
-        writeln!(f)?;
-
-        writeln!(f, "Chain:")?;
-        fmt_chain(f, &self.kind)?;
-
-        writeln!(f)?;
-        writeln!(f, "Spantrace:")?;
-        std::fmt::Display::fmt(&self.context, f)
+        std::fmt::Display::fmt(&self.inner, f)
     }
 }
 
-fn fmt_chain(
-    f: &mut std::fmt::Formatter<'_>,
-    err: &dyn std::error::Error,
-) -> Result<usize, std::fmt::Error> {
-    let count = if let Some(source) = std::error::Error::source(err) {
-        fmt_chain(f, source)?
-    } else {
-        0
-    };
-
-    write!(f, "\t{}. ", count)?;
-    writeln!(f, "{}", err)?;
-
-    Ok(count + 1)
-}
-
 impl std::error::Error for Error {
     fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-        self.kind.source()
+        self.inner.source()
     }
 }
 
@@ -54,8 +35,7 @@ where
 {
     fn from(error: T) -> Self {
         Error {
-            kind: UploadError::from(error),
-            context: SpanTrace::capture(),
+            inner: Report::from(UploadError::from(error)),
         }
     }
 }
@@ -158,25 +138,38 @@ impl From<tokio::sync::AcquireError> for UploadError {
 
 impl ResponseError for Error {
     fn status_code(&self) -> StatusCode {
-        match self.kind {
-            UploadError::DuplicateAlias
-            | UploadError::Limit(_)
-            | UploadError::NoFiles
-            | UploadError::Upload(_) => StatusCode::BAD_REQUEST,
-            UploadError::Sled(crate::repo::sled::SledError::Missing)
-            | UploadError::MissingAlias => StatusCode::NOT_FOUND,
-            UploadError::InvalidToken => StatusCode::FORBIDDEN,
-            UploadError::Range => StatusCode::RANGE_NOT_SATISFIABLE,
+        match self.kind() {
+            Some(
+                UploadError::DuplicateAlias
+                | UploadError::Limit(_)
+                | UploadError::NoFiles
+                | UploadError::Upload(_),
+            ) => StatusCode::BAD_REQUEST,
+            Some(
+                UploadError::Sled(crate::repo::sled::SledError::Missing)
+                | UploadError::MissingAlias,
+            ) => StatusCode::NOT_FOUND,
+            Some(UploadError::InvalidToken) => StatusCode::FORBIDDEN,
+            Some(UploadError::Range) => StatusCode::RANGE_NOT_SATISFIABLE,
             _ => StatusCode::INTERNAL_SERVER_ERROR,
         }
     }
 
     fn error_response(&self) -> HttpResponse {
-        HttpResponse::build(self.status_code())
-            .content_type("application/json")
-            .body(
-                serde_json::to_string(&serde_json::json!({ "msg": self.kind.to_string() }))
-                    .unwrap_or_else(|_| r#"{"msg":"Request failed"}"#.to_string()),
-            )
+        if let Some(kind) = self.kind() {
+            HttpResponse::build(self.status_code())
+                .content_type("application/json")
+                .body(
+                    serde_json::to_string(&serde_json::json!({ "msg": kind.to_string() }))
+                        .unwrap_or_else(|_| r#"{"msg":"Request failed"}"#.to_string()),
+                )
+        } else {
+            HttpResponse::build(self.status_code())
+                .content_type("application/json")
+                .body(
+                    serde_json::to_string(&serde_json::json!({ "msg": "Unknown error" }))
+                        .unwrap_or_else(|_| r#"{"msg":"Request failed"}"#.to_string()),
+                )
+        }
     }
 }
diff --git a/src/init_tracing.rs b/src/init_tracing.rs
index 9840391..c586b41 100644
--- a/src/init_tracing.rs
+++ b/src/init_tracing.rs
@@ -12,7 +12,9 @@ use tracing_subscriber::{
     fmt::format::FmtSpan, layer::SubscriberExt, registry::LookupSpan, Layer, Registry,
 };
 
-pub(super) fn init_tracing(tracing: &Tracing) -> anyhow::Result<()> {
+pub(super) fn init_tracing(tracing: &Tracing) -> color_eyre::Result<()> {
+    color_eyre::install()?;
+
     LogTracer::init()?;
 
     opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
@@ -28,7 +30,7 @@ pub(super) fn init_tracing(tracing: &Tracing) -> anyhow::Result<()> {
     }
 }
 
-fn with_format<F>(format_layer: F, tracing: &Tracing) -> anyhow::Result<()>
+fn with_format<F>(format_layer: F, tracing: &Tracing) -> color_eyre::Result<()>
 where
     F: Layer<Registry> + Send + Sync,
 {
@@ -53,7 +55,7 @@ where
     }
 }
 
-fn with_subscriber<S>(subscriber: S, otel: &OpenTelemetry) -> anyhow::Result<()>
+fn with_subscriber<S>(subscriber: S, otel: &OpenTelemetry) -> color_eyre::Result<()>
 where
     S: SubscriberExt + Send + Sync,
     for<'a> S: LookupSpan<'a>,
diff --git a/src/main.rs b/src/main.rs
index 91f6883..147edc3 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -11,7 +11,7 @@ use futures_util::{
 };
 use once_cell::sync::Lazy;
 use std::{
-    collections::HashSet,
+    collections::BTreeSet,
     future::ready,
     path::PathBuf,
     pin::Pin,
@@ -263,7 +263,7 @@ type ProcessQuery = Vec<(String, String)>;
 fn prepare_process(
     query: web::Query<ProcessQuery>,
     ext: &str,
-    filters: &HashSet<String>,
+    filters: &BTreeSet<String>,
 ) -> Result<(ImageFormat, Alias, PathBuf, Vec<String>), Error> {
     let (alias, operations) =
         query
@@ -306,7 +306,7 @@ async fn process_details<S: Store>(
     ext: web::Path<String>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
-    filters: web::Data<HashSet<String>>,
+    filters: web::Data<BTreeSet<String>>,
 ) -> Result<HttpResponse, Error> {
     let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
 
@@ -330,7 +330,7 @@ async fn process<S: Store + 'static>(
     ext: web::Path<String>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
-    filters: web::Data<HashSet<String>>,
+    filters: web::Data<BTreeSet<String>>,
 ) -> Result<HttpResponse, Error> {
     let (format, alias, thumbnail_path, thumbnail_args) =
         prepare_process(query, ext.as_str(), &filters)?;
@@ -635,7 +635,7 @@ fn build_reqwest_client() -> reqwest::Result<reqwest::Client> {
 async fn launch<S: Store + Clone + 'static>(
     manager: UploadManager,
     store: S,
-) -> anyhow::Result<()> {
+) -> color_eyre::Result<()> {
     // Create a new Multipart Form validator
     //
     // This form is expecting a single array field, 'images' with at most 10 files in it
@@ -769,7 +769,7 @@ async fn migrate_inner<S1>(
     repo: &Repo,
     from: S1,
     to: &config::Store,
-) -> anyhow::Result<()>
+) -> color_eyre::Result<()>
 where
     S1: Store,
 {
@@ -806,7 +806,7 @@ where
 }
 
 #[actix_rt::main]
-async fn main() -> anyhow::Result<()> {
+async fn main() -> color_eyre::Result<()> {
     init_tracing(&CONFIG.tracing)?;
 
     let repo = Repo::open(CONFIG.repo.clone())?;
diff --git a/src/migrate.rs b/src/migrate.rs
index f33ffb5..5548340 100644
--- a/src/migrate.rs
+++ b/src/migrate.rs
@@ -1,14 +1,14 @@
-use crate::UploadError;
+use crate::Error;
 use std::path::PathBuf;
 
 mod s034;
 
-type SledIter = Box<dyn Iterator<Item = Result<(Vec<u8>, Vec<u8>), UploadError>>>;
+type SledIter = Box<dyn Iterator<Item = Result<(Vec<u8>, Vec<u8>), Error>>>;
 
 trait SledDb {
     type SledTree: SledTree;
 
-    fn open_tree(&self, name: &str) -> Result<Self::SledTree, UploadError>;
+    fn open_tree(&self, name: &str) -> Result<Self::SledTree, Error>;
 
     fn self_tree(&self) -> &Self::SledTree;
 }
@@ -19,7 +19,7 @@ where
 {
     type SledTree = T::SledTree;
 
-    fn open_tree(&self, name: &str) -> Result<Self::SledTree, UploadError> {
+    fn open_tree(&self, name: &str) -> Result<Self::SledTree, Error> {
         (*self).open_tree(name)
     }
 
@@ -29,11 +29,11 @@ where
 }
 
 trait SledTree {
-    fn get<K>(&self, key: K) -> Result<Option<Vec<u8>>, UploadError>
+    fn get<K>(&self, key: K) -> Result<Option<Vec<u8>>, Error>
     where
         K: AsRef<[u8]>;
 
-    fn insert<K, V>(&self, key: K, value: V) -> Result<(), UploadError>
+    fn insert<K, V>(&self, key: K, value: V) -> Result<(), Error>
     where
         K: AsRef<[u8]>,
         V: AsRef<[u8]>;
@@ -45,7 +45,7 @@ trait SledTree {
         K: AsRef<[u8]>,
         R: std::ops::RangeBounds<K>;
 
-    fn flush(&self) -> Result<(), UploadError>;
+    fn flush(&self) -> Result<(), Error>;
 }
 
 pub(crate) struct LatestDb {
@@ -60,7 +60,7 @@ impl LatestDb {
         LatestDb { root_dir, version }
     }
 
-    pub(crate) fn migrate(self) -> Result<sled::Db, UploadError> {
+    pub(crate) fn migrate(self) -> Result<sled::Db, Error> {
         let LatestDb { root_dir, version } = self;
 
         loop {
@@ -89,7 +89,7 @@ impl DbVersion {
         DbVersion::Fresh
     }
 
-    fn migrate(self, root: PathBuf) -> Result<sled::Db, UploadError> {
+    fn migrate(self, root: PathBuf) -> Result<sled::Db, Error> {
         match self {
             DbVersion::Sled034 | DbVersion::Fresh => s034::open(root),
         }
diff --git a/src/migrate/s034.rs b/src/migrate/s034.rs
index 5bcad24..fde33a2 100644
--- a/src/migrate/s034.rs
+++ b/src/migrate/s034.rs
@@ -1,6 +1,6 @@
 use crate::{
+    error::Error,
     migrate::{SledDb, SledIter, SledTree},
-    UploadError,
 };
 use sled as sled034;
 use std::path::PathBuf;
@@ -26,7 +26,7 @@ pub(crate) fn migrating(base: PathBuf) -> bool {
     true
 }
 
-pub(crate) fn open(mut base: PathBuf) -> Result<sled034::Db, UploadError> {
+pub(crate) fn open(mut base: PathBuf) -> Result<sled034::Db, Error> {
     base.push("sled");
     base.push(SLED_034);
 
@@ -41,7 +41,7 @@ pub(crate) fn open(mut base: PathBuf) -> Result<sled034::Db, UploadError> {
 impl SledDb for sled034::Db {
     type SledTree = sled034::Tree;
 
-    fn open_tree(&self, name: &str) -> Result<Self::SledTree, UploadError> {
+    fn open_tree(&self, name: &str) -> Result<Self::SledTree, Error> {
         Ok(sled034::Db::open_tree(self, name)?)
     }
 
@@ -51,14 +51,14 @@ impl SledDb for sled034::Db {
 }
 
 impl SledTree for sled034::Tree {
-    fn get<K>(&self, key: K) -> Result<Option<Vec<u8>>, UploadError>
+    fn get<K>(&self, key: K) -> Result<Option<Vec<u8>>, Error>
     where
         K: AsRef<[u8]>,
     {
         Ok(sled034::Tree::get(self, key)?.map(|v| Vec::from(v.as_ref())))
     }
 
-    fn insert<K, V>(&self, key: K, value: V) -> Result<(), UploadError>
+    fn insert<K, V>(&self, key: K, value: V) -> Result<(), Error>
     where
         K: AsRef<[u8]>,
         V: AsRef<[u8]>,
@@ -69,7 +69,7 @@ impl SledTree for sled034::Tree {
     fn iter(&self) -> SledIter {
         Box::new(sled034::Tree::iter(self).map(|res| {
             res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec()))
-                .map_err(UploadError::from)
+                .map_err(Error::from)
         }))
     }
 
@@ -80,13 +80,11 @@ impl SledTree for sled034::Tree {
     {
         Box::new(sled034::Tree::range(self, range).map(|res| {
             res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec()))
-                .map_err(UploadError::from)
+                .map_err(Error::from)
         }))
     }
 
-    fn flush(&self) -> Result<(), UploadError> {
-        sled034::Tree::flush(self)
-            .map(|_| ())
-            .map_err(UploadError::from)
+    fn flush(&self) -> Result<(), Error> {
+        sled034::Tree::flush(self).map(|_| ()).map_err(Error::from)
     }
 }
diff --git a/src/processor.rs b/src/processor.rs
index bf7c749..9e264ec 100644
--- a/src/processor.rs
+++ b/src/processor.rs
@@ -24,7 +24,7 @@ pub(crate) fn build_chain(
     args: &[(String, String)],
     ext: &str,
 ) -> Result<(PathBuf, Vec<String>), Error> {
-    fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, UploadError> {
+    fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, Error> {
         if key == P::NAME {
             return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?));
         }
diff --git a/src/repo.rs b/src/repo.rs
index 802eda6..8cc0115 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -120,7 +120,7 @@ pub(crate) trait AliasRepo {
 }
 
 impl Repo {
-    pub(crate) fn open(config: config::Repo) -> anyhow::Result<Self> {
+    pub(crate) fn open(config: config::Repo) -> color_eyre::Result<Self> {
         match config {
             config::Repo::Sled(config::Sled {
                 mut path,
@@ -139,7 +139,7 @@ impl Repo {
     }
 
     #[tracing::instrument(skip_all)]
-    pub(crate) async fn from_db(&self, db: ::sled::Db) -> anyhow::Result<()> {
+    pub(crate) async fn from_db(&self, db: ::sled::Db) -> color_eyre::Result<()> {
         if self.has_migrated().await? {
             return Ok(());
         }
@@ -161,13 +161,13 @@ impl Repo {
         Ok(())
     }
 
-    async fn has_migrated(&self) -> anyhow::Result<bool> {
+    async fn has_migrated(&self) -> color_eyre::Result<bool> {
         match self {
             Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_O1).await?.is_some()),
         }
     }
 
-    async fn mark_migrated(&self) -> anyhow::Result<()> {
+    async fn mark_migrated(&self) -> color_eyre::Result<()> {
         match self {
             Self::Sled(repo) => {
                 repo.set(REPO_MIGRATION_O1, b"1".to_vec().into()).await?;
@@ -182,7 +182,7 @@ const REPO_MIGRATION_O1: &[u8] = b"repo-migration-01";
 const STORE_MIGRATION_PROGRESS: &[u8] = b"store-migration-progress";
 const GENERATOR_KEY: &[u8] = b"last-path";
 
-async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> anyhow::Result<()>
+async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> color_eyre::Result<()>
 where
     T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo,
 {
diff --git a/src/repo/old.rs b/src/repo/old.rs
index 584db76..e6a3ac2 100644
--- a/src/repo/old.rs
+++ b/src/repo/old.rs
@@ -17,9 +17,19 @@
 // - Settings Tree
 //   - store-migration-progress -> Path Tree Key
 
+use super::{Alias, DeleteToken, Details};
 use std::path::PathBuf;
 
-use super::{Alias, DeleteToken, Details};
+#[derive(Debug)]
+struct OldDbError(&'static str);
+
+impl std::fmt::Display for OldDbError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl std::error::Error for OldDbError {}
 
 pub(super) struct Old {
     alias_tree: ::sled::Tree,
@@ -32,7 +42,7 @@ pub(super) struct Old {
 }
 
 impl Old {
-    pub(super) fn open(db: sled::Db) -> anyhow::Result<Self> {
+    pub(super) fn open(db: sled::Db) -> color_eyre::Result<Self> {
         Ok(Self {
             alias_tree: db.open_tree("alias")?,
             filename_tree: db.open_tree("filename")?,
@@ -44,7 +54,7 @@ impl Old {
         })
     }
 
-    pub(super) fn setting(&self, key: &[u8]) -> anyhow::Result<Option<sled::IVec>> {
+    pub(super) fn setting(&self, key: &[u8]) -> color_eyre::Result<Option<sled::IVec>> {
         Ok(self.settings_tree.get(key)?)
     }
 
@@ -55,11 +65,14 @@ impl Old {
             .filter_map(|res| res.ok())
     }
 
-    pub(super) fn details(&self, hash: &sled::IVec) -> anyhow::Result<Vec<(sled::IVec, Details)>> {
+    pub(super) fn details(
+        &self,
+        hash: &sled::IVec,
+    ) -> color_eyre::Result<Vec<(sled::IVec, Details)>> {
         let filename = self
             .main_tree
             .get(hash)?
-            .ok_or_else(|| anyhow::anyhow!("missing filename"))?;
+            .ok_or(OldDbError("Missing filename"))?;
 
         let filename = String::from_utf8_lossy(&filename);
 
@@ -81,22 +94,26 @@ impl Old {
             .collect())
     }
 
-    pub(super) fn main_identifier(&self, hash: &sled::IVec) -> anyhow::Result<sled::IVec> {
+    pub(super) fn main_identifier(&self, hash: &sled::IVec) -> color_eyre::Result<sled::IVec> {
         let filename = self
             .main_tree
             .get(hash)?
-            .ok_or_else(|| anyhow::anyhow!("Missing filename"))?;
+            .ok_or(OldDbError("Missing filename"))?;
 
-        self.identifier_tree
+        Ok(self
+            .identifier_tree
             .get(filename)?
-            .ok_or_else(|| anyhow::anyhow!("Missing identifier"))
+            .ok_or(OldDbError("Missing identifier"))?)
     }
 
-    pub(super) fn variants(&self, hash: &sled::IVec) -> anyhow::Result<Vec<(PathBuf, sled::IVec)>> {
+    pub(super) fn variants(
+        &self,
+        hash: &sled::IVec,
+    ) -> color_eyre::Result<Vec<(PathBuf, sled::IVec)>> {
         let filename = self
             .main_tree
             .get(hash)?
-            .ok_or_else(|| anyhow::anyhow!("Missing filename"))?;
+            .ok_or(OldDbError("Missing filename"))?;
 
         let filename_string = String::from_utf8_lossy(&filename);
 
@@ -126,11 +143,11 @@ impl Old {
     pub(super) fn motion_identifier(
         &self,
         hash: &sled::IVec,
-    ) -> anyhow::Result<Option<sled::IVec>> {
+    ) -> color_eyre::Result<Option<sled::IVec>> {
         let filename = self
             .main_tree
             .get(hash)?
-            .ok_or_else(|| anyhow::anyhow!("Missing filename"))?;
+            .ok_or(OldDbError("Missing filename"))?;
 
         let filename_string = String::from_utf8_lossy(&filename);
 
@@ -151,7 +168,7 @@ impl Old {
             .collect()
     }
 
-    pub(super) fn delete_token(&self, alias: &Alias) -> anyhow::Result<Option<DeleteToken>> {
+    pub(super) fn delete_token(&self, alias: &Alias) -> color_eyre::Result<Option<DeleteToken>> {
         let key = format!("{}/delete", alias);
 
         if let Some(ivec) = self.alias_tree.get(key)? {
diff --git a/src/store/object_store.rs b/src/store/object_store.rs
index 9734b3a..dcd09f1 100644
--- a/src/store/object_store.rs
+++ b/src/store/object_store.rs
@@ -122,6 +122,9 @@ impl Store for ObjectStore {
 
         let request_span = tracing::info_span!(parent: None, "Get Object");
 
+        // NOTE: isolating reqwest in it's own span is to prevent the request's span from getting
+        // smuggled into a long-lived task. Unfortunately, I am unable to create a minimal
+        // reproduction of this problem so I can't open a bug about it.
         let request = request_span.in_scope(|| {
             Client::request(
                 &self.client,

From dd9ef89709c080a251a60d36a632c3662c866a96 Mon Sep 17 00:00:00 2001
From: "Aode (Lion)" <asonix@asonix.dog>
Date: Mon, 28 Mar 2022 20:48:12 -0500
Subject: [PATCH 14/37] Update tomls

---
 defaults.toml                      |  6 +++---
 dev.toml                           |  6 +++---
 docker/object-storage/pict-rs.toml | 10 ++++++++--
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/defaults.toml b/defaults.toml
index 01834cc..1312337 100644
--- a/defaults.toml
+++ b/defaults.toml
@@ -2,7 +2,7 @@
 address = '0.0.0.0:8080'
 [tracing.logging]
 format = 'normal'
-targets = 'info'
+targets = 'warn,tracing_actix_web=info,actix_server=info,actix_web=info'
 
 [tracing.console]
 buffer_capacity = 102400
@@ -21,10 +21,10 @@ max_area = 40000000
 max_file_size = 40
 enable_silent_video = true
 filters = [
-    'crop',
     'blur',
-    'resize',
+    'crop',
     'identity',
+    'resize',
     'thumbnail',
 ]
 skip_validate_imports = false
diff --git a/dev.toml b/dev.toml
index 91c5b00..9a5748b 100644
--- a/dev.toml
+++ b/dev.toml
@@ -2,7 +2,7 @@
 address = '0.0.0.0:8080'
 [tracing.logging]
 format = 'normal'
-targets = 'info'
+targets = 'warn,tracing_actix_web=info,actix_server=info,actix_web=info'
 
 [tracing.console]
 buffer_capacity = 102400
@@ -21,11 +21,11 @@ max_area = 40000000
 max_file_size = 40
 enable_silent_video = true
 filters = [
+    'blur',
+    'crop',
     'identity',
     'resize',
-    'crop',
     'thumbnail',
-    'blur',
 ]
 skip_validate_imports = false
 
diff --git a/docker/object-storage/pict-rs.toml b/docker/object-storage/pict-rs.toml
index 4ef7d90..980d749 100644
--- a/docker/object-storage/pict-rs.toml
+++ b/docker/object-storage/pict-rs.toml
@@ -2,7 +2,7 @@
 address = '0.0.0.0:8080'
 [tracing.logging]
 format = 'normal'
-targets = 'info'
+targets = 'warn,tracing_actix_web=info,actix_server=info,actix_web=info'
 
 [tracing.console]
 buffer_capacity = 102400
@@ -20,7 +20,13 @@ max_height = 10000
 max_area = 40000000
 max_file_size = 40
 enable_silent_video = true
-filters = ['crop', 'resize', 'thumbnail', 'blur', 'identity']
+filters = [
+    'blur',
+    'crop',
+    'identity',
+    'resize',
+    'thumbnail',
+]
 skip_validate_imports = false
 
 [repo]

From 99d042f36e356642b0beb8f0f9dadc7dcede963b Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Tue, 29 Mar 2022 11:04:56 -0500
Subject: [PATCH 15/37] Allow disabling gif/mp4

---
 src/error.rs                  |  7 +++++-
 src/main.rs                   | 11 ++++++++--
 src/upload_manager/session.rs |  4 ++++
 src/validate.rs               | 41 ++++++++++++++++++++++++-----------
 4 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/src/error.rs b/src/error.rs
index 60b33ac..679f70b 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -90,6 +90,9 @@ pub(crate) enum UploadError {
     #[error("Unsupported image format")]
     UnsupportedFormat,
 
+    #[error("Gif uploads are not enabled")]
+    SilentVideoDisabled,
+
     #[error("Invalid media dimensions")]
     Dimensions,
 
@@ -143,7 +146,9 @@ impl ResponseError for Error {
                 UploadError::DuplicateAlias
                 | UploadError::Limit(_)
                 | UploadError::NoFiles
-                | UploadError::Upload(_),
+                | UploadError::Upload(_)
+                | UploadError::UnsupportedFormat
+                | UploadError::SilentVideoDisabled,
             ) => StatusCode::BAD_REQUEST,
             Some(
                 UploadError::Sled(crate::repo::sled::SledError::Missing)
diff --git a/src/main.rs b/src/main.rs
index 147edc3..87e6957 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -212,7 +212,10 @@ async fn download<S: Store>(
     futures_util::pin_mut!(stream);
 
     let permit = PROCESS_SEMAPHORE.acquire().await?;
-    let session = manager.session((**store).clone()).upload(stream).await?;
+    let session = manager
+        .session((**store).clone())
+        .upload(CONFIG.media.enable_silent_video, stream)
+        .await?;
     let alias = session.alias().unwrap().to_owned();
     drop(permit);
     let delete_token = session.delete_token().await?;
@@ -658,7 +661,10 @@ async fn launch<S: Store + Clone + 'static>(
 
                     let res = manager
                         .session(store)
-                        .upload(map_error::map_crate_error(stream))
+                        .upload(
+                            CONFIG.media.enable_silent_video,
+                            map_error::map_crate_error(stream),
+                        )
                         .await;
 
                     drop(permit);
@@ -694,6 +700,7 @@ async fn launch<S: Store + Clone + 'static>(
                         .import(
                             filename,
                             validate_imports,
+                            CONFIG.media.enable_silent_video,
                             map_error::map_crate_error(stream),
                         )
                         .await;
diff --git a/src/upload_manager/session.rs b/src/upload_manager/session.rs
index 726ac77..390d758 100644
--- a/src/upload_manager/session.rs
+++ b/src/upload_manager/session.rs
@@ -113,6 +113,7 @@ impl<S: Store> UploadManagerSession<S> {
         mut self,
         alias: String,
         validate: bool,
+        enable_silent_video: bool,
         mut stream: impl Stream<Item = Result<web::Bytes, Error>> + Unpin,
     ) -> Result<Self, Error> {
         let mut bytes_mut = actix_web::web::BytesMut::new();
@@ -127,6 +128,7 @@ impl<S: Store> UploadManagerSession<S> {
         let (_, validated_reader) = crate::validate::validate_image_bytes(
             bytes_mut.freeze(),
             self.manager.inner.format,
+            enable_silent_video,
             validate,
         )
         .await?;
@@ -150,6 +152,7 @@ impl<S: Store> UploadManagerSession<S> {
     #[instrument(skip(self, stream))]
     pub(crate) async fn upload(
         mut self,
+        enable_silent_video: bool,
         mut stream: impl Stream<Item = Result<web::Bytes, Error>> + Unpin,
     ) -> Result<Self, Error> {
         let mut bytes_mut = actix_web::web::BytesMut::new();
@@ -164,6 +167,7 @@ impl<S: Store> UploadManagerSession<S> {
         let (input_type, validated_reader) = crate::validate::validate_image_bytes(
             bytes_mut.freeze(),
             self.manager.inner.format,
+            enable_silent_video,
             true,
         )
         .await?;
diff --git a/src/validate.rs b/src/validate.rs
index d6bfa44..0a78159 100644
--- a/src/validate.rs
+++ b/src/validate.rs
@@ -1,5 +1,9 @@
 use crate::{
-    config::ImageFormat, either::Either, error::Error, ffmpeg::InputFormat, magick::ValidInputType,
+    config::ImageFormat,
+    either::Either,
+    error::{Error, UploadError},
+    ffmpeg::InputFormat,
+    magick::ValidInputType,
 };
 use actix_web::web::Bytes;
 use tokio::io::AsyncRead;
@@ -36,6 +40,7 @@ impl AsyncRead for UnvalidatedBytes {
 pub(crate) async fn validate_image_bytes(
     bytes: Bytes,
     prescribed_format: Option<ImageFormat>,
+    enable_silent_video: bool,
     validate: bool,
 ) -> Result<(ValidInputType, impl AsyncRead + Unpin), Error> {
     let input_type = crate::magick::input_type_bytes(bytes.clone()).await?;
@@ -45,18 +50,28 @@ pub(crate) async fn validate_image_bytes(
     }
 
     match (prescribed_format, input_type) {
-        (_, ValidInputType::Gif) => Ok((
-            ValidInputType::Mp4,
-            Either::right(Either::left(
-                crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Gif).await?,
-            )),
-        )),
-        (_, ValidInputType::Mp4) => Ok((
-            ValidInputType::Mp4,
-            Either::right(Either::left(
-                crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Mp4).await?,
-            )),
-        )),
+        (_, ValidInputType::Gif) => {
+            if !enable_silent_video {
+                return Err(UploadError::SilentVideoDisabled.into());
+            }
+            Ok((
+                ValidInputType::Mp4,
+                Either::right(Either::left(
+                    crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Gif).await?,
+                )),
+            ))
+        }
+        (_, ValidInputType::Mp4) => {
+            if !enable_silent_video {
+                return Err(UploadError::SilentVideoDisabled.into());
+            }
+            Ok((
+                ValidInputType::Mp4,
+                Either::right(Either::left(
+                    crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Mp4).await?,
+                )),
+            ))
+        }
         (Some(ImageFormat::Jpeg) | None, ValidInputType::Jpeg) => Ok((
             ValidInputType::Jpeg,
             Either::right(Either::right(Either::left(

From e25a4781a8f0c681142b090a3b72a3a05071541c Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Tue, 29 Mar 2022 11:09:23 -0500
Subject: [PATCH 16/37] Remove unneeded variable

---
 src/main.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 87e6957..6692442 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -677,7 +677,6 @@ async fn launch<S: Store + Clone + 'static>(
     // Create a new Multipart Form validator for internal imports
     //
     // This form is expecting a single array field, 'images' with at most 10 files in it
-    let validate_imports = !CONFIG.media.skip_validate_imports;
     let manager2 = manager.clone();
     let store2 = store.clone();
     let import_form = Form::new()
@@ -699,7 +698,7 @@ async fn launch<S: Store + Clone + 'static>(
                         .session(store)
                         .import(
                             filename,
-                            validate_imports,
+                            !CONFIG.media.skip_validate_imports,
                             CONFIG.media.enable_silent_video,
                             map_error::map_crate_error(stream),
                         )

From 602d1ea93503514d1f68c16fdcd089f79cf97196 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Tue, 29 Mar 2022 12:51:16 -0500
Subject: [PATCH 17/37] Begin implementing queue, put cleanups in it

---
 src/config/commandline.rs |  12 +++-
 src/config/defaults.rs    |   2 +
 src/config/file.rs        |   2 +
 src/main.rs               |  31 ++++++-----
 src/queue.rs              | 113 ++++++++++++++++++++++++++++++++++++++
 src/repo.rs               |  24 +++++---
 src/repo/sled.rs          |  80 +++++++++++++++++++++++----
 src/upload_manager.rs     |  82 +++++----------------------
 8 files changed, 245 insertions(+), 101 deletions(-)
 create mode 100644 src/queue.rs

diff --git a/src/config/commandline.rs b/src/config/commandline.rs
index 08900d3..c335cbb 100644
--- a/src/config/commandline.rs
+++ b/src/config/commandline.rs
@@ -44,6 +44,7 @@ impl Args {
             Command::Run(Run {
                 address,
                 api_key,
+                worker_id,
                 media_skip_validate_imports,
                 media_max_width,
                 media_max_height,
@@ -54,7 +55,11 @@ impl Args {
                 media_format,
                 store,
             }) => {
-                let server = Server { address, api_key };
+                let server = Server {
+                    address,
+                    api_key,
+                    worker_id,
+                };
                 let media = Media {
                     skip_validate_imports: media_skip_validate_imports,
                     max_width: media_max_width,
@@ -240,6 +245,8 @@ struct Server {
     #[serde(skip_serializing_if = "Option::is_none")]
     address: Option<SocketAddr>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    worker_id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     api_key: Option<String>,
 }
 
@@ -372,6 +379,9 @@ struct Run {
     #[clap(long)]
     api_key: Option<String>,
 
+    #[clap(long)]
+    worker_id: Option<String>,
+
     /// Whether to validate media on the "import" endpoint
     #[clap(long)]
     media_skip_validate_imports: Option<bool>,
diff --git a/src/config/defaults.rs b/src/config/defaults.rs
index a459e73..bcedb0d 100644
--- a/src/config/defaults.rs
+++ b/src/config/defaults.rs
@@ -19,6 +19,7 @@ pub(crate) struct Defaults {
 #[serde(rename_all = "snake_case")]
 struct ServerDefaults {
     address: SocketAddr,
+    worker_id: String,
 }
 
 #[derive(Clone, Debug, Default, serde::Serialize)]
@@ -100,6 +101,7 @@ impl Default for ServerDefaults {
     fn default() -> Self {
         ServerDefaults {
             address: "0.0.0.0:8080".parse().expect("Valid address string"),
+            worker_id: String::from("pict-rs-1"),
         }
     }
 }
diff --git a/src/config/file.rs b/src/config/file.rs
index 10cd8dc..d49edc2 100644
--- a/src/config/file.rs
+++ b/src/config/file.rs
@@ -33,6 +33,8 @@ pub(crate) enum Repo {
 pub(crate) struct Server {
     pub(crate) address: SocketAddr,
 
+    pub(crate) worker_id: String,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) api_key: Option<String>,
 }
diff --git a/src/main.rs b/src/main.rs
index 6692442..15b3725 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -39,6 +39,7 @@ mod middleware;
 mod migrate;
 mod process;
 mod processor;
+mod queue;
 mod range;
 mod repo;
 mod serde_str;
@@ -246,9 +247,8 @@ async fn download<S: Store>(
 
 /// Delete aliases and files
 #[instrument(name = "Deleting file", skip(manager))]
-async fn delete<S: Store>(
+async fn delete(
     manager: web::Data<UploadManager>,
-    store: web::Data<S>,
     path_entries: web::Path<(String, String)>,
 ) -> Result<HttpResponse, Error> {
     let (token, alias) = path_entries.into_inner();
@@ -256,7 +256,7 @@ async fn delete<S: Store>(
     let token = DeleteToken::from_existing(&token);
     let alias = Alias::from_existing(&alias);
 
-    manager.delete((**store).clone(), alias, token).await?;
+    manager.delete(alias, token).await?;
 
     Ok(HttpResponse::NoContent().finish())
 }
@@ -308,7 +308,6 @@ async fn process_details<S: Store>(
     query: web::Query<ProcessQuery>,
     ext: web::Path<String>,
     manager: web::Data<UploadManager>,
-    store: web::Data<S>,
     filters: web::Data<BTreeSet<String>>,
 ) -> Result<HttpResponse, Error> {
     let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
@@ -581,17 +580,16 @@ struct AliasQuery {
 }
 
 #[instrument(name = "Purging file", skip(upload_manager))]
-async fn purge<S: Store>(
+async fn purge(
     query: web::Query<AliasQuery>,
     upload_manager: web::Data<UploadManager>,
-    store: web::Data<S>,
 ) -> Result<HttpResponse, Error> {
     let alias = Alias::from_existing(&query.alias);
     let aliases = upload_manager.aliases_by_alias(&alias).await?;
 
     for alias in aliases.iter() {
         upload_manager
-            .delete_without_token((**store).clone(), alias.to_owned())
+            .delete_without_token(alias.to_owned())
             .await?;
     }
 
@@ -602,10 +600,9 @@ async fn purge<S: Store>(
 }
 
 #[instrument(name = "Fetching aliases", skip(upload_manager))]
-async fn aliases<S: Store>(
+async fn aliases(
     query: web::Query<AliasQuery>,
     upload_manager: web::Data<UploadManager>,
-    store: web::Data<S>,
 ) -> Result<HttpResponse, Error> {
     let alias = Alias::from_existing(&query.alias);
     let aliases = upload_manager.aliases_by_alias(&alias).await?;
@@ -639,6 +636,14 @@ async fn launch<S: Store + Clone + 'static>(
     manager: UploadManager,
     store: S,
 ) -> color_eyre::Result<()> {
+    let repo = manager.repo().clone();
+
+    actix_rt::spawn(queue::process_jobs(
+        repo,
+        store.clone(),
+        CONFIG.server.worker_id.as_bytes().to_vec(),
+    ));
+
     // Create a new Multipart Form validator
     //
     // This form is expecting a single array field, 'images' with at most 10 files in it
@@ -730,8 +735,8 @@ async fn launch<S: Store + Clone + 'static>(
                     .service(web::resource("/download").route(web::get().to(download::<S>)))
                     .service(
                         web::resource("/delete/{delete_token}/{filename}")
-                            .route(web::delete().to(delete::<S>))
-                            .route(web::get().to(delete::<S>)),
+                            .route(web::delete().to(delete))
+                            .route(web::get().to(delete)),
                     )
                     .service(web::resource("/original/{filename}").route(web::get().to(serve::<S>)))
                     .service(web::resource("/process.{ext}").route(web::get().to(process::<S>)))
@@ -757,8 +762,8 @@ async fn launch<S: Store + Clone + 'static>(
                             .wrap(import_form.clone())
                             .route(web::post().to(upload::<S>)),
                     )
-                    .service(web::resource("/purge").route(web::post().to(purge::<S>)))
-                    .service(web::resource("/aliases").route(web::get().to(aliases::<S>))),
+                    .service(web::resource("/purge").route(web::post().to(purge)))
+                    .service(web::resource("/aliases").route(web::get().to(aliases))),
             )
     })
     .bind(CONFIG.server.address)?
diff --git a/src/queue.rs b/src/queue.rs
new file mode 100644
index 0000000..04af1b6
--- /dev/null
+++ b/src/queue.rs
@@ -0,0 +1,113 @@
+use crate::{
+    error::Error,
+    repo::{AliasRepo, HashRepo, IdentifierRepo, QueueRepo, Repo},
+    store::Store,
+};
+use tracing::{debug, error, Span};
+
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+enum Job {
+    Cleanup { hash: Vec<u8> },
+}
+
+pub(crate) async fn queue_cleanup<R: QueueRepo>(repo: &R, hash: R::Bytes) -> Result<(), Error> {
+    let job = serde_json::to_vec(&Job::Cleanup {
+        hash: hash.as_ref().to_vec(),
+    })?;
+    repo.push(job.into()).await?;
+    Ok(())
+}
+
+pub(crate) async fn process_jobs<S: Store>(repo: Repo, store: S, worker_id: Vec<u8>) {
+    loop {
+        let res = match repo {
+            Repo::Sled(ref repo) => do_process_jobs(repo, &store, worker_id.clone()).await,
+        };
+
+        if let Err(e) = res {
+            tracing::warn!("Error processing jobs: {}", e);
+            tracing::warn!("{:?}", e);
+            continue;
+        }
+
+        break;
+    }
+}
+
+async fn do_process_jobs<R, S>(repo: &R, store: &S, worker_id: Vec<u8>) -> Result<(), Error>
+where
+    R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
+    R::Bytes: Clone,
+    S: Store,
+{
+    loop {
+        let bytes = repo.pop(worker_id.clone()).await?;
+
+        match serde_json::from_slice(bytes.as_ref()) {
+            Ok(job) => match job {
+                Job::Cleanup { hash } => cleanup(repo, store, hash).await?,
+            },
+            Err(e) => {
+                tracing::warn!("Invalid job: {}", e);
+            }
+        }
+    }
+}
+
+#[tracing::instrument(skip(repo, store))]
+async fn cleanup<R, S>(repo: &R, store: &S, hash: Vec<u8>) -> Result<(), Error>
+where
+    R: HashRepo + IdentifierRepo + AliasRepo,
+    R::Bytes: Clone,
+    S: Store,
+{
+    let hash: R::Bytes = hash.into();
+
+    let aliases = repo.aliases(hash.clone()).await?;
+
+    if !aliases.is_empty() {
+        return Ok(());
+    }
+
+    let variant_idents = repo
+        .variants::<S::Identifier>(hash.clone())
+        .await?
+        .into_iter()
+        .map(|(_, v)| v)
+        .collect::<Vec<_>>();
+    let main_ident = repo.identifier(hash.clone()).await?;
+    let motion_ident = repo.motion_identifier(hash.clone()).await?;
+
+    HashRepo::cleanup(repo, hash).await?;
+
+    let cleanup_span = tracing::info_span!(parent: None, "Cleaning files");
+    cleanup_span.follows_from(Span::current());
+
+    let mut errors = Vec::new();
+
+    for identifier in variant_idents
+        .iter()
+        .chain(&[main_ident])
+        .chain(motion_ident.iter())
+    {
+        debug!("Deleting {:?}", identifier);
+        if let Err(e) = store.remove(identifier).await {
+            errors.push(e);
+        }
+
+        if let Err(e) = IdentifierRepo::cleanup(repo, identifier).await {
+            errors.push(e);
+        }
+    }
+
+    if !errors.is_empty() {
+        let span = tracing::error_span!("Error deleting files");
+        span.in_scope(|| {
+            for error in errors {
+                error!("{}", error);
+            }
+        });
+    }
+
+    Ok(())
+}
diff --git a/src/repo.rs b/src/repo.rs
index 8cc0115..27f3cf0 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -30,17 +30,28 @@ pub(crate) struct DeleteToken {
 
 pub(crate) struct AlreadyExists;
 
-#[async_trait::async_trait(?Send)]
-pub(crate) trait SettingsRepo {
+pub(crate) trait BaseRepo {
     type Bytes: AsRef<[u8]> + From<Vec<u8>>;
+}
 
+#[async_trait::async_trait(?Send)]
+pub(crate) trait QueueRepo: BaseRepo {
+    async fn in_progress(&self, worker_id: Vec<u8>) -> Result<Option<Self::Bytes>, Error>;
+
+    async fn push(&self, job: Self::Bytes) -> Result<(), Error>;
+
+    async fn pop(&self, worker_id: Vec<u8>) -> Result<Self::Bytes, Error>;
+}
+
+#[async_trait::async_trait(?Send)]
+pub(crate) trait SettingsRepo: BaseRepo {
     async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Error>;
     async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Error>;
     async fn remove(&self, key: &'static [u8]) -> Result<(), Error>;
 }
 
 #[async_trait::async_trait(?Send)]
-pub(crate) trait IdentifierRepo {
+pub(crate) trait IdentifierRepo: BaseRepo {
     async fn relate_details<I: Identifier>(
         &self,
         identifier: &I,
@@ -52,8 +63,7 @@ pub(crate) trait IdentifierRepo {
 }
 
 #[async_trait::async_trait(?Send)]
-pub(crate) trait HashRepo {
-    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
+pub(crate) trait HashRepo: BaseRepo {
     type Stream: Stream<Item = Result<Self::Bytes, Error>>;
 
     async fn hashes(&self) -> Self::Stream;
@@ -101,9 +111,7 @@ pub(crate) trait HashRepo {
 }
 
 #[async_trait::async_trait(?Send)]
-pub(crate) trait AliasRepo {
-    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
-
+pub(crate) trait AliasRepo: BaseRepo {
     async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Error>;
 
     async fn relate_delete_token(
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index c1f323f..a049861 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -1,9 +1,15 @@
-use super::{
-    Alias, AliasRepo, AlreadyExists, DeleteToken, Details, HashRepo, Identifier, IdentifierRepo,
-    SettingsRepo,
+use crate::{
+    error::Error,
+    repo::{
+        Alias, AliasRepo, AlreadyExists, DeleteToken, Details, HashRepo, Identifier,
+        IdentifierRepo, QueueRepo, SettingsRepo,
+    },
 };
-use crate::error::Error;
 use sled::{Db, IVec, Tree};
+use std::sync::Arc;
+use tokio::sync::Notify;
+
+use super::BaseRepo;
 
 macro_rules! b {
     ($self:ident.$ident:ident, $expr:expr) => {{
@@ -42,7 +48,10 @@ pub(crate) struct SledRepo {
     aliases: Tree,
     alias_hashes: Tree,
     alias_delete_tokens: Tree,
-    _db: Db,
+    queue: Tree,
+    in_progress_queue: Tree,
+    queue_notifier: Arc<Notify>,
+    db: Db,
 }
 
 impl SledRepo {
@@ -58,15 +67,67 @@ impl SledRepo {
             aliases: db.open_tree("pict-rs-aliases-tree")?,
             alias_hashes: db.open_tree("pict-rs-alias-hashes-tree")?,
             alias_delete_tokens: db.open_tree("pict-rs-alias-delete-tokens-tree")?,
-            _db: db,
+            queue: db.open_tree("pict-rs-queue-tree")?,
+            in_progress_queue: db.open_tree("pict-rs-in-progress-queue-tree")?,
+            queue_notifier: Arc::new(Notify::new()),
+            db,
         })
     }
 }
 
+impl BaseRepo for SledRepo {
+    type Bytes = IVec;
+}
+
+#[async_trait::async_trait(?Send)]
+impl QueueRepo for SledRepo {
+    async fn in_progress(&self, worker_id: Vec<u8>) -> Result<Option<Self::Bytes>, Error> {
+        let opt = b!(self.in_progress_queue, in_progress_queue.get(worker_id));
+
+        Ok(opt)
+    }
+
+    async fn push(&self, job: Self::Bytes) -> Result<(), Error> {
+        let id = self.db.generate_id()?;
+        b!(self.queue, queue.insert(id.to_be_bytes(), job));
+        self.queue_notifier.notify_one();
+        Ok(())
+    }
+
+    async fn pop(&self, worker_id: Vec<u8>) -> Result<Self::Bytes, Error> {
+        let notify = Arc::clone(&self.queue_notifier);
+
+        loop {
+            let in_progress_queue = self.in_progress_queue.clone();
+
+            let worker_id = worker_id.clone();
+            let job = b!(self.queue, {
+                in_progress_queue.remove(&worker_id)?;
+
+                while let Some((key, job)) = queue.iter().find_map(Result::ok) {
+                    in_progress_queue.insert(&worker_id, &job)?;
+
+                    if queue.remove(key)?.is_some() {
+                        return Ok(Some(job));
+                    }
+
+                    in_progress_queue.remove(&worker_id)?;
+                }
+
+                Ok(None) as Result<_, SledError>
+            });
+
+            if let Some(job) = job {
+                return Ok(job);
+            }
+
+            notify.notified().await
+        }
+    }
+}
+
 #[async_trait::async_trait(?Send)]
 impl SettingsRepo for SledRepo {
-    type Bytes = IVec;
-
     #[tracing::instrument(skip(value))]
     async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Error> {
         b!(self.settings, settings.insert(key, value));
@@ -212,7 +273,6 @@ fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> {
 
 #[async_trait::async_trait(?Send)]
 impl HashRepo for SledRepo {
-    type Bytes = IVec;
     type Stream = HashStream;
 
     async fn hashes(&self) -> Self::Stream {
@@ -429,8 +489,6 @@ impl HashRepo for SledRepo {
 
 #[async_trait::async_trait(?Send)]
 impl AliasRepo for SledRepo {
-    type Bytes = sled::IVec;
-
     #[tracing::instrument]
     async fn create(&self, alias: &Alias) -> Result<Result<(), AlreadyExists>, Error> {
         let bytes = alias.to_bytes();
diff --git a/src/upload_manager.rs b/src/upload_manager.rs
index 42edaad..0952b81 100644
--- a/src/upload_manager.rs
+++ b/src/upload_manager.rs
@@ -5,15 +5,15 @@ use crate::{
     ffmpeg::{InputFormat, ThumbnailFormat},
     magick::details_hint,
     repo::{
-        sled::SledRepo, Alias, AliasRepo, DeleteToken, HashRepo, IdentifierRepo, Repo, SettingsRepo,
+        sled::SledRepo, Alias, AliasRepo, BaseRepo, DeleteToken, HashRepo, IdentifierRepo, Repo,
+        SettingsRepo,
     },
     store::{Identifier, Store},
 };
 use futures_util::StreamExt;
 use sha2::Digest;
 use std::sync::Arc;
-use tracing::{debug, error, instrument, Span};
-use tracing_futures::Instrument;
+use tracing::instrument;
 
 mod hasher;
 mod session;
@@ -34,6 +34,10 @@ pub(crate) struct UploadManagerInner {
 }
 
 impl UploadManager {
+    pub(crate) fn repo(&self) -> &Repo {
+        &self.inner.repo
+    }
+
     /// Create a new UploadManager
     pub(crate) async fn new(repo: Repo, format: Option<ImageFormat>) -> Result<Self, Error> {
         let manager = UploadManager {
@@ -229,26 +233,17 @@ impl UploadManager {
     }
 
     /// Delete an alias without a delete token
-    pub(crate) async fn delete_without_token<S: Store + 'static>(
-        &self,
-        store: S,
-        alias: Alias,
-    ) -> Result<(), Error> {
+    pub(crate) async fn delete_without_token(&self, alias: Alias) -> Result<(), Error> {
         let token = match self.inner.repo {
             Repo::Sled(ref sled_repo) => sled_repo.delete_token(&alias).await?,
         };
 
-        self.delete(store, alias, token).await
+        self.delete(alias, token).await
     }
 
     /// Delete the alias, and the file & variants if no more aliases exist
     #[instrument(skip(self, alias, token))]
-    pub(crate) async fn delete<S: Store + 'static>(
-        &self,
-        store: S,
-        alias: Alias,
-        token: DeleteToken,
-    ) -> Result<(), Error> {
+    pub(crate) async fn delete(&self, alias: Alias, token: DeleteToken) -> Result<(), Error> {
         let hash = match self.inner.repo {
             Repo::Sled(ref sled_repo) => {
                 let saved_delete_token = sled_repo.delete_token(&alias).await?;
@@ -262,17 +257,13 @@ impl UploadManager {
             }
         };
 
-        self.check_delete_files(store, hash).await
+        self.check_delete_files(hash).await
     }
 
-    async fn check_delete_files<S: Store + 'static>(
-        &self,
-        store: S,
-        hash: Vec<u8>,
-    ) -> Result<(), Error> {
+    async fn check_delete_files(&self, hash: Vec<u8>) -> Result<(), Error> {
         match self.inner.repo {
             Repo::Sled(ref sled_repo) => {
-                let hash: <SledRepo as HashRepo>::Bytes = hash.into();
+                let hash: <SledRepo as BaseRepo>::Bytes = hash.into();
 
                 let aliases = sled_repo.aliases(hash.clone()).await?;
 
@@ -280,52 +271,7 @@ impl UploadManager {
                     return Ok(());
                 }
 
-                let variant_idents = sled_repo
-                    .variants::<S::Identifier>(hash.clone())
-                    .await?
-                    .into_iter()
-                    .map(|(_, v)| v)
-                    .collect::<Vec<_>>();
-                let main_ident = sled_repo.identifier(hash.clone()).await?;
-                let motion_ident = sled_repo.motion_identifier(hash.clone()).await?;
-
-                let repo = sled_repo.clone();
-
-                HashRepo::cleanup(sled_repo, hash).await?;
-
-                let cleanup_span = tracing::info_span!(parent: None, "Cleaning files");
-                cleanup_span.follows_from(Span::current());
-
-                actix_rt::spawn(
-                    async move {
-                        let mut errors = Vec::new();
-
-                        for identifier in variant_idents
-                            .iter()
-                            .chain(&[main_ident])
-                            .chain(motion_ident.iter())
-                        {
-                            debug!("Deleting {:?}", identifier);
-                            if let Err(e) = store.remove(identifier).await {
-                                errors.push(e);
-                            }
-
-                            if let Err(e) = IdentifierRepo::cleanup(&repo, identifier).await {
-                                errors.push(e);
-                            }
-                        }
-
-                        if !errors.is_empty() {
-                            let span = tracing::error_span!("Error deleting files");
-                            span.in_scope(|| {
-                                for error in errors {
-                                    error!("{}", error);
-                                }
-                            });
-                        }
-                    }
-                    .instrument(cleanup_span),
-                );
+                crate::queue::queue_cleanup(sled_repo, hash).await?;
             }
         }
 

From 8226a3571d6d14f32ac1d9b944ca3592fb6e92d1 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Tue, 29 Mar 2022 13:18:47 -0500
Subject: [PATCH 18/37] Enable multi-threaded job processing

---
 defaults.toml                      |   1 +
 dev.toml                           |   1 +
 docker/object-storage/pict-rs.toml |   1 +
 src/main.rs                        |  69 +++++--------
 src/queue.rs                       | 149 ++++++++++++++++++-----------
 5 files changed, 120 insertions(+), 101 deletions(-)

diff --git a/defaults.toml b/defaults.toml
index 1312337..c43d4f2 100644
--- a/defaults.toml
+++ b/defaults.toml
@@ -1,5 +1,6 @@
 [server]
 address = '0.0.0.0:8080'
+worker_id = 'pict-rs-1'
 [tracing.logging]
 format = 'normal'
 targets = 'warn,tracing_actix_web=info,actix_server=info,actix_web=info'
diff --git a/dev.toml b/dev.toml
index 9a5748b..04e9397 100644
--- a/dev.toml
+++ b/dev.toml
@@ -1,5 +1,6 @@
 [server]
 address = '0.0.0.0:8080'
+worker_id = 'pict-rs-1'
 [tracing.logging]
 format = 'normal'
 targets = 'warn,tracing_actix_web=info,actix_server=info,actix_web=info'
diff --git a/docker/object-storage/pict-rs.toml b/docker/object-storage/pict-rs.toml
index 980d749..f710493 100644
--- a/docker/object-storage/pict-rs.toml
+++ b/docker/object-storage/pict-rs.toml
@@ -1,5 +1,6 @@
 [server]
 address = '0.0.0.0:8080'
+worker_id = 'pict-rs-1'
 [tracing.logging]
 format = 'normal'
 targets = 'warn,tracing_actix_web=info,actix_server=info,actix_web=info'
diff --git a/src/main.rs b/src/main.rs
index 15b3725..f963e38 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -15,11 +15,12 @@ use std::{
     future::ready,
     path::PathBuf,
     pin::Pin,
+    sync::atomic::{AtomicU64, Ordering},
     task::{Context, Poll},
     time::SystemTime,
 };
 use tokio::{io::AsyncReadExt, sync::Semaphore};
-use tracing::{debug, error, info, instrument, Span};
+use tracing::{debug, error, info, instrument};
 use tracing_actix_web::TracingLogger;
 use tracing_awc::Tracing;
 use tracing_futures::Instrument;
@@ -382,38 +383,11 @@ async fn process<S: Store + 'static>(
 
         let details = Details::from_bytes(bytes.clone(), format.as_hint()).await?;
 
-        let save_span = tracing::info_span!(
-            parent: None,
-            "Saving variant information",
-            path = tracing::field::debug(&thumbnail_path),
-            name = tracing::field::display(&alias),
-        );
-        save_span.follows_from(Span::current());
-        let details2 = details.clone();
-        let bytes2 = bytes.clone();
-        let alias2 = alias.clone();
-        actix_rt::spawn(
-            async move {
-                let identifier = match store.save_bytes(bytes2).await {
-                    Ok(identifier) => identifier,
-                    Err(e) => {
-                        tracing::warn!("Failed to generate directory path: {}", e);
-                        return;
-                    }
-                };
-                if let Err(e) = manager.store_details(&identifier, &details2).await {
-                    tracing::warn!("Error saving variant details: {}", e);
-                    return;
-                }
-                if let Err(e) = manager
-                    .store_variant(&alias2, &thumbnail_path, &identifier)
-                    .await
-                {
-                    tracing::warn!("Error saving variant info: {}", e);
-                }
-            }
-            .instrument(save_span),
-        );
+        let identifier = store.save_bytes(bytes.clone()).await?;
+        manager.store_details(&identifier, &details).await?;
+        manager
+            .store_variant(&alias, &thumbnail_path, &identifier)
+            .await?;
 
         Ok((details, bytes)) as Result<(Details, web::Bytes), Error>
     };
@@ -632,18 +606,18 @@ fn build_reqwest_client() -> reqwest::Result<reqwest::Client> {
         .build()
 }
 
+fn next_worker_id() -> String {
+    static WORKER_ID: AtomicU64 = AtomicU64::new(0);
+
+    let next_id = WORKER_ID.fetch_add(1, Ordering::Relaxed);
+
+    format!("{}-{}", CONFIG.server.worker_id, next_id)
+}
+
 async fn launch<S: Store + Clone + 'static>(
     manager: UploadManager,
     store: S,
 ) -> color_eyre::Result<()> {
-    let repo = manager.repo().clone();
-
-    actix_rt::spawn(queue::process_jobs(
-        repo,
-        store.clone(),
-        CONFIG.server.worker_id.as_bytes().to_vec(),
-    ));
-
     // Create a new Multipart Form validator
     //
     // This form is expecting a single array field, 'images' with at most 10 files in it
@@ -717,11 +691,20 @@ async fn launch<S: Store + Clone + 'static>(
         );
 
     HttpServer::new(move || {
+        let manager = manager.clone();
+        let store = store.clone();
+
+        actix_rt::spawn(queue::process_jobs(
+            manager.repo().clone(),
+            store.clone(),
+            next_worker_id(),
+        ));
+
         App::new()
             .wrap(TracingLogger::default())
             .wrap(Deadline)
-            .app_data(web::Data::new(store.clone()))
-            .app_data(web::Data::new(manager.clone()))
+            .app_data(web::Data::new(store))
+            .app_data(web::Data::new(manager))
             .app_data(web::Data::new(build_client()))
             .app_data(web::Data::new(CONFIG.media.filters.clone()))
             .service(
diff --git a/src/queue.rs b/src/queue.rs
index 04af1b6..6579ae2 100644
--- a/src/queue.rs
+++ b/src/queue.rs
@@ -1,103 +1,100 @@
 use crate::{
     error::Error,
     repo::{AliasRepo, HashRepo, IdentifierRepo, QueueRepo, Repo},
-    store::Store,
+    store::{Identifier, Store},
 };
-use tracing::{debug, error, Span};
+use tracing::{debug, error};
 
 #[derive(Debug, serde::Deserialize, serde::Serialize)]
 enum Job {
-    Cleanup { hash: Vec<u8> },
+    CleanupHash { hash: Vec<u8> },
+    CleanupIdentifier { identifier: Vec<u8> },
 }
 
 pub(crate) async fn queue_cleanup<R: QueueRepo>(repo: &R, hash: R::Bytes) -> Result<(), Error> {
-    let job = serde_json::to_vec(&Job::Cleanup {
+    let job = serde_json::to_vec(&Job::CleanupHash {
         hash: hash.as_ref().to_vec(),
     })?;
     repo.push(job.into()).await?;
     Ok(())
 }
 
-pub(crate) async fn process_jobs<S: Store>(repo: Repo, store: S, worker_id: Vec<u8>) {
-    loop {
-        let res = match repo {
-            Repo::Sled(ref repo) => do_process_jobs(repo, &store, worker_id.clone()).await,
-        };
+pub(crate) async fn process_jobs<S: Store>(repo: Repo, store: S, worker_id: String) {
+    match repo {
+        Repo::Sled(ref repo) => {
+            if let Ok(Some(job)) = repo.in_progress(worker_id.as_bytes().to_vec()).await {
+                if let Err(e) = run_job(repo, &store, &job).await {
+                    tracing::warn!("Failed to run previously dropped job: {}", e);
+                    tracing::warn!("{:?}", e);
+                }
+            }
+            loop {
+                let res = job_loop(repo, &store, worker_id.clone()).await;
 
-        if let Err(e) = res {
-            tracing::warn!("Error processing jobs: {}", e);
-            tracing::warn!("{:?}", e);
-            continue;
+                if let Err(e) = res {
+                    tracing::warn!("Error processing jobs: {}", e);
+                    tracing::warn!("{:?}", e);
+                    continue;
+                }
+
+                break;
+            }
         }
-
-        break;
     }
 }
 
-async fn do_process_jobs<R, S>(repo: &R, store: &S, worker_id: Vec<u8>) -> Result<(), Error>
+async fn job_loop<R, S>(repo: &R, store: &S, worker_id: String) -> Result<(), Error>
 where
     R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
     R::Bytes: Clone,
     S: Store,
 {
     loop {
-        let bytes = repo.pop(worker_id.clone()).await?;
+        let bytes = repo.pop(worker_id.as_bytes().to_vec()).await?;
 
-        match serde_json::from_slice(bytes.as_ref()) {
-            Ok(job) => match job {
-                Job::Cleanup { hash } => cleanup(repo, store, hash).await?,
-            },
-            Err(e) => {
-                tracing::warn!("Invalid job: {}", e);
-            }
-        }
+        run_job(repo, store, bytes.as_ref()).await?;
     }
 }
 
-#[tracing::instrument(skip(repo, store))]
-async fn cleanup<R, S>(repo: &R, store: &S, hash: Vec<u8>) -> Result<(), Error>
+async fn run_job<R, S>(repo: &R, store: &S, job: &[u8]) -> Result<(), Error>
 where
-    R: HashRepo + IdentifierRepo + AliasRepo,
+    R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
     R::Bytes: Clone,
     S: Store,
 {
-    let hash: R::Bytes = hash.into();
-
-    let aliases = repo.aliases(hash.clone()).await?;
-
-    if !aliases.is_empty() {
-        return Ok(());
+    match serde_json::from_slice(job) {
+        Ok(job) => match job {
+            Job::CleanupHash { hash } => cleanup_hash::<R, S>(repo, hash).await?,
+            Job::CleanupIdentifier { identifier } => {
+                cleanup_identifier(repo, store, identifier).await?
+            }
+        },
+        Err(e) => {
+            tracing::warn!("Invalid job: {}", e);
+        }
     }
 
-    let variant_idents = repo
-        .variants::<S::Identifier>(hash.clone())
-        .await?
-        .into_iter()
-        .map(|(_, v)| v)
-        .collect::<Vec<_>>();
-    let main_ident = repo.identifier(hash.clone()).await?;
-    let motion_ident = repo.motion_identifier(hash.clone()).await?;
+    Ok(())
+}
 
-    HashRepo::cleanup(repo, hash).await?;
-
-    let cleanup_span = tracing::info_span!(parent: None, "Cleaning files");
-    cleanup_span.follows_from(Span::current());
+#[tracing::instrument(skip(repo, store))]
+async fn cleanup_identifier<R, S>(repo: &R, store: &S, identifier: Vec<u8>) -> Result<(), Error>
+where
+    R: QueueRepo + HashRepo + IdentifierRepo,
+    R::Bytes: Clone,
+    S: Store,
+{
+    let identifier = S::Identifier::from_bytes(identifier)?;
 
     let mut errors = Vec::new();
 
-    for identifier in variant_idents
-        .iter()
-        .chain(&[main_ident])
-        .chain(motion_ident.iter())
-    {
-        debug!("Deleting {:?}", identifier);
-        if let Err(e) = store.remove(identifier).await {
-            errors.push(e);
-        }
+    debug!("Deleting {:?}", identifier);
+    if let Err(e) = store.remove(&identifier).await {
+        errors.push(e);
+    }
 
-        if let Err(e) = IdentifierRepo::cleanup(repo, identifier).await {
-            errors.push(e);
-        }
+    if let Err(e) = IdentifierRepo::cleanup(repo, &identifier).await {
+        errors.push(e);
     }
 
     if !errors.is_empty() {
@@ -111,3 +108,39 @@ where
 
     Ok(())
 }
+
+#[tracing::instrument(skip(repo))]
+async fn cleanup_hash<R, S>(repo: &R, hash: Vec<u8>) -> Result<(), Error>
+where
+    R: QueueRepo + AliasRepo + HashRepo + IdentifierRepo,
+    R::Bytes: Clone,
+    S: Store,
+{
+    let hash: R::Bytes = hash.into();
+
+    let aliases = repo.aliases(hash.clone()).await?;
+
+    if !aliases.is_empty() {
+        return Ok(());
+    }
+
+    let mut idents = repo
+        .variants::<S::Identifier>(hash.clone())
+        .await?
+        .into_iter()
+        .map(|(_, v)| v)
+        .collect::<Vec<_>>();
+    idents.push(repo.identifier(hash.clone()).await?);
+    idents.extend(repo.motion_identifier(hash.clone()).await?);
+
+    for identifier in idents {
+        if let Ok(identifier) = identifier.to_bytes() {
+            let job = serde_json::to_vec(&Job::CleanupIdentifier { identifier })?;
+            repo.push(job.into()).await?;
+        }
+    }
+
+    HashRepo::cleanup(repo, hash).await?;
+
+    Ok(())
+}

From 25f44808092e17aea95cc243e1c8d904ba6ab0cd Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Tue, 29 Mar 2022 14:17:20 -0500
Subject: [PATCH 19/37] Remove custom stream error mapping types

---
 src/main.rs               | 22 ++++++++++++--------
 src/map_error.rs          | 43 ---------------------------------------
 src/store/object_store.rs | 42 ++++++--------------------------------
 3 files changed, 19 insertions(+), 88 deletions(-)
 delete mode 100644 src/map_error.rs

diff --git a/src/main.rs b/src/main.rs
index f963e38..8d5549d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -7,7 +7,7 @@ use actix_web::{
 use awc::Client;
 use futures_util::{
     stream::{empty, once},
-    Stream,
+    Stream, TryStreamExt,
 };
 use once_cell::sync::Lazy;
 use std::{
@@ -35,7 +35,6 @@ mod ffmpeg;
 mod file;
 mod init_tracing;
 mod magick;
-mod map_error;
 mod middleware;
 mod migrate;
 mod process;
@@ -207,7 +206,7 @@ async fn download<S: Store>(
     }
 
     let stream = Limit::new(
-        map_error::map_crate_error(res),
+        res.map_err(Error::from),
         (CONFIG.media.max_file_size * MEGABYTES) as u64,
     );
 
@@ -495,9 +494,11 @@ async fn ranged_file_resp<S: Store>(
                 builder.insert_header(content_range);
                 (
                     builder,
-                    Either::left(Either::left(map_error::map_crate_error(
-                        range::chop_store(range, store, &identifier, len).await?,
-                    ))),
+                    Either::left(Either::left(
+                        range::chop_store(range, store, &identifier, len)
+                            .await?
+                            .map_err(Error::from),
+                    )),
                 )
             } else {
                 (
@@ -510,7 +511,10 @@ async fn ranged_file_resp<S: Store>(
         }
     } else {
         //No Range header in the request - return the entire document
-        let stream = map_error::map_crate_error(store.to_stream(&identifier, None, None).await?);
+        let stream = store
+            .to_stream(&identifier, None, None)
+            .await?
+            .map_err(Error::from);
         (HttpResponse::Ok(), Either::right(stream))
     };
 
@@ -642,7 +646,7 @@ async fn launch<S: Store + Clone + 'static>(
                         .session(store)
                         .upload(
                             CONFIG.media.enable_silent_video,
-                            map_error::map_crate_error(stream),
+                            stream.map_err(Error::from),
                         )
                         .await;
 
@@ -679,7 +683,7 @@ async fn launch<S: Store + Clone + 'static>(
                             filename,
                             !CONFIG.media.skip_validate_imports,
                             CONFIG.media.enable_silent_video,
-                            map_error::map_crate_error(stream),
+                            stream.map_err(Error::from),
                         )
                         .await;
 
diff --git a/src/map_error.rs b/src/map_error.rs
deleted file mode 100644
index b63b9a8..0000000
--- a/src/map_error.rs
+++ /dev/null
@@ -1,43 +0,0 @@
-use crate::error::Error;
-use futures_util::stream::Stream;
-use std::{
-    marker::PhantomData,
-    pin::Pin,
-    task::{Context, Poll},
-};
-
-pin_project_lite::pin_project! {
-    pub(super) struct MapError<E, S> {
-        #[pin]
-        inner: S,
-
-        _error: PhantomData<E>,
-    }
-}
-
-pub(super) fn map_crate_error<S>(inner: S) -> MapError<Error, S> {
-    map_error(inner)
-}
-
-pub(super) fn map_error<S, E>(inner: S) -> MapError<E, S> {
-    MapError {
-        inner,
-        _error: PhantomData,
-    }
-}
-
-impl<T, StreamErr, E, S> Stream for MapError<E, S>
-where
-    S: Stream<Item = Result<T, StreamErr>>,
-    E: From<StreamErr>,
-{
-    type Item = Result<T, E>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        let this = self.as_mut().project();
-
-        this.inner
-            .poll_next(cx)
-            .map(|opt| opt.map(|res| res.map_err(Into::into)))
-    }
-}
diff --git a/src/store/object_store.rs b/src/store/object_store.rs
index dcd09f1..edddd71 100644
--- a/src/store/object_store.rs
+++ b/src/store/object_store.rs
@@ -5,7 +5,7 @@ use crate::{
 };
 use actix_rt::time::Sleep;
 use actix_web::web::Bytes;
-use futures_util::stream::Stream;
+use futures_util::{stream::Stream, TryStreamExt};
 use s3::{
     client::Client, command::Command, creds::Credentials, request_trait::Request, Bucket, Region,
 };
@@ -58,13 +58,6 @@ pub(crate) struct ObjectStore {
     client: reqwest::Client,
 }
 
-pin_project_lite::pin_project! {
-    struct IoError<S> {
-        #[pin]
-        inner: S,
-    }
-}
-
 pin_project_lite::pin_project! {
     struct Timeout<S> {
         sleep: Option<Pin<Box<Sleep>>>,
@@ -146,10 +139,11 @@ impl Store for ObjectStore {
 
         let allotted = allotted.saturating_sub(now.elapsed());
 
-        Ok(
-            request_span
-                .in_scope(|| Box::pin(timeout(allotted, io_error(response.bytes_stream())))),
-        )
+        let stream = response
+            .bytes_stream()
+            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e));
+
+        Ok(request_span.in_scope(|| Box::pin(timeout(allotted, stream))))
     }
 
     #[tracing::instrument(skip(writer))]
@@ -272,14 +266,6 @@ async fn init_generator(repo: &Repo) -> Result<Generator, Error> {
     }
 }
 
-fn io_error<S, T, E>(stream: S) -> impl Stream<Item = std::io::Result<T>>
-where
-    S: Stream<Item = Result<T, E>>,
-    E: Into<Box<dyn std::error::Error + Send + Sync>>,
-{
-    IoError { inner: stream }
-}
-
 fn timeout<S, T>(duration: Duration, stream: S) -> impl Stream<Item = std::io::Result<T>>
 where
     S: Stream<Item = std::io::Result<T>>,
@@ -291,22 +277,6 @@ where
     }
 }
 
-impl<S, T, E> Stream for IoError<S>
-where
-    S: Stream<Item = Result<T, E>>,
-    E: Into<Box<dyn std::error::Error + Send + Sync>>,
-{
-    type Item = std::io::Result<T>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        let this = self.as_mut().project();
-
-        this.inner.poll_next(cx).map(|opt| {
-            opt.map(|res| res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)))
-        })
-    }
-}
-
 struct TimeoutWaker {
     woken: Arc<AtomicBool>,
     inner: Waker,

From 0e490ff54a76bbc846f08a566d08ca763b9cfc22 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Tue, 29 Mar 2022 14:28:08 -0500
Subject: [PATCH 20/37] Remove custom BytesFreezer stream

---
 src/file.rs | 35 ++---------------------------------
 1 file changed, 2 insertions(+), 33 deletions(-)

diff --git a/src/file.rs b/src/file.rs
index 6f011e7..34348a5 100644
--- a/src/file.rs
+++ b/src/file.rs
@@ -8,7 +8,7 @@ pub(crate) use tokio_file::File;
 mod tokio_file {
     use crate::{store::file_store::FileError, Either};
     use actix_web::web::{Bytes, BytesMut};
-    use futures_util::stream::{Stream, StreamExt};
+    use futures_util::{Stream, StreamExt, TryStreamExt};
     use std::{io::SeekFrom, path::Path};
     use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
     use tokio_util::codec::{BytesCodec, FramedRead};
@@ -91,38 +91,7 @@ mod tokio_file {
                 (None, None) => Either::right(self.inner),
             };
 
-            Ok(BytesFreezer::new(FramedRead::new(obj, BytesCodec::new())))
-        }
-    }
-
-    pin_project_lite::pin_project! {
-        struct BytesFreezer<S> {
-            #[pin]
-            inner: S,
-        }
-    }
-
-    impl<S> BytesFreezer<S> {
-        fn new(inner: S) -> Self {
-            BytesFreezer { inner }
-        }
-    }
-
-    impl<S, E> Stream for BytesFreezer<S>
-    where
-        S: Stream<Item = Result<BytesMut, E>> + Unpin,
-    {
-        type Item = Result<Bytes, E>;
-
-        fn poll_next(
-            mut self: std::pin::Pin<&mut Self>,
-            cx: &mut std::task::Context<'_>,
-        ) -> std::task::Poll<Option<Self::Item>> {
-            let this = self.as_mut().project();
-
-            this.inner
-                .poll_next(cx)
-                .map(|opt| opt.map(|res| res.map(BytesMut::freeze)))
+            Ok(FramedRead::new(obj, BytesCodec::new()).map_ok(BytesMut::freeze))
         }
     }
 }

From 5adb3fde897482a6a510c9ae715891cdd3e68bda Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Tue, 29 Mar 2022 15:59:17 -0500
Subject: [PATCH 21/37] Move some Stream implementations into 'stream' module

---
 src/error.rs              |   2 +-
 src/main.rs               |  66 +----------
 src/repo/sled.rs          |  76 ++-----------
 src/store/object_store.rs |  91 ++-------------
 src/stream.rs             | 228 ++++++++++++++++++++++++++++++++++++++
 5 files changed, 253 insertions(+), 210 deletions(-)
 create mode 100644 src/stream.rs

diff --git a/src/error.rs b/src/error.rs
index 679f70b..940f221 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -118,7 +118,7 @@ pub(crate) enum UploadError {
     Range,
 
     #[error("Hit limit")]
-    Limit(#[from] super::LimitError),
+    Limit(#[from] crate::stream::LimitError),
 }
 
 impl From<awc::error::SendRequestError> for UploadError {
diff --git a/src/main.rs b/src/main.rs
index 8d5549d..8286c17 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -14,13 +14,11 @@ use std::{
     collections::BTreeSet,
     future::ready,
     path::PathBuf,
-    pin::Pin,
     sync::atomic::{AtomicU64, Ordering},
-    task::{Context, Poll},
     time::SystemTime,
 };
 use tokio::{io::AsyncReadExt, sync::Semaphore};
-use tracing::{debug, error, info, instrument};
+use tracing::{debug, info, instrument};
 use tracing_actix_web::TracingLogger;
 use tracing_awc::Tracing;
 use tracing_futures::Instrument;
@@ -44,6 +42,7 @@ mod range;
 mod repo;
 mod serde_str;
 mod store;
+mod stream;
 mod tmp_file;
 mod upload_manager;
 mod validate;
@@ -61,6 +60,7 @@ use self::{
     repo::{Alias, DeleteToken, Repo},
     serde_str::Serde,
     store::{file_store::FileStore, object_store::ObjectStore, Store},
+    stream::StreamLimit,
     upload_manager::{UploadManager, UploadManagerSession},
 };
 
@@ -138,59 +138,6 @@ struct UrlQuery {
     url: String,
 }
 
-pin_project_lite::pin_project! {
-    struct Limit<S> {
-        #[pin]
-        inner: S,
-
-        count: u64,
-        limit: u64,
-    }
-}
-
-impl<S> Limit<S> {
-    fn new(inner: S, limit: u64) -> Self {
-        Limit {
-            inner,
-            count: 0,
-            limit,
-        }
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-#[error("Resonse body larger than size limit")]
-struct LimitError;
-
-impl<S, E> Stream for Limit<S>
-where
-    S: Stream<Item = Result<web::Bytes, E>>,
-    E: From<LimitError>,
-{
-    type Item = Result<web::Bytes, E>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        let this = self.as_mut().project();
-
-        let limit = this.limit;
-        let count = this.count;
-        let inner = this.inner;
-
-        inner.poll_next(cx).map(|opt| {
-            opt.map(|res| match res {
-                Ok(bytes) => {
-                    *count += bytes.len() as u64;
-                    if *count > *limit {
-                        return Err(LimitError.into());
-                    }
-                    Ok(bytes)
-                }
-                Err(e) => Err(e),
-            })
-        })
-    }
-}
-
 /// download an image from a URL
 #[instrument(name = "Downloading file", skip(client, manager))]
 async fn download<S: Store>(
@@ -205,10 +152,9 @@ async fn download<S: Store>(
         return Err(UploadError::Download(res.status()).into());
     }
 
-    let stream = Limit::new(
-        res.map_err(Error::from),
-        (CONFIG.media.max_file_size * MEGABYTES) as u64,
-    );
+    let stream = res
+        .map_err(Error::from)
+        .limit((CONFIG.media.max_file_size * MEGABYTES) as u64);
 
     futures_util::pin_mut!(stream);
 
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index a049861..b65b562 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -4,9 +4,11 @@ use crate::{
         Alias, AliasRepo, AlreadyExists, DeleteToken, Details, HashRepo, Identifier,
         IdentifierRepo, QueueRepo, SettingsRepo,
     },
+    stream::from_iterator,
 };
+use futures_util::Stream;
 use sled::{Db, IVec, Tree};
-use std::sync::Arc;
+use std::{pin::Pin, sync::Arc};
 use tokio::sync::Notify;
 
 use super::BaseRepo;
@@ -205,65 +207,8 @@ impl IdentifierRepo for SledRepo {
     }
 }
 
-type BoxIterator<'a, T> = Box<dyn std::iter::Iterator<Item = T> + Send + 'a>;
-
-type HashIterator = BoxIterator<'static, Result<IVec, sled::Error>>;
-
 type StreamItem = Result<IVec, Error>;
-
-type NextFutResult = Result<(HashIterator, Option<StreamItem>), Error>;
-
-pub(crate) struct HashStream {
-    hashes: Option<HashIterator>,
-    next_fut: Option<futures_util::future::LocalBoxFuture<'static, NextFutResult>>,
-}
-
-impl futures_util::Stream for HashStream {
-    type Item = StreamItem;
-
-    fn poll_next(
-        self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Option<Self::Item>> {
-        let this = self.get_mut();
-
-        if let Some(mut fut) = this.next_fut.take() {
-            match fut.as_mut().poll(cx) {
-                std::task::Poll::Ready(Ok((iter, opt))) => {
-                    this.hashes = Some(iter);
-                    std::task::Poll::Ready(opt)
-                }
-                std::task::Poll::Ready(Err(e)) => std::task::Poll::Ready(Some(Err(e))),
-                std::task::Poll::Pending => {
-                    this.next_fut = Some(fut);
-                    std::task::Poll::Pending
-                }
-            }
-        } else if let Some(mut iter) = this.hashes.take() {
-            let fut = Box::pin(async move {
-                actix_rt::task::spawn_blocking(move || {
-                    let opt = iter.next();
-
-                    (iter, opt)
-                })
-                .await
-                .map(|(iter, opt)| {
-                    (
-                        iter,
-                        opt.map(|res| res.map_err(SledError::from).map_err(Error::from)),
-                    )
-                })
-                .map_err(SledError::from)
-                .map_err(Error::from)
-            });
-
-            this.next_fut = Some(fut);
-            std::pin::Pin::new(this).poll_next(cx)
-        } else {
-            std::task::Poll::Ready(None)
-        }
-    }
-}
+type LocalBoxStream<'a, T> = Pin<Box<dyn Stream<Item = T> + 'a>>;
 
 fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> {
     let mut v = hash.to_vec();
@@ -273,15 +218,16 @@ fn hash_alias_key(hash: &IVec, alias: &Alias) -> Vec<u8> {
 
 #[async_trait::async_trait(?Send)]
 impl HashRepo for SledRepo {
-    type Stream = HashStream;
+    type Stream = LocalBoxStream<'static, StreamItem>;
 
     async fn hashes(&self) -> Self::Stream {
-        let iter = self.hashes.iter().keys();
+        let iter = self
+            .hashes
+            .iter()
+            .keys()
+            .map(|res| res.map_err(Error::from));
 
-        HashStream {
-            hashes: Some(Box::new(iter)),
-            next_fut: None,
-        }
+        Box::pin(from_iterator(iter))
     }
 
     #[tracing::instrument]
diff --git a/src/store/object_store.rs b/src/store/object_store.rs
index edddd71..f2bf76b 100644
--- a/src/store/object_store.rs
+++ b/src/store/object_store.rs
@@ -2,22 +2,16 @@ use crate::{
     error::Error,
     repo::{Repo, SettingsRepo},
     store::Store,
+    stream::StreamTimeout,
 };
-use actix_rt::time::Sleep;
 use actix_web::web::Bytes;
-use futures_util::{stream::Stream, TryStreamExt};
+use futures_util::{Stream, StreamExt};
 use s3::{
     client::Client, command::Command, creds::Credentials, request_trait::Request, Bucket, Region,
 };
 use std::{
-    future::Future,
     pin::Pin,
     string::FromUtf8Error,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-    task::{Context, Poll, Wake, Waker},
     time::{Duration, Instant},
 };
 use storage_path_generator::{Generator, Path};
@@ -58,17 +52,6 @@ pub(crate) struct ObjectStore {
     client: reqwest::Client,
 }
 
-pin_project_lite::pin_project! {
-    struct Timeout<S> {
-        sleep: Option<Pin<Box<Sleep>>>,
-
-        woken: Arc<AtomicBool>,
-
-        #[pin]
-        inner: S,
-    }
-}
-
 #[async_trait::async_trait(?Send)]
 impl Store for ObjectStore {
     type Identifier = ObjectId;
@@ -139,11 +122,12 @@ impl Store for ObjectStore {
 
         let allotted = allotted.saturating_sub(now.elapsed());
 
-        let stream = response
-            .bytes_stream()
-            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e));
+        let stream = response.bytes_stream().timeout(allotted).map(|res| {
+            res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
+                .and_then(|res| res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)))
+        });
 
-        Ok(request_span.in_scope(|| Box::pin(timeout(allotted, stream))))
+        Ok(request_span.in_scope(|| Box::pin(stream)))
     }
 
     #[tracing::instrument(skip(writer))]
@@ -266,67 +250,6 @@ async fn init_generator(repo: &Repo) -> Result<Generator, Error> {
     }
 }
 
-fn timeout<S, T>(duration: Duration, stream: S) -> impl Stream<Item = std::io::Result<T>>
-where
-    S: Stream<Item = std::io::Result<T>>,
-{
-    Timeout {
-        sleep: Some(Box::pin(actix_rt::time::sleep(duration))),
-        woken: Arc::new(AtomicBool::new(true)),
-        inner: stream,
-    }
-}
-
-struct TimeoutWaker {
-    woken: Arc<AtomicBool>,
-    inner: Waker,
-}
-
-impl Wake for TimeoutWaker {
-    fn wake(self: Arc<Self>) {
-        self.wake_by_ref()
-    }
-
-    fn wake_by_ref(self: &Arc<Self>) {
-        self.woken.store(true, Ordering::Release);
-        self.inner.wake_by_ref();
-    }
-}
-
-impl<S, T> Stream for Timeout<S>
-where
-    S: Stream<Item = std::io::Result<T>>,
-{
-    type Item = std::io::Result<T>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        let this = self.as_mut().project();
-
-        if this.woken.swap(false, Ordering::Acquire) {
-            if let Some(mut sleep) = this.sleep.take() {
-                let timeout_waker = Arc::new(TimeoutWaker {
-                    woken: Arc::clone(this.woken),
-                    inner: cx.waker().clone(),
-                })
-                .into();
-                let mut timeout_cx = Context::from_waker(&timeout_waker);
-                if let Poll::Ready(()) = sleep.as_mut().poll(&mut timeout_cx) {
-                    return Poll::Ready(Some(Err(std::io::Error::new(
-                        std::io::ErrorKind::Other,
-                        Error::from(ObjectError::Elapsed),
-                    ))));
-                } else {
-                    *this.sleep = Some(sleep);
-                }
-            } else {
-                return Poll::Ready(None);
-            }
-        }
-
-        this.inner.poll_next(cx)
-    }
-}
-
 impl std::fmt::Debug for ObjectStore {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("ObjectStore")
diff --git a/src/stream.rs b/src/stream.rs
new file mode 100644
index 0000000..7ded6bd
--- /dev/null
+++ b/src/stream.rs
@@ -0,0 +1,228 @@
+use actix_rt::{task::JoinHandle, time::Sleep};
+use actix_web::web::Bytes;
+use futures_util::Stream;
+use std::{
+    future::Future,
+    pin::Pin,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    task::{Context, Poll, Wake, Waker},
+    time::Duration,
+};
+
+pub(crate) trait StreamLimit {
+    fn limit(self, limit: u64) -> Limit<Self>
+    where
+        Self: Sized,
+    {
+        Limit {
+            inner: self,
+            count: 0,
+            limit,
+        }
+    }
+}
+
+pub(crate) trait StreamTimeout {
+    fn timeout(self, duration: Duration) -> Timeout<Self>
+    where
+        Self: Sized,
+    {
+        Timeout {
+            sleep: actix_rt::time::sleep(duration),
+            inner: self,
+            expired: false,
+            woken: Arc::new(AtomicBool::new(true)),
+        }
+    }
+}
+
+pub(crate) fn from_iterator<I: IntoIterator + Unpin + Send + 'static>(
+    iterator: I,
+) -> IterStream<I, I::Item> {
+    IterStream {
+        state: IterStreamState::New { iterator },
+    }
+}
+
+impl<S, E> StreamLimit for S where S: Stream<Item = Result<Bytes, E>> {}
+impl<S> StreamTimeout for S where S: Stream {}
+
+pin_project_lite::pin_project! {
+    pub(crate) struct Limit<S> {
+        #[pin]
+        inner: S,
+
+        count: u64,
+        limit: u64,
+    }
+}
+
+pin_project_lite::pin_project! {
+    pub(crate) struct Timeout<S> {
+        #[pin]
+        sleep: Sleep,
+
+        #[pin]
+        inner: S,
+
+        expired: bool,
+        woken: Arc<AtomicBool>,
+    }
+}
+
+enum IterStreamState<I, T> {
+    New {
+        iterator: I,
+    },
+    Running {
+        handle: JoinHandle<()>,
+        receiver: tokio::sync::mpsc::Receiver<T>,
+    },
+    Pending,
+}
+
+pub(crate) struct IterStream<I, T> {
+    state: IterStreamState<I, T>,
+}
+
+struct TimeoutWaker {
+    woken: Arc<AtomicBool>,
+    inner: Waker,
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("Resonse body larger than size limit")]
+pub(crate) struct LimitError;
+
+#[derive(Debug, thiserror::Error)]
+#[error("Timeout in body")]
+pub(crate) struct TimeoutError;
+
+impl<S, E> Stream for Limit<S>
+where
+    S: Stream<Item = Result<Bytes, E>>,
+    E: From<LimitError>,
+{
+    type Item = Result<Bytes, E>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.as_mut().project();
+
+        let limit = this.limit;
+        let count = this.count;
+        let inner = this.inner;
+
+        inner.poll_next(cx).map(|opt| {
+            opt.map(|res| match res {
+                Ok(bytes) => {
+                    *count += bytes.len() as u64;
+                    if *count > *limit {
+                        return Err(LimitError.into());
+                    }
+                    Ok(bytes)
+                }
+                Err(e) => Err(e),
+            })
+        })
+    }
+}
+
+impl Wake for TimeoutWaker {
+    fn wake(self: Arc<Self>) {
+        self.wake_by_ref()
+    }
+
+    fn wake_by_ref(self: &Arc<Self>) {
+        self.woken.store(true, Ordering::Release);
+        self.inner.wake_by_ref();
+    }
+}
+
+impl<S, T> Stream for Timeout<S>
+where
+    S: Stream<Item = T>,
+{
+    type Item = Result<T, TimeoutError>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.as_mut().project();
+
+        if *this.expired {
+            return Poll::Ready(None);
+        }
+
+        if this.woken.swap(false, Ordering::Acquire) {
+            let timeout_waker = Arc::new(TimeoutWaker {
+                woken: Arc::clone(this.woken),
+                inner: cx.waker().clone(),
+            })
+            .into();
+
+            let mut timeout_cx = Context::from_waker(&timeout_waker);
+
+            if this.sleep.poll(&mut timeout_cx).is_ready() {
+                *this.expired = true;
+                return Poll::Ready(Some(Err(TimeoutError)));
+            }
+        }
+
+        this.inner.poll_next(cx).map(|opt| opt.map(Ok))
+    }
+}
+
+impl<I, T> Stream for IterStream<I, T>
+where
+    I: IntoIterator<Item = T> + Send + Unpin + 'static,
+    T: Send + 'static,
+{
+    type Item = T;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.as_mut().get_mut();
+
+        match std::mem::replace(&mut this.state, IterStreamState::Pending) {
+            IterStreamState::New { iterator } => {
+                let (sender, receiver) = tokio::sync::mpsc::channel(1);
+
+                let mut handle = actix_rt::task::spawn_blocking(move || {
+                    let iterator = iterator.into_iter();
+
+                    for item in iterator {
+                        if sender.blocking_send(item).is_err() {
+                            break;
+                        }
+                    }
+                });
+
+                if Pin::new(&mut handle).poll(cx).is_ready() {
+                    return Poll::Ready(None);
+                }
+
+                this.state = IterStreamState::Running { handle, receiver };
+            }
+            IterStreamState::Running {
+                mut handle,
+                mut receiver,
+            } => match Pin::new(&mut receiver).poll_recv(cx) {
+                Poll::Ready(Some(item)) => {
+                    if Pin::new(&mut handle).poll(cx).is_ready() {
+                        return Poll::Ready(Some(item));
+                    }
+
+                    this.state = IterStreamState::Running { handle, receiver };
+                }
+                Poll::Ready(None) => return Poll::Ready(None),
+                Poll::Pending => {
+                    this.state = IterStreamState::Running { handle, receiver };
+                    return Poll::Pending;
+                }
+            },
+            IterStreamState::Pending => return Poll::Ready(None),
+        }
+
+        self.poll_next(cx)
+    }
+}

From 63d66050c8350af5204d66e8850179900815fe1d Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Tue, 29 Mar 2022 16:18:00 -0500
Subject: [PATCH 22/37] Enable buffering iterator items for iterstream

---
 src/repo/sled.rs |  2 +-
 src/stream.rs    | 29 +++++++++++++++++------------
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index b65b562..b34d008 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -227,7 +227,7 @@ impl HashRepo for SledRepo {
             .keys()
             .map(|res| res.map_err(Error::from));
 
-        Box::pin(from_iterator(iter))
+        Box::pin(from_iterator(iter, 8))
     }
 
     #[tracing::instrument]
diff --git a/src/stream.rs b/src/stream.rs
index 7ded6bd..bd14aef 100644
--- a/src/stream.rs
+++ b/src/stream.rs
@@ -41,9 +41,10 @@ pub(crate) trait StreamTimeout {
 
 pub(crate) fn from_iterator<I: IntoIterator + Unpin + Send + 'static>(
     iterator: I,
+    buffer: usize,
 ) -> IterStream<I, I::Item> {
     IterStream {
-        state: IterStreamState::New { iterator },
+        state: IterStreamState::New { iterator, buffer },
     }
 }
 
@@ -76,6 +77,7 @@ pin_project_lite::pin_project! {
 enum IterStreamState<I, T> {
     New {
         iterator: I,
+        buffer: usize,
     },
     Running {
         handle: JoinHandle<()>,
@@ -184,8 +186,8 @@ where
         let this = self.as_mut().get_mut();
 
         match std::mem::replace(&mut this.state, IterStreamState::Pending) {
-            IterStreamState::New { iterator } => {
-                let (sender, receiver) = tokio::sync::mpsc::channel(1);
+            IterStreamState::New { iterator, buffer } => {
+                let (sender, receiver) = tokio::sync::mpsc::channel(buffer);
 
                 let mut handle = actix_rt::task::spawn_blocking(move || {
                     let iterator = iterator.into_iter();
@@ -202,27 +204,30 @@ where
                 }
 
                 this.state = IterStreamState::Running { handle, receiver };
+
+                self.poll_next(cx)
             }
             IterStreamState::Running {
                 mut handle,
                 mut receiver,
             } => match Pin::new(&mut receiver).poll_recv(cx) {
                 Poll::Ready(Some(item)) => {
+                    this.state = IterStreamState::Running { handle, receiver };
+
+                    Poll::Ready(Some(item))
+                }
+                Poll::Ready(None) => Poll::Ready(None),
+                Poll::Pending => {
                     if Pin::new(&mut handle).poll(cx).is_ready() {
-                        return Poll::Ready(Some(item));
+                        return Poll::Ready(None);
                     }
 
                     this.state = IterStreamState::Running { handle, receiver };
-                }
-                Poll::Ready(None) => return Poll::Ready(None),
-                Poll::Pending => {
-                    this.state = IterStreamState::Running { handle, receiver };
-                    return Poll::Pending;
+
+                    Poll::Pending
                 }
             },
-            IterStreamState::Pending => return Poll::Ready(None),
+            IterStreamState::Pending => panic!("Polled after completion"),
         }
-
-        self.poll_next(cx)
     }
 }

From 09281d9ae8f7eac70dcd0fd52afe3a9904c219d3 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Tue, 29 Mar 2022 16:48:26 -0500
Subject: [PATCH 23/37] Move stream timeout to all response bodies

---
 src/error.rs              |  3 +++
 src/main.rs               | 12 ++++++++++--
 src/store/object_store.rs | 29 ++++++++---------------------
 3 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/src/error.rs b/src/error.rs
index 940f221..52a1c70 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -119,6 +119,9 @@ pub(crate) enum UploadError {
 
     #[error("Hit limit")]
     Limit(#[from] crate::stream::LimitError),
+
+    #[error("Response timeout")]
+    Timeout(#[from] crate::stream::TimeoutError),
 }
 
 impl From<awc::error::SendRequestError> for UploadError {
diff --git a/src/main.rs b/src/main.rs
index 8286c17..3a3a868 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -7,7 +7,7 @@ use actix_web::{
 use awc::Client;
 use futures_util::{
     stream::{empty, once},
-    Stream, TryStreamExt,
+    Stream, StreamExt, TryStreamExt,
 };
 use once_cell::sync::Lazy;
 use std::{
@@ -15,7 +15,7 @@ use std::{
     future::ready,
     path::PathBuf,
     sync::atomic::{AtomicU64, Ordering},
-    time::SystemTime,
+    time::{Duration, SystemTime},
 };
 use tokio::{io::AsyncReadExt, sync::Semaphore};
 use tracing::{debug, info, instrument};
@@ -47,6 +47,8 @@ mod tmp_file;
 mod upload_manager;
 mod validate;
 
+use crate::stream::StreamTimeout;
+
 use self::{
     concurrent_processor::CancelSafeProcessor,
     config::{Configuration, ImageFormat, Operation},
@@ -486,6 +488,12 @@ where
     E: std::error::Error + 'static,
     actix_web::Error: From<E>,
 {
+    let stream = stream.timeout(Duration::from_secs(5)).map(|res| match res {
+        Ok(Ok(item)) => Ok(item),
+        Ok(Err(e)) => Err(actix_web::Error::from(e)),
+        Err(e) => Err(Error::from(e).into()),
+    });
+
     builder
         .insert_header(LastModified(modified.into()))
         .insert_header(CacheControl(vec![
diff --git a/src/store/object_store.rs b/src/store/object_store.rs
index f2bf76b..78f490e 100644
--- a/src/store/object_store.rs
+++ b/src/store/object_store.rs
@@ -2,18 +2,13 @@ use crate::{
     error::Error,
     repo::{Repo, SettingsRepo},
     store::Store,
-    stream::StreamTimeout,
 };
 use actix_web::web::Bytes;
-use futures_util::{Stream, StreamExt};
+use futures_util::{Stream, TryStreamExt};
 use s3::{
     client::Client, command::Command, creds::Credentials, request_trait::Request, Bucket, Region,
 };
-use std::{
-    pin::Pin,
-    string::FromUtf8Error,
-    time::{Duration, Instant},
-};
+use std::{pin::Pin, string::FromUtf8Error};
 use storage_path_generator::{Generator, Path};
 use tokio::io::{AsyncRead, AsyncWrite};
 use tracing::Instrument;
@@ -31,9 +26,6 @@ pub(crate) enum ObjectError {
     #[error("Failed to generate path")]
     PathGenerator(#[from] storage_path_generator::PathError),
 
-    #[error("Timeout")]
-    Elapsed,
-
     #[error("Failed to parse string")]
     Utf8(#[from] FromUtf8Error),
 
@@ -110,24 +102,19 @@ impl Store for ObjectStore {
             )
         });
 
-        let now = Instant::now();
-        let allotted = Duration::from_secs(5);
-
         let response = request_span
-            .in_scope(|| tokio::time::timeout(allotted, request.response()))
+            .in_scope(|| request.response())
             .instrument(request_span.clone())
             .await
-            .map_err(|_| ObjectError::Elapsed)?
             .map_err(ObjectError::from)?;
 
-        let allotted = allotted.saturating_sub(now.elapsed());
-
-        let stream = response.bytes_stream().timeout(allotted).map(|res| {
-            res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
-                .and_then(|res| res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)))
+        let stream = request_span.in_scope(|| {
+            response
+                .bytes_stream()
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
         });
 
-        Ok(request_span.in_scope(|| Box::pin(stream)))
+        Ok(Box::pin(stream))
     }
 
     #[tracing::instrument(skip(writer))]

From c0d8e0e8e3460ab04251b75c12667eca5b24057f Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Fri, 1 Apr 2022 11:51:46 -0500
Subject: [PATCH 24/37] Support multiple queues for job processor

---
 src/queue.rs              | 85 +++++----------------------------------
 src/queue/cleanup.rs      | 74 ++++++++++++++++++++++++++++++++++
 src/repo.rs               | 20 ++++-----
 src/repo/sled.rs          | 67 ++++++++++++++++++++++++------
 src/store/file_store.rs   |  2 +-
 src/store/object_store.rs |  2 +-
 src/upload_manager.rs     |  2 +-
 7 files changed, 152 insertions(+), 100 deletions(-)
 create mode 100644 src/queue/cleanup.rs

diff --git a/src/queue.rs b/src/queue.rs
index 6579ae2..9987b8b 100644
--- a/src/queue.rs
+++ b/src/queue.rs
@@ -1,9 +1,12 @@
 use crate::{
     error::Error,
     repo::{AliasRepo, HashRepo, IdentifierRepo, QueueRepo, Repo},
-    store::{Identifier, Store},
+    store::Store,
 };
-use tracing::{debug, error};
+
+mod cleanup;
+
+const CLEANUP_QUEUE: &str = "cleanup";
 
 #[derive(Debug, serde::Deserialize, serde::Serialize)]
 enum Job {
@@ -15,7 +18,7 @@ pub(crate) async fn queue_cleanup<R: QueueRepo>(repo: &R, hash: R::Bytes) -> Res
     let job = serde_json::to_vec(&Job::CleanupHash {
         hash: hash.as_ref().to_vec(),
     })?;
-    repo.push(job.into()).await?;
+    repo.push(CLEANUP_QUEUE, job.into()).await?;
     Ok(())
 }
 
@@ -50,7 +53,9 @@ where
     S: Store,
 {
     loop {
-        let bytes = repo.pop(worker_id.as_bytes().to_vec()).await?;
+        let bytes = repo
+            .pop(CLEANUP_QUEUE, worker_id.as_bytes().to_vec())
+            .await?;
 
         run_job(repo, store, bytes.as_ref()).await?;
     }
@@ -64,9 +69,9 @@ where
 {
     match serde_json::from_slice(job) {
         Ok(job) => match job {
-            Job::CleanupHash { hash } => cleanup_hash::<R, S>(repo, hash).await?,
+            Job::CleanupHash { hash } => cleanup::hash::<R, S>(repo, hash).await?,
             Job::CleanupIdentifier { identifier } => {
-                cleanup_identifier(repo, store, identifier).await?
+                cleanup::identifier(repo, store, identifier).await?
             }
         },
         Err(e) => {
@@ -76,71 +81,3 @@ where
 
     Ok(())
 }
-
-#[tracing::instrument(skip(repo, store))]
-async fn cleanup_identifier<R, S>(repo: &R, store: &S, identifier: Vec<u8>) -> Result<(), Error>
-where
-    R: QueueRepo + HashRepo + IdentifierRepo,
-    R::Bytes: Clone,
-    S: Store,
-{
-    let identifier = S::Identifier::from_bytes(identifier)?;
-
-    let mut errors = Vec::new();
-
-    debug!("Deleting {:?}", identifier);
-    if let Err(e) = store.remove(&identifier).await {
-        errors.push(e);
-    }
-
-    if let Err(e) = IdentifierRepo::cleanup(repo, &identifier).await {
-        errors.push(e);
-    }
-
-    if !errors.is_empty() {
-        let span = tracing::error_span!("Error deleting files");
-        span.in_scope(|| {
-            for error in errors {
-                error!("{}", error);
-            }
-        });
-    }
-
-    Ok(())
-}
-
-#[tracing::instrument(skip(repo))]
-async fn cleanup_hash<R, S>(repo: &R, hash: Vec<u8>) -> Result<(), Error>
-where
-    R: QueueRepo + AliasRepo + HashRepo + IdentifierRepo,
-    R::Bytes: Clone,
-    S: Store,
-{
-    let hash: R::Bytes = hash.into();
-
-    let aliases = repo.aliases(hash.clone()).await?;
-
-    if !aliases.is_empty() {
-        return Ok(());
-    }
-
-    let mut idents = repo
-        .variants::<S::Identifier>(hash.clone())
-        .await?
-        .into_iter()
-        .map(|(_, v)| v)
-        .collect::<Vec<_>>();
-    idents.push(repo.identifier(hash.clone()).await?);
-    idents.extend(repo.motion_identifier(hash.clone()).await?);
-
-    for identifier in idents {
-        if let Ok(identifier) = identifier.to_bytes() {
-            let job = serde_json::to_vec(&Job::CleanupIdentifier { identifier })?;
-            repo.push(job.into()).await?;
-        }
-    }
-
-    HashRepo::cleanup(repo, hash).await?;
-
-    Ok(())
-}
diff --git a/src/queue/cleanup.rs b/src/queue/cleanup.rs
new file mode 100644
index 0000000..9bed6c7
--- /dev/null
+++ b/src/queue/cleanup.rs
@@ -0,0 +1,74 @@
+use crate::{
+    error::Error,
+    queue::{Job, CLEANUP_QUEUE},
+    repo::{AliasRepo, HashRepo, IdentifierRepo, QueueRepo},
+    store::{Identifier, Store},
+};
+use tracing::error;
+
+#[tracing::instrument(skip(repo, store))]
+pub(super) async fn identifier<R, S>(repo: &R, store: &S, identifier: Vec<u8>) -> Result<(), Error>
+where
+    R: QueueRepo + HashRepo + IdentifierRepo,
+    R::Bytes: Clone,
+    S: Store,
+{
+    let identifier = S::Identifier::from_bytes(identifier)?;
+
+    let mut errors = Vec::new();
+
+    if let Err(e) = store.remove(&identifier).await {
+        errors.push(e);
+    }
+
+    if let Err(e) = IdentifierRepo::cleanup(repo, &identifier).await {
+        errors.push(e);
+    }
+
+    if !errors.is_empty() {
+        let span = tracing::error_span!("Error deleting files");
+        span.in_scope(|| {
+            for error in errors {
+                error!("{}", error);
+            }
+        });
+    }
+
+    Ok(())
+}
+
+#[tracing::instrument(skip(repo))]
+pub(super) async fn hash<R, S>(repo: &R, hash: Vec<u8>) -> Result<(), Error>
+where
+    R: QueueRepo + AliasRepo + HashRepo + IdentifierRepo,
+    R::Bytes: Clone,
+    S: Store,
+{
+    let hash: R::Bytes = hash.into();
+
+    let aliases = repo.aliases(hash.clone()).await?;
+
+    if !aliases.is_empty() {
+        return Ok(());
+    }
+
+    let mut idents = repo
+        .variants::<S::Identifier>(hash.clone())
+        .await?
+        .into_iter()
+        .map(|(_, v)| v)
+        .collect::<Vec<_>>();
+    idents.push(repo.identifier(hash.clone()).await?);
+    idents.extend(repo.motion_identifier(hash.clone()).await?);
+
+    for identifier in idents {
+        if let Ok(identifier) = identifier.to_bytes() {
+            let job = serde_json::to_vec(&Job::CleanupIdentifier { identifier })?;
+            repo.push(CLEANUP_QUEUE, job.into()).await?;
+        }
+    }
+
+    HashRepo::cleanup(repo, hash).await?;
+
+    Ok(())
+}
diff --git a/src/repo.rs b/src/repo.rs
index 27f3cf0..145c3e7 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -38,16 +38,16 @@ pub(crate) trait BaseRepo {
 pub(crate) trait QueueRepo: BaseRepo {
     async fn in_progress(&self, worker_id: Vec<u8>) -> Result<Option<Self::Bytes>, Error>;
 
-    async fn push(&self, job: Self::Bytes) -> Result<(), Error>;
+    async fn push(&self, queue: &'static str, job: Self::Bytes) -> Result<(), Error>;
 
-    async fn pop(&self, worker_id: Vec<u8>) -> Result<Self::Bytes, Error>;
+    async fn pop(&self, queue: &'static str, worker_id: Vec<u8>) -> Result<Self::Bytes, Error>;
 }
 
 #[async_trait::async_trait(?Send)]
 pub(crate) trait SettingsRepo: BaseRepo {
-    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Error>;
-    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Error>;
-    async fn remove(&self, key: &'static [u8]) -> Result<(), Error>;
+    async fn set(&self, key: &'static str, value: Self::Bytes) -> Result<(), Error>;
+    async fn get(&self, key: &'static str) -> Result<Option<Self::Bytes>, Error>;
+    async fn remove(&self, key: &'static str) -> Result<(), Error>;
 }
 
 #[async_trait::async_trait(?Send)]
@@ -186,9 +186,9 @@ impl Repo {
     }
 }
 
-const REPO_MIGRATION_O1: &[u8] = b"repo-migration-01";
-const STORE_MIGRATION_PROGRESS: &[u8] = b"store-migration-progress";
-const GENERATOR_KEY: &[u8] = b"last-path";
+const REPO_MIGRATION_O1: &str = "repo-migration-01";
+const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress";
+const GENERATOR_KEY: &str = "last-path";
 
 async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> color_eyre::Result<()>
 where
@@ -233,12 +233,12 @@ where
         let _ = repo.relate_details(&identifier.to_vec(), &details).await;
     }
 
-    if let Ok(Some(value)) = old.setting(STORE_MIGRATION_PROGRESS) {
+    if let Ok(Some(value)) = old.setting(STORE_MIGRATION_PROGRESS.as_bytes()) {
         repo.set(STORE_MIGRATION_PROGRESS, value.to_vec().into())
             .await?;
     }
 
-    if let Ok(Some(value)) = old.setting(GENERATOR_KEY) {
+    if let Ok(Some(value)) = old.setting(GENERATOR_KEY.as_bytes()) {
         repo.set(GENERATOR_KEY, value.to_vec().into()).await?;
     }
 
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index b34d008..c699158 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -8,7 +8,11 @@ use crate::{
 };
 use futures_util::Stream;
 use sled::{Db, IVec, Tree};
-use std::{pin::Pin, sync::Arc};
+use std::{
+    collections::HashMap,
+    pin::Pin,
+    sync::{Arc, RwLock},
+};
 use tokio::sync::Notify;
 
 use super::BaseRepo;
@@ -52,7 +56,7 @@ pub(crate) struct SledRepo {
     alias_delete_tokens: Tree,
     queue: Tree,
     in_progress_queue: Tree,
-    queue_notifier: Arc<Notify>,
+    queue_notifier: Arc<RwLock<HashMap<&'static str, Arc<Notify>>>>,
     db: Db,
 }
 
@@ -71,7 +75,7 @@ impl SledRepo {
             alias_delete_tokens: db.open_tree("pict-rs-alias-delete-tokens-tree")?,
             queue: db.open_tree("pict-rs-queue-tree")?,
             in_progress_queue: db.open_tree("pict-rs-in-progress-queue-tree")?,
-            queue_notifier: Arc::new(Notify::new()),
+            queue_notifier: Arc::new(RwLock::new(HashMap::new())),
             db,
         })
     }
@@ -89,16 +93,33 @@ impl QueueRepo for SledRepo {
         Ok(opt)
     }
 
-    async fn push(&self, job: Self::Bytes) -> Result<(), Error> {
+    async fn push(&self, queue: &'static str, job: Self::Bytes) -> Result<(), Error> {
         let id = self.db.generate_id()?;
-        b!(self.queue, queue.insert(id.to_be_bytes(), job));
-        self.queue_notifier.notify_one();
+        let mut key = queue.as_bytes().to_vec();
+        key.extend(id.to_be_bytes());
+
+        b!(self.queue, queue.insert(key, job));
+
+        if let Some(notifier) = self.queue_notifier.read().unwrap().get(&queue) {
+            notifier.notify_one();
+            return Ok(());
+        }
+
+        self.queue_notifier
+            .write()
+            .unwrap()
+            .entry(queue)
+            .or_insert_with(|| Arc::new(Notify::new()))
+            .notify_one();
+
         Ok(())
     }
 
-    async fn pop(&self, worker_id: Vec<u8>) -> Result<Self::Bytes, Error> {
-        let notify = Arc::clone(&self.queue_notifier);
-
+    async fn pop(
+        &self,
+        queue_name: &'static str,
+        worker_id: Vec<u8>,
+    ) -> Result<Self::Bytes, Error> {
         loop {
             let in_progress_queue = self.in_progress_queue.clone();
 
@@ -106,7 +127,10 @@ impl QueueRepo for SledRepo {
             let job = b!(self.queue, {
                 in_progress_queue.remove(&worker_id)?;
 
-                while let Some((key, job)) = queue.iter().find_map(Result::ok) {
+                while let Some((key, job)) = queue
+                    .scan_prefix(queue_name.as_bytes())
+                    .find_map(Result::ok)
+                {
                     in_progress_queue.insert(&worker_id, &job)?;
 
                     if queue.remove(key)?.is_some() {
@@ -123,6 +147,23 @@ impl QueueRepo for SledRepo {
                 return Ok(job);
             }
 
+            let opt = self
+                .queue_notifier
+                .read()
+                .unwrap()
+                .get(&queue_name)
+                .map(Arc::clone);
+
+            let notify = if let Some(notify) = opt {
+                notify
+            } else {
+                let mut guard = self.queue_notifier.write().unwrap();
+                let entry = guard
+                    .entry(queue_name)
+                    .or_insert_with(|| Arc::new(Notify::new()));
+                Arc::clone(entry)
+            };
+
             notify.notified().await
         }
     }
@@ -131,21 +172,21 @@ impl QueueRepo for SledRepo {
 #[async_trait::async_trait(?Send)]
 impl SettingsRepo for SledRepo {
     #[tracing::instrument(skip(value))]
-    async fn set(&self, key: &'static [u8], value: Self::Bytes) -> Result<(), Error> {
+    async fn set(&self, key: &'static str, value: Self::Bytes) -> Result<(), Error> {
         b!(self.settings, settings.insert(key, value));
 
         Ok(())
     }
 
     #[tracing::instrument]
-    async fn get(&self, key: &'static [u8]) -> Result<Option<Self::Bytes>, Error> {
+    async fn get(&self, key: &'static str) -> Result<Option<Self::Bytes>, Error> {
         let opt = b!(self.settings, settings.get(key));
 
         Ok(opt)
     }
 
     #[tracing::instrument]
-    async fn remove(&self, key: &'static [u8]) -> Result<(), Error> {
+    async fn remove(&self, key: &'static str) -> Result<(), Error> {
         b!(self.settings, settings.remove(key));
 
         Ok(())
diff --git a/src/store/file_store.rs b/src/store/file_store.rs
index 4145d98..dede6f3 100644
--- a/src/store/file_store.rs
+++ b/src/store/file_store.rs
@@ -20,7 +20,7 @@ pub(crate) use file_id::FileId;
 // - Settings Tree
 //   - last-path -> last generated path
 
-const GENERATOR_KEY: &[u8] = b"last-path";
+const GENERATOR_KEY: &str = "last-path";
 
 #[derive(Debug, thiserror::Error)]
 pub(crate) enum FileError {
diff --git a/src/store/object_store.rs b/src/store/object_store.rs
index 78f490e..41bf188 100644
--- a/src/store/object_store.rs
+++ b/src/store/object_store.rs
@@ -19,7 +19,7 @@ pub(crate) use object_id::ObjectId;
 // - Settings Tree
 //   - last-path -> last generated path
 
-const GENERATOR_KEY: &[u8] = b"last-path";
+const GENERATOR_KEY: &str = "last-path";
 
 #[derive(Debug, thiserror::Error)]
 pub(crate) enum ObjectError {
diff --git a/src/upload_manager.rs b/src/upload_manager.rs
index 0952b81..1d1f48c 100644
--- a/src/upload_manager.rs
+++ b/src/upload_manager.rs
@@ -20,7 +20,7 @@ mod session;
 
 pub(super) use session::UploadManagerSession;
 
-const STORE_MIGRATION_PROGRESS: &[u8] = b"store-migration-progress";
+const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress";
 
 #[derive(Clone)]
 pub(crate) struct UploadManager {

From 6ed592c432614b75f96b1c7ba3b1fde9762bde5a Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Fri, 1 Apr 2022 16:51:12 -0500
Subject: [PATCH 25/37] In Progress: process jobs

- Is this a good idea? it definitely will make interacting with pict-rs harder. Maybe it's best not to do this
---
 src/details.rs        |   2 +-
 src/magick.rs         |   4 +-
 src/main.rs           |  31 +++++----
 src/process.rs        |   2 +-
 src/queue.rs          | 144 ++++++++++++++++++++++++++++--------------
 src/queue/cleanup.rs  |  35 ++++++++--
 src/queue/process.rs  |  87 +++++++++++++++++++++++++
 src/repo.rs           |  42 ++++++++++++
 src/store.rs          |  54 +++++++++++++++-
 src/upload_manager.rs |   2 +-
 10 files changed, 329 insertions(+), 74 deletions(-)
 create mode 100644 src/queue/process.rs

diff --git a/src/details.rs b/src/details.rs
index 1a9e9b8..37ddc50 100644
--- a/src/details.rs
+++ b/src/details.rs
@@ -30,7 +30,7 @@ impl Details {
     }
 
     #[tracing::instrument("Details from store")]
-    pub(crate) async fn from_store<S: Store>(
+    pub(crate) async fn from_store<S: Store + 'static>(
         store: S,
         identifier: S::Identifier,
         expected_format: Option<ValidInputType>,
diff --git a/src/magick.rs b/src/magick.rs
index 8efbf23..e0c1c0d 100644
--- a/src/magick.rs
+++ b/src/magick.rs
@@ -140,7 +140,7 @@ pub(crate) async fn details_bytes(
 }
 
 #[tracing::instrument(skip(store))]
-pub(crate) async fn details_store<S: Store>(
+pub(crate) async fn details_store<S: Store + 'static>(
     store: S,
     identifier: S::Identifier,
     hint: Option<ValidInputType>,
@@ -255,7 +255,7 @@ pub(crate) async fn input_type_bytes(input: Bytes) -> Result<ValidInputType, Err
 }
 
 #[instrument(name = "Spawning process command")]
-pub(crate) fn process_image_store_read<S: Store>(
+pub(crate) fn process_image_store_read<S: Store + 'static>(
     store: S,
     identifier: S::Identifier,
     args: Vec<String>,
diff --git a/src/main.rs b/src/main.rs
index 3a3a868..f243f58 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -11,7 +11,6 @@ use futures_util::{
 };
 use once_cell::sync::Lazy;
 use std::{
-    collections::BTreeSet,
     future::ready,
     path::PathBuf,
     sync::atomic::{AtomicU64, Ordering},
@@ -142,7 +141,7 @@ struct UrlQuery {
 
 /// download an image from a URL
 #[instrument(name = "Downloading file", skip(client, manager))]
-async fn download<S: Store>(
+async fn download<S: Store + 'static>(
     client: web::Data<Client>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
@@ -214,7 +213,6 @@ type ProcessQuery = Vec<(String, String)>;
 fn prepare_process(
     query: web::Query<ProcessQuery>,
     ext: &str,
-    filters: &BTreeSet<String>,
 ) -> Result<(ImageFormat, Alias, PathBuf, Vec<String>), Error> {
     let (alias, operations) =
         query
@@ -237,7 +235,7 @@ fn prepare_process(
 
     let operations = operations
         .into_iter()
-        .filter(|(k, _)| filters.contains(&k.to_lowercase()))
+        .filter(|(k, _)| CONFIG.media.filters.contains(&k.to_lowercase()))
         .collect::<Vec<_>>();
 
     let format = ext
@@ -251,14 +249,13 @@ fn prepare_process(
     Ok((format, alias, thumbnail_path, thumbnail_args))
 }
 
-#[instrument(name = "Fetching derived details", skip(manager, filters))]
+#[instrument(name = "Fetching derived details", skip(manager))]
 async fn process_details<S: Store>(
     query: web::Query<ProcessQuery>,
     ext: web::Path<String>,
     manager: web::Data<UploadManager>,
-    filters: web::Data<BTreeSet<String>>,
 ) -> Result<HttpResponse, Error> {
-    let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
+    let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str())?;
 
     let identifier = manager
         .variant_identifier::<S>(&alias, &thumbnail_path)
@@ -273,17 +270,15 @@ async fn process_details<S: Store>(
 }
 
 /// Process files
-#[instrument(name = "Serving processed image", skip(manager, filters))]
+#[instrument(name = "Serving processed image", skip(manager))]
 async fn process<S: Store + 'static>(
     range: Option<web::Header<Range>>,
     query: web::Query<ProcessQuery>,
     ext: web::Path<String>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
-    filters: web::Data<BTreeSet<String>>,
 ) -> Result<HttpResponse, Error> {
-    let (format, alias, thumbnail_path, thumbnail_args) =
-        prepare_process(query, ext.as_str(), &filters)?;
+    let (format, alias, thumbnail_path, thumbnail_args) = prepare_process(query, ext.as_str())?;
 
     let identifier_opt = manager
         .variant_identifier::<S>(&alias, &thumbnail_path)
@@ -376,7 +371,7 @@ async fn process<S: Store + 'static>(
 
 /// Fetch file details
 #[instrument(name = "Fetching details", skip(manager))]
-async fn details<S: Store>(
+async fn details<S: Store + 'static>(
     alias: web::Path<String>,
     manager: web::Data<UploadManager>,
     store: web::Data<S>,
@@ -402,7 +397,7 @@ async fn details<S: Store>(
 
 /// Serve files
 #[instrument(name = "Serving file", skip(manager))]
-async fn serve<S: Store>(
+async fn serve<S: Store + 'static>(
     range: Option<web::Header<Range>>,
     alias: web::Path<String>,
     manager: web::Data<UploadManager>,
@@ -426,7 +421,7 @@ async fn serve<S: Store>(
     ranged_file_resp(&**store, identifier, range, details).await
 }
 
-async fn ranged_file_resp<S: Store>(
+async fn ranged_file_resp<S: Store + 'static>(
     store: &S,
     identifier: S::Identifier,
     range: Option<web::Header<Range>>,
@@ -652,7 +647,12 @@ async fn launch<S: Store + Clone + 'static>(
         let manager = manager.clone();
         let store = store.clone();
 
-        actix_rt::spawn(queue::process_jobs(
+        actix_rt::spawn(queue::process_cleanup(
+            manager.repo().clone(),
+            store.clone(),
+            next_worker_id(),
+        ));
+        actix_rt::spawn(queue::process_images(
             manager.repo().clone(),
             store.clone(),
             next_worker_id(),
@@ -664,7 +664,6 @@ async fn launch<S: Store + Clone + 'static>(
             .app_data(web::Data::new(store))
             .app_data(web::Data::new(manager))
             .app_data(web::Data::new(build_client()))
-            .app_data(web::Data::new(CONFIG.media.filters.clone()))
             .service(
                 web::scope("/image")
                     .service(
diff --git a/src/process.rs b/src/process.rs
index 057d809..4d5b628 100644
--- a/src/process.rs
+++ b/src/process.rs
@@ -144,7 +144,7 @@ impl Process {
         })
     }
 
-    pub(crate) fn store_read<S: Store>(
+    pub(crate) fn store_read<S: Store + 'static>(
         mut self,
         store: S,
         identifier: S::Identifier,
diff --git a/src/queue.rs b/src/queue.rs
index 9987b8b..9eceb7c 100644
--- a/src/queue.rs
+++ b/src/queue.rs
@@ -1,83 +1,135 @@
 use crate::{
+    config::ImageFormat,
     error::Error,
-    repo::{AliasRepo, HashRepo, IdentifierRepo, QueueRepo, Repo},
+    repo::{Alias, AliasRepo, HashRepo, IdentifierRepo, QueueRepo, Repo},
+    serde_str::Serde,
     store::Store,
 };
+use std::{future::Future, path::PathBuf, pin::Pin};
+use uuid::Uuid;
 
 mod cleanup;
+mod process;
 
 const CLEANUP_QUEUE: &str = "cleanup";
+const PROCESS_QUEUE: &str = "process";
 
 #[derive(Debug, serde::Deserialize, serde::Serialize)]
-enum Job {
+enum Cleanup {
     CleanupHash { hash: Vec<u8> },
     CleanupIdentifier { identifier: Vec<u8> },
 }
 
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+enum Process {
+    Ingest {
+        identifier: Vec<u8>,
+        upload_id: Uuid,
+        declared_alias: Option<Serde<Alias>>,
+        should_validate: bool,
+    },
+    Generate {
+        target_format: ImageFormat,
+        source: Serde<Alias>,
+        process_path: PathBuf,
+        process_args: Vec<String>,
+    },
+}
+
 pub(crate) async fn queue_cleanup<R: QueueRepo>(repo: &R, hash: R::Bytes) -> Result<(), Error> {
-    let job = serde_json::to_vec(&Job::CleanupHash {
+    let job = serde_json::to_vec(&Cleanup::CleanupHash {
         hash: hash.as_ref().to_vec(),
     })?;
     repo.push(CLEANUP_QUEUE, job.into()).await?;
     Ok(())
 }
 
-pub(crate) async fn process_jobs<S: Store>(repo: Repo, store: S, worker_id: String) {
+pub(crate) async fn queue_ingest<R: QueueRepo>(
+    repo: &R,
+    identifier: Vec<u8>,
+    upload_id: Uuid,
+    declared_alias: Option<Alias>,
+    should_validate: bool,
+) -> Result<(), Error> {
+    let job = serde_json::to_vec(&Process::Ingest {
+        identifier,
+        declared_alias: declared_alias.map(Serde::new),
+        upload_id,
+        should_validate,
+    })?;
+    repo.push(PROCESS_QUEUE, job.into()).await?;
+    Ok(())
+}
+
+pub(crate) async fn queue_generate<R: QueueRepo>(
+    repo: &R,
+    target_format: ImageFormat,
+    source: Alias,
+    process_path: PathBuf,
+    process_args: Vec<String>,
+) -> Result<(), Error> {
+    let job = serde_json::to_vec(&Process::Generate {
+        target_format,
+        source: Serde::new(source),
+        process_path,
+        process_args,
+    })?;
+    repo.push(PROCESS_QUEUE, job.into()).await?;
+    Ok(())
+}
+
+pub(crate) async fn process_cleanup<S: Store>(repo: Repo, store: S, worker_id: String) {
     match repo {
-        Repo::Sled(ref repo) => {
-            if let Ok(Some(job)) = repo.in_progress(worker_id.as_bytes().to_vec()).await {
-                if let Err(e) = run_job(repo, &store, &job).await {
-                    tracing::warn!("Failed to run previously dropped job: {}", e);
-                    tracing::warn!("{:?}", e);
-                }
-            }
-            loop {
-                let res = job_loop(repo, &store, worker_id.clone()).await;
-
-                if let Err(e) = res {
-                    tracing::warn!("Error processing jobs: {}", e);
-                    tracing::warn!("{:?}", e);
-                    continue;
-                }
-
-                break;
-            }
-        }
+        Repo::Sled(repo) => process_jobs(&repo, &store, worker_id, cleanup::perform).await,
     }
 }
 
-async fn job_loop<R, S>(repo: &R, store: &S, worker_id: String) -> Result<(), Error>
+pub(crate) async fn process_images<S: Store>(repo: Repo, store: S, worker_id: String) {
+    match repo {
+        Repo::Sled(repo) => process_jobs(&repo, &store, worker_id, process::perform).await,
+    }
+}
+
+type LocalBoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + 'a>>;
+
+async fn process_jobs<R, S, F>(repo: &R, store: &S, worker_id: String, callback: F)
 where
     R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
     R::Bytes: Clone,
     S: Store,
+    for<'a> F: Fn(&'a R, &'a S, &'a [u8]) -> LocalBoxFuture<'a, Result<(), Error>> + Copy,
+{
+    if let Ok(Some(job)) = repo.in_progress(worker_id.as_bytes().to_vec()).await {
+        if let Err(e) = (callback)(repo, store, job.as_ref()).await {
+            tracing::warn!("Failed to run previously dropped job: {}", e);
+            tracing::warn!("{:?}", e);
+        }
+    }
+    loop {
+        let res = job_loop(repo, store, worker_id.clone(), callback).await;
+
+        if let Err(e) = res {
+            tracing::warn!("Error processing jobs: {}", e);
+            tracing::warn!("{:?}", e);
+            continue;
+        }
+
+        break;
+    }
+}
+
+async fn job_loop<R, S, F>(repo: &R, store: &S, worker_id: String, callback: F) -> Result<(), Error>
+where
+    R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
+    R::Bytes: Clone,
+    S: Store,
+    for<'a> F: Fn(&'a R, &'a S, &'a [u8]) -> LocalBoxFuture<'a, Result<(), Error>> + Copy,
 {
     loop {
         let bytes = repo
             .pop(CLEANUP_QUEUE, worker_id.as_bytes().to_vec())
             .await?;
 
-        run_job(repo, store, bytes.as_ref()).await?;
+        (callback)(repo, store, bytes.as_ref()).await?;
     }
 }
-
-async fn run_job<R, S>(repo: &R, store: &S, job: &[u8]) -> Result<(), Error>
-where
-    R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
-    R::Bytes: Clone,
-    S: Store,
-{
-    match serde_json::from_slice(job) {
-        Ok(job) => match job {
-            Job::CleanupHash { hash } => cleanup::hash::<R, S>(repo, hash).await?,
-            Job::CleanupIdentifier { identifier } => {
-                cleanup::identifier(repo, store, identifier).await?
-            }
-        },
-        Err(e) => {
-            tracing::warn!("Invalid job: {}", e);
-        }
-    }
-
-    Ok(())
-}
diff --git a/src/queue/cleanup.rs b/src/queue/cleanup.rs
index 9bed6c7..1abec81 100644
--- a/src/queue/cleanup.rs
+++ b/src/queue/cleanup.rs
@@ -1,13 +1,40 @@
 use crate::{
     error::Error,
-    queue::{Job, CLEANUP_QUEUE},
+    queue::{Cleanup, LocalBoxFuture, CLEANUP_QUEUE},
     repo::{AliasRepo, HashRepo, IdentifierRepo, QueueRepo},
     store::{Identifier, Store},
 };
 use tracing::error;
 
+pub(super) fn perform<'a, R, S>(
+    repo: &'a R,
+    store: &'a S,
+    job: &'a [u8],
+) -> LocalBoxFuture<'a, Result<(), Error>>
+where
+    R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
+    R::Bytes: Clone,
+    S: Store,
+{
+    Box::pin(async move {
+        match serde_json::from_slice(job) {
+            Ok(job) => match job {
+                Cleanup::CleanupHash { hash: in_hash } => hash::<R, S>(repo, in_hash).await?,
+                Cleanup::CleanupIdentifier {
+                    identifier: in_identifier,
+                } => identifier(repo, &store, in_identifier).await?,
+            },
+            Err(e) => {
+                tracing::warn!("Invalid job: {}", e);
+            }
+        }
+
+        Ok(())
+    })
+}
+
 #[tracing::instrument(skip(repo, store))]
-pub(super) async fn identifier<R, S>(repo: &R, store: &S, identifier: Vec<u8>) -> Result<(), Error>
+async fn identifier<R, S>(repo: &R, store: &S, identifier: Vec<u8>) -> Result<(), Error>
 where
     R: QueueRepo + HashRepo + IdentifierRepo,
     R::Bytes: Clone,
@@ -38,7 +65,7 @@ where
 }
 
 #[tracing::instrument(skip(repo))]
-pub(super) async fn hash<R, S>(repo: &R, hash: Vec<u8>) -> Result<(), Error>
+async fn hash<R, S>(repo: &R, hash: Vec<u8>) -> Result<(), Error>
 where
     R: QueueRepo + AliasRepo + HashRepo + IdentifierRepo,
     R::Bytes: Clone,
@@ -63,7 +90,7 @@ where
 
     for identifier in idents {
         if let Ok(identifier) = identifier.to_bytes() {
-            let job = serde_json::to_vec(&Job::CleanupIdentifier { identifier })?;
+            let job = serde_json::to_vec(&Cleanup::CleanupIdentifier { identifier })?;
             repo.push(CLEANUP_QUEUE, job.into()).await?;
         }
     }
diff --git a/src/queue/process.rs b/src/queue/process.rs
new file mode 100644
index 0000000..261f095
--- /dev/null
+++ b/src/queue/process.rs
@@ -0,0 +1,87 @@
+use crate::{
+    config::ImageFormat,
+    error::Error,
+    queue::{LocalBoxFuture, Process},
+    repo::{Alias, AliasRepo, HashRepo, IdentifierRepo, QueueRepo},
+    serde_str::Serde,
+    store::Store,
+};
+use std::path::PathBuf;
+use uuid::Uuid;
+
+pub(super) fn perform<'a, R, S>(
+    repo: &'a R,
+    store: &'a S,
+    job: &'a [u8],
+) -> LocalBoxFuture<'a, Result<(), Error>>
+where
+    R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
+    R::Bytes: Clone,
+    S: Store,
+{
+    Box::pin(async move {
+        match serde_json::from_slice(job) {
+            Ok(job) => match job {
+                Process::Ingest {
+                    identifier,
+                    upload_id,
+                    declared_alias,
+                    should_validate,
+                } => {
+                    ingest(
+                        repo,
+                        store,
+                        identifier,
+                        upload_id,
+                        declared_alias.map(Serde::into_inner),
+                        should_validate,
+                    )
+                    .await?
+                }
+                Process::Generate {
+                    target_format,
+                    source,
+                    process_path,
+                    process_args,
+                } => {
+                    generate(
+                        repo,
+                        store,
+                        target_format,
+                        Serde::into_inner(source),
+                        process_path,
+                        process_args,
+                    )
+                    .await?
+                }
+            },
+            Err(e) => {
+                tracing::warn!("Invalid job: {}", e);
+            }
+        }
+
+        Ok(())
+    })
+}
+
+async fn ingest<R, S>(
+    repo: &R,
+    store: &S,
+    identifier: Vec<u8>,
+    upload_id: Uuid,
+    declared_alias: Option<Alias>,
+    should_validate: bool,
+) -> Result<(), Error> {
+    unimplemented!("do this")
+}
+
+async fn generate<R, S>(
+    repo: &R,
+    store: &S,
+    target_format: ImageFormat,
+    source: Alias,
+    process_path: PathBuf,
+    process_args: Vec<String>,
+) -> Result<(), Error> {
+    unimplemented!("do this")
+}
diff --git a/src/repo.rs b/src/repo.rs
index 145c3e7..e77a482 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -30,10 +30,29 @@ pub(crate) struct DeleteToken {
 
 pub(crate) struct AlreadyExists;
 
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub(crate) struct UploadId {
+    id: Uuid,
+}
+
+pub(crate) enum UploadResult {
+    Success { alias: Alias, token: DeleteToken },
+    Failure { message: String },
+}
+
 pub(crate) trait BaseRepo {
     type Bytes: AsRef<[u8]> + From<Vec<u8>>;
 }
 
+#[async_trait::async_trait(?Send)]
+pub(crate) trait UploadRepo: BaseRepo {
+    async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, Error>;
+
+    async fn claim(&self, upload_id: UploadId) -> Result<(), Error>;
+
+    async fn complete(&self, upload_id: UploadId, result: UploadResult) -> Result<(), Error>;
+}
+
 #[async_trait::async_trait(?Send)]
 pub(crate) trait QueueRepo: BaseRepo {
     async fn in_progress(&self, worker_id: Vec<u8>) -> Result<Option<Self::Bytes>, Error>;
@@ -362,6 +381,21 @@ impl DeleteToken {
     }
 }
 
+impl UploadId {
+    pub(crate) fn generate() -> Self {
+        Self { id: Uuid::new_v4() }
+    }
+
+    pub(crate) fn as_bytes(&self) -> &[u8] {
+        &self.id.as_bytes()[..]
+    }
+
+    pub(crate) fn from_bytes(&self, bytes: &[u8]) -> Option<Self> {
+        let id = Uuid::from_slice(bytes).ok()?;
+        Some(Self { id })
+    }
+}
+
 impl std::fmt::Display for MaybeUuid {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -377,6 +411,14 @@ impl std::fmt::Display for DeleteToken {
     }
 }
 
+impl std::str::FromStr for Alias {
+    type Err = std::convert::Infallible;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Alias::from_existing(s))
+    }
+}
+
 impl std::fmt::Display for Alias {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         if let Some(ext) = self.extension() {
diff --git a/src/store.rs b/src/store.rs
index fea8374..4f8f0b0 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -16,9 +16,9 @@ pub(crate) trait Identifier: Send + Sync + Clone + Debug {
 }
 
 #[async_trait::async_trait(?Send)]
-pub(crate) trait Store: Send + Sync + Clone + Debug + 'static {
-    type Identifier: Identifier;
-    type Stream: Stream<Item = std::io::Result<Bytes>>;
+pub(crate) trait Store: Send + Sync + Clone + Debug {
+    type Identifier: Identifier + 'static;
+    type Stream: Stream<Item = std::io::Result<Bytes>> + 'static;
 
     async fn save_async_read<Reader>(&self, reader: &mut Reader) -> Result<Self::Identifier, Error>
     where
@@ -45,3 +45,51 @@ pub(crate) trait Store: Send + Sync + Clone + Debug + 'static {
 
     async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error>;
 }
+
+#[async_trait::async_trait(?Send)]
+impl<'a, T> Store for &'a T
+where
+    T: Store,
+{
+    type Identifier = T::Identifier;
+    type Stream = T::Stream;
+
+    async fn save_async_read<Reader>(&self, reader: &mut Reader) -> Result<Self::Identifier, Error>
+    where
+        Reader: AsyncRead + Unpin,
+    {
+        T::save_async_read(self, reader).await
+    }
+
+    async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Error> {
+        T::save_bytes(self, bytes).await
+    }
+
+    async fn to_stream(
+        &self,
+        identifier: &Self::Identifier,
+        from_start: Option<u64>,
+        len: Option<u64>,
+    ) -> Result<Self::Stream, Error> {
+        T::to_stream(self, identifier, from_start, len).await
+    }
+
+    async fn read_into<Writer>(
+        &self,
+        identifier: &Self::Identifier,
+        writer: &mut Writer,
+    ) -> Result<(), std::io::Error>
+    where
+        Writer: AsyncWrite + Send + Unpin,
+    {
+        T::read_into(self, identifier, writer).await
+    }
+
+    async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Error> {
+        T::len(self, identifier).await
+    }
+
+    async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Error> {
+        T::remove(self, identifier).await
+    }
+}
diff --git a/src/upload_manager.rs b/src/upload_manager.rs
index 1d1f48c..a5c916c 100644
--- a/src/upload_manager.rs
+++ b/src/upload_manager.rs
@@ -61,7 +61,7 @@ impl UploadManager {
         }
     }
 
-    pub(crate) async fn still_identifier_from_alias<S: Store + Clone>(
+    pub(crate) async fn still_identifier_from_alias<S: Store + Clone + 'static>(
         &self,
         store: S,
         alias: &Alias,

From 09f53b9ce68d224d16f993280bc29fc251a0f0fa Mon Sep 17 00:00:00 2001
From: "Aode (Lion)" <asonix@asonix.dog>
Date: Sat, 2 Apr 2022 16:44:03 -0500
Subject: [PATCH 26/37] Move away from UploadManager to direct repo & store
 actions

---
 src/ingest.rs                            | 218 +++++++++++++
 src/{upload_manager => ingest}/hasher.rs |  30 +-
 src/main.rs                              | 372 ++++++++++++++---------
 src/queue.rs                             |  60 +++-
 src/queue/cleanup.rs                     |  57 +++-
 src/queue/process.rs                     |  63 +++-
 src/repo.rs                              |  58 +++-
 src/repo/sled.rs                         |  23 +-
 src/upload_manager.rs                    | 366 ----------------------
 src/upload_manager/session.rs            | 266 ----------------
 10 files changed, 666 insertions(+), 847 deletions(-)
 create mode 100644 src/ingest.rs
 rename src/{upload_manager => ingest}/hasher.rs (79%)
 delete mode 100644 src/upload_manager.rs
 delete mode 100644 src/upload_manager/session.rs

diff --git a/src/ingest.rs b/src/ingest.rs
new file mode 100644
index 0000000..bae7bb6
--- /dev/null
+++ b/src/ingest.rs
@@ -0,0 +1,218 @@
+use crate::{
+    error::{Error, UploadError},
+    magick::ValidInputType,
+    repo::{Alias, AliasRepo, DeleteToken, FullRepo, HashRepo},
+    store::Store,
+    CONFIG,
+};
+use actix_web::web::{Bytes, BytesMut};
+use futures_util::{Stream, StreamExt};
+use once_cell::sync::Lazy;
+use sha2::{Digest, Sha256};
+use tokio::sync::Semaphore;
+use tracing::debug;
+
+mod hasher;
+use hasher::Hasher;
+
+static PROCESS_SEMAPHORE: Lazy<Semaphore> = Lazy::new(|| Semaphore::new(num_cpus::get()));
+
+pub(crate) struct Session<R, S>
+where
+    R: FullRepo + 'static,
+    S: Store,
+{
+    repo: R,
+    hash: Option<Vec<u8>>,
+    alias: Option<Alias>,
+    identifier: Option<S::Identifier>,
+}
+
+pub(crate) async fn ingest<R, S>(
+    repo: &R,
+    store: &S,
+    stream: impl Stream<Item = Result<Bytes, Error>>,
+    declared_alias: Option<Alias>,
+    should_validate: bool,
+) -> Result<Session<R, S>, Error>
+where
+    R: FullRepo + 'static,
+    S: Store,
+{
+    let permit = PROCESS_SEMAPHORE.acquire().await;
+
+    let mut bytes_mut = BytesMut::new();
+
+    futures_util::pin_mut!(stream);
+
+    debug!("Reading stream to memory");
+    while let Some(res) = stream.next().await {
+        let bytes = res?;
+        bytes_mut.extend_from_slice(&bytes);
+    }
+
+    debug!("Validating bytes");
+    let (input_type, validated_reader) = crate::validate::validate_image_bytes(
+        bytes_mut.freeze(),
+        CONFIG.media.format,
+        CONFIG.media.enable_silent_video,
+        should_validate,
+    )
+    .await?;
+
+    let mut hasher_reader = Hasher::new(validated_reader, Sha256::new());
+
+    let identifier = store.save_async_read(&mut hasher_reader).await?;
+
+    drop(permit);
+
+    let mut session = Session {
+        repo: repo.clone(),
+        hash: None,
+        alias: None,
+        identifier: Some(identifier.clone()),
+    };
+
+    let hash = hasher_reader.finalize_reset().await?;
+
+    session.hash = Some(hash.clone());
+
+    debug!("Saving upload");
+
+    save_upload(repo, store, &hash, &identifier).await?;
+
+    debug!("Adding alias");
+
+    if let Some(alias) = declared_alias {
+        session.add_existing_alias(&hash, alias).await?
+    } else {
+        session.create_alias(&hash, input_type).await?;
+    }
+
+    Ok(session)
+}
+
+async fn save_upload<R, S>(
+    repo: &R,
+    store: &S,
+    hash: &[u8],
+    identifier: &S::Identifier,
+) -> Result<(), Error>
+where
+    S: Store,
+    R: FullRepo,
+{
+    if HashRepo::create(repo, hash.to_vec().into()).await?.is_err() {
+        store.remove(identifier).await?;
+        return Ok(());
+    }
+
+    repo.relate_identifier(hash.to_vec().into(), identifier)
+        .await?;
+
+    Ok(())
+}
+
+impl<R, S> Session<R, S>
+where
+    R: FullRepo + 'static,
+    S: Store,
+{
+    pub(crate) fn disarm(&mut self) {
+        let _ = self.alias.take();
+        let _ = self.identifier.take();
+    }
+
+    pub(crate) fn alias(&self) -> Option<&Alias> {
+        self.alias.as_ref()
+    }
+
+    pub(crate) async fn delete_token(&self) -> Result<DeleteToken, Error> {
+        let alias = self.alias.clone().ok_or(UploadError::MissingAlias)?;
+
+        debug!("Generating delete token");
+        let delete_token = DeleteToken::generate();
+
+        debug!("Saving delete token");
+        let res = self.repo.relate_delete_token(&alias, &delete_token).await?;
+
+        if res.is_err() {
+            let delete_token = self.repo.delete_token(&alias).await?;
+            debug!("Returning existing delete token, {:?}", delete_token);
+            return Ok(delete_token);
+        }
+
+        debug!("Returning new delete token, {:?}", delete_token);
+        Ok(delete_token)
+    }
+
+    async fn add_existing_alias(&mut self, hash: &[u8], alias: Alias) -> Result<(), Error> {
+        AliasRepo::create(&self.repo, &alias)
+            .await?
+            .map_err(|_| UploadError::DuplicateAlias)?;
+
+        self.alias = Some(alias.clone());
+
+        self.repo.relate_hash(&alias, hash.to_vec().into()).await?;
+        self.repo.relate_alias(hash.to_vec().into(), &alias).await?;
+
+        Ok(())
+    }
+
+    async fn create_alias(&mut self, hash: &[u8], input_type: ValidInputType) -> Result<(), Error> {
+        debug!("Alias gen loop");
+
+        loop {
+            let alias = Alias::generate(input_type.as_ext().to_string());
+
+            if AliasRepo::create(&self.repo, &alias).await?.is_ok() {
+                self.alias = Some(alias.clone());
+
+                self.repo.relate_hash(&alias, hash.to_vec().into()).await?;
+                self.repo.relate_alias(hash.to_vec().into(), &alias).await?;
+
+                return Ok(());
+            }
+
+            debug!("Alias exists, regenerating");
+        }
+    }
+}
+
+impl<R, S> Drop for Session<R, S>
+where
+    R: FullRepo + 'static,
+    S: Store,
+{
+    fn drop(&mut self) {
+        if let Some(hash) = self.hash.take() {
+            let repo = self.repo.clone();
+            actix_rt::spawn(async move {
+                let _ = crate::queue::cleanup_hash(&repo, hash.into()).await;
+            });
+        }
+
+        if let Some(alias) = self.alias.take() {
+            let repo = self.repo.clone();
+
+            actix_rt::spawn(async move {
+                if let Ok(token) = repo.delete_token(&alias).await {
+                    let _ = crate::queue::cleanup_alias(&repo, alias, token).await;
+                } else {
+                    let token = DeleteToken::generate();
+                    if let Ok(Ok(())) = repo.relate_delete_token(&alias, &token).await {
+                        let _ = crate::queue::cleanup_alias(&repo, alias, token).await;
+                    }
+                }
+            });
+        }
+
+        if let Some(identifier) = self.identifier.take() {
+            let repo = self.repo.clone();
+
+            actix_rt::spawn(async move {
+                let _ = crate::queue::cleanup_identifier(&repo, identifier).await;
+            });
+        }
+    }
+}
diff --git a/src/upload_manager/hasher.rs b/src/ingest/hasher.rs
similarity index 79%
rename from src/upload_manager/hasher.rs
rename to src/ingest/hasher.rs
index 0ae9a8b..e1a1551 100644
--- a/src/upload_manager/hasher.rs
+++ b/src/ingest/hasher.rs
@@ -16,10 +16,6 @@ pin_project_lite::pin_project! {
     }
 }
 
-pub(super) struct Hash {
-    inner: Vec<u8>,
-}
-
 impl<I, D> Hasher<I, D>
 where
     D: Digest + FixedOutputReset + Send + 'static,
@@ -31,27 +27,13 @@ where
         }
     }
 
-    pub(super) async fn finalize_reset(self) -> Result<Hash, Error> {
+    pub(super) async fn finalize_reset(self) -> Result<Vec<u8>, Error> {
         let mut hasher = self.hasher;
-        let hash = web::block(move || Hash::new(hasher.finalize_reset().to_vec())).await?;
+        let hash = web::block(move || hasher.finalize_reset().to_vec()).await?;
         Ok(hash)
     }
 }
 
-impl Hash {
-    fn new(inner: Vec<u8>) -> Self {
-        Hash { inner }
-    }
-
-    pub(super) fn as_slice(&self) -> &[u8] {
-        &self.inner
-    }
-
-    pub(super) fn into_inner(self) -> Vec<u8> {
-        self.inner
-    }
-}
-
 impl<I, D> AsyncRead for Hasher<I, D>
 where
     I: AsyncRead,
@@ -77,12 +59,6 @@ where
     }
 }
 
-impl std::fmt::Debug for Hash {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "{}", base64::encode(&self.inner))
-    }
-}
-
 #[cfg(test)]
 mod test {
     use super::Hasher;
@@ -127,6 +103,6 @@ mod test {
         hasher.update(vec);
         let correct_hash = hasher.finalize_reset().to_vec();
 
-        assert_eq!(hash.inner, correct_hash);
+        assert_eq!(hash, correct_hash);
     }
 }
diff --git a/src/main.rs b/src/main.rs
index f243f58..15f635a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -30,6 +30,7 @@ mod error;
 mod exiftool;
 mod ffmpeg;
 mod file;
+mod ingest;
 mod init_tracing;
 mod magick;
 mod middleware;
@@ -43,26 +44,24 @@ mod serde_str;
 mod store;
 mod stream;
 mod tmp_file;
-mod upload_manager;
 mod validate;
 
-use crate::stream::StreamTimeout;
-
 use self::{
     concurrent_processor::CancelSafeProcessor,
     config::{Configuration, ImageFormat, Operation},
     details::Details,
     either::Either,
     error::{Error, UploadError},
+    ffmpeg::{InputFormat, ThumbnailFormat},
+    ingest::Session,
     init_tracing::init_tracing,
     magick::details_hint,
     middleware::{Deadline, Internal},
     migrate::LatestDb,
-    repo::{Alias, DeleteToken, Repo},
+    repo::{Alias, DeleteToken, FullRepo, HashRepo, IdentifierRepo, Repo, SettingsRepo},
     serde_str::Serde,
-    store::{file_store::FileStore, object_store::ObjectStore, Store},
-    stream::StreamLimit,
-    upload_manager::{UploadManager, UploadManagerSession},
+    store::{file_store::FileStore, object_store::ObjectStore, Identifier, Store},
+    stream::{StreamLimit, StreamTimeout},
 };
 
 const MEGABYTES: usize = 1024 * 1024;
@@ -78,10 +77,10 @@ static PROCESS_SEMAPHORE: Lazy<Semaphore> =
     Lazy::new(|| Semaphore::new(num_cpus::get().saturating_sub(1).max(1)));
 
 /// Handle responding to succesful uploads
-#[instrument(name = "Uploaded files", skip(value, manager))]
-async fn upload<S: Store>(
-    value: Value<UploadManagerSession<S>>,
-    manager: web::Data<UploadManager>,
+#[instrument(name = "Uploaded files", skip(value))]
+async fn upload<R: FullRepo, S: Store + 'static>(
+    value: Value<Session<R, S>>,
+    repo: web::Data<R>,
     store: web::Data<S>,
 ) -> Result<HttpResponse, Error> {
     let images = value
@@ -100,8 +99,8 @@ async fn upload<S: Store>(
             info!("Uploaded {} as {:?}", image.filename, alias);
             let delete_token = image.result.delete_token().await?;
 
-            let identifier = manager.identifier_from_alias::<S>(alias).await?;
-            let details = manager.details(&identifier).await?;
+            let identifier = repo.identifier_from_alias::<S::Identifier>(alias).await?;
+            let details = repo.details(&identifier).await?;
 
             let details = if let Some(details) = details {
                 debug!("details exist");
@@ -112,7 +111,7 @@ async fn upload<S: Store>(
                 let new_details =
                     Details::from_store((**store).clone(), identifier.clone(), hint).await?;
                 debug!("storing details for {:?}", identifier);
-                manager.store_details(&identifier, &new_details).await?;
+                repo.relate_details(&identifier, &new_details).await?;
                 debug!("stored");
                 new_details
             };
@@ -125,8 +124,8 @@ async fn upload<S: Store>(
         }
     }
 
-    for image in images {
-        image.result.succeed();
+    for mut image in images {
+        image.result.disarm();
     }
     Ok(HttpResponse::Created().json(&serde_json::json!({
         "msg": "ok",
@@ -140,10 +139,10 @@ struct UrlQuery {
 }
 
 /// download an image from a URL
-#[instrument(name = "Downloading file", skip(client, manager))]
-async fn download<S: Store + 'static>(
+#[instrument(name = "Downloading file", skip(client, repo))]
+async fn download<R: FullRepo + 'static, S: Store + 'static>(
     client: web::Data<Client>,
-    manager: web::Data<UploadManager>,
+    repo: web::Data<R>,
     store: web::Data<S>,
     query: web::Query<UrlQuery>,
 ) -> Result<HttpResponse, Error> {
@@ -157,31 +156,25 @@ async fn download<S: Store + 'static>(
         .map_err(Error::from)
         .limit((CONFIG.media.max_file_size * MEGABYTES) as u64);
 
-    futures_util::pin_mut!(stream);
+    let mut session = ingest::ingest(&**repo, &**store, stream, None, true).await?;
 
-    let permit = PROCESS_SEMAPHORE.acquire().await?;
-    let session = manager
-        .session((**store).clone())
-        .upload(CONFIG.media.enable_silent_video, stream)
-        .await?;
-    let alias = session.alias().unwrap().to_owned();
-    drop(permit);
+    let alias = session.alias().expect("alias should exist").to_owned();
     let delete_token = session.delete_token().await?;
 
-    let identifier = manager.identifier_from_alias::<S>(&alias).await?;
+    let identifier = repo.identifier_from_alias::<S::Identifier>(&alias).await?;
 
-    let details = manager.details(&identifier).await?;
+    let details = repo.details(&identifier).await?;
 
     let details = if let Some(details) = details {
         details
     } else {
         let hint = details_hint(&alias);
         let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
-        manager.store_details(&identifier, &new_details).await?;
+        repo.relate_details(&identifier, &new_details).await?;
         new_details
     };
 
-    session.succeed();
+    session.disarm();
     Ok(HttpResponse::Created().json(&serde_json::json!({
         "msg": "ok",
         "files": [{
@@ -193,9 +186,9 @@ async fn download<S: Store + 'static>(
 }
 
 /// Delete aliases and files
-#[instrument(name = "Deleting file", skip(manager))]
-async fn delete(
-    manager: web::Data<UploadManager>,
+#[instrument(name = "Deleting file", skip(repo))]
+async fn delete<R: FullRepo>(
+    repo: web::Data<R>,
     path_entries: web::Path<(String, String)>,
 ) -> Result<HttpResponse, Error> {
     let (token, alias) = path_entries.into_inner();
@@ -203,7 +196,7 @@ async fn delete(
     let token = DeleteToken::from_existing(&token);
     let alias = Alias::from_existing(&alias);
 
-    manager.delete(alias, token).await?;
+    queue::cleanup_alias(&**repo, alias, token).await?;
 
     Ok(HttpResponse::NoContent().finish())
 }
@@ -249,20 +242,21 @@ fn prepare_process(
     Ok((format, alias, thumbnail_path, thumbnail_args))
 }
 
-#[instrument(name = "Fetching derived details", skip(manager))]
-async fn process_details<S: Store>(
+#[instrument(name = "Fetching derived details", skip(repo))]
+async fn process_details<R: FullRepo, S: Store>(
     query: web::Query<ProcessQuery>,
     ext: web::Path<String>,
-    manager: web::Data<UploadManager>,
+    repo: web::Data<R>,
 ) -> Result<HttpResponse, Error> {
     let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str())?;
 
-    let identifier = manager
-        .variant_identifier::<S>(&alias, &thumbnail_path)
+    let hash = repo.hash(&alias).await?;
+    let identifier = repo
+        .variant_identifier::<S::Identifier>(hash, thumbnail_path.to_string_lossy().to_string())
         .await?
         .ok_or(UploadError::MissingAlias)?;
 
-    let details = manager.details(&identifier).await?;
+    let details = repo.details(&identifier).await?;
 
     let details = details.ok_or(UploadError::NoFiles)?;
 
@@ -270,38 +264,60 @@ async fn process_details<S: Store>(
 }
 
 /// Process files
-#[instrument(name = "Serving processed image", skip(manager))]
-async fn process<S: Store + 'static>(
+#[instrument(name = "Serving processed image", skip(repo))]
+async fn process<R: FullRepo, S: Store + 'static>(
     range: Option<web::Header<Range>>,
     query: web::Query<ProcessQuery>,
     ext: web::Path<String>,
-    manager: web::Data<UploadManager>,
+    repo: web::Data<R>,
     store: web::Data<S>,
 ) -> Result<HttpResponse, Error> {
     let (format, alias, thumbnail_path, thumbnail_args) = prepare_process(query, ext.as_str())?;
 
-    let identifier_opt = manager
-        .variant_identifier::<S>(&alias, &thumbnail_path)
+    let path_string = thumbnail_path.to_string_lossy().to_string();
+    let hash = repo.hash(&alias).await?;
+    let identifier_opt = repo
+        .variant_identifier::<S::Identifier>(hash.clone(), path_string)
         .await?;
 
     if let Some(identifier) = identifier_opt {
-        let details_opt = manager.details(&identifier).await?;
+        let details_opt = repo.details(&identifier).await?;
 
         let details = if let Some(details) = details_opt {
             details
         } else {
             let hint = details_hint(&alias);
             let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
-            manager.store_details(&identifier, &details).await?;
+            repo.relate_details(&identifier, &details).await?;
             details
         };
 
         return ranged_file_resp(&**store, identifier, range, details).await;
     }
 
-    let identifier = manager
-        .still_identifier_from_alias((**store).clone(), &alias)
+    let identifier = if let Some(identifier) = repo
+        .still_identifier_from_alias::<S::Identifier>(&alias)
+        .await?
+    {
+        identifier
+    } else {
+        let identifier = repo.identifier(hash.clone()).await?;
+        let permit = PROCESS_SEMAPHORE.acquire().await;
+        let mut reader = crate::ffmpeg::thumbnail(
+            (**store).clone(),
+            identifier,
+            InputFormat::Mp4,
+            ThumbnailFormat::Jpeg,
+        )
         .await?;
+        let motion_identifier = store.save_async_read(&mut reader).await?;
+        drop(permit);
+
+        repo.relate_motion_identifier(hash.clone(), &motion_identifier)
+            .await?;
+
+        motion_identifier
+    };
 
     let thumbnail_path2 = thumbnail_path.clone();
     let identifier2 = identifier.clone();
@@ -326,10 +342,13 @@ async fn process<S: Store + 'static>(
         let details = Details::from_bytes(bytes.clone(), format.as_hint()).await?;
 
         let identifier = store.save_bytes(bytes.clone()).await?;
-        manager.store_details(&identifier, &details).await?;
-        manager
-            .store_variant(&alias, &thumbnail_path, &identifier)
-            .await?;
+        repo.relate_details(&identifier, &details).await?;
+        repo.relate_variant_identifier(
+            hash,
+            thumbnail_path.to_string_lossy().to_string(),
+            &identifier,
+        )
+        .await?;
 
         Ok((details, bytes)) as Result<(Details, web::Bytes), Error>
     };
@@ -370,25 +389,25 @@ async fn process<S: Store + 'static>(
 }
 
 /// Fetch file details
-#[instrument(name = "Fetching details", skip(manager))]
-async fn details<S: Store + 'static>(
+#[instrument(name = "Fetching details", skip(repo))]
+async fn details<R: FullRepo, S: Store + 'static>(
     alias: web::Path<String>,
-    manager: web::Data<UploadManager>,
+    repo: web::Data<R>,
     store: web::Data<S>,
 ) -> Result<HttpResponse, Error> {
     let alias = alias.into_inner();
     let alias = Alias::from_existing(&alias);
 
-    let identifier = manager.identifier_from_alias::<S>(&alias).await?;
+    let identifier = repo.identifier_from_alias::<S::Identifier>(&alias).await?;
 
-    let details = manager.details(&identifier).await?;
+    let details = repo.details(&identifier).await?;
 
     let details = if let Some(details) = details {
         details
     } else {
         let hint = details_hint(&alias);
         let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
-        manager.store_details(&identifier, &new_details).await?;
+        repo.relate_details(&identifier, &new_details).await?;
         new_details
     };
 
@@ -396,25 +415,25 @@ async fn details<S: Store + 'static>(
 }
 
 /// Serve files
-#[instrument(name = "Serving file", skip(manager))]
-async fn serve<S: Store + 'static>(
+#[instrument(name = "Serving file", skip(repo))]
+async fn serve<R: FullRepo, S: Store + 'static>(
     range: Option<web::Header<Range>>,
     alias: web::Path<String>,
-    manager: web::Data<UploadManager>,
+    repo: web::Data<R>,
     store: web::Data<S>,
 ) -> Result<HttpResponse, Error> {
     let alias = alias.into_inner();
     let alias = Alias::from_existing(&alias);
-    let identifier = manager.identifier_from_alias::<S>(&alias).await?;
+    let identifier = repo.identifier_from_alias::<S::Identifier>(&alias).await?;
 
-    let details = manager.details(&identifier).await?;
+    let details = repo.details(&identifier).await?;
 
     let details = if let Some(details) = details {
         details
     } else {
         let hint = details_hint(&alias);
         let details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
-        manager.store_details(&identifier, &details).await?;
+        repo.relate_details(&identifier, &details).await?;
         details
     };
 
@@ -506,18 +525,17 @@ struct AliasQuery {
     alias: String,
 }
 
-#[instrument(name = "Purging file", skip(upload_manager))]
-async fn purge(
+#[instrument(name = "Purging file", skip(repo))]
+async fn purge<R: FullRepo>(
     query: web::Query<AliasQuery>,
-    upload_manager: web::Data<UploadManager>,
+    repo: web::Data<R>,
 ) -> Result<HttpResponse, Error> {
     let alias = Alias::from_existing(&query.alias);
-    let aliases = upload_manager.aliases_by_alias(&alias).await?;
+    let aliases = repo.aliases_from_alias(&alias).await?;
 
     for alias in aliases.iter() {
-        upload_manager
-            .delete_without_token(alias.to_owned())
-            .await?;
+        let token = repo.delete_token(alias).await?;
+        queue::cleanup_alias(&**repo, alias.clone(), token).await?;
     }
 
     Ok(HttpResponse::Ok().json(&serde_json::json!({
@@ -526,13 +544,13 @@ async fn purge(
     })))
 }
 
-#[instrument(name = "Fetching aliases", skip(upload_manager))]
-async fn aliases(
+#[instrument(name = "Fetching aliases", skip(repo))]
+async fn aliases<R: FullRepo>(
     query: web::Query<AliasQuery>,
-    upload_manager: web::Data<UploadManager>,
+    repo: web::Data<R>,
 ) -> Result<HttpResponse, Error> {
     let alias = Alias::from_existing(&query.alias);
-    let aliases = upload_manager.aliases_by_alias(&alias).await?;
+    let aliases = repo.aliases_from_alias(&alias).await?;
 
     Ok(HttpResponse::Ok().json(&serde_json::json!({
         "msg": "ok",
@@ -567,14 +585,14 @@ fn next_worker_id() -> String {
     format!("{}-{}", CONFIG.server.worker_id, next_id)
 }
 
-async fn launch<S: Store + Clone + 'static>(
-    manager: UploadManager,
+async fn launch<R: FullRepo + Clone + 'static, S: Store + Clone + 'static>(
+    repo: R,
     store: S,
 ) -> color_eyre::Result<()> {
     // Create a new Multipart Form validator
     //
     // This form is expecting a single array field, 'images' with at most 10 files in it
-    let manager2 = manager.clone();
+    let repo2 = repo.clone();
     let store2 = store.clone();
     let form = Form::new()
         .max_files(10)
@@ -583,33 +601,24 @@ async fn launch<S: Store + Clone + 'static>(
         .field(
             "images",
             Field::array(Field::file(move |filename, _, stream| {
+                let repo = repo2.clone();
                 let store = store2.clone();
-                let manager = manager2.clone();
 
                 let span = tracing::info_span!("file-upload", ?filename);
 
-                async move {
-                    let permit = PROCESS_SEMAPHORE.acquire().await?;
+                let stream = stream.map_err(Error::from);
 
-                    let res = manager
-                        .session(store)
-                        .upload(
-                            CONFIG.media.enable_silent_video,
-                            stream.map_err(Error::from),
-                        )
-                        .await;
-
-                    drop(permit);
-                    res
-                }
-                .instrument(span)
+                Box::pin(
+                    async move { ingest::ingest(&repo, &store, stream, None, true).await }
+                        .instrument(span),
+                )
             })),
         );
 
     // Create a new Multipart Form validator for internal imports
     //
     // This form is expecting a single array field, 'images' with at most 10 files in it
-    let manager2 = manager.clone();
+    let repo2 = repo.clone();
     let store2 = store.clone();
     let import_form = Form::new()
         .max_files(10)
@@ -618,42 +627,40 @@ async fn launch<S: Store + Clone + 'static>(
         .field(
             "images",
             Field::array(Field::file(move |filename, _, stream| {
+                let repo = repo2.clone();
                 let store = store2.clone();
-                let manager = manager2.clone();
 
                 let span = tracing::info_span!("file-import", ?filename);
 
-                async move {
-                    let permit = PROCESS_SEMAPHORE.acquire().await?;
+                let stream = stream.map_err(Error::from);
 
-                    let res = manager
-                        .session(store)
-                        .import(
-                            filename,
+                Box::pin(
+                    async move {
+                        ingest::ingest(
+                            &repo,
+                            &store,
+                            stream,
+                            Some(Alias::from_existing(&filename)),
                             !CONFIG.media.skip_validate_imports,
-                            CONFIG.media.enable_silent_video,
-                            stream.map_err(Error::from),
                         )
-                        .await;
-
-                    drop(permit);
-                    res
-                }
-                .instrument(span)
+                        .await
+                    }
+                    .instrument(span),
+                )
             })),
         );
 
     HttpServer::new(move || {
-        let manager = manager.clone();
         let store = store.clone();
+        let repo = repo.clone();
 
         actix_rt::spawn(queue::process_cleanup(
-            manager.repo().clone(),
+            repo.clone(),
             store.clone(),
             next_worker_id(),
         ));
         actix_rt::spawn(queue::process_images(
-            manager.repo().clone(),
+            repo.clone(),
             store.clone(),
             next_worker_id(),
         ));
@@ -661,8 +668,8 @@ async fn launch<S: Store + Clone + 'static>(
         App::new()
             .wrap(TracingLogger::default())
             .wrap(Deadline)
+            .app_data(web::Data::new(repo))
             .app_data(web::Data::new(store))
-            .app_data(web::Data::new(manager))
             .app_data(web::Data::new(build_client()))
             .service(
                 web::scope("/image")
@@ -670,25 +677,27 @@ async fn launch<S: Store + Clone + 'static>(
                         web::resource("")
                             .guard(guard::Post())
                             .wrap(form.clone())
-                            .route(web::post().to(upload::<S>)),
+                            .route(web::post().to(upload::<R, S>)),
                     )
-                    .service(web::resource("/download").route(web::get().to(download::<S>)))
+                    .service(web::resource("/download").route(web::get().to(download::<R, S>)))
                     .service(
                         web::resource("/delete/{delete_token}/{filename}")
-                            .route(web::delete().to(delete))
-                            .route(web::get().to(delete)),
+                            .route(web::delete().to(delete::<R>))
+                            .route(web::get().to(delete::<R>)),
                     )
-                    .service(web::resource("/original/{filename}").route(web::get().to(serve::<S>)))
-                    .service(web::resource("/process.{ext}").route(web::get().to(process::<S>)))
+                    .service(
+                        web::resource("/original/{filename}").route(web::get().to(serve::<R, S>)),
+                    )
+                    .service(web::resource("/process.{ext}").route(web::get().to(process::<R, S>)))
                     .service(
                         web::scope("/details")
                             .service(
                                 web::resource("/original/{filename}")
-                                    .route(web::get().to(details::<S>)),
+                                    .route(web::get().to(details::<R, S>)),
                             )
                             .service(
                                 web::resource("/process.{ext}")
-                                    .route(web::get().to(process_details::<S>)),
+                                    .route(web::get().to(process_details::<R, S>)),
                             ),
                     ),
             )
@@ -700,10 +709,10 @@ async fn launch<S: Store + Clone + 'static>(
                     .service(
                         web::resource("/import")
                             .wrap(import_form.clone())
-                            .route(web::post().to(upload::<S>)),
+                            .route(web::post().to(upload::<R, S>)),
                     )
-                    .service(web::resource("/purge").route(web::post().to(purge)))
-                    .service(web::resource("/aliases").route(web::get().to(aliases))),
+                    .service(web::resource("/purge").route(web::post().to(purge::<R>)))
+                    .service(web::resource("/aliases").route(web::get().to(aliases::<R>))),
             )
     })
     .bind(CONFIG.server.address)?
@@ -715,19 +724,16 @@ async fn launch<S: Store + Clone + 'static>(
     Ok(())
 }
 
-async fn migrate_inner<S1>(
-    manager: &UploadManager,
-    repo: &Repo,
-    from: S1,
-    to: &config::Store,
-) -> color_eyre::Result<()>
+async fn migrate_inner<S1>(repo: &Repo, from: S1, to: &config::Store) -> color_eyre::Result<()>
 where
     S1: Store,
 {
     match to {
         config::Store::Filesystem(config::Filesystem { path }) => {
             let to = FileStore::build(path.clone(), repo.clone()).await?;
-            manager.migrate_store::<S1, FileStore>(from, to).await?;
+            match repo {
+                Repo::Sled(repo) => migrate_store(repo, from, to).await?,
+            }
         }
         config::Store::ObjectStorage(config::ObjectStorage {
             bucket_name,
@@ -749,7 +755,9 @@ where
             )
             .await?;
 
-            manager.migrate_store::<S1, ObjectStore>(from, to).await?;
+            match repo {
+                Repo::Sled(repo) => migrate_store(repo, from, to).await?,
+            }
         }
     }
 
@@ -765,15 +773,13 @@ async fn main() -> color_eyre::Result<()> {
     let db = LatestDb::exists(CONFIG.old_db.path.clone()).migrate()?;
     repo.from_db(db).await?;
 
-    let manager = UploadManager::new(repo.clone(), CONFIG.media.format).await?;
-
     match (*OPERATION).clone() {
         Operation::Run => (),
         Operation::MigrateStore { from, to } => {
             match from {
                 config::Store::Filesystem(config::Filesystem { path }) => {
                     let from = FileStore::build(path.clone(), repo.clone()).await?;
-                    migrate_inner(&manager, &repo, from, &to).await?;
+                    migrate_inner(&repo, from, &to).await?;
                 }
                 config::Store::ObjectStorage(config::ObjectStorage {
                     bucket_name,
@@ -795,7 +801,7 @@ async fn main() -> color_eyre::Result<()> {
                     )
                     .await?;
 
-                    migrate_inner(&manager, &repo, from, &to).await?;
+                    migrate_inner(&repo, from, &to).await?;
                 }
             }
 
@@ -805,8 +811,10 @@ async fn main() -> color_eyre::Result<()> {
 
     match CONFIG.store.clone() {
         config::Store::Filesystem(config::Filesystem { path }) => {
-            let store = FileStore::build(path, repo).await?;
-            launch(manager, store).await
+            let store = FileStore::build(path, repo.clone()).await?;
+            match repo {
+                Repo::Sled(sled_repo) => launch(sled_repo, store).await,
+            }
         }
         config::Store::ObjectStorage(config::ObjectStorage {
             bucket_name,
@@ -823,12 +831,92 @@ async fn main() -> color_eyre::Result<()> {
                 Some(secret_key),
                 security_token,
                 session_token,
-                repo,
+                repo.clone(),
                 build_reqwest_client()?,
             )
             .await?;
 
-            launch(manager, store).await
+            match repo {
+                Repo::Sled(sled_repo) => launch(sled_repo, store).await,
+            }
         }
     }
 }
+
+const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress";
+
+async fn migrate_store<R, S1, S2>(repo: &R, from: S1, to: S2) -> Result<(), Error>
+where
+    S1: Store,
+    S2: Store,
+    R: IdentifierRepo + HashRepo + SettingsRepo,
+{
+    let stream = repo.hashes().await;
+    let mut stream = Box::pin(stream);
+
+    while let Some(hash) = stream.next().await {
+        let hash = hash?;
+        if let Some(identifier) = repo
+            .motion_identifier(hash.as_ref().to_vec().into())
+            .await?
+        {
+            let new_identifier = migrate_file(&from, &to, &identifier).await?;
+            migrate_details(repo, identifier, &new_identifier).await?;
+            repo.relate_motion_identifier(hash.as_ref().to_vec().into(), &new_identifier)
+                .await?;
+        }
+
+        for (variant, identifier) in repo.variants(hash.as_ref().to_vec().into()).await? {
+            let new_identifier = migrate_file(&from, &to, &identifier).await?;
+            migrate_details(repo, identifier, &new_identifier).await?;
+            repo.relate_variant_identifier(hash.as_ref().to_vec().into(), variant, &new_identifier)
+                .await?;
+        }
+
+        let identifier = repo.identifier(hash.as_ref().to_vec().into()).await?;
+        let new_identifier = migrate_file(&from, &to, &identifier).await?;
+        migrate_details(repo, identifier, &new_identifier).await?;
+        repo.relate_identifier(hash.as_ref().to_vec().into(), &new_identifier)
+            .await?;
+
+        repo.set(STORE_MIGRATION_PROGRESS, hash.as_ref().to_vec().into())
+            .await?;
+    }
+
+    // clean up the migration key to avoid interfering with future migrations
+    repo.remove(STORE_MIGRATION_PROGRESS).await?;
+
+    Ok(())
+}
+
+async fn migrate_file<S1, S2>(
+    from: &S1,
+    to: &S2,
+    identifier: &S1::Identifier,
+) -> Result<S2::Identifier, Error>
+where
+    S1: Store,
+    S2: Store,
+{
+    let stream = from.to_stream(identifier, None, None).await?;
+    futures_util::pin_mut!(stream);
+    let mut reader = tokio_util::io::StreamReader::new(stream);
+
+    let new_identifier = to.save_async_read(&mut reader).await?;
+
+    Ok(new_identifier)
+}
+
+async fn migrate_details<R, I1, I2>(repo: &R, from: I1, to: &I2) -> Result<(), Error>
+where
+    R: IdentifierRepo,
+    I1: Identifier,
+    I2: Identifier,
+{
+    if let Some(details) = repo.details(&from).await? {
+        repo.relate_details(to, &details).await?;
+        repo.cleanup(&from).await?;
+    }
+
+    Ok(())
+}
diff --git a/src/queue.rs b/src/queue.rs
index 9eceb7c..8398f23 100644
--- a/src/queue.rs
+++ b/src/queue.rs
@@ -1,9 +1,9 @@
 use crate::{
     config::ImageFormat,
     error::Error,
-    repo::{Alias, AliasRepo, HashRepo, IdentifierRepo, QueueRepo, Repo},
+    repo::{Alias, AliasRepo, DeleteToken, FullRepo, HashRepo, IdentifierRepo, QueueRepo},
     serde_str::Serde,
-    store::Store,
+    store::{Identifier, Store},
 };
 use std::{future::Future, path::PathBuf, pin::Pin};
 use uuid::Uuid;
@@ -16,8 +16,16 @@ const PROCESS_QUEUE: &str = "process";
 
 #[derive(Debug, serde::Deserialize, serde::Serialize)]
 enum Cleanup {
-    CleanupHash { hash: Vec<u8> },
-    CleanupIdentifier { identifier: Vec<u8> },
+    Hash {
+        hash: Vec<u8>,
+    },
+    Identifier {
+        identifier: Vec<u8>,
+    },
+    Alias {
+        alias: Serde<Alias>,
+        token: Serde<DeleteToken>,
+    },
 }
 
 #[derive(Debug, serde::Deserialize, serde::Serialize)]
@@ -36,14 +44,38 @@ enum Process {
     },
 }
 
-pub(crate) async fn queue_cleanup<R: QueueRepo>(repo: &R, hash: R::Bytes) -> Result<(), Error> {
-    let job = serde_json::to_vec(&Cleanup::CleanupHash {
+pub(crate) async fn cleanup_alias<R: QueueRepo>(
+    repo: &R,
+    alias: Alias,
+    token: DeleteToken,
+) -> Result<(), Error> {
+    let job = serde_json::to_vec(&Cleanup::Alias {
+        alias: Serde::new(alias),
+        token: Serde::new(token),
+    })?;
+    repo.push(CLEANUP_QUEUE, job.into()).await?;
+    Ok(())
+}
+
+pub(crate) async fn cleanup_hash<R: QueueRepo>(repo: &R, hash: R::Bytes) -> Result<(), Error> {
+    let job = serde_json::to_vec(&Cleanup::Hash {
         hash: hash.as_ref().to_vec(),
     })?;
     repo.push(CLEANUP_QUEUE, job.into()).await?;
     Ok(())
 }
 
+pub(crate) async fn cleanup_identifier<R: QueueRepo, I: Identifier>(
+    repo: &R,
+    identifier: I,
+) -> Result<(), Error> {
+    let job = serde_json::to_vec(&Cleanup::Identifier {
+        identifier: identifier.to_bytes()?,
+    })?;
+    repo.push(CLEANUP_QUEUE, job.into()).await?;
+    Ok(())
+}
+
 pub(crate) async fn queue_ingest<R: QueueRepo>(
     repo: &R,
     identifier: Vec<u8>,
@@ -78,16 +110,16 @@ pub(crate) async fn queue_generate<R: QueueRepo>(
     Ok(())
 }
 
-pub(crate) async fn process_cleanup<S: Store>(repo: Repo, store: S, worker_id: String) {
-    match repo {
-        Repo::Sled(repo) => process_jobs(&repo, &store, worker_id, cleanup::perform).await,
-    }
+pub(crate) async fn process_cleanup<R: FullRepo, S: Store>(repo: R, store: S, worker_id: String) {
+    process_jobs(&repo, &store, worker_id, cleanup::perform).await
 }
 
-pub(crate) async fn process_images<S: Store>(repo: Repo, store: S, worker_id: String) {
-    match repo {
-        Repo::Sled(repo) => process_jobs(&repo, &store, worker_id, process::perform).await,
-    }
+pub(crate) async fn process_images<R: FullRepo + 'static, S: Store>(
+    repo: R,
+    store: S,
+    worker_id: String,
+) {
+    process_jobs(&repo, &store, worker_id, process::perform).await
 }
 
 type LocalBoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + 'a>>;
diff --git a/src/queue/cleanup.rs b/src/queue/cleanup.rs
index 1abec81..0496695 100644
--- a/src/queue/cleanup.rs
+++ b/src/queue/cleanup.rs
@@ -1,7 +1,8 @@
 use crate::{
-    error::Error,
-    queue::{Cleanup, LocalBoxFuture, CLEANUP_QUEUE},
-    repo::{AliasRepo, HashRepo, IdentifierRepo, QueueRepo},
+    error::{Error, UploadError},
+    queue::{Cleanup, LocalBoxFuture},
+    repo::{Alias, AliasRepo, DeleteToken, FullRepo, HashRepo, IdentifierRepo},
+    serde_str::Serde,
     store::{Identifier, Store},
 };
 use tracing::error;
@@ -12,17 +13,27 @@ pub(super) fn perform<'a, R, S>(
     job: &'a [u8],
 ) -> LocalBoxFuture<'a, Result<(), Error>>
 where
-    R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
-    R::Bytes: Clone,
+    R: FullRepo,
     S: Store,
 {
     Box::pin(async move {
         match serde_json::from_slice(job) {
             Ok(job) => match job {
-                Cleanup::CleanupHash { hash: in_hash } => hash::<R, S>(repo, in_hash).await?,
-                Cleanup::CleanupIdentifier {
+                Cleanup::Hash { hash: in_hash } => hash::<R, S>(repo, in_hash).await?,
+                Cleanup::Identifier {
                     identifier: in_identifier,
                 } => identifier(repo, &store, in_identifier).await?,
+                Cleanup::Alias {
+                    alias: stored_alias,
+                    token,
+                } => {
+                    alias(
+                        repo,
+                        Serde::into_inner(stored_alias),
+                        Serde::into_inner(token),
+                    )
+                    .await?
+                }
             },
             Err(e) => {
                 tracing::warn!("Invalid job: {}", e);
@@ -36,8 +47,7 @@ where
 #[tracing::instrument(skip(repo, store))]
 async fn identifier<R, S>(repo: &R, store: &S, identifier: Vec<u8>) -> Result<(), Error>
 where
-    R: QueueRepo + HashRepo + IdentifierRepo,
-    R::Bytes: Clone,
+    R: FullRepo,
     S: Store,
 {
     let identifier = S::Identifier::from_bytes(identifier)?;
@@ -67,8 +77,7 @@ where
 #[tracing::instrument(skip(repo))]
 async fn hash<R, S>(repo: &R, hash: Vec<u8>) -> Result<(), Error>
 where
-    R: QueueRepo + AliasRepo + HashRepo + IdentifierRepo,
-    R::Bytes: Clone,
+    R: FullRepo,
     S: Store,
 {
     let hash: R::Bytes = hash.into();
@@ -89,13 +98,31 @@ where
     idents.extend(repo.motion_identifier(hash.clone()).await?);
 
     for identifier in idents {
-        if let Ok(identifier) = identifier.to_bytes() {
-            let job = serde_json::to_vec(&Cleanup::CleanupIdentifier { identifier })?;
-            repo.push(CLEANUP_QUEUE, job.into()).await?;
-        }
+        let _ = crate::queue::cleanup_identifier(repo, identifier).await;
     }
 
     HashRepo::cleanup(repo, hash).await?;
 
     Ok(())
 }
+
+async fn alias<R>(repo: &R, alias: Alias, token: DeleteToken) -> Result<(), Error>
+where
+    R: FullRepo,
+{
+    let saved_delete_token = repo.delete_token(&alias).await?;
+    if saved_delete_token != token {
+        return Err(UploadError::InvalidToken.into());
+    }
+
+    let hash = repo.hash(&alias).await?;
+
+    AliasRepo::cleanup(repo, &alias).await?;
+    repo.remove_alias(hash.clone(), &alias).await?;
+
+    if repo.aliases(hash.clone()).await?.is_empty() {
+        crate::queue::cleanup_hash(repo, hash).await?;
+    }
+
+    Ok(())
+}
diff --git a/src/queue/process.rs b/src/queue/process.rs
index 261f095..76e4c8d 100644
--- a/src/queue/process.rs
+++ b/src/queue/process.rs
@@ -1,13 +1,14 @@
 use crate::{
     config::ImageFormat,
     error::Error,
+    ingest::Session,
     queue::{LocalBoxFuture, Process},
-    repo::{Alias, AliasRepo, HashRepo, IdentifierRepo, QueueRepo},
+    repo::{Alias, DeleteToken, FullRepo, UploadId, UploadResult},
     serde_str::Serde,
-    store::Store,
+    store::{Identifier, Store},
 };
+use futures_util::TryStreamExt;
 use std::path::PathBuf;
-use uuid::Uuid;
 
 pub(super) fn perform<'a, R, S>(
     repo: &'a R,
@@ -15,8 +16,7 @@ pub(super) fn perform<'a, R, S>(
     job: &'a [u8],
 ) -> LocalBoxFuture<'a, Result<(), Error>>
 where
-    R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
-    R::Bytes: Clone,
+    R: FullRepo + 'static,
     S: Store,
 {
     Box::pin(async move {
@@ -28,11 +28,11 @@ where
                     declared_alias,
                     should_validate,
                 } => {
-                    ingest(
+                    process_ingest(
                         repo,
                         store,
                         identifier,
-                        upload_id,
+                        upload_id.into(),
                         declared_alias.map(Serde::into_inner),
                         should_validate,
                     )
@@ -64,15 +64,54 @@ where
     })
 }
 
-async fn ingest<R, S>(
+#[tracing::instrument(skip(repo, store))]
+async fn process_ingest<R, S>(
     repo: &R,
     store: &S,
-    identifier: Vec<u8>,
-    upload_id: Uuid,
+    unprocessed_identifier: Vec<u8>,
+    upload_id: UploadId,
     declared_alias: Option<Alias>,
     should_validate: bool,
-) -> Result<(), Error> {
-    unimplemented!("do this")
+) -> Result<(), Error>
+where
+    R: FullRepo + 'static,
+    S: Store,
+{
+    let fut = async {
+        let unprocessed_identifier = S::Identifier::from_bytes(unprocessed_identifier)?;
+
+        let stream = store
+            .to_stream(&unprocessed_identifier, None, None)
+            .await?
+            .map_err(Error::from);
+
+        let session =
+            crate::ingest::ingest(repo, store, stream, declared_alias, should_validate).await?;
+
+        let token = session.delete_token().await?;
+
+        Ok((session, token)) as Result<(Session<R, S>, DeleteToken), Error>
+    };
+
+    let result = match fut.await {
+        Ok((mut session, token)) => {
+            let alias = session.alias().take().expect("Alias should exist").clone();
+            let result = UploadResult::Success { alias, token };
+            session.disarm();
+            result
+        }
+        Err(e) => {
+            tracing::warn!("Failed to ingest {}, {:?}", e, e);
+
+            UploadResult::Failure {
+                message: e.to_string(),
+            }
+        }
+    };
+
+    repo.complete(upload_id, result).await?;
+
+    Ok(())
 }
 
 async fn generate<R, S>(
diff --git a/src/repo.rs b/src/repo.rs
index e77a482..934576e 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -1,5 +1,6 @@
 use crate::{config, details::Details, error::Error, store::Identifier};
 use futures_util::Stream;
+use std::fmt::Debug;
 use tracing::debug;
 use uuid::Uuid;
 
@@ -40,8 +41,49 @@ pub(crate) enum UploadResult {
     Failure { message: String },
 }
 
+#[async_trait::async_trait(?Send)]
+pub(crate) trait FullRepo:
+    UploadRepo
+    + SettingsRepo
+    + IdentifierRepo
+    + AliasRepo
+    + QueueRepo
+    + HashRepo
+    + Send
+    + Sync
+    + Clone
+    + Debug
+{
+    async fn identifier_from_alias<I: Identifier + 'static>(
+        &self,
+        alias: &Alias,
+    ) -> Result<I, Error> {
+        let hash = self.hash(alias).await?;
+        self.identifier(hash).await
+    }
+
+    async fn aliases_from_alias(&self, alias: &Alias) -> Result<Vec<Alias>, Error> {
+        let hash = self.hash(alias).await?;
+        self.aliases(hash).await
+    }
+
+    async fn still_identifier_from_alias<I: Identifier + 'static>(
+        &self,
+        alias: &Alias,
+    ) -> Result<Option<I>, Error> {
+        let hash = self.hash(alias).await?;
+        let identifier = self.identifier::<I>(hash.clone()).await?;
+
+        match self.details(&identifier).await? {
+            Some(details) if details.is_motion() => self.motion_identifier::<I>(hash).await,
+            Some(_) => Ok(Some(identifier)),
+            None => Ok(None),
+        }
+    }
+}
+
 pub(crate) trait BaseRepo {
-    type Bytes: AsRef<[u8]> + From<Vec<u8>>;
+    type Bytes: AsRef<[u8]> + From<Vec<u8>> + Clone;
 }
 
 #[async_trait::async_trait(?Send)]
@@ -396,6 +438,12 @@ impl UploadId {
     }
 }
 
+impl From<Uuid> for UploadId {
+    fn from(id: Uuid) -> Self {
+        Self { id }
+    }
+}
+
 impl std::fmt::Display for MaybeUuid {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -405,6 +453,14 @@ impl std::fmt::Display for MaybeUuid {
     }
 }
 
+impl std::str::FromStr for DeleteToken {
+    type Err = std::convert::Infallible;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(DeleteToken::from_existing(s))
+    }
+}
+
 impl std::fmt::Display for DeleteToken {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.id)
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index c699158..262fa81 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -1,8 +1,8 @@
 use crate::{
     error::Error,
     repo::{
-        Alias, AliasRepo, AlreadyExists, DeleteToken, Details, HashRepo, Identifier,
-        IdentifierRepo, QueueRepo, SettingsRepo,
+        Alias, AliasRepo, AlreadyExists, BaseRepo, DeleteToken, Details, FullRepo, HashRepo,
+        Identifier, IdentifierRepo, QueueRepo, SettingsRepo, UploadId, UploadRepo, UploadResult,
     },
     stream::from_iterator,
 };
@@ -15,8 +15,6 @@ use std::{
 };
 use tokio::sync::Notify;
 
-use super::BaseRepo;
-
 macro_rules! b {
     ($self:ident.$ident:ident, $expr:expr) => {{
         let $ident = $self.$ident.clone();
@@ -85,6 +83,23 @@ impl BaseRepo for SledRepo {
     type Bytes = IVec;
 }
 
+impl FullRepo for SledRepo {}
+
+#[async_trait::async_trait(?Send)]
+impl UploadRepo for SledRepo {
+    async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, Error> {
+        unimplemented!("DO THIS")
+    }
+
+    async fn claim(&self, upload_id: UploadId) -> Result<(), Error> {
+        unimplemented!("DO THIS")
+    }
+
+    async fn complete(&self, upload_id: UploadId, result: UploadResult) -> Result<(), Error> {
+        unimplemented!("DO THIS")
+    }
+}
+
 #[async_trait::async_trait(?Send)]
 impl QueueRepo for SledRepo {
     async fn in_progress(&self, worker_id: Vec<u8>) -> Result<Option<Self::Bytes>, Error> {
diff --git a/src/upload_manager.rs b/src/upload_manager.rs
deleted file mode 100644
index a5c916c..0000000
--- a/src/upload_manager.rs
+++ /dev/null
@@ -1,366 +0,0 @@
-use crate::{
-    config::ImageFormat,
-    details::Details,
-    error::{Error, UploadError},
-    ffmpeg::{InputFormat, ThumbnailFormat},
-    magick::details_hint,
-    repo::{
-        sled::SledRepo, Alias, AliasRepo, BaseRepo, DeleteToken, HashRepo, IdentifierRepo, Repo,
-        SettingsRepo,
-    },
-    store::{Identifier, Store},
-};
-use futures_util::StreamExt;
-use sha2::Digest;
-use std::sync::Arc;
-use tracing::instrument;
-
-mod hasher;
-mod session;
-
-pub(super) use session::UploadManagerSession;
-
-const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress";
-
-#[derive(Clone)]
-pub(crate) struct UploadManager {
-    inner: Arc<UploadManagerInner>,
-}
-
-pub(crate) struct UploadManagerInner {
-    format: Option<ImageFormat>,
-    hasher: sha2::Sha256,
-    repo: Repo,
-}
-
-impl UploadManager {
-    pub(crate) fn repo(&self) -> &Repo {
-        &self.inner.repo
-    }
-
-    /// Create a new UploadManager
-    pub(crate) async fn new(repo: Repo, format: Option<ImageFormat>) -> Result<Self, Error> {
-        let manager = UploadManager {
-            inner: Arc::new(UploadManagerInner {
-                format,
-                hasher: sha2::Sha256::new(),
-                repo,
-            }),
-        };
-
-        Ok(manager)
-    }
-
-    pub(crate) async fn migrate_store<S1, S2>(&self, from: S1, to: S2) -> Result<(), Error>
-    where
-        S1: Store,
-        S2: Store,
-    {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => do_migrate_store(sled_repo, from, to).await,
-        }
-    }
-
-    pub(crate) async fn still_identifier_from_alias<S: Store + Clone + 'static>(
-        &self,
-        store: S,
-        alias: &Alias,
-    ) -> Result<S::Identifier, Error> {
-        let identifier = self.identifier_from_alias::<S>(alias).await?;
-        let details = if let Some(details) = self.details(&identifier).await? {
-            details
-        } else {
-            let hint = details_hint(alias);
-            Details::from_store(store.clone(), identifier.clone(), hint).await?
-        };
-
-        if !details.is_motion() {
-            return Ok(identifier);
-        }
-
-        if let Some(motion_identifier) = self.motion_identifier::<S>(alias).await? {
-            return Ok(motion_identifier);
-        }
-
-        let permit = crate::PROCESS_SEMAPHORE.acquire().await;
-        let mut reader = crate::ffmpeg::thumbnail(
-            store.clone(),
-            identifier,
-            InputFormat::Mp4,
-            ThumbnailFormat::Jpeg,
-        )
-        .await?;
-        let motion_identifier = store.save_async_read(&mut reader).await?;
-        drop(permit);
-
-        self.store_motion_identifier(alias, &motion_identifier)
-            .await?;
-        Ok(motion_identifier)
-    }
-
-    async fn motion_identifier<S: Store>(
-        &self,
-        alias: &Alias,
-    ) -> Result<Option<S::Identifier>, Error> {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let hash = sled_repo.hash(alias).await?;
-                Ok(sled_repo.motion_identifier(hash).await?)
-            }
-        }
-    }
-
-    async fn store_motion_identifier<I: Identifier + 'static>(
-        &self,
-        alias: &Alias,
-        identifier: &I,
-    ) -> Result<(), Error> {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let hash = sled_repo.hash(alias).await?;
-                Ok(sled_repo.relate_motion_identifier(hash, identifier).await?)
-            }
-        }
-    }
-
-    #[instrument(skip(self))]
-    pub(crate) async fn identifier_from_alias<S: Store>(
-        &self,
-        alias: &Alias,
-    ) -> Result<S::Identifier, Error> {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let hash = sled_repo.hash(alias).await?;
-                Ok(sled_repo.identifier(hash).await?)
-            }
-        }
-    }
-
-    #[instrument(skip(self))]
-    async fn store_identifier<I: Identifier>(
-        &self,
-        hash: Vec<u8>,
-        identifier: &I,
-    ) -> Result<(), Error> {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                Ok(sled_repo.relate_identifier(hash.into(), identifier).await?)
-            }
-        }
-    }
-
-    #[instrument(skip(self))]
-    pub(crate) async fn variant_identifier<S: Store>(
-        &self,
-        alias: &Alias,
-        process_path: &std::path::Path,
-    ) -> Result<Option<S::Identifier>, Error> {
-        let variant = process_path.to_string_lossy().to_string();
-
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let hash = sled_repo.hash(alias).await?;
-                Ok(sled_repo.variant_identifier(hash, variant).await?)
-            }
-        }
-    }
-
-    /// Store the path to a generated image variant so we can easily clean it up later
-    #[instrument(skip(self))]
-    pub(crate) async fn store_full_res<I: Identifier>(
-        &self,
-        alias: &Alias,
-        identifier: &I,
-    ) -> Result<(), Error> {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let hash = sled_repo.hash(alias).await?;
-                Ok(sled_repo.relate_identifier(hash, identifier).await?)
-            }
-        }
-    }
-
-    /// Store the path to a generated image variant so we can easily clean it up later
-    #[instrument(skip(self))]
-    pub(crate) async fn store_variant<I: Identifier>(
-        &self,
-        alias: &Alias,
-        variant_process_path: &std::path::Path,
-        identifier: &I,
-    ) -> Result<(), Error> {
-        let variant = variant_process_path.to_string_lossy().to_string();
-
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let hash = sled_repo.hash(alias).await?;
-                Ok(sled_repo
-                    .relate_variant_identifier(hash, variant, identifier)
-                    .await?)
-            }
-        }
-    }
-
-    /// Get the image details for a given variant
-    #[instrument(skip(self))]
-    pub(crate) async fn details<I: Identifier>(
-        &self,
-        identifier: &I,
-    ) -> Result<Option<Details>, Error> {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => Ok(sled_repo.details(identifier).await?),
-        }
-    }
-
-    #[instrument(skip(self))]
-    pub(crate) async fn store_details<I: Identifier>(
-        &self,
-        identifier: &I,
-        details: &Details,
-    ) -> Result<(), Error> {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => Ok(sled_repo.relate_details(identifier, details).await?),
-        }
-    }
-
-    /// Get a list of aliases for a given alias
-    pub(crate) async fn aliases_by_alias(&self, alias: &Alias) -> Result<Vec<Alias>, Error> {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let hash = sled_repo.hash(alias).await?;
-                Ok(sled_repo.aliases(hash).await?)
-            }
-        }
-    }
-
-    /// Delete an alias without a delete token
-    pub(crate) async fn delete_without_token(&self, alias: Alias) -> Result<(), Error> {
-        let token = match self.inner.repo {
-            Repo::Sled(ref sled_repo) => sled_repo.delete_token(&alias).await?,
-        };
-
-        self.delete(alias, token).await
-    }
-
-    /// Delete the alias, and the file & variants if no more aliases exist
-    #[instrument(skip(self, alias, token))]
-    pub(crate) async fn delete(&self, alias: Alias, token: DeleteToken) -> Result<(), Error> {
-        let hash = match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let saved_delete_token = sled_repo.delete_token(&alias).await?;
-                if saved_delete_token != token {
-                    return Err(UploadError::InvalidToken.into());
-                }
-                let hash = sled_repo.hash(&alias).await?;
-                AliasRepo::cleanup(sled_repo, &alias).await?;
-                sled_repo.remove_alias(hash.clone(), &alias).await?;
-                hash.to_vec()
-            }
-        };
-
-        self.check_delete_files(hash).await
-    }
-
-    async fn check_delete_files(&self, hash: Vec<u8>) -> Result<(), Error> {
-        match self.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let hash: <SledRepo as BaseRepo>::Bytes = hash.into();
-
-                let aliases = sled_repo.aliases(hash.clone()).await?;
-
-                if !aliases.is_empty() {
-                    return Ok(());
-                }
-
-                crate::queue::queue_cleanup(sled_repo, hash).await?;
-            }
-        }
-
-        Ok(())
-    }
-
-    pub(crate) fn session<S: Store + Clone + 'static>(&self, store: S) -> UploadManagerSession<S> {
-        UploadManagerSession::new(self.clone(), store)
-    }
-}
-
-async fn migrate_file<S1, S2>(
-    from: &S1,
-    to: &S2,
-    identifier: &S1::Identifier,
-) -> Result<S2::Identifier, Error>
-where
-    S1: Store,
-    S2: Store,
-{
-    let stream = from.to_stream(identifier, None, None).await?;
-    futures_util::pin_mut!(stream);
-    let mut reader = tokio_util::io::StreamReader::new(stream);
-
-    let new_identifier = to.save_async_read(&mut reader).await?;
-
-    Ok(new_identifier)
-}
-
-async fn migrate_details<R, I1, I2>(repo: &R, from: I1, to: &I2) -> Result<(), Error>
-where
-    R: IdentifierRepo,
-    I1: Identifier,
-    I2: Identifier,
-{
-    if let Some(details) = repo.details(&from).await? {
-        repo.relate_details(to, &details).await?;
-        repo.cleanup(&from).await?;
-    }
-
-    Ok(())
-}
-
-async fn do_migrate_store<R, S1, S2>(repo: &R, from: S1, to: S2) -> Result<(), Error>
-where
-    S1: Store,
-    S2: Store,
-    R: IdentifierRepo + HashRepo + SettingsRepo,
-{
-    let stream = repo.hashes().await;
-    let mut stream = Box::pin(stream);
-
-    while let Some(hash) = stream.next().await {
-        let hash = hash?;
-        if let Some(identifier) = repo
-            .motion_identifier(hash.as_ref().to_vec().into())
-            .await?
-        {
-            let new_identifier = migrate_file(&from, &to, &identifier).await?;
-            migrate_details(repo, identifier, &new_identifier).await?;
-            repo.relate_motion_identifier(hash.as_ref().to_vec().into(), &new_identifier)
-                .await?;
-        }
-
-        for (variant, identifier) in repo.variants(hash.as_ref().to_vec().into()).await? {
-            let new_identifier = migrate_file(&from, &to, &identifier).await?;
-            migrate_details(repo, identifier, &new_identifier).await?;
-            repo.relate_variant_identifier(hash.as_ref().to_vec().into(), variant, &new_identifier)
-                .await?;
-        }
-
-        let identifier = repo.identifier(hash.as_ref().to_vec().into()).await?;
-        let new_identifier = migrate_file(&from, &to, &identifier).await?;
-        migrate_details(repo, identifier, &new_identifier).await?;
-        repo.relate_identifier(hash.as_ref().to_vec().into(), &new_identifier)
-            .await?;
-
-        repo.set(STORE_MIGRATION_PROGRESS, hash.as_ref().to_vec().into())
-            .await?;
-    }
-
-    // clean up the migration key to avoid interfering with future migrations
-    repo.remove(STORE_MIGRATION_PROGRESS).await?;
-
-    Ok(())
-}
-
-impl std::fmt::Debug for UploadManager {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        f.debug_struct("UploadManager").finish()
-    }
-}
diff --git a/src/upload_manager/session.rs b/src/upload_manager/session.rs
deleted file mode 100644
index 390d758..0000000
--- a/src/upload_manager/session.rs
+++ /dev/null
@@ -1,266 +0,0 @@
-use crate::{
-    error::{Error, UploadError},
-    magick::ValidInputType,
-    repo::{Alias, AliasRepo, AlreadyExists, DeleteToken, HashRepo, IdentifierRepo, Repo},
-    store::Store,
-    upload_manager::{
-        hasher::{Hash, Hasher},
-        UploadManager,
-    },
-};
-use actix_web::web;
-use futures_util::stream::{Stream, StreamExt};
-use tracing::{debug, instrument, Span};
-use tracing_futures::Instrument;
-
-pub(crate) struct UploadManagerSession<S: Store + Clone + 'static> {
-    store: S,
-    manager: UploadManager,
-    alias: Option<Alias>,
-    finished: bool,
-}
-
-impl<S: Store + Clone + 'static> UploadManagerSession<S> {
-    pub(super) fn new(manager: UploadManager, store: S) -> Self {
-        UploadManagerSession {
-            store,
-            manager,
-            alias: None,
-            finished: false,
-        }
-    }
-
-    pub(crate) fn succeed(mut self) {
-        self.finished = true;
-    }
-
-    pub(crate) fn alias(&self) -> Option<&Alias> {
-        self.alias.as_ref()
-    }
-}
-
-impl<S: Store + Clone + 'static> Drop for UploadManagerSession<S> {
-    fn drop(&mut self) {
-        if self.finished {
-            return;
-        }
-
-        if let Some(alias) = self.alias.take() {
-            let store = self.store.clone();
-            let manager = self.manager.clone();
-            let cleanup_span = tracing::info_span!(
-                parent: None,
-                "Upload cleanup",
-                alias = &tracing::field::display(&alias),
-            );
-            cleanup_span.follows_from(Span::current());
-            actix_rt::spawn(
-                async move {
-                    // undo alias -> hash mapping
-                    match manager.inner.repo {
-                        Repo::Sled(ref sled_repo) => {
-                            if let Ok(hash) = sled_repo.hash(&alias).await {
-                                debug!("Clean alias repo");
-                                let _ = AliasRepo::cleanup(sled_repo, &alias).await;
-
-                                if let Ok(identifier) = sled_repo.identifier(hash.clone()).await {
-                                    debug!("Clean identifier repo");
-                                    let _ = IdentifierRepo::cleanup(sled_repo, &identifier).await;
-
-                                    debug!("Remove stored files");
-                                    let _ = store.remove(&identifier).await;
-                                }
-                                debug!("Clean hash repo");
-                                let _ = HashRepo::cleanup(sled_repo, hash).await;
-                            }
-                        }
-                    }
-                }
-                .instrument(cleanup_span),
-            );
-        }
-    }
-}
-
-impl<S: Store> UploadManagerSession<S> {
-    /// Generate a delete token for an alias
-    #[instrument(skip(self))]
-    pub(crate) async fn delete_token(&self) -> Result<DeleteToken, Error> {
-        let alias = self.alias.clone().ok_or(UploadError::MissingAlias)?;
-
-        debug!("Generating delete token");
-        let delete_token = DeleteToken::generate();
-
-        debug!("Saving delete token");
-        match self.manager.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                let res = sled_repo.relate_delete_token(&alias, &delete_token).await?;
-
-                Ok(if res.is_err() {
-                    let delete_token = sled_repo.delete_token(&alias).await?;
-                    debug!("Returning existing delete token, {:?}", delete_token);
-                    delete_token
-                } else {
-                    debug!("Returning new delete token, {:?}", delete_token);
-                    delete_token
-                })
-            }
-        }
-    }
-
-    /// Import the file, discarding bytes if it's already present, or saving if it's new
-    pub(crate) async fn import(
-        mut self,
-        alias: String,
-        validate: bool,
-        enable_silent_video: bool,
-        mut stream: impl Stream<Item = Result<web::Bytes, Error>> + Unpin,
-    ) -> Result<Self, Error> {
-        let mut bytes_mut = actix_web::web::BytesMut::new();
-
-        debug!("Reading stream to memory");
-        while let Some(res) = stream.next().await {
-            let bytes = res?;
-            bytes_mut.extend_from_slice(&bytes);
-        }
-
-        debug!("Validating bytes");
-        let (_, validated_reader) = crate::validate::validate_image_bytes(
-            bytes_mut.freeze(),
-            self.manager.inner.format,
-            enable_silent_video,
-            validate,
-        )
-        .await?;
-
-        let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
-
-        let identifier = self.store.save_async_read(&mut hasher_reader).await?;
-        let hash = hasher_reader.finalize_reset().await?;
-
-        debug!("Adding alias");
-        self.add_existing_alias(&hash, alias).await?;
-
-        debug!("Saving file");
-        self.save_upload(&identifier, hash).await?;
-
-        // Return alias to file
-        Ok(self)
-    }
-
-    /// Upload the file, discarding bytes if it's already present, or saving if it's new
-    #[instrument(skip(self, stream))]
-    pub(crate) async fn upload(
-        mut self,
-        enable_silent_video: bool,
-        mut stream: impl Stream<Item = Result<web::Bytes, Error>> + Unpin,
-    ) -> Result<Self, Error> {
-        let mut bytes_mut = actix_web::web::BytesMut::new();
-
-        debug!("Reading stream to memory");
-        while let Some(res) = stream.next().await {
-            let bytes = res?;
-            bytes_mut.extend_from_slice(&bytes);
-        }
-
-        debug!("Validating bytes");
-        let (input_type, validated_reader) = crate::validate::validate_image_bytes(
-            bytes_mut.freeze(),
-            self.manager.inner.format,
-            enable_silent_video,
-            true,
-        )
-        .await?;
-
-        let mut hasher_reader = Hasher::new(validated_reader, self.manager.inner.hasher.clone());
-
-        let identifier = self.store.save_async_read(&mut hasher_reader).await?;
-        let hash = hasher_reader.finalize_reset().await?;
-
-        debug!("Adding alias");
-        self.add_alias(&hash, input_type).await?;
-
-        debug!("Saving file");
-        self.save_upload(&identifier, hash).await?;
-
-        // Return alias to file
-        Ok(self)
-    }
-
-    // check duplicates & store image if new
-    #[instrument(skip(self, hash))]
-    async fn save_upload(&self, identifier: &S::Identifier, hash: Hash) -> Result<(), Error> {
-        let res = self.check_duplicate(&hash).await?;
-
-        // bail early with alias to existing file if this is a duplicate
-        if res.is_err() {
-            debug!("Duplicate exists, removing file");
-
-            self.store.remove(identifier).await?;
-            return Ok(());
-        }
-
-        self.manager
-            .store_identifier(hash.into_inner(), identifier)
-            .await?;
-
-        Ok(())
-    }
-
-    // check for an already-uploaded image with this hash, returning the path to the target file
-    #[instrument(skip(self, hash))]
-    async fn check_duplicate(&self, hash: &Hash) -> Result<Result<(), AlreadyExists>, Error> {
-        let hash = hash.as_slice().to_vec();
-
-        match self.manager.inner.repo {
-            Repo::Sled(ref sled_repo) => Ok(HashRepo::create(sled_repo, hash.into()).await?),
-        }
-    }
-
-    // Add an alias from an existing filename
-    async fn add_existing_alias(&mut self, hash: &Hash, filename: String) -> Result<(), Error> {
-        let alias = Alias::from_existing(&filename);
-
-        match self.manager.inner.repo {
-            Repo::Sled(ref sled_repo) => {
-                AliasRepo::create(sled_repo, &alias)
-                    .await?
-                    .map_err(|_| UploadError::DuplicateAlias)?;
-                self.alias = Some(alias.clone());
-
-                let hash = hash.as_slice().to_vec();
-                sled_repo.relate_hash(&alias, hash.clone().into()).await?;
-                sled_repo.relate_alias(hash.into(), &alias).await?;
-            }
-        }
-
-        Ok(())
-    }
-
-    // Add an alias to an existing file
-    //
-    // This will help if multiple 'users' upload the same file, and one of them wants to delete it
-    #[instrument(skip(self, hash, input_type))]
-    async fn add_alias(&mut self, hash: &Hash, input_type: ValidInputType) -> Result<(), Error> {
-        loop {
-            debug!("Alias gen loop");
-            let alias = Alias::generate(input_type.as_ext().to_string());
-
-            match self.manager.inner.repo {
-                Repo::Sled(ref sled_repo) => {
-                    let res = AliasRepo::create(sled_repo, &alias).await?;
-
-                    if res.is_ok() {
-                        self.alias = Some(alias.clone());
-                        let hash = hash.as_slice().to_vec();
-                        sled_repo.relate_hash(&alias, hash.clone().into()).await?;
-                        sled_repo.relate_alias(hash.into(), &alias).await?;
-                        return Ok(());
-                    }
-                }
-            };
-
-            debug!("Alias exists, regenning");
-        }
-    }
-}

From 132e395e5c3f7503b0349b0480cbd15f3f01b1c6 Mon Sep 17 00:00:00 2001
From: "Aode (Lion)" <asonix@asonix.dog>
Date: Sat, 2 Apr 2022 17:40:04 -0500
Subject: [PATCH 27/37] Multiple items:

- Reduce duplicate work in generate job
- Use hash instead of identifier for unique processing
- Move motion ident generation behind concurrent processor lock
---
 src/concurrent_processor.rs | 15 +++---
 src/generate.rs             | 93 +++++++++++++++++++++++++++++++++++++
 src/ingest.rs               |  6 +--
 src/main.rs                 | 76 ++++++------------------------
 src/queue.rs                |  2 +-
 src/queue/process.rs        | 28 +++++++++--
 6 files changed, 139 insertions(+), 81 deletions(-)
 create mode 100644 src/generate.rs

diff --git a/src/concurrent_processor.rs b/src/concurrent_processor.rs
index 3db0e6b..92a8cf5 100644
--- a/src/concurrent_processor.rs
+++ b/src/concurrent_processor.rs
@@ -1,7 +1,6 @@
 use crate::{
     details::Details,
     error::{Error, UploadError},
-    store::Identifier,
 };
 use actix_web::web;
 use dashmap::{mapref::entry::Entry, DashMap};
@@ -42,10 +41,8 @@ impl<F> CancelSafeProcessor<F>
 where
     F: Future<Output = Result<(Details, web::Bytes), Error>>,
 {
-    pub(super) fn new<I: Identifier>(identifier: I, path: PathBuf, fut: F) -> Result<Self, Error> {
-        let id_bytes = identifier.to_bytes()?;
-
-        let key = (id_bytes, path.clone());
+    pub(super) fn new(hash: &[u8], path: PathBuf, fut: F) -> Self {
+        let key = (hash.to_vec(), path.clone());
 
         let entry = PROCESS_MAP.entry(key.clone());
 
@@ -54,7 +51,7 @@ where
                 vacant.insert(Vec::new());
                 let span = tracing::info_span!(
                     "Processing image",
-                    identifier = &tracing::field::debug(&identifier),
+                    hash = &tracing::field::debug(&hash),
                     path = &tracing::field::debug(&path),
                     completed = &tracing::field::Empty,
                 );
@@ -65,21 +62,21 @@ where
                 occupied.get_mut().push(tx);
                 let span = tracing::info_span!(
                     "Waiting for processed image",
-                    identifier = &tracing::field::debug(&identifier),
+                    hash = &tracing::field::debug(&hash),
                     path = &tracing::field::debug(&path),
                 );
                 (Some(rx), span)
             }
         };
 
-        Ok(CancelSafeProcessor {
+        CancelSafeProcessor {
             cancel_token: CancelToken {
                 span,
                 key,
                 receiver,
             },
             fut,
-        })
+        }
     }
 }
 
diff --git a/src/generate.rs b/src/generate.rs
new file mode 100644
index 0000000..bffac6f
--- /dev/null
+++ b/src/generate.rs
@@ -0,0 +1,93 @@
+use crate::{
+    concurrent_processor::CancelSafeProcessor,
+    config::ImageFormat,
+    details::Details,
+    error::Error,
+    ffmpeg::{InputFormat, ThumbnailFormat},
+    repo::{Alias, FullRepo},
+    store::Store,
+};
+use actix_web::web::Bytes;
+use std::path::PathBuf;
+use tokio::io::AsyncReadExt;
+
+pub(crate) async fn generate<R: FullRepo, S: Store + 'static>(
+    repo: &R,
+    store: &S,
+    format: ImageFormat,
+    alias: Alias,
+    thumbnail_path: PathBuf,
+    thumbnail_args: Vec<String>,
+    hash: R::Bytes,
+) -> Result<(Details, Bytes), Error> {
+    let process_fut = process(
+        repo,
+        store,
+        format,
+        alias,
+        thumbnail_path.clone(),
+        thumbnail_args,
+        hash.clone(),
+    );
+
+    let (details, bytes) =
+        CancelSafeProcessor::new(hash.as_ref(), thumbnail_path, process_fut).await?;
+
+    Ok((details, bytes))
+}
+
+async fn process<R: FullRepo, S: Store + 'static>(
+    repo: &R,
+    store: &S,
+    format: ImageFormat,
+    alias: Alias,
+    thumbnail_path: PathBuf,
+    thumbnail_args: Vec<String>,
+    hash: R::Bytes,
+) -> Result<(Details, Bytes), Error> {
+    let permit = crate::PROCESS_SEMAPHORE.acquire().await?;
+
+    let identifier = if let Some(identifier) = repo
+        .still_identifier_from_alias::<S::Identifier>(&alias)
+        .await?
+    {
+        identifier
+    } else {
+        let identifier = repo.identifier(hash.clone()).await?;
+        let mut reader = crate::ffmpeg::thumbnail(
+            store.clone(),
+            identifier,
+            InputFormat::Mp4,
+            ThumbnailFormat::Jpeg,
+        )
+        .await?;
+        let motion_identifier = store.save_async_read(&mut reader).await?;
+
+        repo.relate_motion_identifier(hash.clone(), &motion_identifier)
+            .await?;
+
+        motion_identifier
+    };
+
+    let mut processed_reader =
+        crate::magick::process_image_store_read(store.clone(), identifier, thumbnail_args, format)?;
+
+    let mut vec = Vec::new();
+    processed_reader.read_to_end(&mut vec).await?;
+    let bytes = Bytes::from(vec);
+
+    drop(permit);
+
+    let details = Details::from_bytes(bytes.clone(), format.as_hint()).await?;
+
+    let identifier = store.save_bytes(bytes.clone()).await?;
+    repo.relate_details(&identifier, &details).await?;
+    repo.relate_variant_identifier(
+        hash,
+        thumbnail_path.to_string_lossy().to_string(),
+        &identifier,
+    )
+    .await?;
+
+    Ok((details, bytes)) as Result<(Details, Bytes), Error>
+}
diff --git a/src/ingest.rs b/src/ingest.rs
index bae7bb6..613bac3 100644
--- a/src/ingest.rs
+++ b/src/ingest.rs
@@ -7,16 +7,12 @@ use crate::{
 };
 use actix_web::web::{Bytes, BytesMut};
 use futures_util::{Stream, StreamExt};
-use once_cell::sync::Lazy;
 use sha2::{Digest, Sha256};
-use tokio::sync::Semaphore;
 use tracing::debug;
 
 mod hasher;
 use hasher::Hasher;
 
-static PROCESS_SEMAPHORE: Lazy<Semaphore> = Lazy::new(|| Semaphore::new(num_cpus::get()));
-
 pub(crate) struct Session<R, S>
 where
     R: FullRepo + 'static,
@@ -39,7 +35,7 @@ where
     R: FullRepo + 'static,
     S: Store,
 {
-    let permit = PROCESS_SEMAPHORE.acquire().await;
+    let permit = crate::PROCESS_SEMAPHORE.acquire().await;
 
     let mut bytes_mut = BytesMut::new();
 
diff --git a/src/main.rs b/src/main.rs
index 15f635a..ee3fd54 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -16,7 +16,7 @@ use std::{
     sync::atomic::{AtomicU64, Ordering},
     time::{Duration, SystemTime},
 };
-use tokio::{io::AsyncReadExt, sync::Semaphore};
+use tokio::sync::Semaphore;
 use tracing::{debug, info, instrument};
 use tracing_actix_web::TracingLogger;
 use tracing_awc::Tracing;
@@ -30,6 +30,7 @@ mod error;
 mod exiftool;
 mod ffmpeg;
 mod file;
+mod generate;
 mod ingest;
 mod init_tracing;
 mod magick;
@@ -47,12 +48,10 @@ mod tmp_file;
 mod validate;
 
 use self::{
-    concurrent_processor::CancelSafeProcessor,
     config::{Configuration, ImageFormat, Operation},
     details::Details,
     either::Either,
     error::{Error, UploadError},
-    ffmpeg::{InputFormat, ThumbnailFormat},
     ingest::Session,
     init_tracing::init_tracing,
     magick::details_hint,
@@ -94,6 +93,7 @@ async fn upload<R: FullRepo, S: Store + 'static>(
         .into_iter()
         .filter_map(|i| i.file())
         .collect::<Vec<_>>();
+
     for image in &images {
         if let Some(alias) = image.result.alias() {
             info!("Uploaded {} as {:?}", image.filename, alias);
@@ -295,66 +295,16 @@ async fn process<R: FullRepo, S: Store + 'static>(
         return ranged_file_resp(&**store, identifier, range, details).await;
     }
 
-    let identifier = if let Some(identifier) = repo
-        .still_identifier_from_alias::<S::Identifier>(&alias)
-        .await?
-    {
-        identifier
-    } else {
-        let identifier = repo.identifier(hash.clone()).await?;
-        let permit = PROCESS_SEMAPHORE.acquire().await;
-        let mut reader = crate::ffmpeg::thumbnail(
-            (**store).clone(),
-            identifier,
-            InputFormat::Mp4,
-            ThumbnailFormat::Jpeg,
-        )
-        .await?;
-        let motion_identifier = store.save_async_read(&mut reader).await?;
-        drop(permit);
-
-        repo.relate_motion_identifier(hash.clone(), &motion_identifier)
-            .await?;
-
-        motion_identifier
-    };
-
-    let thumbnail_path2 = thumbnail_path.clone();
-    let identifier2 = identifier.clone();
-    let process_fut = async {
-        let thumbnail_path = thumbnail_path2;
-
-        let permit = PROCESS_SEMAPHORE.acquire().await?;
-
-        let mut processed_reader = crate::magick::process_image_store_read(
-            (**store).clone(),
-            identifier2,
-            thumbnail_args,
-            format,
-        )?;
-
-        let mut vec = Vec::new();
-        processed_reader.read_to_end(&mut vec).await?;
-        let bytes = web::Bytes::from(vec);
-
-        drop(permit);
-
-        let details = Details::from_bytes(bytes.clone(), format.as_hint()).await?;
-
-        let identifier = store.save_bytes(bytes.clone()).await?;
-        repo.relate_details(&identifier, &details).await?;
-        repo.relate_variant_identifier(
-            hash,
-            thumbnail_path.to_string_lossy().to_string(),
-            &identifier,
-        )
-        .await?;
-
-        Ok((details, bytes)) as Result<(Details, web::Bytes), Error>
-    };
-
-    let (details, bytes) =
-        CancelSafeProcessor::new(identifier, thumbnail_path.clone(), process_fut)?.await?;
+    let (details, bytes) = generate::generate(
+        &**repo,
+        &**store,
+        format,
+        alias,
+        thumbnail_path,
+        thumbnail_args,
+        hash,
+    )
+    .await?;
 
     let (builder, stream) = if let Some(web::Header(range_header)) = range {
         if let Some(range) = range::single_bytes_range(&range_header) {
diff --git a/src/queue.rs b/src/queue.rs
index 8398f23..6c404eb 100644
--- a/src/queue.rs
+++ b/src/queue.rs
@@ -114,7 +114,7 @@ pub(crate) async fn process_cleanup<R: FullRepo, S: Store>(repo: R, store: S, wo
     process_jobs(&repo, &store, worker_id, cleanup::perform).await
 }
 
-pub(crate) async fn process_images<R: FullRepo + 'static, S: Store>(
+pub(crate) async fn process_images<R: FullRepo + 'static, S: Store + 'static>(
     repo: R,
     store: S,
     worker_id: String,
diff --git a/src/queue/process.rs b/src/queue/process.rs
index 76e4c8d..24f7e30 100644
--- a/src/queue/process.rs
+++ b/src/queue/process.rs
@@ -17,7 +17,7 @@ pub(super) fn perform<'a, R, S>(
 ) -> LocalBoxFuture<'a, Result<(), Error>>
 where
     R: FullRepo + 'static,
-    S: Store,
+    S: Store + 'static,
 {
     Box::pin(async move {
         match serde_json::from_slice(job) {
@@ -114,7 +114,7 @@ where
     Ok(())
 }
 
-async fn generate<R, S>(
+async fn generate<R: FullRepo, S: Store + 'static>(
     repo: &R,
     store: &S,
     target_format: ImageFormat,
@@ -122,5 +122,27 @@ async fn generate<R, S>(
     process_path: PathBuf,
     process_args: Vec<String>,
 ) -> Result<(), Error> {
-    unimplemented!("do this")
+    let hash = repo.hash(&source).await?;
+
+    let path_string = process_path.to_string_lossy().to_string();
+    let identifier_opt = repo
+        .variant_identifier::<S::Identifier>(hash.clone(), path_string)
+        .await?;
+
+    if identifier_opt.is_some() {
+        return Ok(());
+    }
+
+    crate::generate::generate(
+        repo,
+        store,
+        target_format,
+        source,
+        process_path,
+        process_args,
+        hash,
+    )
+    .await?;
+
+    Ok(())
 }

From 77a400c7ca8f82f4d3ff21adf226d4bf8f9e8c4a Mon Sep 17 00:00:00 2001
From: "Aode (Lion)" <asonix@asonix.dog>
Date: Sat, 2 Apr 2022 17:41:00 -0500
Subject: [PATCH 28/37] Implement UploadRepo

---
 src/repo.rs      |  5 ----
 src/repo/sled.rs | 61 ++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/src/repo.rs b/src/repo.rs
index 934576e..7137b65 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -431,11 +431,6 @@ impl UploadId {
     pub(crate) fn as_bytes(&self) -> &[u8] {
         &self.id.as_bytes()[..]
     }
-
-    pub(crate) fn from_bytes(&self, bytes: &[u8]) -> Option<Self> {
-        let id = Uuid::from_slice(bytes).ok()?;
-        Some(Self { id })
-    }
 }
 
 impl From<Uuid> for UploadId {
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index 262fa81..99850b8 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -1,9 +1,10 @@
 use crate::{
-    error::Error,
+    error::{Error, UploadError},
     repo::{
         Alias, AliasRepo, AlreadyExists, BaseRepo, DeleteToken, Details, FullRepo, HashRepo,
         Identifier, IdentifierRepo, QueueRepo, SettingsRepo, UploadId, UploadRepo, UploadResult,
     },
+    serde_str::Serde,
     stream::from_iterator,
 };
 use futures_util::Stream;
@@ -55,6 +56,7 @@ pub(crate) struct SledRepo {
     queue: Tree,
     in_progress_queue: Tree,
     queue_notifier: Arc<RwLock<HashMap<&'static str, Arc<Notify>>>>,
+    uploads: Tree,
     db: Db,
 }
 
@@ -74,6 +76,7 @@ impl SledRepo {
             queue: db.open_tree("pict-rs-queue-tree")?,
             in_progress_queue: db.open_tree("pict-rs-in-progress-queue-tree")?,
             queue_notifier: Arc::new(RwLock::new(HashMap::new())),
+            uploads: db.open_tree("pict-rs-uploads-tree")?,
             db,
         })
     }
@@ -85,18 +88,68 @@ impl BaseRepo for SledRepo {
 
 impl FullRepo for SledRepo {}
 
+#[derive(serde::Deserialize, serde::Serialize)]
+enum InnerUploadResult {
+    Success {
+        alias: Serde<Alias>,
+        token: Serde<DeleteToken>,
+    },
+    Failure {
+        message: String,
+    },
+}
+
+impl From<UploadResult> for InnerUploadResult {
+    fn from(u: UploadResult) -> Self {
+        match u {
+            UploadResult::Success { alias, token } => InnerUploadResult::Success {
+                alias: Serde::new(alias),
+                token: Serde::new(token),
+            },
+            UploadResult::Failure { message } => InnerUploadResult::Failure { message },
+        }
+    }
+}
+
+impl From<InnerUploadResult> for UploadResult {
+    fn from(i: InnerUploadResult) -> Self {
+        match i {
+            InnerUploadResult::Success { alias, token } => UploadResult::Success {
+                alias: Serde::into_inner(alias),
+                token: Serde::into_inner(token),
+            },
+            InnerUploadResult::Failure { message } => UploadResult::Failure { message },
+        }
+    }
+}
+
 #[async_trait::async_trait(?Send)]
 impl UploadRepo for SledRepo {
     async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, Error> {
-        unimplemented!("DO THIS")
+        let mut subscriber = self.uploads.watch_prefix(upload_id.as_bytes());
+
+        while let Some(event) = (&mut subscriber).await {
+            if let sled::Event::Insert { value, .. } = event {
+                let result: InnerUploadResult = serde_json::from_slice(&value)?;
+                return Ok(result.into());
+            }
+        }
+
+        Err(UploadError::Canceled.into())
     }
 
     async fn claim(&self, upload_id: UploadId) -> Result<(), Error> {
-        unimplemented!("DO THIS")
+        b!(self.uploads, uploads.remove(upload_id.as_bytes()));
+        Ok(())
     }
 
     async fn complete(&self, upload_id: UploadId, result: UploadResult) -> Result<(), Error> {
-        unimplemented!("DO THIS")
+        let result: InnerUploadResult = result.into();
+        let result = serde_json::to_vec(&result)?;
+
+        b!(self.uploads, uploads.insert(upload_id.as_bytes(), result));
+
+        Ok(())
     }
 }
 

From c4d014597e0e6d01cb1e7b1ca3195c86b9337671 Mon Sep 17 00:00:00 2001
From: "Aode (Lion)" <asonix@asonix.dog>
Date: Sat, 2 Apr 2022 18:53:03 -0500
Subject: [PATCH 29/37] Better job recovery

---
 src/main.rs      |  3 +++
 src/queue.rs     | 40 ++++++++++++++++++-------------
 src/repo.rs      |  2 +-
 src/repo/sled.rs | 61 +++++++++++++++++++++++++++++++++++++++++-------
 4 files changed, 81 insertions(+), 25 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index ee3fd54..a022a46 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -127,6 +127,7 @@ async fn upload<R: FullRepo, S: Store + 'static>(
     for mut image in images {
         image.result.disarm();
     }
+
     Ok(HttpResponse::Created().json(&serde_json::json!({
         "msg": "ok",
         "files": files
@@ -539,6 +540,8 @@ async fn launch<R: FullRepo + Clone + 'static, S: Store + Clone + 'static>(
     repo: R,
     store: S,
 ) -> color_eyre::Result<()> {
+    repo.requeue_in_progress(CONFIG.server.worker_id.as_bytes().to_vec())
+        .await?;
     // Create a new Multipart Form validator
     //
     // This form is expecting a single array field, 'images' with at most 10 files in it
diff --git a/src/queue.rs b/src/queue.rs
index 6c404eb..fbfc1ae 100644
--- a/src/queue.rs
+++ b/src/queue.rs
@@ -6,6 +6,7 @@ use crate::{
     store::{Identifier, Store},
 };
 use std::{future::Future, path::PathBuf, pin::Pin};
+use tracing::Instrument;
 use uuid::Uuid;
 
 mod cleanup;
@@ -111,7 +112,7 @@ pub(crate) async fn queue_generate<R: QueueRepo>(
 }
 
 pub(crate) async fn process_cleanup<R: FullRepo, S: Store>(repo: R, store: S, worker_id: String) {
-    process_jobs(&repo, &store, worker_id, cleanup::perform).await
+    process_jobs(&repo, &store, worker_id, CLEANUP_QUEUE, cleanup::perform).await
 }
 
 pub(crate) async fn process_images<R: FullRepo + 'static, S: Store + 'static>(
@@ -119,26 +120,25 @@ pub(crate) async fn process_images<R: FullRepo + 'static, S: Store + 'static>(
     store: S,
     worker_id: String,
 ) {
-    process_jobs(&repo, &store, worker_id, process::perform).await
+    process_jobs(&repo, &store, worker_id, PROCESS_QUEUE, process::perform).await
 }
 
 type LocalBoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + 'a>>;
 
-async fn process_jobs<R, S, F>(repo: &R, store: &S, worker_id: String, callback: F)
-where
+async fn process_jobs<R, S, F>(
+    repo: &R,
+    store: &S,
+    worker_id: String,
+    queue: &'static str,
+    callback: F,
+) where
     R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
     R::Bytes: Clone,
     S: Store,
     for<'a> F: Fn(&'a R, &'a S, &'a [u8]) -> LocalBoxFuture<'a, Result<(), Error>> + Copy,
 {
-    if let Ok(Some(job)) = repo.in_progress(worker_id.as_bytes().to_vec()).await {
-        if let Err(e) = (callback)(repo, store, job.as_ref()).await {
-            tracing::warn!("Failed to run previously dropped job: {}", e);
-            tracing::warn!("{:?}", e);
-        }
-    }
     loop {
-        let res = job_loop(repo, store, worker_id.clone(), callback).await;
+        let res = job_loop(repo, store, worker_id.clone(), queue, callback).await;
 
         if let Err(e) = res {
             tracing::warn!("Error processing jobs: {}", e);
@@ -150,7 +150,13 @@ where
     }
 }
 
-async fn job_loop<R, S, F>(repo: &R, store: &S, worker_id: String, callback: F) -> Result<(), Error>
+async fn job_loop<R, S, F>(
+    repo: &R,
+    store: &S,
+    worker_id: String,
+    queue: &'static str,
+    callback: F,
+) -> Result<(), Error>
 where
     R: QueueRepo + HashRepo + IdentifierRepo + AliasRepo,
     R::Bytes: Clone,
@@ -158,10 +164,12 @@ where
     for<'a> F: Fn(&'a R, &'a S, &'a [u8]) -> LocalBoxFuture<'a, Result<(), Error>> + Copy,
 {
     loop {
-        let bytes = repo
-            .pop(CLEANUP_QUEUE, worker_id.as_bytes().to_vec())
-            .await?;
+        let bytes = repo.pop(queue, worker_id.as_bytes().to_vec()).await?;
 
-        (callback)(repo, store, bytes.as_ref()).await?;
+        let span = tracing::info_span!("Running Job", worker_id = ?worker_id);
+
+        span.in_scope(|| (callback)(repo, store, bytes.as_ref()))
+            .instrument(span)
+            .await?;
     }
 }
diff --git a/src/repo.rs b/src/repo.rs
index 7137b65..e823bcf 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -97,7 +97,7 @@ pub(crate) trait UploadRepo: BaseRepo {
 
 #[async_trait::async_trait(?Send)]
 pub(crate) trait QueueRepo: BaseRepo {
-    async fn in_progress(&self, worker_id: Vec<u8>) -> Result<Option<Self::Bytes>, Error>;
+    async fn requeue_in_progress(&self, worker_prefix: Vec<u8>) -> Result<(), Error>;
 
     async fn push(&self, queue: &'static str, job: Self::Bytes) -> Result<(), Error>;
 
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index 99850b8..ab568b0 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -155,20 +155,60 @@ impl UploadRepo for SledRepo {
 
 #[async_trait::async_trait(?Send)]
 impl QueueRepo for SledRepo {
-    async fn in_progress(&self, worker_id: Vec<u8>) -> Result<Option<Self::Bytes>, Error> {
-        let opt = b!(self.in_progress_queue, in_progress_queue.get(worker_id));
+    #[tracing::instrument(skip_all, fields(worker_id = %String::from_utf8_lossy(&worker_prefix)))]
+    async fn requeue_in_progress(&self, worker_prefix: Vec<u8>) -> Result<(), Error> {
+        let vec: Vec<(String, IVec)> = b!(self.in_progress_queue, {
+            let vec = in_progress_queue
+                .scan_prefix(worker_prefix)
+                .values()
+                .filter_map(Result::ok)
+                .filter_map(|ivec| {
+                    let index = ivec.as_ref().iter().enumerate().find_map(|(index, byte)| {
+                        if *byte == 0 {
+                            Some(index)
+                        } else {
+                            None
+                        }
+                    })?;
 
-        Ok(opt)
+                    let (queue, job) = ivec.split_at(index);
+                    if queue.is_empty() || job.len() <= 1 {
+                        return None;
+                    }
+                    let job = &job[1..];
+
+                    Some((String::from_utf8_lossy(queue).to_string(), IVec::from(job)))
+                })
+                .collect::<Vec<(String, IVec)>>();
+
+            Ok(vec) as Result<_, Error>
+        });
+
+        let db = self.db.clone();
+        b!(self.queue, {
+            for (queue_name, job) in vec {
+                let id = db.generate_id()?;
+                let mut key = queue_name.as_bytes().to_vec();
+                key.extend(id.to_be_bytes());
+
+                queue.insert(key, job)?;
+            }
+
+            Ok(()) as Result<(), Error>
+        });
+
+        Ok(())
     }
 
-    async fn push(&self, queue: &'static str, job: Self::Bytes) -> Result<(), Error> {
+    #[tracing::instrument(skip(self, job), fields(worker_id = %String::from_utf8_lossy(&job)))]
+    async fn push(&self, queue_name: &'static str, job: Self::Bytes) -> Result<(), Error> {
         let id = self.db.generate_id()?;
-        let mut key = queue.as_bytes().to_vec();
+        let mut key = queue_name.as_bytes().to_vec();
         key.extend(id.to_be_bytes());
 
         b!(self.queue, queue.insert(key, job));
 
-        if let Some(notifier) = self.queue_notifier.read().unwrap().get(&queue) {
+        if let Some(notifier) = self.queue_notifier.read().unwrap().get(&queue_name) {
             notifier.notify_one();
             return Ok(());
         }
@@ -176,13 +216,14 @@ impl QueueRepo for SledRepo {
         self.queue_notifier
             .write()
             .unwrap()
-            .entry(queue)
+            .entry(queue_name)
             .or_insert_with(|| Arc::new(Notify::new()))
             .notify_one();
 
         Ok(())
     }
 
+    #[tracing::instrument(skip(self, worker_id), fields(worker_id = %String::from_utf8_lossy(&worker_id)))]
     async fn pop(
         &self,
         queue_name: &'static str,
@@ -199,7 +240,11 @@ impl QueueRepo for SledRepo {
                     .scan_prefix(queue_name.as_bytes())
                     .find_map(Result::ok)
                 {
-                    in_progress_queue.insert(&worker_id, &job)?;
+                    let mut in_progress_value = queue_name.as_bytes().to_vec();
+                    in_progress_value.push(0);
+                    in_progress_value.extend_from_slice(&job);
+
+                    in_progress_queue.insert(&worker_id, in_progress_value)?;
 
                     if queue.remove(key)?.is_some() {
                         return Ok(Some(job));

From 8734dfbdc72a17771e070c26287dd43f9676d271 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sat, 2 Apr 2022 20:56:29 -0500
Subject: [PATCH 30/37] Add backgrounding routes

- Accept backgrounded uploads
- Allow backgrounded processing

Still TODO:
- Endpoint for waiting on/claiming an upload
---
 src/backgrounded.rs  |  92 ++++++++++++++++++++++++++++++++++++++
 src/main.rs          | 103 +++++++++++++++++++++++++++++++++++++++++++
 src/queue.rs         |  11 ++---
 src/queue/process.rs |   2 +-
 src/repo.rs          |  24 +++++++++-
 src/repo/sled.rs     |  30 +++++++++++--
 6 files changed, 252 insertions(+), 10 deletions(-)
 create mode 100644 src/backgrounded.rs

diff --git a/src/backgrounded.rs b/src/backgrounded.rs
new file mode 100644
index 0000000..58b3b04
--- /dev/null
+++ b/src/backgrounded.rs
@@ -0,0 +1,92 @@
+use crate::{
+    error::Error,
+    repo::{FullRepo, UploadId, UploadRepo},
+    store::Store,
+};
+use actix_web::web::Bytes;
+use futures_util::{Stream, TryStreamExt};
+use tokio_util::io::StreamReader;
+
+pub(crate) struct Backgrounded<R, S>
+where
+    R: FullRepo + 'static,
+    S: Store,
+{
+    repo: R,
+    identifier: Option<S::Identifier>,
+    upload_id: Option<UploadId>,
+}
+
+impl<R, S> Backgrounded<R, S>
+where
+    R: FullRepo + 'static,
+    S: Store,
+{
+    pub(crate) fn disarm(mut self) {
+        let _ = self.identifier.take();
+        let _ = self.upload_id.take();
+    }
+
+    pub(crate) fn upload_id(&self) -> Option<UploadId> {
+        self.upload_id
+    }
+
+    pub(crate) fn identifier(&self) -> Option<&S::Identifier> {
+        self.identifier.as_ref()
+    }
+
+    pub(crate) async fn proxy<P>(repo: R, store: S, stream: P) -> Result<Self, Error>
+    where
+        P: Stream<Item = Result<Bytes, Error>>,
+    {
+        let mut this = Self {
+            repo,
+            identifier: None,
+            upload_id: Some(UploadId::generate()),
+        };
+
+        this.do_proxy(store, stream).await?;
+
+        Ok(this)
+    }
+
+    async fn do_proxy<P>(&mut self, store: S, stream: P) -> Result<(), Error>
+    where
+        P: Stream<Item = Result<Bytes, Error>>,
+    {
+        UploadRepo::create(&self.repo, self.upload_id.expect("Upload id exists")).await?;
+
+        let stream = stream.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e));
+        let mut reader = StreamReader::new(Box::pin(stream));
+
+        let identifier = store.save_async_read(&mut reader).await?;
+
+        self.identifier = Some(identifier.clone());
+
+        Ok(())
+    }
+}
+
+impl<R, S> Drop for Backgrounded<R, S>
+where
+    R: FullRepo + 'static,
+    S: Store,
+{
+    fn drop(&mut self) {
+        if let Some(identifier) = self.identifier.take() {
+            let repo = self.repo.clone();
+
+            actix_rt::spawn(async move {
+                let _ = crate::queue::cleanup_identifier(&repo, identifier).await;
+            });
+        }
+
+        if let Some(upload_id) = self.upload_id {
+            let repo = self.repo.clone();
+
+            actix_rt::spawn(async move {
+                let _ = repo.claim(upload_id).await;
+            });
+        }
+    }
+}
diff --git a/src/main.rs b/src/main.rs
index a022a46..e096f62 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -22,6 +22,7 @@ use tracing_actix_web::TracingLogger;
 use tracing_awc::Tracing;
 use tracing_futures::Instrument;
 
+mod backgrounded;
 mod concurrent_processor;
 mod config;
 mod details;
@@ -48,6 +49,7 @@ mod tmp_file;
 mod validate;
 
 use self::{
+    backgrounded::Backgrounded,
     config::{Configuration, ImageFormat, Operation},
     details::Details,
     either::Either,
@@ -57,6 +59,7 @@ use self::{
     magick::details_hint,
     middleware::{Deadline, Internal},
     migrate::LatestDb,
+    queue::queue_generate,
     repo::{Alias, DeleteToken, FullRepo, HashRepo, IdentifierRepo, Repo, SettingsRepo},
     serde_str::Serde,
     store::{file_store::FileStore, object_store::ObjectStore, Identifier, Store},
@@ -134,6 +137,48 @@ async fn upload<R: FullRepo, S: Store + 'static>(
     })))
 }
 
+#[instrument(name = "Uploaded files", skip(value))]
+async fn upload_backgrounded<R: FullRepo, S: Store>(
+    value: Value<Backgrounded<R, S>>,
+    repo: web::Data<R>,
+) -> Result<HttpResponse, Error> {
+    let images = value
+        .map()
+        .and_then(|mut m| m.remove("images"))
+        .and_then(|images| images.array())
+        .ok_or(UploadError::NoFiles)?;
+
+    let mut files = Vec::new();
+    let images = images
+        .into_iter()
+        .filter_map(|i| i.file())
+        .collect::<Vec<_>>();
+
+    for image in &images {
+        let upload_id = image.result.upload_id().expect("Upload ID exists");
+        let identifier = image
+            .result
+            .identifier()
+            .expect("Identifier exists")
+            .to_bytes()?;
+
+        queue::queue_ingest(&**repo, identifier, upload_id, None, true).await?;
+
+        files.push(serde_json::json!({
+            "file": upload_id.to_string(),
+        }));
+    }
+
+    for image in images {
+        image.result.disarm();
+    }
+
+    Ok(HttpResponse::Created().json(&serde_json::json!({
+        "msg": "ok",
+        "files": files
+    })))
+}
+
 #[derive(Debug, serde::Deserialize)]
 struct UrlQuery {
     url: String,
@@ -339,6 +384,30 @@ async fn process<R: FullRepo, S: Store + 'static>(
     ))
 }
 
+/// Process files
+#[instrument(name = "Spawning image process", skip(repo))]
+async fn process_backgrounded<R: FullRepo, S: Store>(
+    query: web::Query<ProcessQuery>,
+    ext: web::Path<String>,
+    repo: web::Data<R>,
+) -> Result<HttpResponse, Error> {
+    let (target_format, source, process_path, process_args) = prepare_process(query, ext.as_str())?;
+
+    let path_string = process_path.to_string_lossy().to_string();
+    let hash = repo.hash(&source).await?;
+    let identifier_opt = repo
+        .variant_identifier::<S::Identifier>(hash.clone(), path_string)
+        .await?;
+
+    if identifier_opt.is_some() {
+        return Ok(HttpResponse::Accepted().finish());
+    }
+
+    queue_generate(&**repo, target_format, source, process_path, process_args).await?;
+
+    Ok(HttpResponse::Accepted().finish())
+}
+
 /// Fetch file details
 #[instrument(name = "Fetching details", skip(repo))]
 async fn details<R: FullRepo, S: Store + 'static>(
@@ -603,6 +672,31 @@ async fn launch<R: FullRepo + Clone + 'static, S: Store + Clone + 'static>(
             })),
         );
 
+    // Create a new Multipart Form validator for backgrounded uploads
+    //
+    // This form is expecting a single array field, 'images' with at most 10 files in it
+    let repo2 = repo.clone();
+    let store2 = store.clone();
+    let backgrounded_form = Form::new()
+        .max_files(10)
+        .max_file_size(CONFIG.media.max_file_size * MEGABYTES)
+        .transform_error(transform_error)
+        .field(
+            "images",
+            Field::array(Field::file(move |filename, _, stream| {
+                let repo = repo2.clone();
+                let store = store2.clone();
+
+                let span = tracing::info_span!("file-proxy", ?filename);
+
+                let stream = stream.map_err(Error::from);
+
+                Box::pin(
+                    async move { Backgrounded::proxy(repo, store, stream).await }.instrument(span),
+                )
+            })),
+        );
+
     HttpServer::new(move || {
         let store = store.clone();
         let repo = repo.clone();
@@ -632,6 +726,11 @@ async fn launch<R: FullRepo + Clone + 'static, S: Store + Clone + 'static>(
                             .wrap(form.clone())
                             .route(web::post().to(upload::<R, S>)),
                     )
+                    .service(
+                        web::resource("/backgrounded")
+                            .wrap(backgrounded_form.clone())
+                            .route(web::post().to(upload_backgrounded::<R, S>)),
+                    )
                     .service(web::resource("/download").route(web::get().to(download::<R, S>)))
                     .service(
                         web::resource("/delete/{delete_token}/{filename}")
@@ -642,6 +741,10 @@ async fn launch<R: FullRepo + Clone + 'static, S: Store + Clone + 'static>(
                         web::resource("/original/{filename}").route(web::get().to(serve::<R, S>)),
                     )
                     .service(web::resource("/process.{ext}").route(web::get().to(process::<R, S>)))
+                    .service(
+                        web::resource("/process_backgrounded.{ext}")
+                            .route(web::get().to(process_backgrounded::<R, S>)),
+                    )
                     .service(
                         web::scope("/details")
                             .service(
diff --git a/src/queue.rs b/src/queue.rs
index fbfc1ae..22e095f 100644
--- a/src/queue.rs
+++ b/src/queue.rs
@@ -1,13 +1,14 @@
 use crate::{
     config::ImageFormat,
     error::Error,
-    repo::{Alias, AliasRepo, DeleteToken, FullRepo, HashRepo, IdentifierRepo, QueueRepo},
+    repo::{
+        Alias, AliasRepo, DeleteToken, FullRepo, HashRepo, IdentifierRepo, QueueRepo, UploadId,
+    },
     serde_str::Serde,
     store::{Identifier, Store},
 };
 use std::{future::Future, path::PathBuf, pin::Pin};
 use tracing::Instrument;
-use uuid::Uuid;
 
 mod cleanup;
 mod process;
@@ -33,7 +34,7 @@ enum Cleanup {
 enum Process {
     Ingest {
         identifier: Vec<u8>,
-        upload_id: Uuid,
+        upload_id: Serde<UploadId>,
         declared_alias: Option<Serde<Alias>>,
         should_validate: bool,
     },
@@ -80,14 +81,14 @@ pub(crate) async fn cleanup_identifier<R: QueueRepo, I: Identifier>(
 pub(crate) async fn queue_ingest<R: QueueRepo>(
     repo: &R,
     identifier: Vec<u8>,
-    upload_id: Uuid,
+    upload_id: UploadId,
     declared_alias: Option<Alias>,
     should_validate: bool,
 ) -> Result<(), Error> {
     let job = serde_json::to_vec(&Process::Ingest {
         identifier,
         declared_alias: declared_alias.map(Serde::new),
-        upload_id,
+        upload_id: Serde::new(upload_id),
         should_validate,
     })?;
     repo.push(PROCESS_QUEUE, job.into()).await?;
diff --git a/src/queue/process.rs b/src/queue/process.rs
index 24f7e30..d8be535 100644
--- a/src/queue/process.rs
+++ b/src/queue/process.rs
@@ -32,7 +32,7 @@ where
                         repo,
                         store,
                         identifier,
-                        upload_id.into(),
+                        Serde::into_inner(upload_id),
                         declared_alias.map(Serde::into_inner),
                         should_validate,
                     )
diff --git a/src/repo.rs b/src/repo.rs
index e823bcf..a90755b 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -31,7 +31,7 @@ pub(crate) struct DeleteToken {
 
 pub(crate) struct AlreadyExists;
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub(crate) struct UploadId {
     id: Uuid,
 }
@@ -88,6 +88,8 @@ pub(crate) trait BaseRepo {
 
 #[async_trait::async_trait(?Send)]
 pub(crate) trait UploadRepo: BaseRepo {
+    async fn create(&self, upload_id: UploadId) -> Result<(), Error>;
+
     async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, Error>;
 
     async fn claim(&self, upload_id: UploadId) -> Result<(), Error>;
@@ -439,6 +441,26 @@ impl From<Uuid> for UploadId {
     }
 }
 
+impl From<UploadId> for Uuid {
+    fn from(uid: UploadId) -> Self {
+        uid.id
+    }
+}
+
+impl std::str::FromStr for UploadId {
+    type Err = <Uuid as std::str::FromStr>::Err;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(UploadId { id: s.parse()? })
+    }
+}
+
+impl std::fmt::Display for UploadId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.id, f)
+    }
+}
+
 impl std::fmt::Display for MaybeUuid {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
diff --git a/src/repo/sled.rs b/src/repo/sled.rs
index ab568b0..7955309 100644
--- a/src/repo/sled.rs
+++ b/src/repo/sled.rs
@@ -125,14 +125,38 @@ impl From<InnerUploadResult> for UploadResult {
 
 #[async_trait::async_trait(?Send)]
 impl UploadRepo for SledRepo {
+    async fn create(&self, upload_id: UploadId) -> Result<(), Error> {
+        b!(self.uploads, uploads.insert(upload_id.as_bytes(), b"1"));
+        Ok(())
+    }
+
     async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, Error> {
         let mut subscriber = self.uploads.watch_prefix(upload_id.as_bytes());
 
-        while let Some(event) = (&mut subscriber).await {
-            if let sled::Event::Insert { value, .. } = event {
-                let result: InnerUploadResult = serde_json::from_slice(&value)?;
+        let bytes = upload_id.as_bytes().to_vec();
+        let opt = b!(self.uploads, uploads.get(bytes));
+
+        if let Some(bytes) = opt {
+            if bytes != b"1" {
+                let result: InnerUploadResult = serde_json::from_slice(&bytes)?;
                 return Ok(result.into());
             }
+        } else {
+            return Err(UploadError::NoFiles.into());
+        }
+
+        while let Some(event) = (&mut subscriber).await {
+            match event {
+                sled::Event::Remove { .. } => {
+                    return Err(UploadError::NoFiles.into());
+                }
+                sled::Event::Insert { value, .. } => {
+                    if value != b"1" {
+                        let result: InnerUploadResult = serde_json::from_slice(&value)?;
+                        return Ok(result.into());
+                    }
+                }
+            }
         }
 
         Err(UploadError::Canceled.into())

From 29f07743312fcfd88efe0911c75cf6f23b36f21d Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sat, 2 Apr 2022 21:15:39 -0500
Subject: [PATCH 31/37] Add ability to claim uploads

---
 src/backgrounded.rs  |  2 +-
 src/main.rs          | 62 +++++++++++++++++++++++++++++++++++++++-----
 src/queue/process.rs |  2 ++
 3 files changed, 58 insertions(+), 8 deletions(-)

diff --git a/src/backgrounded.rs b/src/backgrounded.rs
index 58b3b04..e696161 100644
--- a/src/backgrounded.rs
+++ b/src/backgrounded.rs
@@ -61,7 +61,7 @@ where
 
         let identifier = store.save_async_read(&mut reader).await?;
 
-        self.identifier = Some(identifier.clone());
+        self.identifier = Some(identifier);
 
         Ok(())
     }
diff --git a/src/main.rs b/src/main.rs
index e096f62..66b9a3d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -48,6 +48,8 @@ mod stream;
 mod tmp_file;
 mod validate;
 
+use crate::repo::UploadResult;
+
 use self::{
     backgrounded::Backgrounded,
     config::{Configuration, ImageFormat, Operation},
@@ -60,7 +62,7 @@ use self::{
     middleware::{Deadline, Internal},
     migrate::LatestDb,
     queue::queue_generate,
-    repo::{Alias, DeleteToken, FullRepo, HashRepo, IdentifierRepo, Repo, SettingsRepo},
+    repo::{Alias, DeleteToken, FullRepo, HashRepo, IdentifierRepo, Repo, SettingsRepo, UploadId},
     serde_str::Serde,
     store::{file_store::FileStore, object_store::ObjectStore, Identifier, Store},
     stream::{StreamLimit, StreamTimeout},
@@ -165,7 +167,7 @@ async fn upload_backgrounded<R: FullRepo, S: Store>(
         queue::queue_ingest(&**repo, identifier, upload_id, None, true).await?;
 
         files.push(serde_json::json!({
-            "file": upload_id.to_string(),
+            "upload_id": upload_id.to_string(),
         }));
     }
 
@@ -173,12 +175,51 @@ async fn upload_backgrounded<R: FullRepo, S: Store>(
         image.result.disarm();
     }
 
-    Ok(HttpResponse::Created().json(&serde_json::json!({
+    Ok(HttpResponse::Accepted().json(&serde_json::json!({
         "msg": "ok",
-        "files": files
+        "uploads": files
     })))
 }
 
+#[derive(Debug, serde::Deserialize)]
+struct ClaimQuery {
+    upload_id: Serde<UploadId>,
+}
+
+/// Claim a backgrounded upload
+#[instrument(name = "Waiting on upload", skip(repo))]
+async fn claim_upload<R: FullRepo>(
+    repo: web::Data<R>,
+    query: web::Query<ClaimQuery>,
+) -> Result<HttpResponse, Error> {
+    let upload_id = Serde::into_inner(query.into_inner().upload_id);
+
+    match actix_rt::time::timeout(Duration::from_secs(10), repo.wait(upload_id)).await {
+        Ok(wait_res) => {
+            let upload_result = wait_res?;
+            repo.claim(upload_id).await?;
+
+            match upload_result {
+                UploadResult::Success { alias, token } => {
+                    Ok(HttpResponse::Ok().json(&serde_json::json!({
+                        "msg": "ok",
+                        "files": [{
+                            "file": alias.to_string(),
+                            "delete_token": token.to_string(),
+                        }]
+                    })))
+                }
+                UploadResult::Failure { message } => Ok(HttpResponse::UnprocessableEntity().json(
+                    &serde_json::json!({
+                        "msg": message,
+                    }),
+                )),
+            }
+        }
+        Err(_) => Ok(HttpResponse::NoContent().finish()),
+    }
+}
+
 #[derive(Debug, serde::Deserialize)]
 struct UrlQuery {
     url: String,
@@ -727,9 +768,16 @@ async fn launch<R: FullRepo + Clone + 'static, S: Store + Clone + 'static>(
                             .route(web::post().to(upload::<R, S>)),
                     )
                     .service(
-                        web::resource("/backgrounded")
-                            .wrap(backgrounded_form.clone())
-                            .route(web::post().to(upload_backgrounded::<R, S>)),
+                        web::scope("/backgrounded")
+                            .service(
+                                web::resource("")
+                                    .guard(guard::Post())
+                                    .wrap(backgrounded_form.clone())
+                                    .route(web::post().to(upload_backgrounded::<R, S>)),
+                            )
+                            .service(
+                                web::resource("/claim").route(web::get().to(claim_upload::<R>)),
+                            ),
                     )
                     .service(web::resource("/download").route(web::get().to(download::<R, S>)))
                     .service(
diff --git a/src/queue/process.rs b/src/queue/process.rs
index d8be535..6b31a78 100644
--- a/src/queue/process.rs
+++ b/src/queue/process.rs
@@ -90,6 +90,8 @@ where
 
         let token = session.delete_token().await?;
 
+        store.remove(&unprocessed_identifier).await?;
+
         Ok((session, token)) as Result<(Session<R, S>, DeleteToken), Error>
     };
 

From 409b7d4c5495aee7f2e8b27640e3b8af38cc2b5f Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sun, 3 Apr 2022 12:51:21 -0500
Subject: [PATCH 32/37] Remove unused conversions

---
 src/repo.rs | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/src/repo.rs b/src/repo.rs
index a90755b..3985c12 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -435,18 +435,6 @@ impl UploadId {
     }
 }
 
-impl From<Uuid> for UploadId {
-    fn from(id: Uuid) -> Self {
-        Self { id }
-    }
-}
-
-impl From<UploadId> for Uuid {
-    fn from(uid: UploadId) -> Self {
-        uid.id
-    }
-}
-
 impl std::str::FromStr for UploadId {
     type Err = <Uuid as std::str::FromStr>::Err;
 

From 044d24c92dd8a6c863404820522274cf05409a6c Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sun, 3 Apr 2022 12:53:14 -0500
Subject: [PATCH 33/37] Add bash client example for testing backgrounded
 operations

---
 client-examples/bash/upload.sh | 38 ++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100755 client-examples/bash/upload.sh

diff --git a/client-examples/bash/upload.sh b/client-examples/bash/upload.sh
new file mode 100755
index 0000000..c800176
--- /dev/null
+++ b/client-examples/bash/upload.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -xe
+
+upload_ids=$(
+  curl \
+    -F "images[]=@../cat.jpg" \
+    -F "images[]=@../earth.gif" \
+    -F "images[]=@../scene.webp" \
+    -F "images[]=@../test.png" \
+    -F "images[]=@../earth.gif" \
+    'http://localhost:8080/image/backgrounded' | \
+  jq '.uploads[].upload_id' | \
+  sed 's/"//g'
+)
+
+for upload in $(echo $upload_ids)
+do
+  echo "Processing for $upload"
+
+  json=$(curl "http://localhost:8080/image/backgrounded/claim?upload_id=$upload") 
+  delete_token=$(echo $json | jq '.files[0].delete_token' | sed 's/"//g')
+  filename=$(echo $json | jq '.files[0].file' | sed 's/"//g')
+
+  details=$(curl "http://localhost:8080/image/details/original/$filename")
+  mime_type=$(echo $details | jq '.content_type' | sed 's/"//g')
+
+  echo "Original mime: $mime_type"
+
+  curl "http://localhost:8080/image/process_backgrounded.webp?src=$filename&resize=200"
+  sleep 1
+  details=$(curl "http://localhost:8080/image/details/process.webp?src=$filename&resize=200")
+  mime_type=$(echo $details | jq '.content_type' | sed 's/"//g')
+
+  echo "Processed mime: $mime_type"
+
+  curl "http://localhost:8080/image/delete/$delete_token/$filename"
+done

From 82b7caf60e3870835b0faa8e5f0142609d9c7f89 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sun, 3 Apr 2022 12:55:16 -0500
Subject: [PATCH 34/37] bash example: Add more duplicate uploads

---
 client-examples/bash/upload.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/client-examples/bash/upload.sh b/client-examples/bash/upload.sh
index c800176..0520ac0 100755
--- a/client-examples/bash/upload.sh
+++ b/client-examples/bash/upload.sh
@@ -9,6 +9,9 @@ upload_ids=$(
     -F "images[]=@../scene.webp" \
     -F "images[]=@../test.png" \
     -F "images[]=@../earth.gif" \
+    -F "images[]=@../test.png" \
+    -F "images[]=@../cat.jpg" \
+    -F "images[]=@../scene.webp" \
     'http://localhost:8080/image/backgrounded' | \
   jq '.uploads[].upload_id' | \
   sed 's/"//g'

From d16fa711b897b49cf1c5e3446be6e5a02826b4d6 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sun, 3 Apr 2022 13:31:52 -0500
Subject: [PATCH 35/37] Update readme

---
 README.md | 189 ++++++++++++++++++++++--------------------------------
 1 file changed, 78 insertions(+), 111 deletions(-)

diff --git a/README.md b/README.md
index c45163a..e970c18 100644
--- a/README.md
+++ b/README.md
@@ -9,149 +9,116 @@ _a simple image hosting service_
 ## Usage
 ### Running
 ```
-pict-rs 
+pict-rs 0.4.0-alpha.1
+asonix <asonix@asonix.dog>
+A simple image hosting service
 
 USAGE:
     pict-rs [OPTIONS] <SUBCOMMAND>
 
 OPTIONS:
-    -a, --addr <ADDR>
-            The address and port the server binds to.
-
-        --api-key <API_KEY>
-            An optional string to be checked on requests to privileged endpoints
-
     -c, --config-file <CONFIG_FILE>
             Path to the pict-rs configuration file
 
+        --console-address <CONSOLE_ADDRESS>
+            Address and port to expose tokio-console metrics
+
         --console-buffer-capacity <CONSOLE_BUFFER_CAPACITY>
-            Specify the number of events the console subscriber is allowed to buffer
-
-    -f, --filters <FILTERS>
-            An optional list of filters to permit, supports 'identity', 'thumbnail', 'resize',
-            'crop', and 'blur'
-
-        --filesystem-storage-path <FILESYSTEM_STORAGE_PATH>
-            Path in which pict-rs will create it's 'files' directory
+            Capacity of the console-subscriber Event Buffer
 
     -h, --help
             Print help information
 
-    -i, --image-format <IMAGE_FORMAT>
-            An optional image format to convert all uploaded files into, supports 'jpg', 'png', and
-            'webp'
+        --log-format <LOG_FORMAT>
+            Format of logs printed to stdout
 
-    -m, --max-file-size <MAX_FILE_SIZE>
-            Specify the maximum allowed uploaded file size (in Megabytes)
+        --log-targets <LOG_TARGETS>
+            Log levels to print to stdout, respects RUST_LOG formatting
 
-        --max-image-area <MAX_IMAGE_AREA>
-            Specify the maximum area in pixels allowed in an image
+        --old-db-path <OLD_DB_PATH>
+            Path to the old pict-rs sled database
 
-        --max-image-height <MAX_IMAGE_HEIGHT>
-            Specify the maximum width in pixels allowed on an image
+        --opentelemetry-service-name <OPENTELEMETRY_SERVICE_NAME>
+            Service Name to use for OpenTelemetry
 
-        --max-image-width <MAX_IMAGE_WIDTH>
-            Specify the maximum width in pixels allowed on an image
+        --opentelemetry-targets <OPENTELEMETRY_TARGETS>
+            Log levels to use for OpenTelemetry, respects RUST_LOG formatting
 
-    -o, --opentelemetry-url <OPENTELEMETRY_URL>
-            Enable OpenTelemetry Tracing exports to the given OpenTelemetry collector
+        --opentelemetry-url <OPENTELEMETRY_URL>
+            URL to send OpenTelemetry metrics
 
-        --object-store-access-key <OBJECT_STORE_ACCESS_KEY>
-            
+        --save-to <SAVE_TO>
+            File to save the current configuration for reproducible runs
 
-        --object-store-bucket-name <OBJECT_STORE_BUCKET_NAME>
-            Name of the bucket in which pict-rs will store images
-
-        --object-store-region <OBJECT_STORE_REGION>
-            Region in which the bucket exists, can be an http endpoint
-
-        --object-store-secret-key <OBJECT_STORE_SECRET_KEY>
-            
-
-        --object-store-security-token <OBJECT_STORE_SECURITY_TOKEN>
-            
-
-        --object-store-session-token <OBJECT_STORE_SESSION_TOKEN>
-            
-
-    -p, --path <PATH>
-            The path to the data directory, e.g. data/
-
-    -R, --repo <REPO>
-            Set the database implementation. Available options are 'sled'. Default is 'sled'
-
-    -s, --skip-validate-imports
-            Whether to skip validating images uploaded via the internal import API
-
-    -S, --store <STORE>
-            Set the image store. Available options are 'object-storage' or 'filesystem'. Default is
-            'filesystem'
-
-        --sled-cache-capacity <SLED_CACHE_CAPACITY>
-            The number of bytes sled is allowed to use for it's in-memory cache
-
-        --sled-path <SLED_PATH>
-            Path in which pict-rs will create it's 'repo' directory
+    -V, --version
+            Print version information
 
 SUBCOMMANDS:
-    dump             
-    help             Print this message or the help of the given subcommand(s)
-    migrate-repo     
-    migrate-store    
-    run
-```
-
-```
-pict-rs-dump 
-
-USAGE:
-    pict-rs dump <PATH>
-
-ARGS:
-    <PATH>    
-
-OPTIONS:
-    -h, --help    Print help information
-```
-
-```
-pict-rs-migrate-repo 
-
-USAGE:
-    pict-rs migrate-repo <TO>
-
-ARGS:
-    <TO>    
-
-OPTIONS:
-    -h, --help    Print help information
-```
-
-```
-pict-rs-migrate-store 
-
-USAGE:
-    pict-rs migrate-store <TO>
-
-ARGS:
-    <TO>    
-
-OPTIONS:
-    -h, --help    Print help information
+    filesystem        Migrate from the provided filesystem storage
+    help              Print this message or the help of the given subcommand(s)
+    object-storage    Migrate from the provided object storage
+    run               Runs the pict-rs web server
 ```
 
 ```
 pict-rs-run 
+Runs the pict-rs web server
 
 USAGE:
-    pict-rs run
+    pict-rs run [OPTIONS] [SUBCOMMAND]
 
 OPTIONS:
-    -h, --help    Print help information
+    -a, --address <ADDRESS>
+            The address and port to bind the pict-rs web server
+
+        --api-key <API_KEY>
+            The API KEY required to access restricted routes
+
+    -h, --help
+            Print help information
+
+        --media-enable-silent-video <MEDIA_ENABLE_SILENT_VIDEO>
+            Whether to enable GIF and silent MP4 uploads. Full videos are unsupported
+
+        --media-filters <MEDIA_FILTERS>
+            Which media filters should be enabled on the `process` endpoint
+
+        --media-format <MEDIA_FORMAT>
+            Enforce uploaded media is transcoded to the provided format
+
+        --media-max-area <MEDIA_MAX_AREA>
+            The maximum area, in pixels, for uploaded media
+
+        --media-max-file-size <MEDIA_MAX_FILE_SIZE>
+            The maximum size, in megabytes, for uploaded media
+
+        --media-max-height <MEDIA_MAX_HEIGHT>
+            The maximum height, in pixels, for uploaded media
+
+        --media-max-width <MEDIA_MAX_WIDTH>
+            The maximum width, in pixels, for uploaded media
+
+        --media-skip-validate-imports <MEDIA_SKIP_VALIDATE_IMPORTS>
+            Whether to validate media on the "import" endpoint
+
+        --worker-id <WORKER_ID>
+            
+
+SUBCOMMANDS:
+    filesystem        Run pict-rs with filesystem storage
+    help              Print this message or the help of the given subcommand(s)
+    object-storage    Run pict-rs with object storage
 ```
 
-See [`pict-rs.toml`](https://git.asonix.dog/asonix/pict-rs/src/branch/main/pict-rs.toml) and 
-[`migrate.toml`](https://git.asonix.dog/asonix/pict-rs/src/branch/main/migrate.toml) for more
+Try running `help` commands for more runtime configuration options
+```
+$ pict-rs run filesystem -h
+$ pict-rs run object-storage -h
+$ pict-rs run filesystem sled -h
+$ pict-rs run object-storage sled -h
+```
+
+See [`pict-rs.toml`](https://git.asonix.dog/asonix/pict-rs/src/branch/main/pict-rs.toml) for more
 configuration
 
 #### Example:

From b0670c2f666442b82d534e512224bb3803104eb3 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sun, 3 Apr 2022 15:07:31 -0500
Subject: [PATCH 36/37] Move migrate to old

---
 src/main.rs                        | 6 +-----
 src/repo.rs                        | 5 +++--
 src/repo/old.rs                    | 6 +++++-
 src/{ => repo/old}/migrate.rs      | 0
 src/{ => repo/old}/migrate/s034.rs | 2 +-
 5 files changed, 10 insertions(+), 9 deletions(-)
 rename src/{ => repo/old}/migrate.rs (100%)
 rename src/{ => repo/old}/migrate/s034.rs (97%)

diff --git a/src/main.rs b/src/main.rs
index 66b9a3d..6c2d6a5 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -36,7 +36,6 @@ mod ingest;
 mod init_tracing;
 mod magick;
 mod middleware;
-mod migrate;
 mod process;
 mod processor;
 mod queue;
@@ -60,7 +59,6 @@ use self::{
     init_tracing::init_tracing,
     magick::details_hint,
     middleware::{Deadline, Internal},
-    migrate::LatestDb,
     queue::queue_generate,
     repo::{Alias, DeleteToken, FullRepo, HashRepo, IdentifierRepo, Repo, SettingsRepo, UploadId},
     serde_str::Serde,
@@ -873,9 +871,7 @@ async fn main() -> color_eyre::Result<()> {
     init_tracing(&CONFIG.tracing)?;
 
     let repo = Repo::open(CONFIG.repo.clone())?;
-
-    let db = LatestDb::exists(CONFIG.old_db.path.clone()).migrate()?;
-    repo.from_db(db).await?;
+    repo.from_db(CONFIG.old_db.path.clone()).await?;
 
     match (*OPERATION).clone() {
         Operation::Run => (),
diff --git a/src/repo.rs b/src/repo.rs
index 3985c12..f356a97 100644
--- a/src/repo.rs
+++ b/src/repo.rs
@@ -1,6 +1,7 @@
 use crate::{config, details::Details, error::Error, store::Identifier};
 use futures_util::Stream;
 use std::fmt::Debug;
+use std::path::PathBuf;
 use tracing::debug;
 use uuid::Uuid;
 
@@ -210,12 +211,12 @@ impl Repo {
     }
 
     #[tracing::instrument(skip_all)]
-    pub(crate) async fn from_db(&self, db: ::sled::Db) -> color_eyre::Result<()> {
+    pub(crate) async fn from_db(&self, path: PathBuf) -> color_eyre::Result<()> {
         if self.has_migrated().await? {
             return Ok(());
         }
 
-        let old = self::old::Old::open(db)?;
+        let old = self::old::Old::open(path)?;
 
         for hash in old.hashes() {
             match self {
diff --git a/src/repo/old.rs b/src/repo/old.rs
index e6a3ac2..c555a17 100644
--- a/src/repo/old.rs
+++ b/src/repo/old.rs
@@ -20,6 +20,8 @@
 use super::{Alias, DeleteToken, Details};
 use std::path::PathBuf;
 
+mod migrate;
+
 #[derive(Debug)]
 struct OldDbError(&'static str);
 
@@ -42,7 +44,9 @@ pub(super) struct Old {
 }
 
 impl Old {
-    pub(super) fn open(db: sled::Db) -> color_eyre::Result<Self> {
+    pub(super) fn open(path: PathBuf) -> color_eyre::Result<Self> {
+        let db = migrate::LatestDb::exists(path).migrate()?;
+
         Ok(Self {
             alias_tree: db.open_tree("alias")?,
             filename_tree: db.open_tree("filename")?,
diff --git a/src/migrate.rs b/src/repo/old/migrate.rs
similarity index 100%
rename from src/migrate.rs
rename to src/repo/old/migrate.rs
diff --git a/src/migrate/s034.rs b/src/repo/old/migrate/s034.rs
similarity index 97%
rename from src/migrate/s034.rs
rename to src/repo/old/migrate/s034.rs
index fde33a2..16340ed 100644
--- a/src/migrate/s034.rs
+++ b/src/repo/old/migrate/s034.rs
@@ -1,6 +1,6 @@
 use crate::{
     error::Error,
-    migrate::{SledDb, SledIter, SledTree},
+    repo::old::migrate::{SledDb, SledIter, SledTree},
 };
 use sled as sled034;
 use std::path::PathBuf;

From f2d7c685a6bc3ad7261a0b5388169c3b48c605d2 Mon Sep 17 00:00:00 2001
From: "Aode (lion)" <asonix@asonix.dog>
Date: Sun, 3 Apr 2022 15:21:14 -0500
Subject: [PATCH 37/37] Update deps

---
 Cargo.lock | 175 ++++++++++++++++++++---------------------------------
 Cargo.toml |   2 +-
 2 files changed, 67 insertions(+), 110 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index fee71d6..6458c45 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -16,7 +16,7 @@ dependencies = [
  "memchr",
  "pin-project-lite",
  "tokio",
- "tokio-util 0.7.0",
+ "tokio-util 0.7.1",
 ]
 
 [[package]]
@@ -45,7 +45,7 @@ dependencies = [
  "actix-rt",
  "actix-service",
  "actix-utils",
- "ahash 0.7.6",
+ "ahash",
  "base64",
  "bitflags",
  "bytes",
@@ -168,7 +168,7 @@ dependencies = [
  "log",
  "pin-project-lite",
  "tokio-rustls",
- "tokio-util 0.7.0",
+ "tokio-util 0.7.1",
  "webpki-roots",
 ]
 
@@ -195,7 +195,7 @@ dependencies = [
  "actix-server",
  "actix-service",
  "actix-utils",
- "ahash 0.7.6",
+ "ahash",
  "bytes",
  "bytestring",
  "cfg-if",
@@ -234,12 +234,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
-[[package]]
-name = "ahash"
-version = "0.4.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
-
 [[package]]
 name = "ahash"
 version = "0.7.6"
@@ -298,9 +292,9 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.52"
+version = "0.1.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3"
+checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -336,7 +330,7 @@ dependencies = [
  "actix-service",
  "actix-tls",
  "actix-utils",
- "ahash 0.7.6",
+ "ahash",
  "base64",
  "bytes",
  "cfg-if",
@@ -365,7 +359,7 @@ source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#943
 dependencies = [
  "anyhow",
  "dirs",
- "rust-ini 0.18.0",
+ "rust-ini",
  "serde",
  "serde-xml-rs",
  "serde_derive",
@@ -494,9 +488,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clap"
-version = "3.1.6"
+version = "3.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8c93436c21e4698bacadf42917db28b23017027a4deccb35dbe47a7e7840123"
+checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c"
 dependencies = [
  "atty",
  "bitflags",
@@ -511,9 +505,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "3.1.4"
+version = "3.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da95d038ede1a964ce99f49cbe27a7fb538d1da595e4b4f70b8c8f338d17bf16"
+checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1"
 dependencies = [
  "heck 0.4.0",
  "proc-macro-error",
@@ -551,9 +545,9 @@ dependencies = [
 
 [[package]]
 name = "config"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54ad70579325f1a38ea4c13412b82241c5900700a69785d73e2736bd65a33f86"
+checksum = "a623a46970097d353e2c8154fa527d8ce45cd0e02cc1812969b889c49b3728e8"
 dependencies = [
  "async-trait",
  "json5",
@@ -561,7 +555,7 @@ dependencies = [
  "nom",
  "pathdiff",
  "ron",
- "rust-ini 0.17.0",
+ "rust-ini",
  "serde",
  "serde_json",
  "toml",
@@ -755,15 +749,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "dlv-list"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68df3f2b690c1b86e65ef7830956aededf3cb0a16f898f79b9a6f421a7b6211b"
-dependencies = [
- "rand",
-]
-
 [[package]]
 name = "dlv-list"
 version = "0.3.0"
@@ -978,9 +963,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
+checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
 dependencies = [
  "cfg-if",
  "libc",
@@ -995,9 +980,9 @@ checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
 
 [[package]]
 name = "h2"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"
+checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57"
 dependencies = [
  "bytes",
  "fnv",
@@ -1008,19 +993,10 @@ dependencies = [
  "indexmap",
  "slab",
  "tokio",
- "tokio-util 0.6.9",
+ "tokio-util 0.7.1",
  "tracing",
 ]
 
-[[package]]
-name = "hashbrown"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
-dependencies = [
- "ahash 0.4.7",
-]
-
 [[package]]
 name = "hashbrown"
 version = "0.11.2"
@@ -1033,7 +1009,7 @@ version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758"
 dependencies = [
- "ahash 0.7.6",
+ "ahash",
 ]
 
 [[package]]
@@ -1197,9 +1173,9 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
 
 [[package]]
 name = "indexmap"
-version = "1.8.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
+checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
 dependencies = [
  "autocfg",
  "hashbrown 0.11.2",
@@ -1309,10 +1285,11 @@ checksum = "902eb695eb0591864543cbfbf6d742510642a605a61fc5e97fe6ceb5a30ac4fb"
 
 [[package]]
 name = "lock_api"
-version = "0.4.6"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b"
+checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
 dependencies = [
+ "autocfg",
  "scopeguard",
 ]
 
@@ -1542,23 +1519,13 @@ dependencies = [
  "tonic-build",
 ]
 
-[[package]]
-name = "ordered-multimap"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485"
-dependencies = [
- "dlv-list 0.2.3",
- "hashbrown 0.9.1",
-]
-
 [[package]]
 name = "ordered-multimap"
 version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
 dependencies = [
- "dlv-list 0.3.0",
+ "dlv-list",
  "hashbrown 0.12.0",
 ]
 
@@ -1595,7 +1562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"
 dependencies = [
  "lock_api",
- "parking_lot_core 0.9.1",
+ "parking_lot_core 0.9.2",
 ]
 
 [[package]]
@@ -1614,9 +1581,9 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954"
+checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37"
 dependencies = [
  "cfg-if",
  "libc",
@@ -1627,9 +1594,9 @@ dependencies = [
 
 [[package]]
 name = "paste"
-version = "1.0.6"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5"
+checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
 
 [[package]]
 name = "pathdiff"
@@ -1731,7 +1698,7 @@ dependencies = [
  "time",
  "tokio",
  "tokio-uring",
- "tokio-util 0.7.0",
+ "tokio-util 0.7.1",
  "toml",
  "tracing",
  "tracing-actix-web",
@@ -1871,9 +1838,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.16"
+version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57"
+checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58"
 dependencies = [
  "proc-macro2",
 ]
@@ -1910,18 +1877,18 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.12"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0"
+checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
 dependencies = [
  "bitflags",
 ]
 
 [[package]]
 name = "redox_users"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55"
+checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
 dependencies = [
  "getrandom",
  "redox_syscall",
@@ -2037,16 +2004,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "rust-ini"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22"
-dependencies = [
- "cfg-if",
- "ordered-multimap 0.3.1",
-]
-
 [[package]]
 name = "rust-ini"
 version = "0.18.0"
@@ -2054,7 +2011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
 dependencies = [
  "cfg-if",
- "ordered-multimap 0.4.3",
+ "ordered-multimap",
 ]
 
 [[package]]
@@ -2152,9 +2109,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.6"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d"
+checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4"
 
 [[package]]
 name = "serde"
@@ -2278,9 +2235,9 @@ dependencies = [
 
 [[package]]
 name = "slab"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
+checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
 
 [[package]]
 name = "sled"
@@ -2344,9 +2301,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.89"
+version = "1.0.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54"
+checksum = "704df27628939572cd88d33f171cd6f896f4eaca85252c6e0a72d8d8287ee86f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2540,16 +2497,16 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1"
+checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764"
 dependencies = [
  "bytes",
  "futures-core",
  "futures-sink",
- "log",
  "pin-project-lite",
  "tokio",
+ "tracing",
 ]
 
 [[package]]
@@ -2618,7 +2575,7 @@ dependencies = [
  "rand",
  "slab",
  "tokio",
- "tokio-util 0.7.0",
+ "tokio-util 0.7.1",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -2694,9 +2651,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-core"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c"
+checksum = "90442985ee2f57c9e1b548ee72ae842f4a9a20e3f417cc38dbc5dc684d9bb4ee"
 dependencies = [
  "lazy_static",
  "valuable",
@@ -2758,9 +2715,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.9"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e0ab7bdc962035a87fba73f3acca9b8a8d0034c2e6f60b84aeaaddddc155dce"
+checksum = "b9df98b037d039d03400d9dd06b0f8ce05486b5f25e9a2d7d36196e142ebbc52"
 dependencies = [
  "ansi_term",
  "lazy_static",
@@ -3040,9 +2997,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windows-sys"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6"
+checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825"
 dependencies = [
  "windows_aarch64_msvc",
  "windows_i686_gnu",
@@ -3053,33 +3010,33 @@ dependencies = [
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5"
+checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615"
+checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172"
+checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc"
+checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"
+checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9"
 
 [[package]]
 name = "winreg"
diff --git a/Cargo.toml b/Cargo.toml
index d27e3dd..63b2f5d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -29,7 +29,7 @@ awc = { version = "3.0.0", default-features = false, features = ["rustls"] }
 base64 = "0.13.0"
 clap = { version = "3.1.6", features = ["derive"] }
 color-eyre = "0.6"
-config = "0.12.0"
+config = "0.13.0"
 console-subscriber = "0.1"
 dashmap = "5.1.0"
 futures-util = "0.3.17"