Improve migration from 0.3 formats

Cargo.lock

@@ -265,9 +265,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.56"
+version = "1.0.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
+checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
 
 [[package]]
 name = "async-stream"
@@ -377,9 +377,9 @@ dependencies = [
 
 [[package]]
 name = "axum"
-version = "0.5.1"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25"
+checksum = "f523b4e98ba6897ae90994bc18423d9877c54f9047b06a00ddc8122a957b1c70"
 dependencies = [
  "async-trait",
  "axum-core",
@@ -406,9 +406,9 @@ dependencies = [
 
 [[package]]
 name = "axum-core"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a671c9ae99531afdd5d3ee8340b8da547779430689947144c140fc74a740244"
+checksum = "d3ddbd16eabff8b45f21b98671fddcc93daaa7ac4c84f8473693437226040de5"
 dependencies = [
  "async-trait",
  "bytes",
@@ -420,15 +420,15 @@ dependencies = [
 
 [[package]]
 name = "backtrace"
-version = "0.3.64"
+version = "0.3.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f"
+checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61"
 dependencies = [
  "addr2line",
  "cc",
  "cfg-if",
  "libc",
- "miniz_oxide 0.4.4",
+ "miniz_oxide",
  "object",
  "rustc-demangle",
 ]
@@ -531,16 +531,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clap"
-version = "3.1.8"
+version = "3.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c"
+checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db"
 dependencies = [
  "atty",
  "bitflags",
  "clap_derive",
+ "clap_lex",
  "indexmap",
  "lazy_static",
- "os_str_bytes",
  "strsim",
  "termcolor",
  "textwrap",
@@ -559,6 +559,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "clap_lex"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669"
+dependencies = [
+ "os_str_bytes",
+]
+
 [[package]]
 name = "color-eyre"
 version = "0.6.1"
@@ -858,7 +867,7 @@ dependencies = [
  "cfg-if",
  "crc32fast",
  "libc",
- "miniz_oxide 0.5.1",
+ "miniz_oxide",
 ]
 
 [[package]]
@@ -1256,9 +1265,9 @@ dependencies = [
 
 [[package]]
 name = "ipnet"
-version = "2.4.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c"
+checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
 
 [[package]]
 name = "itertools"
@@ -1309,9 +1318,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.123"
+version = "0.2.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd"
+checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
 
 [[package]]
 name = "linked-hash-map"
@@ -1427,16 +1436,6 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
-[[package]]
-name = "miniz_oxide"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
-dependencies = [
- "adler",
- "autocfg",
-]
-
 [[package]]
 name = "miniz_oxide"
 version = "0.5.1"
@@ -1524,9 +1523,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.27.1"
+version = "0.28.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9"
+checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456"
 dependencies = [
  "memchr",
 ]
@@ -1603,15 +1602,12 @@ name = "os_str_bytes"
 version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
-dependencies = [
- "memchr",
-]
 
 [[package]]
 name = "owo-colors"
-version = "3.3.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e72e30578e0d0993c8ae20823dd9cff2bc5517d2f586a8aef462a581e8a03eb"
+checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b"
 
 [[package]]
 name = "parking_lot"
@@ -2848,9 +2844,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-log"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3"
+checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"
 dependencies = [
  "lazy_static",
  "log",

src/main.rs

@@ -947,6 +947,8 @@ async fn main() -> color_eyre::Result<()> {
 
     match CONFIG.store.clone() {
         config::Store::Filesystem(config::Filesystem { path }) => {
+            repo.migrate_identifiers().await?;
+
             let store = FileStore::build(path, repo.clone()).await?;
             match repo {
                 Repo::Sled(sled_repo) => launch(sled_repo, store).await,

src/repo.rs

@@ -1,8 +1,11 @@
-use crate::{config, details::Details, error::Error, store::Identifier};
+use crate::{
+    config,
+    details::Details,
+    error::Error,
+    store::{file_store::FileId, Identifier},
+};
 use futures_util::Stream;
-use std::fmt::Debug;
+use std::{fmt::Debug, path::PathBuf};
-use std::path::PathBuf;
-use tracing::debug;
 use uuid::Uuid;
 
 mod old;
@@ -240,6 +243,8 @@ impl Repo {
             return Ok(());
         }
 
+        tracing::warn!("Migrating Database from 0.3 layout to 0.4 layout");
+
         let old = self::old::Old::open(path)?;
 
         for hash in old.hashes() {
@@ -257,12 +262,45 @@ impl Repo {
         Ok(())
     }
 
+    #[tracing::instrument(skip_all)]
+    pub(crate) async fn migrate_identifiers(&self) -> color_eyre::Result<()> {
+        if self.has_migrated_identifiers().await? {
+            return Ok(());
+        }
+
+        tracing::warn!("Migrating File Identifiers from 0.3 format to 0.4 format");
+
+        match self {
+            Self::Sled(repo) => {
+                use futures_util::StreamExt;
+                let mut hashes = repo.hashes().await;
+
+                while let Some(res) = hashes.next().await {
+                    let hash = res?;
+                    if let Err(e) = migrate_identifiers_for_hash(repo, hash).await {
+                        tracing::error!("Failed to migrate identifiers for hash: {}", e);
+                    }
+                }
+            }
+        }
+
+        self.mark_migrated_identifiers().await?;
+
+        Ok(())
+    }
+
     async fn has_migrated(&self) -> color_eyre::Result<bool> {
         match self {
             Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_O1).await?.is_some()),
         }
     }
 
+    async fn has_migrated_identifiers(&self) -> color_eyre::Result<bool> {
+        match self {
+            Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_02).await?.is_some()),
+        }
+    }
+
     async fn mark_migrated(&self) -> color_eyre::Result<()> {
         match self {
             Self::Sled(repo) => {
@@ -272,18 +310,80 @@ impl Repo {
 
         Ok(())
     }
+
+    async fn mark_migrated_identifiers(&self) -> color_eyre::Result<()> {
+        match self {
+            Self::Sled(repo) => {
+                repo.set(REPO_MIGRATION_02, b"1".to_vec().into()).await?;
+            }
+        }
+
+        Ok(())
+    }
 }
 
 const REPO_MIGRATION_O1: &str = "repo-migration-01";
+const REPO_MIGRATION_02: &str = "repo-migration-02";
 const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress";
 const GENERATOR_KEY: &str = "last-path";
 
+#[tracing::instrument]
+async fn migrate_identifiers_for_hash<T>(repo: &T, hash: ::sled::IVec) -> color_eyre::Result<()>
+where
+    T: FullRepo,
+{
+    let hash: T::Bytes = hash.to_vec().into();
+
+    if let Some(motion_identifier) = repo.motion_identifier::<FileId>(hash.clone()).await? {
+        if let Some(new_motion_identifier) = motion_identifier.normalize_for_migration() {
+            migrate_identifier_details(repo, &motion_identifier, &new_motion_identifier).await?;
+            repo.relate_motion_identifier(hash.clone(), &new_motion_identifier)
+                .await?;
+        }
+    }
+
+    for (variant_path, variant_identifier) in repo.variants::<FileId>(hash.clone()).await? {
+        if let Some(new_variant_identifier) = variant_identifier.normalize_for_migration() {
+            migrate_identifier_details(repo, &variant_identifier, &new_variant_identifier).await?;
+            repo.relate_variant_identifier(hash.clone(), variant_path, &new_variant_identifier)
+                .await?;
+        }
+    }
+
+    let main_identifier = repo.identifier::<FileId>(hash.clone()).await?;
+
+    if let Some(new_main_identifier) = main_identifier.normalize_for_migration() {
+        migrate_identifier_details(repo, &main_identifier, &new_main_identifier).await?;
+        repo.relate_identifier(hash, &new_main_identifier).await?;
+    }
+
+    Ok(())
+}
+
+#[tracing::instrument]
+async fn migrate_identifier_details<T>(
+    repo: &T,
+    old: &FileId,
+    new: &FileId,
+) -> color_eyre::Result<()>
+where
+    T: FullRepo,
+{
+    if let Some(details) = repo.details(old).await? {
+        repo.relate_details(new, &details).await?;
+        IdentifierRepo::cleanup(repo, old).await?;
+    }
+
+    Ok(())
+}
+
+#[tracing::instrument(skip(old))]
 async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> color_eyre::Result<()>
 where
-    T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo,
+    T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo + Debug,
 {
     if HashRepo::create(repo, hash.to_vec().into()).await?.is_err() {
-        debug!("Duplicate hash detected");
+        tracing::debug!("Duplicate hash detected");
         return Ok(());
     }
 

src/store/file_store/file_id.rs

@@ -34,6 +34,16 @@ impl Identifier for FileId {
     }
 }
 
+impl FileId {
+    pub(crate) fn normalize_for_migration(&self) -> Option<Self> {
+        if self.0.starts_with("files") {
+            Some(Self(self.0.components().skip(1).collect::<PathBuf>()))
+        } else {
+            None
+        }
+    }
+}
+
 impl FileStore {
     pub(super) fn file_id_from_path(&self, path: PathBuf) -> Result<FileId, FileError> {
         let stripped = path