From 4ac20546ce36694ce2e19ad84ecb03aef6173b1b Mon Sep 17 00:00:00 2001 From: "Aode (Lion)" Date: Sun, 24 Apr 2022 17:10:15 -0500 Subject: [PATCH] Improve migration from 0.3 formats --- Cargo.lock | 68 +++++++++---------- src/main.rs | 2 + src/repo.rs | 112 ++++++++++++++++++++++++++++++-- src/store/file_store/file_id.rs | 10 +++ 4 files changed, 150 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb2c22a..c5a3b47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" [[package]] name = "async-stream" @@ -377,9 +377,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25" +checksum = "f523b4e98ba6897ae90994bc18423d9877c54f9047b06a00ddc8122a957b1c70" dependencies = [ "async-trait", "axum-core", @@ -406,9 +406,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a671c9ae99531afdd5d3ee8340b8da547779430689947144c140fc74a740244" +checksum = "d3ddbd16eabff8b45f21b98671fddcc93daaa7ac4c84f8473693437226040de5" dependencies = [ "async-trait", "bytes", @@ -420,15 +420,15 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" +checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61" dependencies = [ "addr2line", "cc", "cfg-if", "libc", - "miniz_oxide 0.4.4", + "miniz_oxide", "object", "rustc-demangle", ] @@ -531,16 +531,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "3.1.8" +version = "3.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c" +checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db" dependencies = [ "atty", "bitflags", "clap_derive", + "clap_lex", "indexmap", "lazy_static", - "os_str_bytes", "strsim", "termcolor", "textwrap", @@ -559,6 +559,15 @@ dependencies = [ "syn", ] +[[package]] +name = "clap_lex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "color-eyre" version = "0.6.1" @@ -858,7 +867,7 @@ dependencies = [ "cfg-if", "crc32fast", "libc", - "miniz_oxide 0.5.1", + "miniz_oxide", ] [[package]] @@ -1256,9 +1265,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "itertools" @@ -1309,9 +1318,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.123" +version = "0.2.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd" +checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" [[package]] name = "linked-hash-map" @@ -1427,16 +1436,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" -dependencies = [ - "adler", - "autocfg", -] - [[package]] name = "miniz_oxide" version = "0.5.1" @@ -1524,9 +1523,9 @@ dependencies = [ [[package]] name = "object" -version = "0.27.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456" dependencies = [ "memchr", ] @@ -1603,15 +1602,12 @@ name = "os_str_bytes" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" -dependencies = [ - "memchr", -] [[package]] name = "owo-colors" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e72e30578e0d0993c8ae20823dd9cff2bc5517d2f586a8aef462a581e8a03eb" +checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b" [[package]] name = "parking_lot" @@ -2848,9 +2844,9 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", "log", diff --git a/src/main.rs b/src/main.rs index 73e726a..d12644f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -947,6 +947,8 @@ async fn main() -> color_eyre::Result<()> { match CONFIG.store.clone() { config::Store::Filesystem(config::Filesystem { path }) => { + repo.migrate_identifiers().await?; + let store = FileStore::build(path, repo.clone()).await?; match repo { Repo::Sled(sled_repo) => launch(sled_repo, store).await, diff --git a/src/repo.rs b/src/repo.rs index 3f32e30..6f8d06d 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -1,8 +1,11 @@ -use crate::{config, details::Details, error::Error, store::Identifier}; +use crate::{ + config, + details::Details, + error::Error, + store::{file_store::FileId, Identifier}, +}; use futures_util::Stream; -use std::fmt::Debug; -use std::path::PathBuf; -use tracing::debug; +use std::{fmt::Debug, path::PathBuf}; use uuid::Uuid; mod old; @@ -240,6 +243,8 @@ impl Repo { return Ok(()); } + tracing::warn!("Migrating Database from 0.3 layout to 0.4 layout"); + let old = self::old::Old::open(path)?; for hash in old.hashes() { @@ -257,12 +262,45 @@ impl Repo { Ok(()) } + #[tracing::instrument(skip_all)] + pub(crate) async fn migrate_identifiers(&self) -> color_eyre::Result<()> { + if self.has_migrated_identifiers().await? { + return Ok(()); + } + + tracing::warn!("Migrating File Identifiers from 0.3 format to 0.4 format"); + + match self { + Self::Sled(repo) => { + use futures_util::StreamExt; + let mut hashes = repo.hashes().await; + + while let Some(res) = hashes.next().await { + let hash = res?; + if let Err(e) = migrate_identifiers_for_hash(repo, hash).await { + tracing::error!("Failed to migrate identifiers for hash: {}", e); + } + } + } + } + + self.mark_migrated_identifiers().await?; + + Ok(()) + } + async fn has_migrated(&self) -> color_eyre::Result { match self { Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_O1).await?.is_some()), } } + async fn has_migrated_identifiers(&self) -> color_eyre::Result { + match self { + Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_02).await?.is_some()), + } + } + async fn mark_migrated(&self) -> color_eyre::Result<()> { match self { Self::Sled(repo) => { @@ -272,18 +310,80 @@ impl Repo { Ok(()) } + + async fn mark_migrated_identifiers(&self) -> color_eyre::Result<()> { + match self { + Self::Sled(repo) => { + repo.set(REPO_MIGRATION_02, b"1".to_vec().into()).await?; + } + } + + Ok(()) + } } const REPO_MIGRATION_O1: &str = "repo-migration-01"; +const REPO_MIGRATION_02: &str = "repo-migration-02"; const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress"; const GENERATOR_KEY: &str = "last-path"; +#[tracing::instrument] +async fn migrate_identifiers_for_hash(repo: &T, hash: ::sled::IVec) -> color_eyre::Result<()> +where + T: FullRepo, +{ + let hash: T::Bytes = hash.to_vec().into(); + + if let Some(motion_identifier) = repo.motion_identifier::(hash.clone()).await? { + if let Some(new_motion_identifier) = motion_identifier.normalize_for_migration() { + migrate_identifier_details(repo, &motion_identifier, &new_motion_identifier).await?; + repo.relate_motion_identifier(hash.clone(), &new_motion_identifier) + .await?; + } + } + + for (variant_path, variant_identifier) in repo.variants::(hash.clone()).await? { + if let Some(new_variant_identifier) = variant_identifier.normalize_for_migration() { + migrate_identifier_details(repo, &variant_identifier, &new_variant_identifier).await?; + repo.relate_variant_identifier(hash.clone(), variant_path, &new_variant_identifier) + .await?; + } + } + + let main_identifier = repo.identifier::(hash.clone()).await?; + + if let Some(new_main_identifier) = main_identifier.normalize_for_migration() { + migrate_identifier_details(repo, &main_identifier, &new_main_identifier).await?; + repo.relate_identifier(hash, &new_main_identifier).await?; + } + + Ok(()) +} + +#[tracing::instrument] +async fn migrate_identifier_details( + repo: &T, + old: &FileId, + new: &FileId, +) -> color_eyre::Result<()> +where + T: FullRepo, +{ + if let Some(details) = repo.details(old).await? { + repo.relate_details(new, &details).await?; + IdentifierRepo::cleanup(repo, old).await?; + } + + Ok(()) +} + +#[tracing::instrument(skip(old))] async fn migrate_hash(repo: &T, old: &old::Old, hash: ::sled::IVec) -> color_eyre::Result<()> where - T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo, + T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo + Debug, { if HashRepo::create(repo, hash.to_vec().into()).await?.is_err() { - debug!("Duplicate hash detected"); + tracing::debug!("Duplicate hash detected"); return Ok(()); } diff --git a/src/store/file_store/file_id.rs b/src/store/file_store/file_id.rs index bb7a3ec..cb6ea4e 100644 --- a/src/store/file_store/file_id.rs +++ b/src/store/file_store/file_id.rs @@ -34,6 +34,16 @@ impl Identifier for FileId { } } +impl FileId { + pub(crate) fn normalize_for_migration(&self) -> Option { + if self.0.starts_with("files") { + Some(Self(self.0.components().skip(1).collect::())) + } else { + None + } + } +} + impl FileStore { pub(super) fn file_id_from_path(&self, path: PathBuf) -> Result { let stripped = path