2
0
Fork 0
mirror of https://git.asonix.dog/asonix/pict-rs synced 2024-12-23 03:41:23 +00:00

Improve migration from 0.3 formats

This commit is contained in:
Aode (Lion) 2022-04-24 17:10:15 -05:00
parent 4474850469
commit 4ac20546ce
4 changed files with 150 additions and 42 deletions

68
Cargo.lock generated
View file

@ -265,9 +265,9 @@ dependencies = [
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.56" version = "1.0.57"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
[[package]] [[package]]
name = "async-stream" name = "async-stream"
@ -377,9 +377,9 @@ dependencies = [
[[package]] [[package]]
name = "axum" name = "axum"
version = "0.5.1" version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25" checksum = "f523b4e98ba6897ae90994bc18423d9877c54f9047b06a00ddc8122a957b1c70"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"axum-core", "axum-core",
@ -406,9 +406,9 @@ dependencies = [
[[package]] [[package]]
name = "axum-core" name = "axum-core"
version = "0.2.1" version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a671c9ae99531afdd5d3ee8340b8da547779430689947144c140fc74a740244" checksum = "d3ddbd16eabff8b45f21b98671fddcc93daaa7ac4c84f8473693437226040de5"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"bytes", "bytes",
@ -420,15 +420,15 @@ dependencies = [
[[package]] [[package]]
name = "backtrace" name = "backtrace"
version = "0.3.64" version = "0.3.65"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61"
dependencies = [ dependencies = [
"addr2line", "addr2line",
"cc", "cc",
"cfg-if", "cfg-if",
"libc", "libc",
"miniz_oxide 0.4.4", "miniz_oxide",
"object", "object",
"rustc-demangle", "rustc-demangle",
] ]
@ -531,16 +531,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]] [[package]]
name = "clap" name = "clap"
version = "3.1.8" version = "3.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c" checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db"
dependencies = [ dependencies = [
"atty", "atty",
"bitflags", "bitflags",
"clap_derive", "clap_derive",
"clap_lex",
"indexmap", "indexmap",
"lazy_static", "lazy_static",
"os_str_bytes",
"strsim", "strsim",
"termcolor", "termcolor",
"textwrap", "textwrap",
@ -559,6 +559,15 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "clap_lex"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669"
dependencies = [
"os_str_bytes",
]
[[package]] [[package]]
name = "color-eyre" name = "color-eyre"
version = "0.6.1" version = "0.6.1"
@ -858,7 +867,7 @@ dependencies = [
"cfg-if", "cfg-if",
"crc32fast", "crc32fast",
"libc", "libc",
"miniz_oxide 0.5.1", "miniz_oxide",
] ]
[[package]] [[package]]
@ -1256,9 +1265,9 @@ dependencies = [
[[package]] [[package]]
name = "ipnet" name = "ipnet"
version = "2.4.0" version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
[[package]] [[package]]
name = "itertools" name = "itertools"
@ -1309,9 +1318,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.123" version = "0.2.124"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd" checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
[[package]] [[package]]
name = "linked-hash-map" name = "linked-hash-map"
@ -1427,16 +1436,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
dependencies = [
"adler",
"autocfg",
]
[[package]] [[package]]
name = "miniz_oxide" name = "miniz_oxide"
version = "0.5.1" version = "0.5.1"
@ -1524,9 +1523,9 @@ dependencies = [
[[package]] [[package]]
name = "object" name = "object"
version = "0.27.1" version = "0.28.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@ -1603,15 +1602,12 @@ name = "os_str_bytes"
version = "6.0.0" version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "owo-colors" name = "owo-colors"
version = "3.3.0" version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e72e30578e0d0993c8ae20823dd9cff2bc5517d2f586a8aef462a581e8a03eb" checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b"
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
@ -2848,9 +2844,9 @@ dependencies = [
[[package]] [[package]]
name = "tracing-log" name = "tracing-log"
version = "0.1.2" version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"
dependencies = [ dependencies = [
"lazy_static", "lazy_static",
"log", "log",

View file

@ -947,6 +947,8 @@ async fn main() -> color_eyre::Result<()> {
match CONFIG.store.clone() { match CONFIG.store.clone() {
config::Store::Filesystem(config::Filesystem { path }) => { config::Store::Filesystem(config::Filesystem { path }) => {
repo.migrate_identifiers().await?;
let store = FileStore::build(path, repo.clone()).await?; let store = FileStore::build(path, repo.clone()).await?;
match repo { match repo {
Repo::Sled(sled_repo) => launch(sled_repo, store).await, Repo::Sled(sled_repo) => launch(sled_repo, store).await,

View file

@ -1,8 +1,11 @@
use crate::{config, details::Details, error::Error, store::Identifier}; use crate::{
config,
details::Details,
error::Error,
store::{file_store::FileId, Identifier},
};
use futures_util::Stream; use futures_util::Stream;
use std::fmt::Debug; use std::{fmt::Debug, path::PathBuf};
use std::path::PathBuf;
use tracing::debug;
use uuid::Uuid; use uuid::Uuid;
mod old; mod old;
@ -240,6 +243,8 @@ impl Repo {
return Ok(()); return Ok(());
} }
tracing::warn!("Migrating Database from 0.3 layout to 0.4 layout");
let old = self::old::Old::open(path)?; let old = self::old::Old::open(path)?;
for hash in old.hashes() { for hash in old.hashes() {
@ -257,12 +262,45 @@ impl Repo {
Ok(()) Ok(())
} }
#[tracing::instrument(skip_all)]
pub(crate) async fn migrate_identifiers(&self) -> color_eyre::Result<()> {
if self.has_migrated_identifiers().await? {
return Ok(());
}
tracing::warn!("Migrating File Identifiers from 0.3 format to 0.4 format");
match self {
Self::Sled(repo) => {
use futures_util::StreamExt;
let mut hashes = repo.hashes().await;
while let Some(res) = hashes.next().await {
let hash = res?;
if let Err(e) = migrate_identifiers_for_hash(repo, hash).await {
tracing::error!("Failed to migrate identifiers for hash: {}", e);
}
}
}
}
self.mark_migrated_identifiers().await?;
Ok(())
}
async fn has_migrated(&self) -> color_eyre::Result<bool> { async fn has_migrated(&self) -> color_eyre::Result<bool> {
match self { match self {
Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_O1).await?.is_some()), Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_O1).await?.is_some()),
} }
} }
async fn has_migrated_identifiers(&self) -> color_eyre::Result<bool> {
match self {
Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_02).await?.is_some()),
}
}
async fn mark_migrated(&self) -> color_eyre::Result<()> { async fn mark_migrated(&self) -> color_eyre::Result<()> {
match self { match self {
Self::Sled(repo) => { Self::Sled(repo) => {
@ -272,18 +310,80 @@ impl Repo {
Ok(()) Ok(())
} }
async fn mark_migrated_identifiers(&self) -> color_eyre::Result<()> {
match self {
Self::Sled(repo) => {
repo.set(REPO_MIGRATION_02, b"1".to_vec().into()).await?;
}
}
Ok(())
}
} }
const REPO_MIGRATION_O1: &str = "repo-migration-01"; const REPO_MIGRATION_O1: &str = "repo-migration-01";
const REPO_MIGRATION_02: &str = "repo-migration-02";
const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress"; const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress";
const GENERATOR_KEY: &str = "last-path"; const GENERATOR_KEY: &str = "last-path";
#[tracing::instrument]
async fn migrate_identifiers_for_hash<T>(repo: &T, hash: ::sled::IVec) -> color_eyre::Result<()>
where
T: FullRepo,
{
let hash: T::Bytes = hash.to_vec().into();
if let Some(motion_identifier) = repo.motion_identifier::<FileId>(hash.clone()).await? {
if let Some(new_motion_identifier) = motion_identifier.normalize_for_migration() {
migrate_identifier_details(repo, &motion_identifier, &new_motion_identifier).await?;
repo.relate_motion_identifier(hash.clone(), &new_motion_identifier)
.await?;
}
}
for (variant_path, variant_identifier) in repo.variants::<FileId>(hash.clone()).await? {
if let Some(new_variant_identifier) = variant_identifier.normalize_for_migration() {
migrate_identifier_details(repo, &variant_identifier, &new_variant_identifier).await?;
repo.relate_variant_identifier(hash.clone(), variant_path, &new_variant_identifier)
.await?;
}
}
let main_identifier = repo.identifier::<FileId>(hash.clone()).await?;
if let Some(new_main_identifier) = main_identifier.normalize_for_migration() {
migrate_identifier_details(repo, &main_identifier, &new_main_identifier).await?;
repo.relate_identifier(hash, &new_main_identifier).await?;
}
Ok(())
}
#[tracing::instrument]
async fn migrate_identifier_details<T>(
repo: &T,
old: &FileId,
new: &FileId,
) -> color_eyre::Result<()>
where
T: FullRepo,
{
if let Some(details) = repo.details(old).await? {
repo.relate_details(new, &details).await?;
IdentifierRepo::cleanup(repo, old).await?;
}
Ok(())
}
#[tracing::instrument(skip(old))]
async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> color_eyre::Result<()> async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> color_eyre::Result<()>
where where
T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo, T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo + Debug,
{ {
if HashRepo::create(repo, hash.to_vec().into()).await?.is_err() { if HashRepo::create(repo, hash.to_vec().into()).await?.is_err() {
debug!("Duplicate hash detected"); tracing::debug!("Duplicate hash detected");
return Ok(()); return Ok(());
} }

View file

@ -34,6 +34,16 @@ impl Identifier for FileId {
} }
} }
impl FileId {
pub(crate) fn normalize_for_migration(&self) -> Option<Self> {
if self.0.starts_with("files") {
Some(Self(self.0.components().skip(1).collect::<PathBuf>()))
} else {
None
}
}
}
impl FileStore { impl FileStore {
pub(super) fn file_id_from_path(&self, path: PathBuf) -> Result<FileId, FileError> { pub(super) fn file_id_from_path(&self, path: PathBuf) -> Result<FileId, FileError> {
let stripped = path let stripped = path