mirror of
https://git.asonix.dog/asonix/pict-rs
synced 2024-12-22 19:31:35 +00:00
Improve migration from 0.3 formats
This commit is contained in:
parent
4474850469
commit
4ac20546ce
4 changed files with 150 additions and 42 deletions
68
Cargo.lock
generated
68
Cargo.lock
generated
|
|
@ -265,9 +265,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.56"
|
||||
version = "1.0.57"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
|
||||
checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
|
||||
|
||||
[[package]]
|
||||
name = "async-stream"
|
||||
|
|
@ -377,9 +377,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.5.1"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25"
|
||||
checksum = "f523b4e98ba6897ae90994bc18423d9877c54f9047b06a00ddc8122a957b1c70"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core",
|
||||
|
|
@ -406,9 +406,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.2.1"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a671c9ae99531afdd5d3ee8340b8da547779430689947144c140fc74a740244"
|
||||
checksum = "d3ddbd16eabff8b45f21b98671fddcc93daaa7ac4c84f8473693437226040de5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
|
|
@ -420,15 +420,15 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.64"
|
||||
version = "0.3.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f"
|
||||
checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61"
|
||||
dependencies = [
|
||||
"addr2line",
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide 0.4.4",
|
||||
"miniz_oxide",
|
||||
"object",
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
|
@ -531,16 +531,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
|||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "3.1.8"
|
||||
version = "3.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c"
|
||||
checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"bitflags",
|
||||
"clap_derive",
|
||||
"clap_lex",
|
||||
"indexmap",
|
||||
"lazy_static",
|
||||
"os_str_bytes",
|
||||
"strsim",
|
||||
"termcolor",
|
||||
"textwrap",
|
||||
|
|
@ -559,6 +559,15 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669"
|
||||
dependencies = [
|
||||
"os_str_bytes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "color-eyre"
|
||||
version = "0.6.1"
|
||||
|
|
@ -858,7 +867,7 @@ dependencies = [
|
|||
"cfg-if",
|
||||
"crc32fast",
|
||||
"libc",
|
||||
"miniz_oxide 0.5.1",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1256,9 +1265,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.4.0"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c"
|
||||
checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
|
|
@ -1309,9 +1318,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
|||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.123"
|
||||
version = "0.2.124"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd"
|
||||
checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
|
||||
|
||||
[[package]]
|
||||
name = "linked-hash-map"
|
||||
|
|
@ -1427,16 +1436,6 @@ version = "0.2.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
|
||||
dependencies = [
|
||||
"adler",
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.5.1"
|
||||
|
|
@ -1524,9 +1523,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.27.1"
|
||||
version = "0.28.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9"
|
||||
checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
|
@ -1603,15 +1602,12 @@ name = "os_str_bytes"
|
|||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "owo-colors"
|
||||
version = "3.3.0"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e72e30578e0d0993c8ae20823dd9cff2bc5517d2f586a8aef462a581e8a03eb"
|
||||
checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
|
|
@ -2848,9 +2844,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tracing-log"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3"
|
||||
checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"log",
|
||||
|
|
|
|||
|
|
@ -947,6 +947,8 @@ async fn main() -> color_eyre::Result<()> {
|
|||
|
||||
match CONFIG.store.clone() {
|
||||
config::Store::Filesystem(config::Filesystem { path }) => {
|
||||
repo.migrate_identifiers().await?;
|
||||
|
||||
let store = FileStore::build(path, repo.clone()).await?;
|
||||
match repo {
|
||||
Repo::Sled(sled_repo) => launch(sled_repo, store).await,
|
||||
|
|
|
|||
112
src/repo.rs
112
src/repo.rs
|
|
@ -1,8 +1,11 @@
|
|||
use crate::{config, details::Details, error::Error, store::Identifier};
|
||||
use crate::{
|
||||
config,
|
||||
details::Details,
|
||||
error::Error,
|
||||
store::{file_store::FileId, Identifier},
|
||||
};
|
||||
use futures_util::Stream;
|
||||
use std::fmt::Debug;
|
||||
use std::path::PathBuf;
|
||||
use tracing::debug;
|
||||
use std::{fmt::Debug, path::PathBuf};
|
||||
use uuid::Uuid;
|
||||
|
||||
mod old;
|
||||
|
|
@ -240,6 +243,8 @@ impl Repo {
|
|||
return Ok(());
|
||||
}
|
||||
|
||||
tracing::warn!("Migrating Database from 0.3 layout to 0.4 layout");
|
||||
|
||||
let old = self::old::Old::open(path)?;
|
||||
|
||||
for hash in old.hashes() {
|
||||
|
|
@ -257,12 +262,45 @@ impl Repo {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub(crate) async fn migrate_identifiers(&self) -> color_eyre::Result<()> {
|
||||
if self.has_migrated_identifiers().await? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
tracing::warn!("Migrating File Identifiers from 0.3 format to 0.4 format");
|
||||
|
||||
match self {
|
||||
Self::Sled(repo) => {
|
||||
use futures_util::StreamExt;
|
||||
let mut hashes = repo.hashes().await;
|
||||
|
||||
while let Some(res) = hashes.next().await {
|
||||
let hash = res?;
|
||||
if let Err(e) = migrate_identifiers_for_hash(repo, hash).await {
|
||||
tracing::error!("Failed to migrate identifiers for hash: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.mark_migrated_identifiers().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn has_migrated(&self) -> color_eyre::Result<bool> {
|
||||
match self {
|
||||
Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_O1).await?.is_some()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn has_migrated_identifiers(&self) -> color_eyre::Result<bool> {
|
||||
match self {
|
||||
Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_02).await?.is_some()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn mark_migrated(&self) -> color_eyre::Result<()> {
|
||||
match self {
|
||||
Self::Sled(repo) => {
|
||||
|
|
@ -272,18 +310,80 @@ impl Repo {
|
|||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_migrated_identifiers(&self) -> color_eyre::Result<()> {
|
||||
match self {
|
||||
Self::Sled(repo) => {
|
||||
repo.set(REPO_MIGRATION_02, b"1".to_vec().into()).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
const REPO_MIGRATION_O1: &str = "repo-migration-01";
|
||||
const REPO_MIGRATION_02: &str = "repo-migration-02";
|
||||
const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress";
|
||||
const GENERATOR_KEY: &str = "last-path";
|
||||
|
||||
#[tracing::instrument]
|
||||
async fn migrate_identifiers_for_hash<T>(repo: &T, hash: ::sled::IVec) -> color_eyre::Result<()>
|
||||
where
|
||||
T: FullRepo,
|
||||
{
|
||||
let hash: T::Bytes = hash.to_vec().into();
|
||||
|
||||
if let Some(motion_identifier) = repo.motion_identifier::<FileId>(hash.clone()).await? {
|
||||
if let Some(new_motion_identifier) = motion_identifier.normalize_for_migration() {
|
||||
migrate_identifier_details(repo, &motion_identifier, &new_motion_identifier).await?;
|
||||
repo.relate_motion_identifier(hash.clone(), &new_motion_identifier)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
for (variant_path, variant_identifier) in repo.variants::<FileId>(hash.clone()).await? {
|
||||
if let Some(new_variant_identifier) = variant_identifier.normalize_for_migration() {
|
||||
migrate_identifier_details(repo, &variant_identifier, &new_variant_identifier).await?;
|
||||
repo.relate_variant_identifier(hash.clone(), variant_path, &new_variant_identifier)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
let main_identifier = repo.identifier::<FileId>(hash.clone()).await?;
|
||||
|
||||
if let Some(new_main_identifier) = main_identifier.normalize_for_migration() {
|
||||
migrate_identifier_details(repo, &main_identifier, &new_main_identifier).await?;
|
||||
repo.relate_identifier(hash, &new_main_identifier).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument]
|
||||
async fn migrate_identifier_details<T>(
|
||||
repo: &T,
|
||||
old: &FileId,
|
||||
new: &FileId,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
T: FullRepo,
|
||||
{
|
||||
if let Some(details) = repo.details(old).await? {
|
||||
repo.relate_details(new, &details).await?;
|
||||
IdentifierRepo::cleanup(repo, old).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(old))]
|
||||
async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> color_eyre::Result<()>
|
||||
where
|
||||
T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo,
|
||||
T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo + Debug,
|
||||
{
|
||||
if HashRepo::create(repo, hash.to_vec().into()).await?.is_err() {
|
||||
debug!("Duplicate hash detected");
|
||||
tracing::debug!("Duplicate hash detected");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,16 @@ impl Identifier for FileId {
|
|||
}
|
||||
}
|
||||
|
||||
impl FileId {
|
||||
pub(crate) fn normalize_for_migration(&self) -> Option<Self> {
|
||||
if self.0.starts_with("files") {
|
||||
Some(Self(self.0.components().skip(1).collect::<PathBuf>()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FileStore {
|
||||
pub(super) fn file_id_from_path(&self, path: PathBuf) -> Result<FileId, FileError> {
|
||||
let stripped = path
|
||||
|
|
|
|||
Loading…
Reference in a new issue