LemmyNet/pict-rs
2
0
Fork 0
mirror of https://git.asonix.dog/asonix/pict-rs synced 2024-12-22 11:21:24 +00:00
Code Releases Activity

Improve migration from 0.3 formats

Browse source
This commit is contained in:
Aode (Lion) 2022-04-24 17:10:15 -05:00
parent 4474850469
commit 4ac20546ce
4 changed files with 150 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
Cargo.lock generated
View file

@ -265,9 +265,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.56"
version = "1.0.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
[[package]]
name = "async-stream"
@ -377,9 +377,9 @@ dependencies = [
[[package]]
name = "axum"
version = "0.5.1"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25"
checksum = "f523b4e98ba6897ae90994bc18423d9877c54f9047b06a00ddc8122a957b1c70"
dependencies = [
"async-trait",
"axum-core",
@ -406,9 +406,9 @@ dependencies = [
[[package]]
name = "axum-core"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a671c9ae99531afdd5d3ee8340b8da547779430689947144c140fc74a740244"
checksum = "d3ddbd16eabff8b45f21b98671fddcc93daaa7ac4c84f8473693437226040de5"
dependencies = [
"async-trait",
"bytes",
@ -420,15 +420,15 @@ dependencies = [
[[package]]
name = "backtrace"
version = "0.3.64"
version = "0.3.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f"
checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61"
dependencies = [
"addr2line",
"cc",
"cfg-if",
"libc",
"miniz_oxide 0.4.4",
"miniz_oxide",
"object",
"rustc-demangle",
]
@ -531,16 +531,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clap"
version = "3.1.8"
version = "3.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c"
checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db"
dependencies = [
"atty",
"bitflags",
"clap_derive",
"clap_lex",
"indexmap",
"lazy_static",
"os_str_bytes",
"strsim",
"termcolor",
"textwrap",
@ -559,6 +559,15 @@ dependencies = [
"syn",
]
[[package]]
name = "clap_lex"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669"
dependencies = [
"os_str_bytes",
]
[[package]]
name = "color-eyre"
version = "0.6.1"
@ -858,7 +867,7 @@ dependencies = [
"cfg-if",
"crc32fast",
"libc",
"miniz_oxide 0.5.1",
"miniz_oxide",
]
[[package]]
@ -1256,9 +1265,9 @@ dependencies = [
[[package]]
name = "ipnet"
version = "2.4.0"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c"
checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
[[package]]
name = "itertools"
@ -1309,9 +1318,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.123"
version = "0.2.124"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd"
checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
[[package]]
name = "linked-hash-map"
@ -1427,16 +1436,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
dependencies = [
"adler",
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.5.1"
@ -1524,9 +1523,9 @@ dependencies = [
[[package]]
name = "object"
version = "0.27.1"
version = "0.28.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9"
checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456"
dependencies = [
"memchr",
]
@ -1603,15 +1602,12 @@ name = "os_str_bytes"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
dependencies = [
"memchr",
]
[[package]]
name = "owo-colors"
version = "3.3.0"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e72e30578e0d0993c8ae20823dd9cff2bc5517d2f586a8aef462a581e8a03eb"
checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b"
[[package]]
name = "parking_lot"
@ -2848,9 +2844,9 @@ dependencies = [
[[package]]
name = "tracing-log"
version = "0.1.2"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3"
checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"
dependencies = [
"lazy_static",
"log",

2
src/main.rs
View file

@ -947,6 +947,8 @@ async fn main() -> color_eyre::Result<()> {
match CONFIG.store.clone() {
config::Store::Filesystem(config::Filesystem { path }) => {
repo.migrate_identifiers().await?;
let store = FileStore::build(path, repo.clone()).await?;
match repo {
Repo::Sled(sled_repo) => launch(sled_repo, store).await,

112
src/repo.rs
View file

@ -1,8 +1,11 @@
use crate::{config, details::Details, error::Error, store::Identifier};
use crate::{
config,
details::Details,
error::Error,
store::{file_store::FileId, Identifier},
};
use futures_util::Stream;
use std::fmt::Debug;
use std::path::PathBuf;
use tracing::debug;
use std::{fmt::Debug, path::PathBuf};
use uuid::Uuid;
mod old;
@ -240,6 +243,8 @@ impl Repo {
return Ok(());
}
tracing::warn!("Migrating Database from 0.3 layout to 0.4 layout");
let old = self::old::Old::open(path)?;
for hash in old.hashes() {
@ -257,12 +262,45 @@ impl Repo {
Ok(())
}
#[tracing::instrument(skip_all)]
pub(crate) async fn migrate_identifiers(&self) -> color_eyre::Result<()> {
if self.has_migrated_identifiers().await? {
return Ok(());
}
tracing::warn!("Migrating File Identifiers from 0.3 format to 0.4 format");
match self {
Self::Sled(repo) => {
use futures_util::StreamExt;
let mut hashes = repo.hashes().await;
while let Some(res) = hashes.next().await {
let hash = res?;
if let Err(e) = migrate_identifiers_for_hash(repo, hash).await {
tracing::error!("Failed to migrate identifiers for hash: {}", e);
}
}
}
}
self.mark_migrated_identifiers().await?;
Ok(())
}
async fn has_migrated(&self) -> color_eyre::Result<bool> {
match self {
Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_O1).await?.is_some()),
}
}
async fn has_migrated_identifiers(&self) -> color_eyre::Result<bool> {
match self {
Self::Sled(repo) => Ok(repo.get(REPO_MIGRATION_02).await?.is_some()),
}
}
async fn mark_migrated(&self) -> color_eyre::Result<()> {
match self {
Self::Sled(repo) => {
@ -272,18 +310,80 @@ impl Repo {
Ok(())
}
async fn mark_migrated_identifiers(&self) -> color_eyre::Result<()> {
match self {
Self::Sled(repo) => {
repo.set(REPO_MIGRATION_02, b"1".to_vec().into()).await?;
}
}
Ok(())
}
}
const REPO_MIGRATION_O1: &str = "repo-migration-01";
const REPO_MIGRATION_02: &str = "repo-migration-02";
const STORE_MIGRATION_PROGRESS: &str = "store-migration-progress";
const GENERATOR_KEY: &str = "last-path";
#[tracing::instrument]
async fn migrate_identifiers_for_hash<T>(repo: &T, hash: ::sled::IVec) -> color_eyre::Result<()>
where
T: FullRepo,
{
let hash: T::Bytes = hash.to_vec().into();
if let Some(motion_identifier) = repo.motion_identifier::<FileId>(hash.clone()).await? {
if let Some(new_motion_identifier) = motion_identifier.normalize_for_migration() {
migrate_identifier_details(repo, &motion_identifier, &new_motion_identifier).await?;
repo.relate_motion_identifier(hash.clone(), &new_motion_identifier)
.await?;
}
}
for (variant_path, variant_identifier) in repo.variants::<FileId>(hash.clone()).await? {
if let Some(new_variant_identifier) = variant_identifier.normalize_for_migration() {
migrate_identifier_details(repo, &variant_identifier, &new_variant_identifier).await?;
repo.relate_variant_identifier(hash.clone(), variant_path, &new_variant_identifier)
.await?;
}
}
let main_identifier = repo.identifier::<FileId>(hash.clone()).await?;
if let Some(new_main_identifier) = main_identifier.normalize_for_migration() {
migrate_identifier_details(repo, &main_identifier, &new_main_identifier).await?;
repo.relate_identifier(hash, &new_main_identifier).await?;
}
Ok(())
}
#[tracing::instrument]
async fn migrate_identifier_details<T>(
repo: &T,
old: &FileId,
new: &FileId,
) -> color_eyre::Result<()>
where
T: FullRepo,
{
if let Some(details) = repo.details(old).await? {
repo.relate_details(new, &details).await?;
IdentifierRepo::cleanup(repo, old).await?;
}
Ok(())
}
#[tracing::instrument(skip(old))]
async fn migrate_hash<T>(repo: &T, old: &old::Old, hash: ::sled::IVec) -> color_eyre::Result<()>
where
T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo,
T: IdentifierRepo + HashRepo + AliasRepo + SettingsRepo + Debug,
{
if HashRepo::create(repo, hash.to_vec().into()).await?.is_err() {
debug!("Duplicate hash detected");
tracing::debug!("Duplicate hash detected");
return Ok(());
}

10
src/store/file_store/file_id.rs
View file

@ -34,6 +34,16 @@ impl Identifier for FileId {
}
}
impl FileId {
pub(crate) fn normalize_for_migration(&self) -> Option<Self> {
if self.0.starts_with("files") {
Some(Self(self.0.components().skip(1).collect::<PathBuf>()))
} else {
None
}
}
}
impl FileStore {
pub(super) fn file_id_from_path(&self, path: PathBuf) -> Result<FileId, FileError> {
let stripped = path