Rewrite library
* Remove old code * Rewrite with tests This implements a fassade pattern for ref library With the fassade, we can specify the hasher in a rather easy way, which is not possible with default generics for traits. The "default" part in "default generic type" is not properly implemented yet (as visible in the tests), as I don't know how to realize this. For simplicity, the `hasher` module exports a `default` module with a `DefaultHasher` type, which resolves to the `Sha1Hasher`. Signed-off-by: Matthias Beyer <mail@beyermatthias.de>
This commit is contained in:
parent
a41479c0ec
commit
db7121ccba
7 changed files with 484 additions and 548 deletions
|
|
@ -42,7 +42,7 @@ libimagentryref does store the following data:
|
|||
```toml
|
||||
[ref]
|
||||
filehash.sha1 = "<sha1 hash of the file>"
|
||||
relpath = "/Psy_trance_2018_yearmix.mp3"
|
||||
relpath = "Psy_trance_2018_yearmix.mp3"
|
||||
collection = "music"
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -22,25 +22,17 @@ maintenance = { status = "actively-developed" }
|
|||
[dependencies]
|
||||
itertools = "0.7"
|
||||
log = "0.4.0"
|
||||
failure = "0.1"
|
||||
sha-1 = "0.8"
|
||||
toml = "0.4"
|
||||
toml-query = "0.8"
|
||||
failure = "0.1"
|
||||
sha-1 = { version = "0.7", optional = true }
|
||||
sha2 = { version = "0.7", optional = true }
|
||||
sha3 = { version = "0.7", optional = true }
|
||||
hex = { version = "0.3", optional = true }
|
||||
serde = "1"
|
||||
serde_derive = "1"
|
||||
|
||||
libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
|
||||
libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
|
||||
libimagentryutil = { version = "0.10.0", path = "../../../lib/entry/libimagentryutil" }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
generators = []
|
||||
generators-sha1 = ["sha-1", "hex"]
|
||||
generators-sha224 = ["sha2", "hex"]
|
||||
generators-sha256 = ["sha2", "hex"]
|
||||
generators-sha384 = ["sha2", "hex"]
|
||||
generators-sha512 = ["sha2", "hex"]
|
||||
generators-sha3 = ["sha3", "hex"]
|
||||
[dev-dependencies]
|
||||
env_logger = "0.5"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,276 +0,0 @@
|
|||
//
|
||||
// imag - the personal information management suite for the commandline
|
||||
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
|
||||
//
|
||||
// This library is free software; you can redistribute it and/or
|
||||
// modify it under the terms of the GNU Lesser General Public
|
||||
// License as published by the Free Software Foundation; version
|
||||
// 2.1 of the License.
|
||||
//
|
||||
// This library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
// Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public
|
||||
// License along with this library; if not, write to the Free Software
|
||||
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
//
|
||||
|
||||
//! Default generators
|
||||
//!
|
||||
//! This module provides a number of default `UniqueRefPathGenerator`s
|
||||
//! which can be used for generating references.
|
||||
//!
|
||||
//! These generators are _NOT_ domain specific. So there won't be a "UniqueMailRefPathGenerator" in
|
||||
//! here, for example.
|
||||
//!
|
||||
//! All these generators use "ref" as collection name.
|
||||
//! They can be overridden using the `make_unique_ref_path_generator!()` convenience macro.
|
||||
//!
|
||||
//! # Note
|
||||
//!
|
||||
//! You must enable the appropriate crate feature to use any of the provided generators. With the
|
||||
//! `generators` feature, you only get the convenience macro `make_unique_ref_path_generator!()`.
|
||||
//!
|
||||
|
||||
/// A convenience macro for wrapping a generator in a new one, reusing the functionality from the
|
||||
/// underlying generator
|
||||
///
|
||||
/// The UniqueRefPathGenerator must be in scope.
|
||||
///
|
||||
/// The macro creates a new struct `$name` over `$underlying` and changes the collection name to
|
||||
/// `$collectionname`.
|
||||
/// If passed, the new implementation is used (defaults to the implementation from the underlying
|
||||
/// generator).
|
||||
/// If passed, the new postprocessing is used (defaults to not changing the StoreId)
|
||||
///
|
||||
#[macro_export]
|
||||
macro_rules! make_unique_ref_path_generator {
|
||||
(
|
||||
$name:ident
|
||||
over $underlying:ty
|
||||
=> with collection name $collectionname:expr
|
||||
) => {
|
||||
struct $name;
|
||||
|
||||
impl $crate::refstore::UniqueRefPathGenerator for $name {
|
||||
|
||||
fn collection() -> &'static str {
|
||||
$collectionname
|
||||
}
|
||||
|
||||
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String> {
|
||||
$underlying::unique_hash(path)
|
||||
}
|
||||
|
||||
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
|
||||
-> Result<::libimagstore::storeid::StoreId>
|
||||
{
|
||||
Ok(sid)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
(
|
||||
$name:ident
|
||||
over $underlying:ty
|
||||
=> with collection name $collectionname:expr
|
||||
=> $impl:expr
|
||||
) => {
|
||||
struct $name;
|
||||
|
||||
impl $crate::refstore::UniqueRefPathGenerator for $name {
|
||||
type Error = $errtype;
|
||||
|
||||
fn collection() -> &'static str {
|
||||
$collectionname
|
||||
}
|
||||
|
||||
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String> {
|
||||
debug!("Making unique hash for path: {:?}", path.as_ref());
|
||||
$impl(path)
|
||||
}
|
||||
|
||||
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
|
||||
-> Result<::libimagstore::storeid::StoreId>
|
||||
{
|
||||
Ok(sid)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
(
|
||||
pub $name:ident
|
||||
over $underlying:ty
|
||||
=> with collection name $collectionname:expr
|
||||
=> $impl:expr
|
||||
) => {
|
||||
make_unique_ref_path_generator!(
|
||||
pub $name
|
||||
over $underlying
|
||||
=> with collection name $collectionname
|
||||
=> $impl => |sid| { Ok(sid) }
|
||||
);
|
||||
};
|
||||
|
||||
(
|
||||
pub $name:ident
|
||||
over $underlying:ty
|
||||
=> with collection name $collectionname:expr
|
||||
=> $impl:expr
|
||||
=> $postproc:expr
|
||||
) => {
|
||||
pub struct $name;
|
||||
|
||||
impl $crate::refstore::UniqueRefPathGenerator for $name {
|
||||
fn collection() -> &'static str {
|
||||
$collectionname
|
||||
}
|
||||
|
||||
fn unique_hash<A: AsRef<Path>>(path: A) -> ::failure::Fallible<String> {
|
||||
debug!("Making unique hash for path: {:?}", path.as_ref());
|
||||
$impl(path)
|
||||
}
|
||||
|
||||
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
|
||||
-> ::failure::Fallible<::libimagstore::storeid::StoreId>
|
||||
{
|
||||
$postproc(sid)
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
#[cfg(any(
|
||||
feature = "generators-sha1",
|
||||
feature = "generators-sha224",
|
||||
feature = "generators-sha256",
|
||||
feature = "generators-sha384",
|
||||
feature = "generators-sha512",
|
||||
feature = "generators-sha3",
|
||||
))]
|
||||
mod base;
|
||||
|
||||
/// Helper macro for generating implementations for the various Sha algorithms
|
||||
macro_rules! make_sha_mod {
|
||||
{
|
||||
$modname:ident,
|
||||
$hashname:ident,
|
||||
$hashingimpl:expr
|
||||
} => {
|
||||
pub mod $modname {
|
||||
use std::path::Path;
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Read;
|
||||
|
||||
use hex;
|
||||
make_unique_ref_path_generator! (
|
||||
pub $hashname
|
||||
over generators::base::Base
|
||||
=> with collection name "ref"
|
||||
=> |path| {
|
||||
OpenOptions::new()
|
||||
.read(true)
|
||||
.write(false)
|
||||
.create(false)
|
||||
.open(path)
|
||||
.map_err(::failure::Error::from)
|
||||
.and_then(|mut file| {
|
||||
let mut buffer = String::new();
|
||||
let _ = file.read_to_string(&mut buffer)?;
|
||||
$hashingimpl(buffer)
|
||||
})
|
||||
}
|
||||
);
|
||||
|
||||
impl $hashname {
|
||||
|
||||
/// Function which can be used by a wrapping UniqueRefPathGenerator to hash only N bytes.
|
||||
pub fn hash_n_bytes<A: AsRef<Path>>(path: A, n: usize) -> ::failure::Fallible<String> {
|
||||
debug!("Opening '{}' for hashing", path.as_ref().display());
|
||||
OpenOptions::new()
|
||||
.read(true)
|
||||
.write(false)
|
||||
.create(false)
|
||||
.open(path)
|
||||
.map_err(::failure::Error::from)
|
||||
.and_then(|mut file| {
|
||||
let mut buffer = vec![0; n];
|
||||
debug!("Allocated {} bytes", buffer.capacity());
|
||||
|
||||
match file.read_exact(&mut buffer) {
|
||||
Ok(_) => { /* yay */ Ok(()) },
|
||||
Err(e) => if e.kind() == ::std::io::ErrorKind::UnexpectedEof {
|
||||
debug!("Ignoring unexpected EOF before {} bytes were read", n);
|
||||
Ok(())
|
||||
} else {
|
||||
Err(e)
|
||||
}
|
||||
}?;
|
||||
|
||||
let buffer = String::from_utf8(buffer)?;
|
||||
$hashingimpl(buffer)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "generators-sha1")]
|
||||
make_sha_mod! {
|
||||
sha1, Sha1, |buffer: String| {
|
||||
use sha1::{Sha1, Digest};
|
||||
|
||||
trace!("Hashing: '{:?}'", buffer);
|
||||
let res = hex::encode(Sha1::digest(buffer.as_bytes()));
|
||||
trace!("Hash => '{:?}'", res);
|
||||
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "generators-sha224")]
|
||||
make_sha_mod! {
|
||||
sha224, Sha224, |buffer: String| {
|
||||
use sha2::{Sha224, Digest};
|
||||
Ok(hex::encode(Sha224::digest(buffer.as_bytes())))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "generators-sha256")]
|
||||
make_sha_mod! {
|
||||
sha256, Sha256, |buffer: String| {
|
||||
use sha2::{Sha256, Digest};
|
||||
Ok(hex::encode(Sha256::digest(buffer.as_bytes())))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "generators-sha384")]
|
||||
make_sha_mod! {
|
||||
sha384, Sha384, |buffer: String| {
|
||||
use sha2::{Sha384, Digest};
|
||||
Ok(hex::encode(Sha384::digest(buffer.as_bytes())))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "generators-sha512")]
|
||||
make_sha_mod! {
|
||||
sha512, Sha512, |buffer: String| {
|
||||
use sha2::{Sha512, Digest};
|
||||
Ok(hex::encode(Sha512::digest(buffer.as_bytes())))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "generators-sha3")]
|
||||
make_sha_mod! {
|
||||
sha3, Sha3, |buffer: String| {
|
||||
use sha3::{Sha3_256, Digest};
|
||||
Ok(hex::encode(Sha3_256::digest(buffer.as_bytes())))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -19,29 +19,37 @@
|
|||
|
||||
use std::path::Path;
|
||||
|
||||
use libimagstore::storeid::StoreId;
|
||||
|
||||
use refstore::UniqueRefPathGenerator;
|
||||
|
||||
use failure::Fallible as Result;
|
||||
|
||||
/// A base UniqueRefPathGenerator which must be overridden by the actual UniqueRefPathGenerator
|
||||
/// which is provided by this crate
|
||||
#[allow(dead_code)]
|
||||
pub struct Base;
|
||||
pub trait Hasher {
|
||||
const NAME: &'static str;
|
||||
|
||||
/// hash the file at path `path`
|
||||
fn hash<P: AsRef<Path>>(path: P) -> Result<String>;
|
||||
}
|
||||
|
||||
pub mod default {
|
||||
pub use super::sha1::Sha1Hasher as DefaultHasher;
|
||||
}
|
||||
|
||||
pub mod sha1 {
|
||||
use std::path::Path;
|
||||
|
||||
use failure::Fallible as Result;
|
||||
use sha1::{Sha1, Digest};
|
||||
|
||||
use hasher::Hasher;
|
||||
|
||||
pub struct Sha1Hasher;
|
||||
|
||||
impl Hasher for Sha1Hasher {
|
||||
const NAME : &'static str = "sha1";
|
||||
|
||||
fn hash<P: AsRef<Path>>(path: P) -> Result<String> {
|
||||
let digest = Sha1::digest(::std::fs::read_to_string(path)?.as_bytes());
|
||||
Ok(format!("{:x}", digest)) // TODO: Ugh...
|
||||
}
|
||||
}
|
||||
|
||||
impl UniqueRefPathGenerator for Base {
|
||||
fn collection() -> &'static str {
|
||||
"ref"
|
||||
}
|
||||
|
||||
fn unique_hash<A: AsRef<Path>>(_path: A) -> Result<String> {
|
||||
// This has to be overridden
|
||||
panic!("Not overridden base functionality. This is a BUG!")
|
||||
}
|
||||
|
||||
fn postprocess_storeid(sid: StoreId) -> Result<StoreId> {
|
||||
Ok(sid) // default implementation
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -41,41 +41,17 @@
|
|||
extern crate itertools;
|
||||
extern crate toml;
|
||||
extern crate toml_query;
|
||||
|
||||
#[macro_use] extern crate libimagstore;
|
||||
extern crate libimagerror;
|
||||
#[macro_use] extern crate libimagentryutil;
|
||||
extern crate failure;
|
||||
|
||||
module_entry_path_mod!("ref");
|
||||
|
||||
pub mod reference;
|
||||
pub mod refstore;
|
||||
|
||||
#[cfg(feature = "generators-sha1")]
|
||||
#[macro_use] extern crate serde_derive;
|
||||
extern crate sha1;
|
||||
|
||||
#[cfg(any(
|
||||
feature = "generators-sha224",
|
||||
feature = "generators-sha256",
|
||||
feature = "generators-sha384",
|
||||
feature = "generators-sha512",
|
||||
))]
|
||||
extern crate sha2;
|
||||
extern crate libimagstore;
|
||||
extern crate libimagerror;
|
||||
#[macro_use] extern crate libimagentryutil;
|
||||
#[macro_use] extern crate failure;
|
||||
|
||||
#[cfg(feature = "generators-sha3")]
|
||||
extern crate sha3;
|
||||
#[cfg(test)]
|
||||
extern crate env_logger;
|
||||
|
||||
#[cfg(any(
|
||||
feature = "generators-sha1",
|
||||
feature = "generators-sha224",
|
||||
feature = "generators-sha256",
|
||||
feature = "generators-sha384",
|
||||
feature = "generators-sha512",
|
||||
feature = "generators-sha3",
|
||||
))]
|
||||
extern crate hex;
|
||||
|
||||
#[cfg(feature = "generators")]
|
||||
pub mod generators;
|
||||
pub mod hasher;
|
||||
pub mod reference;
|
||||
|
||||
|
|
|
|||
|
|
@ -17,15 +17,13 @@
|
|||
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
//
|
||||
|
||||
//! The Ref object is a helper over the link functionality, so one is able to create references to
|
||||
//! files outside of the imag store.
|
||||
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::collections::BTreeMap;
|
||||
use std::ops::Deref;
|
||||
|
||||
use libimagentryutil::isa::Is;
|
||||
use libimagentryutil::isa::IsKindHeaderPathProvider;
|
||||
use libimagstore::store::Entry;
|
||||
use libimagerror::errors::ErrorMsg as EM;
|
||||
|
||||
use toml::Value;
|
||||
|
|
@ -34,8 +32,106 @@ use toml_query::delete::TomlValueDeleteExt;
|
|||
use toml_query::insert::TomlValueInsertExt;
|
||||
use failure::Fallible as Result;
|
||||
use failure::Error;
|
||||
use failure::err_msg;
|
||||
use failure::ResultExt;
|
||||
|
||||
use hasher::Hasher;
|
||||
|
||||
/// A configuration of "collection name" -> "collection path" mappings
|
||||
///
|
||||
/// Should be deserializeable from the configuration file right away, because we expect a
|
||||
/// configuration like this in the config file:
|
||||
///
|
||||
/// ```toml
|
||||
/// [ref.collections]
|
||||
/// music = "/home/alice/music"
|
||||
/// documents = "/home/alice/doc"
|
||||
/// ```
|
||||
///
|
||||
/// for example.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct Config(BTreeMap<String, PathBuf>);
|
||||
|
||||
impl Config {
|
||||
pub fn new(map: BTreeMap<String, PathBuf>) -> Self {
|
||||
Config(map)
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for Config {
|
||||
type Target = BTreeMap<String, PathBuf>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
provide_kindflag_path!(pub IsRef, "ref.is_ref");
|
||||
|
||||
/// Fassade module
|
||||
///
|
||||
/// This module is necessary to build a generic fassade around the "entry with a (default) hasher to
|
||||
/// represent the entry as a ref".
|
||||
///
|
||||
/// The module is for code-structuring only, all types in the module are exported publicly in the
|
||||
/// supermodule.
|
||||
pub mod fassade {
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use libimagstore::store::Entry;
|
||||
use libimagentryutil::isa::Is;
|
||||
|
||||
use failure::Fallible as Result;
|
||||
use failure::Error;
|
||||
|
||||
use hasher::sha1::Sha1Hasher;
|
||||
use hasher::Hasher;
|
||||
use super::IsRef;
|
||||
|
||||
pub trait RefFassade {
|
||||
fn is_ref(&self) -> Result<bool>;
|
||||
fn as_ref_with_hasher<H: Hasher>(&self) -> RefWithHasher<H>;
|
||||
fn as_ref_with_hasher_mut<H: Hasher>(&mut self) -> MutRefWithHasher<H>;
|
||||
}
|
||||
|
||||
impl RefFassade for Entry {
|
||||
/// Check whether the underlying object is actually a ref
|
||||
fn is_ref(&self) -> Result<bool> {
|
||||
self.is::<IsRef>().map_err(Error::from)
|
||||
}
|
||||
|
||||
fn as_ref_with_hasher<H: Hasher>(&self) -> RefWithHasher<H> {
|
||||
RefWithHasher::new(self)
|
||||
}
|
||||
|
||||
fn as_ref_with_hasher_mut<H: Hasher>(&mut self) -> MutRefWithHasher<H> {
|
||||
MutRefWithHasher::new(self)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
pub struct RefWithHasher<'a, H: Hasher = Sha1Hasher>(pub(crate) &'a Entry, PhantomData<H>);
|
||||
|
||||
impl<'a, H> RefWithHasher<'a, H>
|
||||
where H: Hasher
|
||||
{
|
||||
fn new(entry: &'a Entry) -> Self {
|
||||
RefWithHasher(entry, PhantomData)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MutRefWithHasher<'a, H: Hasher = Sha1Hasher>(pub(crate) &'a mut Entry, PhantomData<H>);
|
||||
|
||||
impl<'a, H> MutRefWithHasher<'a, H>
|
||||
where H: Hasher
|
||||
{
|
||||
fn new(entry: &'a mut Entry) -> Self {
|
||||
MutRefWithHasher(entry, PhantomData)
|
||||
}
|
||||
}
|
||||
}
|
||||
pub use self::fassade::*;
|
||||
|
||||
use refstore::UniqueRefPathGenerator;
|
||||
|
||||
pub trait Ref {
|
||||
|
||||
|
|
@ -43,76 +139,41 @@ pub trait Ref {
|
|||
fn is_ref(&self) -> Result<bool>;
|
||||
|
||||
/// Get the stored hash.
|
||||
///
|
||||
/// Does not need a `UniqueRefPathGenerator` as it reads the hash stored in the header
|
||||
fn get_hash(&self) -> Result<&str>;
|
||||
|
||||
/// Make this object a ref
|
||||
fn make_ref<P: AsRef<Path>>(&mut self, hash: String, path: P) -> Result<()>;
|
||||
|
||||
/// Get the referenced path.
|
||||
///
|
||||
/// Does not need a `UniqueRefPathGenerator` as it reads the path stored in the header.
|
||||
fn get_path(&self) -> Result<PathBuf>;
|
||||
|
||||
/// Get the stored hash.
|
||||
fn get_hash(&self) -> Result<&str>;
|
||||
|
||||
/// Check whether the referenced file still matches its hash
|
||||
fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> Result<bool>;
|
||||
|
||||
fn remove_ref(&mut self) -> Result<()>;
|
||||
|
||||
/// Alias for `r.fs_link_exists() && r.deref().is_file()`
|
||||
fn is_ref_to_file(&self) -> Result<bool> {
|
||||
self.get_path().map(|p| p.is_file())
|
||||
}
|
||||
|
||||
/// Alias for `r.fs_link_exists() && r.deref().is_dir()`
|
||||
fn is_ref_to_dir(&self) -> Result<bool> {
|
||||
self.get_path().map(|p| p.is_dir())
|
||||
}
|
||||
|
||||
/// Alias for `!Ref::fs_link_exists()`
|
||||
fn is_dangling(&self) -> Result<bool> {
|
||||
self.get_path().map(|p| !p.exists())
|
||||
}
|
||||
|
||||
fn hash_valid(&self, config: &Config) -> Result<bool>;
|
||||
}
|
||||
|
||||
provide_kindflag_path!(pub IsRef, "ref.is_ref");
|
||||
|
||||
impl Ref for Entry {
|
||||
impl<'a, H: Hasher> Ref for RefWithHasher<'a, H> {
|
||||
|
||||
/// Check whether the underlying object is actually a ref
|
||||
fn is_ref(&self) -> Result<bool> {
|
||||
self.is::<IsRef>().map_err(Error::from)
|
||||
self.0.is::<IsRef>().map_err(Error::from)
|
||||
}
|
||||
|
||||
fn get_hash(&self) -> Result<&str> {
|
||||
self.get_header()
|
||||
.read("ref.hash")
|
||||
let header_path = format!("ref.hash.{}", H::NAME);
|
||||
self.0
|
||||
.get_header()
|
||||
.read(&header_path)
|
||||
.map_err(Error::from)?
|
||||
.ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.hash")))
|
||||
.ok_or_else(|| {
|
||||
Error::from(EM::EntryHeaderFieldMissing("ref.hash.<hash>"))
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.as_str().ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash", "string")))
|
||||
v.as_str().ok_or_else(|| {
|
||||
Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string"))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn make_ref<P: AsRef<Path>>(&mut self, hash: String, path: P) -> Result<()> {
|
||||
let path_str : String = path
|
||||
.as_ref()
|
||||
.to_str()
|
||||
.map(String::from)
|
||||
.ok_or_else(|| EM::UTF8Error)?;
|
||||
|
||||
let _ = self.set_isflag::<IsRef>()?;
|
||||
let hdr = self.get_header_mut();
|
||||
hdr.insert("ref.path", Value::String(String::from(path_str)))?;
|
||||
hdr.insert("ref.hash", Value::String(hash))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_path(&self) -> Result<PathBuf> {
|
||||
self.get_header()
|
||||
self.0
|
||||
.get_header()
|
||||
.read("ref.path")
|
||||
.map_err(Error::from)?
|
||||
.ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.path")))
|
||||
|
|
@ -124,20 +185,323 @@ impl Ref for Entry {
|
|||
.map(PathBuf::from)
|
||||
}
|
||||
|
||||
fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> Result<bool> {
|
||||
self.get_path()
|
||||
.map(PathBuf::from)
|
||||
.map_err(Error::from)
|
||||
.and_then(|pb| RPG::unique_hash(pb))
|
||||
.and_then(|h| Ok(h == self.get_hash()?))
|
||||
}
|
||||
fn hash_valid(&self, config: &Config) -> Result<bool> {
|
||||
let ref_header = self.0
|
||||
.get_header()
|
||||
.read("ref")?
|
||||
.ok_or_else(|| err_msg("Header missing at 'ref'"))?;
|
||||
|
||||
fn remove_ref(&mut self) -> Result<()> {
|
||||
let hdr = self.get_header_mut();
|
||||
let _ = hdr.delete("ref.hash")?;
|
||||
let _ = hdr.delete("ref.path")?;
|
||||
let _ = hdr.delete("ref")?;
|
||||
Ok(())
|
||||
let collection_name = ref_header
|
||||
.read("collection")
|
||||
.map_err(Error::from)?
|
||||
.ok_or_else(|| err_msg("Header missing at 'ref.collection'"))?
|
||||
.as_str()
|
||||
.ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string")))?;
|
||||
|
||||
let path = ref_header
|
||||
.read("path")
|
||||
.map_err(Error::from)?
|
||||
.ok_or_else(|| err_msg("Header missing at 'ref.path'"))?
|
||||
.as_str()
|
||||
.map(PathBuf::from)
|
||||
.ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string")))?;
|
||||
|
||||
|
||||
let file_path = get_file_path(config, collection_name.as_ref(), &path)?;
|
||||
|
||||
ref_header
|
||||
.read(H::NAME)
|
||||
.map_err(Error::from)?
|
||||
.ok_or_else(|| format_err!("Header missing at 'ref.{}'", H::NAME))
|
||||
.and_then(|v| {
|
||||
v.as_str().ok_or_else(|| {
|
||||
Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string"))
|
||||
})
|
||||
})
|
||||
.and_then(|hash| H::hash(file_path).map(|h| h == hash))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
pub trait MutRef {
|
||||
fn remove_ref(&mut self) -> Result<()>;
|
||||
|
||||
/// Make a ref out of a normal (non-ref) entry.
|
||||
///
|
||||
/// If the entry is already a ref, this fails if `force` is false
|
||||
fn make_ref<P, Coll>(&mut self, path: P, collection_name: Coll, config: &Config, force: bool)
|
||||
-> Result<()>
|
||||
where P: AsRef<Path>,
|
||||
Coll: AsRef<str>;
|
||||
}
|
||||
|
||||
|
||||
impl<'a, H> MutRef for MutRefWithHasher<'a, H>
|
||||
where H: Hasher
|
||||
{
|
||||
|
||||
fn remove_ref(&mut self) -> Result<()> {
|
||||
debug!("Removing 'ref' header section");
|
||||
{
|
||||
let header = self.0.get_header_mut();
|
||||
trace!("header = {:?}", header);
|
||||
|
||||
let _ = header.delete("ref.relpath").context("Removing ref.relpath")?;
|
||||
|
||||
if let Some(hash_tbl) = header.read_mut("ref.hash")? {
|
||||
match hash_tbl {
|
||||
Value::Table(ref mut tbl) => *tbl = BTreeMap::new(),
|
||||
_ => {
|
||||
// should not happen
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let _ = header.delete("ref.hash").context("Removing ref.hash")?;
|
||||
let _ = header.delete("ref.collection").context("Removing ref.collection")?;
|
||||
}
|
||||
|
||||
debug!("Removing 'ref' header marker");
|
||||
self.0.remove_isflag::<IsRef>().context("Removing ref")?;
|
||||
|
||||
let _ = self.0
|
||||
.get_header_mut()
|
||||
.delete("ref")
|
||||
.context("Removing ref")?;
|
||||
|
||||
trace!("header = {:?}", self.0.get_header());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Make a ref out of a normal (non-ref) entry.
|
||||
///
|
||||
/// `path` is the path to refer to,
|
||||
///
|
||||
/// # Warning
|
||||
///
|
||||
/// If the entry is already a ref, this fails if `force` is false
|
||||
///
|
||||
fn make_ref<P, Coll>(&mut self, path: P, collection_name: Coll, config: &Config, force: bool)
|
||||
-> Result<()>
|
||||
where P: AsRef<Path>,
|
||||
Coll: AsRef<str>
|
||||
{
|
||||
trace!("Making ref out of {:?}", self.0);
|
||||
trace!("Making ref with collection name {:?}", collection_name.as_ref());
|
||||
trace!("Making ref with config {:?}", config);
|
||||
trace!("Making ref forced = {}", force);
|
||||
|
||||
if self.0.get_header().read("ref.is_ref")?.is_some() && !force {
|
||||
debug!("Entry is already a Ref!");
|
||||
let _ = Err(err_msg("Entry is already a reference")).context("Making ref out of entry")?;
|
||||
}
|
||||
|
||||
let file_path = get_file_path(config, collection_name.as_ref(), &path)?;
|
||||
|
||||
if !file_path.exists() {
|
||||
let msg = format_err!("File '{:?}' does not exist", file_path);
|
||||
let _ = Err(msg).context("Making ref out of entry")?;
|
||||
}
|
||||
|
||||
debug!("Entry hashing...");
|
||||
let _ = H::hash(&file_path)
|
||||
.and_then(|hash| make_header_section(hash, H::NAME, path, collection_name))
|
||||
.and_then(|h| self.0.get_header_mut().insert("ref", h).map_err(Error::from))
|
||||
.and_then(|_| self.0.set_isflag::<IsRef>())
|
||||
.context("Making ref out of entry")?;
|
||||
|
||||
debug!("Setting is-ref flag");
|
||||
self.0
|
||||
.set_isflag::<IsRef>()
|
||||
.context("Setting ref-flag")
|
||||
.map_err(Error::from)
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// Create a new header section for a "ref".
|
||||
///
|
||||
/// # Warning
|
||||
///
|
||||
/// The `relpath` _must_ be relative to the configured path for that collection.
|
||||
pub(crate) fn make_header_section<P, C, H>(hash: String, hashname: H, relpath: P, collection: C)
|
||||
-> Result<Value>
|
||||
where P: AsRef<Path>,
|
||||
C: AsRef<str>,
|
||||
H: AsRef<str>,
|
||||
{
|
||||
let mut header_section = Value::Table(BTreeMap::new());
|
||||
{
|
||||
let relpath = relpath
|
||||
.as_ref()
|
||||
.to_str()
|
||||
.map(String::from)
|
||||
.ok_or_else(|| {
|
||||
let msg = format_err!("UTF Error in '{:?}'", relpath.as_ref());
|
||||
Error::from(msg)
|
||||
})?;
|
||||
|
||||
let _ = header_section.insert("relpath", Value::String(relpath))?;
|
||||
}
|
||||
|
||||
{
|
||||
let mut hash_table = Value::Table(BTreeMap::new());
|
||||
let _ = hash_table.insert(hashname.as_ref(), Value::String(hash))?;
|
||||
let _ = header_section.insert("hash", hash_table)?;
|
||||
}
|
||||
|
||||
let _ = header_section.insert("collection", Value::String(String::from(collection.as_ref())));
|
||||
|
||||
Ok(header_section)
|
||||
}
|
||||
|
||||
fn get_file_path<P>(config: &Config, collection_name: &str, path: P) -> Result<PathBuf>
|
||||
where P: AsRef<Path>
|
||||
{
|
||||
config
|
||||
.get(collection_name)
|
||||
.map(PathBuf::clone)
|
||||
.ok_or_else(|| {
|
||||
format_err!("Configuration missing for collection: '{}'", collection_name)
|
||||
})
|
||||
.context("Making ref out of entry")
|
||||
.map_err(Error::from)
|
||||
.map(|p| {
|
||||
let filepath = p.join(&path);
|
||||
trace!("Found filepath: {:?}", filepath.display());
|
||||
filepath
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::path::PathBuf;
|
||||
|
||||
use libimagstore::store::Store;
|
||||
use libimagstore::store::Entry;
|
||||
|
||||
use super::*;
|
||||
use hasher::Hasher;
|
||||
|
||||
fn setup_logging() {
|
||||
let _ = ::env_logger::try_init();
|
||||
}
|
||||
|
||||
pub fn get_store() -> Store {
|
||||
Store::new_inmemory(PathBuf::from("/"), &None).unwrap()
|
||||
}
|
||||
|
||||
struct TestHasher;
|
||||
impl Hasher for TestHasher {
|
||||
const NAME: &'static str = "Testhasher";
|
||||
|
||||
fn hash<P: AsRef<Path>>(path: P) -> Result<String> {
|
||||
path.as_ref()
|
||||
.to_str()
|
||||
.map(String::from)
|
||||
.ok_or_else(|| Error::from(err_msg("Failed to create test hash")))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[test]
|
||||
fn test_isref() {
|
||||
setup_logging();
|
||||
let store = get_store();
|
||||
let entry = store.retrieve(PathBuf::from("test_isref")).unwrap();
|
||||
|
||||
assert!(!entry.is_ref().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_makeref() {
|
||||
setup_logging();
|
||||
let store = get_store();
|
||||
let mut entry = store.retrieve(PathBuf::from("test_makeref")).unwrap();
|
||||
let file = PathBuf::from("/"); // has to exist
|
||||
let collection_name = "some_collection";
|
||||
let config = Config({
|
||||
let mut c = BTreeMap::new();
|
||||
c.insert(String::from("some_collection"), PathBuf::from("/tmp"));
|
||||
c
|
||||
});
|
||||
|
||||
let r = entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false);
|
||||
assert!(r.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_makeref_isref() {
|
||||
setup_logging();
|
||||
let store = get_store();
|
||||
let mut entry = store.retrieve(PathBuf::from("test_makeref_isref")).unwrap();
|
||||
let file = PathBuf::from("/"); // has to exists
|
||||
let collection_name = "some_collection";
|
||||
let config = Config({
|
||||
let mut c = BTreeMap::new();
|
||||
c.insert(String::from("some_collection"), PathBuf::from("/tmp"));
|
||||
c
|
||||
});
|
||||
|
||||
let res = entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false);
|
||||
assert!(res.is_ok(), "Expected to be ok: {:?}", res);
|
||||
|
||||
assert!(entry.as_ref_with_hasher::<TestHasher>().is_ref().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_makeref_is_ref_with_testhash() {
|
||||
setup_logging();
|
||||
let store = get_store();
|
||||
let mut entry = store.retrieve(PathBuf::from("test_makeref_is_ref_with_testhash")).unwrap();
|
||||
let file = PathBuf::from("/"); // has to exist
|
||||
let collection_name = "some_collection";
|
||||
let config = Config({
|
||||
let mut c = BTreeMap::new();
|
||||
c.insert(String::from("some_collection"), PathBuf::from("/"));
|
||||
c
|
||||
});
|
||||
|
||||
assert!(entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false).is_ok());
|
||||
|
||||
let check_isstr = |entry: &Entry, location, shouldbe| {
|
||||
let var = entry.get_header().read(location);
|
||||
|
||||
assert!(var.is_ok(), "{} is not Ok(_): {:?}", location, var);
|
||||
let var = var.unwrap();
|
||||
|
||||
assert!(var.is_some(), "{} is not Some(_): {:?}", location, var);
|
||||
let var = var.unwrap().as_str();
|
||||
|
||||
assert!(var.is_some(), "{} is not String: {:?}", location, var);
|
||||
assert_eq!(var.unwrap(), shouldbe, "{} is not == {}", location, shouldbe);
|
||||
};
|
||||
|
||||
check_isstr(&entry, "ref.relpath", "/");
|
||||
check_isstr(&entry, "ref.hash.Testhasher", "/"); // TestHasher hashes by returning the path itself
|
||||
check_isstr(&entry, "ref.collection", "some_collection");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_makeref_remref() {
|
||||
setup_logging();
|
||||
let store = get_store();
|
||||
let mut entry = store.retrieve(PathBuf::from("test_makeref_remref")).unwrap();
|
||||
let file = PathBuf::from("/"); // has to exist
|
||||
let collection_name = "some_collection";
|
||||
let config = Config({
|
||||
let mut c = BTreeMap::new();
|
||||
c.insert(String::from("some_collection"), PathBuf::from("/"));
|
||||
c
|
||||
});
|
||||
|
||||
assert!(entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false).is_ok());
|
||||
assert!(entry.as_ref_with_hasher::<TestHasher>().is_ref().unwrap());
|
||||
let res = entry.as_ref_with_hasher_mut::<TestHasher>().remove_ref();
|
||||
assert!(res.is_ok(), "Expected to be ok: {:?}", res);
|
||||
assert!(!entry.as_ref_with_hasher::<TestHasher>().is_ref().unwrap());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,128 +0,0 @@
|
|||
//
|
||||
// imag - the personal information management suite for the commandline
|
||||
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
|
||||
//
|
||||
// This library is free software; you can redistribute it and/or
|
||||
// modify it under the terms of the GNU Lesser General Public
|
||||
// License as published by the Free Software Foundation; version
|
||||
// 2.1 of the License.
|
||||
//
|
||||
// This library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
// Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public
|
||||
// License along with this library; if not, write to the Free Software
|
||||
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
//
|
||||
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use libimagstore::store::FileLockEntry;
|
||||
use libimagstore::store::Store;
|
||||
use libimagstore::storeid::StoreId;
|
||||
|
||||
use reference::Ref;
|
||||
|
||||
use failure::Fallible as Result;
|
||||
use failure::Error;
|
||||
|
||||
/// A UniqueRefPathGenerator generates unique Pathes
|
||||
///
|
||||
/// It is basically a functor which generates a StoreId from a &Path.
|
||||
/// For more information have a look at the documentation of RefStore.
|
||||
pub trait UniqueRefPathGenerator {
|
||||
/// The collection the `StoreId` should be created for
|
||||
fn collection() -> &'static str {
|
||||
"ref"
|
||||
}
|
||||
|
||||
/// A function which should generate a unique string for a Path
|
||||
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String>;
|
||||
|
||||
/// Postprocess the generated `StoreId` object
|
||||
fn postprocess_storeid(sid: StoreId) -> Result<StoreId> {
|
||||
Ok(sid)
|
||||
}
|
||||
}
|
||||
|
||||
/// A extensions for the `Store` to handle `Ref` objects
|
||||
///
|
||||
/// The RefStore handles refs using a `UniqueRefPathGenerator`. The `UniqueRefPathGenerator`, as it
|
||||
/// name suggests, generates unique `StoreId`s for a `&Path`. It is a functor `&Path -> StoreId`.
|
||||
///
|
||||
/// It provides three functions which are called in the following sequence:
|
||||
///
|
||||
/// * The `UniqueRefPathGenerator::collection()` function is used for get the collection a `StoreId`
|
||||
/// should be in (The first element of the `StoreId` path)
|
||||
/// * The `UniqueRefPathGenerator::unique_hash()` gets the `&Path` which it then should generate a
|
||||
/// unique String for. How this is done does not matter. It can hash the Path itself, read the
|
||||
/// file and hash that or something else. It should be reproduceable, though.
|
||||
/// * These two parts are joined and put into a `StoreId` which the
|
||||
/// `UniqueRefPathGenerator::postprocess_storeid()` function is then allowed to postprocess (for
|
||||
/// example add more parts to the StoreId). The default implementation does nothing.
|
||||
///
|
||||
/// The StoreId which is generated is then used to carry out the actual action (reading, creating
|
||||
/// ...).
|
||||
/// If a entry is created, header information is set (that it is a ref, the hash which was just
|
||||
/// generated and the path of the referenced file)
|
||||
///
|
||||
/// # Details
|
||||
///
|
||||
/// The `UniqueRefPathGenerator` is passed as type parameter to enforce some semantics:
|
||||
///
|
||||
/// * The used `UniqueRefPathGenerator` is defined by the implementation rather than by the runtime
|
||||
/// of the program or some environment. Of course this is only a small hurdle to enforce this, but
|
||||
/// a hint.
|
||||
/// * The `UniqueRefPathGenerator` is a functor which does not carry state.
|
||||
///
|
||||
pub trait RefStore<'a> {
|
||||
|
||||
fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H) -> Result<Option<FileLockEntry<'a>>>;
|
||||
fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>>;
|
||||
fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>>;
|
||||
|
||||
}
|
||||
|
||||
impl<'a> RefStore<'a> for Store {
|
||||
|
||||
fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H)
|
||||
-> Result<Option<FileLockEntry<'a>>>
|
||||
{
|
||||
let sid = StoreId::new(PathBuf::from(format!("{}/{}", RPG::collection(), hash.as_ref())))
|
||||
.map_err(Error::from)?;
|
||||
|
||||
debug!("Getting: {:?}", sid);
|
||||
self.get(sid)
|
||||
.map_err(Error::from)
|
||||
}
|
||||
|
||||
fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
|
||||
-> Result<FileLockEntry<'a>>
|
||||
{
|
||||
let hash = RPG::unique_hash(&path)?;
|
||||
let pathbuf = PathBuf::from(format!("{}/{}", RPG::collection(), hash));
|
||||
let sid = StoreId::new(pathbuf.clone())?;
|
||||
|
||||
debug!("Creating: {:?}", sid);
|
||||
self.create(sid)
|
||||
.map_err(Error::from)
|
||||
.and_then(|mut fle| {
|
||||
fle.make_ref(hash, path)?;
|
||||
Ok(fle)
|
||||
})
|
||||
}
|
||||
|
||||
fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
|
||||
-> Result<FileLockEntry<'a>>
|
||||
{
|
||||
match self.get_ref::<RPG, String>(RPG::unique_hash(path.as_ref())?)? {
|
||||
Some(r) => Ok(r),
|
||||
None => self.create_ref::<RPG, A>(path),
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Loading…
Reference in a new issue