matthiasbeyer/imag
1
0
Fork 0
Code Releases Activity

Rewrite library

Browse source 
* Remove old code
* Rewrite with tests

This implements a fassade pattern for ref library

With the fassade, we can specify the hasher in a rather easy way, which
is not possible with default generics for traits.

The "default" part in "default generic type" is not properly implemented
yet (as visible in the tests), as I don't know how to realize this.

For simplicity, the `hasher` module exports a `default` module with a
`DefaultHasher` type, which resolves to the `Sha1Hasher`.

Signed-off-by: Matthias Beyer <mail@beyermatthias.de>
This commit is contained in:
Matthias Beyer 2018-10-23 15:39:23 +02:00
parent a41479c0ec
commit db7121ccba
7 changed files with 484 additions and 548 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
doc/src/05100-lib-entryref.md
View file

@ -42,7 +42,7 @@ libimagentryref does store the following data:
```toml ```toml
[ref] [ref]
filehash.sha1 = "<sha1 hash of the file>" filehash.sha1 = "<sha1 hash of the file>"
relpath = "/Psy_trance_2018_yearmix.mp3" relpath = "Psy_trance_2018_yearmix.mp3"
collection = "music" collection = "music"
``` ```

20
lib/entry/libimagentryref/Cargo.toml
View file

@ -22,25 +22,17 @@ maintenance = { status = "actively-developed" }
[dependencies] [dependencies]
itertools = "0.7" itertools = "0.7"
log = "0.4.0" log = "0.4.0"
failure = "0.1"
sha-1 = "0.8"
toml = "0.4" toml = "0.4"
toml-query = "0.8" toml-query = "0.8"
failure = "0.1" serde = "1"
sha-1 = { version = "0.7", optional = true } serde_derive = "1"
sha2 = { version = "0.7", optional = true }
sha3 = { version = "0.7", optional = true }
hex = { version = "0.3", optional = true }
libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" } libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" } libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
libimagentryutil = { version = "0.10.0", path = "../../../lib/entry/libimagentryutil" } libimagentryutil = { version = "0.10.0", path = "../../../lib/entry/libimagentryutil" }
[features] [dev-dependencies]
default = [] env_logger = "0.5"
generators = []
generators-sha1 = ["sha-1", "hex"]
generators-sha224 = ["sha2", "hex"]
generators-sha256 = ["sha2", "hex"]
generators-sha384 = ["sha2", "hex"]
generators-sha512 = ["sha2", "hex"]
generators-sha3 = ["sha3", "hex"]

276
lib/entry/libimagentryref/src/generators/mod.rs
View file

@ -1,276 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
//! Default generators
//!
//! This module provides a number of default `UniqueRefPathGenerator`s
//! which can be used for generating references.
//!
//! These generators are _NOT_ domain specific. So there won't be a "UniqueMailRefPathGenerator" in
//! here, for example.
//!
//! All these generators use "ref" as collection name.
//! They can be overridden using the `make_unique_ref_path_generator!()` convenience macro.
//!
//! # Note
//!
//! You must enable the appropriate crate feature to use any of the provided generators. With the
//! `generators` feature, you only get the convenience macro `make_unique_ref_path_generator!()`.
//!
/// A convenience macro for wrapping a generator in a new one, reusing the functionality from the
/// underlying generator
///
/// The UniqueRefPathGenerator must be in scope.
///
/// The macro creates a new struct `$name` over `$underlying` and changes the collection name to
/// `$collectionname`.
/// If passed, the new implementation is used (defaults to the implementation from the underlying
/// generator).
/// If passed, the new postprocessing is used (defaults to not changing the StoreId)
///
#[macro_export]
macro_rules! make_unique_ref_path_generator {
(
$name:ident
over $underlying:ty
=> with collection name $collectionname:expr
) => {
struct $name;
impl $crate::refstore::UniqueRefPathGenerator for $name {
fn collection() -> &'static str {
$collectionname
}
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String> {
$underlying::unique_hash(path)
}
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
-> Result<::libimagstore::storeid::StoreId>
{
Ok(sid)
}
}
};
(
$name:ident
over $underlying:ty
=> with collection name $collectionname:expr
=> $impl:expr
) => {
struct $name;
impl $crate::refstore::UniqueRefPathGenerator for $name {
type Error = $errtype;
fn collection() -> &'static str {
$collectionname
}
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String> {
debug!("Making unique hash for path: {:?}", path.as_ref());
$impl(path)
}
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
-> Result<::libimagstore::storeid::StoreId>
{
Ok(sid)
}
}
};
(
pub $name:ident
over $underlying:ty
=> with collection name $collectionname:expr
=> $impl:expr
) => {
make_unique_ref_path_generator!(
pub $name
over $underlying
=> with collection name $collectionname
=> $impl => |sid| { Ok(sid) }
);
};
(
pub $name:ident
over $underlying:ty
=> with collection name $collectionname:expr
=> $impl:expr
=> $postproc:expr
) => {
pub struct $name;
impl $crate::refstore::UniqueRefPathGenerator for $name {
fn collection() -> &'static str {
$collectionname
}
fn unique_hash<A: AsRef<Path>>(path: A) -> ::failure::Fallible<String> {
debug!("Making unique hash for path: {:?}", path.as_ref());
$impl(path)
}
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
-> ::failure::Fallible<::libimagstore::storeid::StoreId>
{
$postproc(sid)
}
}
};
}
#[cfg(any(
feature = "generators-sha1",
feature = "generators-sha224",
feature = "generators-sha256",
feature = "generators-sha384",
feature = "generators-sha512",
feature = "generators-sha3",
))]
mod base;
/// Helper macro for generating implementations for the various Sha algorithms
macro_rules! make_sha_mod {
{
$modname:ident,
$hashname:ident,
$hashingimpl:expr
} => {
pub mod $modname {
use std::path::Path;
use std::fs::OpenOptions;
use std::io::Read;
use hex;
make_unique_ref_path_generator! (
pub $hashname
over generators::base::Base
=> with collection name "ref"
=> |path| {
OpenOptions::new()
.read(true)
.write(false)
.create(false)
.open(path)
.map_err(::failure::Error::from)
.and_then(|mut file| {
let mut buffer = String::new();
let _ = file.read_to_string(&mut buffer)?;
$hashingimpl(buffer)
})
}
);
impl $hashname {
/// Function which can be used by a wrapping UniqueRefPathGenerator to hash only N bytes.
pub fn hash_n_bytes<A: AsRef<Path>>(path: A, n: usize) -> ::failure::Fallible<String> {
debug!("Opening '{}' for hashing", path.as_ref().display());
OpenOptions::new()
.read(true)
.write(false)
.create(false)
.open(path)
.map_err(::failure::Error::from)
.and_then(|mut file| {
let mut buffer = vec![0; n];
debug!("Allocated {} bytes", buffer.capacity());
match file.read_exact(&mut buffer) {
Ok(_) => { /* yay */ Ok(()) },
Err(e) => if e.kind() == ::std::io::ErrorKind::UnexpectedEof {
debug!("Ignoring unexpected EOF before {} bytes were read", n);
Ok(())
} else {
Err(e)
}
}?;
let buffer = String::from_utf8(buffer)?;
$hashingimpl(buffer)
})
}
}
}
}
}
#[cfg(feature = "generators-sha1")]
make_sha_mod! {
sha1, Sha1, |buffer: String| {
use sha1::{Sha1, Digest};
trace!("Hashing: '{:?}'", buffer);
let res = hex::encode(Sha1::digest(buffer.as_bytes()));
trace!("Hash => '{:?}'", res);
Ok(res)
}
}
#[cfg(feature = "generators-sha224")]
make_sha_mod! {
sha224, Sha224, |buffer: String| {
use sha2::{Sha224, Digest};
Ok(hex::encode(Sha224::digest(buffer.as_bytes())))
}
}
#[cfg(feature = "generators-sha256")]
make_sha_mod! {
sha256, Sha256, |buffer: String| {
use sha2::{Sha256, Digest};
Ok(hex::encode(Sha256::digest(buffer.as_bytes())))
}
}
#[cfg(feature = "generators-sha384")]
make_sha_mod! {
sha384, Sha384, |buffer: String| {
use sha2::{Sha384, Digest};
Ok(hex::encode(Sha384::digest(buffer.as_bytes())))
}
}
#[cfg(feature = "generators-sha512")]
make_sha_mod! {
sha512, Sha512, |buffer: String| {
use sha2::{Sha512, Digest};
Ok(hex::encode(Sha512::digest(buffer.as_bytes())))
}
}
#[cfg(feature = "generators-sha3")]
make_sha_mod! {
sha3, Sha3, |buffer: String| {
use sha3::{Sha3_256, Digest};
Ok(hex::encode(Sha3_256::digest(buffer.as_bytes())))
}
}

40
lib/entry/libimagentryref/src/generators/base.rs → lib/entry/libimagentryref/src/hasher.rs
View file

@ -19,29 +19,37 @@
use std::path::Path; use std::path::Path;
use libimagstore::storeid::StoreId;
use refstore::UniqueRefPathGenerator;
use failure::Fallible as Result; use failure::Fallible as Result;
/// A base UniqueRefPathGenerator which must be overridden by the actual UniqueRefPathGenerator pub trait Hasher {
/// which is provided by this crate const NAME: &'static str;
#[allow(dead_code)]
pub struct Base;
impl UniqueRefPathGenerator for Base { /// hash the file at path `path`
fn collection() -> &'static str { fn hash<P: AsRef<Path>>(path: P) -> Result<String>;
"ref"
} }
fn unique_hash<A: AsRef<Path>>(_path: A) -> Result<String> { pub mod default {
// This has to be overridden pub use super::sha1::Sha1Hasher as DefaultHasher;
panic!("Not overridden base functionality. This is a BUG!")
} }
fn postprocess_storeid(sid: StoreId) -> Result<StoreId> { pub mod sha1 {
Ok(sid) // default implementation use std::path::Path;
use failure::Fallible as Result;
use sha1::{Sha1, Digest};
use hasher::Hasher;
pub struct Sha1Hasher;
impl Hasher for Sha1Hasher {
const NAME : &'static str = "sha1";
fn hash<P: AsRef<Path>>(path: P) -> Result<String> {
let digest = Sha1::digest(::std::fs::read_to_string(path)?.as_bytes());
Ok(format!("{:x}", digest)) // TODO: Ugh...
} }
} }
}

42
lib/entry/libimagentryref/src/lib.rs
View file

@ -41,41 +41,17 @@
extern crate itertools; extern crate itertools;
extern crate toml; extern crate toml;
extern crate toml_query; extern crate toml_query;
#[macro_use] extern crate serde_derive;
#[macro_use] extern crate libimagstore;
extern crate libimagerror;
#[macro_use] extern crate libimagentryutil;
extern crate failure;
module_entry_path_mod!("ref");
pub mod reference;
pub mod refstore;
#[cfg(feature = "generators-sha1")]
extern crate sha1; extern crate sha1;
#[cfg(any( extern crate libimagstore;
feature = "generators-sha224", extern crate libimagerror;
feature = "generators-sha256", #[macro_use] extern crate libimagentryutil;
feature = "generators-sha384", #[macro_use] extern crate failure;
feature = "generators-sha512",
))]
extern crate sha2;
#[cfg(feature = "generators-sha3")] #[cfg(test)]
extern crate sha3; extern crate env_logger;
#[cfg(any( pub mod hasher;
feature = "generators-sha1", pub mod reference;
feature = "generators-sha224",
feature = "generators-sha256",
feature = "generators-sha384",
feature = "generators-sha512",
feature = "generators-sha3",
))]
extern crate hex;
#[cfg(feature = "generators")]
pub mod generators;

498
lib/entry/libimagentryref/src/reference.rs
View file

@ -17,15 +17,13 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// //
//! The Ref object is a helper over the link functionality, so one is able to create references to
//! files outside of the imag store.
use std::path::Path; use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use std::collections::BTreeMap;
use std::ops::Deref;
use libimagentryutil::isa::Is; use libimagentryutil::isa::Is;
use libimagentryutil::isa::IsKindHeaderPathProvider; use libimagentryutil::isa::IsKindHeaderPathProvider;
use libimagstore::store::Entry;
use libimagerror::errors::ErrorMsg as EM; use libimagerror::errors::ErrorMsg as EM;
use toml::Value; use toml::Value;
@ -34,8 +32,106 @@ use toml_query::delete::TomlValueDeleteExt;
use toml_query::insert::TomlValueInsertExt; use toml_query::insert::TomlValueInsertExt;
use failure::Fallible as Result; use failure::Fallible as Result;
use failure::Error; use failure::Error;
use failure::err_msg;
use failure::ResultExt;
use hasher::Hasher;
/// A configuration of "collection name" -> "collection path" mappings
///
/// Should be deserializeable from the configuration file right away, because we expect a
/// configuration like this in the config file:
///
/// ```toml
/// [ref.collections]
/// music = "/home/alice/music"
/// documents = "/home/alice/doc"
/// ```
///
/// for example.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Config(BTreeMap<String, PathBuf>);
impl Config {
pub fn new(map: BTreeMap<String, PathBuf>) -> Self {
Config(map)
}
}
impl Deref for Config {
type Target = BTreeMap<String, PathBuf>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
provide_kindflag_path!(pub IsRef, "ref.is_ref");
/// Fassade module
///
/// This module is necessary to build a generic fassade around the "entry with a (default) hasher to
/// represent the entry as a ref".
///
/// The module is for code-structuring only, all types in the module are exported publicly in the
/// supermodule.
pub mod fassade {
use std::marker::PhantomData;
use libimagstore::store::Entry;
use libimagentryutil::isa::Is;
use failure::Fallible as Result;
use failure::Error;
use hasher::sha1::Sha1Hasher;
use hasher::Hasher;
use super::IsRef;
pub trait RefFassade {
fn is_ref(&self) -> Result<bool>;
fn as_ref_with_hasher<H: Hasher>(&self) -> RefWithHasher<H>;
fn as_ref_with_hasher_mut<H: Hasher>(&mut self) -> MutRefWithHasher<H>;
}
impl RefFassade for Entry {
/// Check whether the underlying object is actually a ref
fn is_ref(&self) -> Result<bool> {
self.is::<IsRef>().map_err(Error::from)
}
fn as_ref_with_hasher<H: Hasher>(&self) -> RefWithHasher<H> {
RefWithHasher::new(self)
}
fn as_ref_with_hasher_mut<H: Hasher>(&mut self) -> MutRefWithHasher<H> {
MutRefWithHasher::new(self)
}
}
pub struct RefWithHasher<'a, H: Hasher = Sha1Hasher>(pub(crate) &'a Entry, PhantomData<H>);
impl<'a, H> RefWithHasher<'a, H>
where H: Hasher
{
fn new(entry: &'a Entry) -> Self {
RefWithHasher(entry, PhantomData)
}
}
pub struct MutRefWithHasher<'a, H: Hasher = Sha1Hasher>(pub(crate) &'a mut Entry, PhantomData<H>);
impl<'a, H> MutRefWithHasher<'a, H>
where H: Hasher
{
fn new(entry: &'a mut Entry) -> Self {
MutRefWithHasher(entry, PhantomData)
}
}
}
pub use self::fassade::*;
use refstore::UniqueRefPathGenerator;
pub trait Ref { pub trait Ref {
@ -43,76 +139,41 @@ pub trait Ref {
fn is_ref(&self) -> Result<bool>; fn is_ref(&self) -> Result<bool>;
/// Get the stored hash. /// Get the stored hash.
///
/// Does not need a `UniqueRefPathGenerator` as it reads the hash stored in the header
fn get_hash(&self) -> Result<&str>;
/// Make this object a ref
fn make_ref<P: AsRef<Path>>(&mut self, hash: String, path: P) -> Result<()>;
/// Get the referenced path.
///
/// Does not need a `UniqueRefPathGenerator` as it reads the path stored in the header.
fn get_path(&self) -> Result<PathBuf>; fn get_path(&self) -> Result<PathBuf>;
/// Get the stored hash.
fn get_hash(&self) -> Result<&str>;
/// Check whether the referenced file still matches its hash /// Check whether the referenced file still matches its hash
fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> Result<bool>; fn hash_valid(&self, config: &Config) -> Result<bool>;
fn remove_ref(&mut self) -> Result<()>;
/// Alias for `r.fs_link_exists() && r.deref().is_file()`
fn is_ref_to_file(&self) -> Result<bool> {
self.get_path().map(|p| p.is_file())
} }
/// Alias for `r.fs_link_exists() && r.deref().is_dir()` impl<'a, H: Hasher> Ref for RefWithHasher<'a, H> {
fn is_ref_to_dir(&self) -> Result<bool> {
self.get_path().map(|p| p.is_dir())
}
/// Alias for `!Ref::fs_link_exists()`
fn is_dangling(&self) -> Result<bool> {
self.get_path().map(|p| !p.exists())
}
}
provide_kindflag_path!(pub IsRef, "ref.is_ref");
impl Ref for Entry {
/// Check whether the underlying object is actually a ref /// Check whether the underlying object is actually a ref
fn is_ref(&self) -> Result<bool> { fn is_ref(&self) -> Result<bool> {
self.is::<IsRef>().map_err(Error::from) self.0.is::<IsRef>().map_err(Error::from)
} }
fn get_hash(&self) -> Result<&str> { fn get_hash(&self) -> Result<&str> {
self.get_header() let header_path = format!("ref.hash.{}", H::NAME);
.read("ref.hash") self.0
.get_header()
.read(&header_path)
.map_err(Error::from)? .map_err(Error::from)?
.ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.hash"))) .ok_or_else(|| {
Error::from(EM::EntryHeaderFieldMissing("ref.hash.<hash>"))
})
.and_then(|v| { .and_then(|v| {
v.as_str().ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash", "string"))) v.as_str().ok_or_else(|| {
Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string"))
})
}) })
} }
fn make_ref<P: AsRef<Path>>(&mut self, hash: String, path: P) -> Result<()> {
let path_str : String = path
.as_ref()
.to_str()
.map(String::from)
.ok_or_else(|| EM::UTF8Error)?;
let _ = self.set_isflag::<IsRef>()?;
let hdr = self.get_header_mut();
hdr.insert("ref.path", Value::String(String::from(path_str)))?;
hdr.insert("ref.hash", Value::String(hash))?;
Ok(())
}
fn get_path(&self) -> Result<PathBuf> { fn get_path(&self) -> Result<PathBuf> {
self.get_header() self.0
.get_header()
.read("ref.path") .read("ref.path")
.map_err(Error::from)? .map_err(Error::from)?
.ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.path"))) .ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.path")))
@ -124,21 +185,324 @@ impl Ref for Entry {
.map(PathBuf::from) .map(PathBuf::from)
} }
fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> Result<bool> { fn hash_valid(&self, config: &Config) -> Result<bool> {
self.get_path() let ref_header = self.0
.get_header()
.read("ref")?
.ok_or_else(|| err_msg("Header missing at 'ref'"))?;
let collection_name = ref_header
.read("collection")
.map_err(Error::from)?
.ok_or_else(|| err_msg("Header missing at 'ref.collection'"))?
.as_str()
.ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string")))?;
let path = ref_header
.read("path")
.map_err(Error::from)?
.ok_or_else(|| err_msg("Header missing at 'ref.path'"))?
.as_str()
.map(PathBuf::from) .map(PathBuf::from)
.map_err(Error::from) .ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string")))?;
.and_then(|pb| RPG::unique_hash(pb))
.and_then(|h| Ok(h == self.get_hash()?))
let file_path = get_file_path(config, collection_name.as_ref(), &path)?;
ref_header
.read(H::NAME)
.map_err(Error::from)?
.ok_or_else(|| format_err!("Header missing at 'ref.{}'", H::NAME))
.and_then(|v| {
v.as_str().ok_or_else(|| {
Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string"))
})
})
.and_then(|hash| H::hash(file_path).map(|h| h == hash))
} }
}
pub trait MutRef {
fn remove_ref(&mut self) -> Result<()>;
/// Make a ref out of a normal (non-ref) entry.
///
/// If the entry is already a ref, this fails if `force` is false
fn make_ref<P, Coll>(&mut self, path: P, collection_name: Coll, config: &Config, force: bool)
-> Result<()>
where P: AsRef<Path>,
Coll: AsRef<str>;
}
impl<'a, H> MutRef for MutRefWithHasher<'a, H>
where H: Hasher
{
fn remove_ref(&mut self) -> Result<()> { fn remove_ref(&mut self) -> Result<()> {
let hdr = self.get_header_mut(); debug!("Removing 'ref' header section");
let _ = hdr.delete("ref.hash")?; {
let _ = hdr.delete("ref.path")?; let header = self.0.get_header_mut();
let _ = hdr.delete("ref")?; trace!("header = {:?}", header);
let _ = header.delete("ref.relpath").context("Removing ref.relpath")?;
if let Some(hash_tbl) = header.read_mut("ref.hash")? {
match hash_tbl {
Value::Table(ref mut tbl) => *tbl = BTreeMap::new(),
_ => {
// should not happen
}
}
}
let _ = header.delete("ref.hash").context("Removing ref.hash")?;
let _ = header.delete("ref.collection").context("Removing ref.collection")?;
}
debug!("Removing 'ref' header marker");
self.0.remove_isflag::<IsRef>().context("Removing ref")?;
let _ = self.0
.get_header_mut()
.delete("ref")
.context("Removing ref")?;
trace!("header = {:?}", self.0.get_header());
Ok(()) Ok(())
} }
/// Make a ref out of a normal (non-ref) entry.
///
/// `path` is the path to refer to,
///
/// # Warning
///
/// If the entry is already a ref, this fails if `force` is false
///
fn make_ref<P, Coll>(&mut self, path: P, collection_name: Coll, config: &Config, force: bool)
-> Result<()>
where P: AsRef<Path>,
Coll: AsRef<str>
{
trace!("Making ref out of {:?}", self.0);
trace!("Making ref with collection name {:?}", collection_name.as_ref());
trace!("Making ref with config {:?}", config);
trace!("Making ref forced = {}", force);
if self.0.get_header().read("ref.is_ref")?.is_some() && !force {
debug!("Entry is already a Ref!");
let _ = Err(err_msg("Entry is already a reference")).context("Making ref out of entry")?;
}
let file_path = get_file_path(config, collection_name.as_ref(), &path)?;
if !file_path.exists() {
let msg = format_err!("File '{:?}' does not exist", file_path);
let _ = Err(msg).context("Making ref out of entry")?;
}
debug!("Entry hashing...");
let _ = H::hash(&file_path)
.and_then(|hash| make_header_section(hash, H::NAME, path, collection_name))
.and_then(|h| self.0.get_header_mut().insert("ref", h).map_err(Error::from))
.and_then(|_| self.0.set_isflag::<IsRef>())
.context("Making ref out of entry")?;
debug!("Setting is-ref flag");
self.0
.set_isflag::<IsRef>()
.context("Setting ref-flag")
.map_err(Error::from)
.map(|_| ())
}
}
/// Create a new header section for a "ref".
///
/// # Warning
///
/// The `relpath` _must_ be relative to the configured path for that collection.
pub(crate) fn make_header_section<P, C, H>(hash: String, hashname: H, relpath: P, collection: C)
-> Result<Value>
where P: AsRef<Path>,
C: AsRef<str>,
H: AsRef<str>,
{
let mut header_section = Value::Table(BTreeMap::new());
{
let relpath = relpath
.as_ref()
.to_str()
.map(String::from)
.ok_or_else(|| {
let msg = format_err!("UTF Error in '{:?}'", relpath.as_ref());
Error::from(msg)
})?;
let _ = header_section.insert("relpath", Value::String(relpath))?;
}
{
let mut hash_table = Value::Table(BTreeMap::new());
let _ = hash_table.insert(hashname.as_ref(), Value::String(hash))?;
let _ = header_section.insert("hash", hash_table)?;
}
let _ = header_section.insert("collection", Value::String(String::from(collection.as_ref())));
Ok(header_section)
}
fn get_file_path<P>(config: &Config, collection_name: &str, path: P) -> Result<PathBuf>
where P: AsRef<Path>
{
config
.get(collection_name)
.map(PathBuf::clone)
.ok_or_else(|| {
format_err!("Configuration missing for collection: '{}'", collection_name)
})
.context("Making ref out of entry")
.map_err(Error::from)
.map(|p| {
let filepath = p.join(&path);
trace!("Found filepath: {:?}", filepath.display());
filepath
})
}
#[cfg(test)]
mod test {
use std::path::PathBuf;
use libimagstore::store::Store;
use libimagstore::store::Entry;
use super::*;
use hasher::Hasher;
fn setup_logging() {
let _ = ::env_logger::try_init();
}
pub fn get_store() -> Store {
Store::new_inmemory(PathBuf::from("/"), &None).unwrap()
}
struct TestHasher;
impl Hasher for TestHasher {
const NAME: &'static str = "Testhasher";
fn hash<P: AsRef<Path>>(path: P) -> Result<String> {
path.as_ref()
.to_str()
.map(String::from)
.ok_or_else(|| Error::from(err_msg("Failed to create test hash")))
}
}
#[test]
fn test_isref() {
setup_logging();
let store = get_store();
let entry = store.retrieve(PathBuf::from("test_isref")).unwrap();
assert!(!entry.is_ref().unwrap());
}
#[test]
fn test_makeref() {
setup_logging();
let store = get_store();
let mut entry = store.retrieve(PathBuf::from("test_makeref")).unwrap();
let file = PathBuf::from("/"); // has to exist
let collection_name = "some_collection";
let config = Config({
let mut c = BTreeMap::new();
c.insert(String::from("some_collection"), PathBuf::from("/tmp"));
c
});
let r = entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false);
assert!(r.is_ok());
}
#[test]
fn test_makeref_isref() {
setup_logging();
let store = get_store();
let mut entry = store.retrieve(PathBuf::from("test_makeref_isref")).unwrap();
let file = PathBuf::from("/"); // has to exists
let collection_name = "some_collection";
let config = Config({
let mut c = BTreeMap::new();
c.insert(String::from("some_collection"), PathBuf::from("/tmp"));
c
});
let res = entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false);
assert!(res.is_ok(), "Expected to be ok: {:?}", res);
assert!(entry.as_ref_with_hasher::<TestHasher>().is_ref().unwrap());
}
#[test]
fn test_makeref_is_ref_with_testhash() {
setup_logging();
let store = get_store();
let mut entry = store.retrieve(PathBuf::from("test_makeref_is_ref_with_testhash")).unwrap();
let file = PathBuf::from("/"); // has to exist
let collection_name = "some_collection";
let config = Config({
let mut c = BTreeMap::new();
c.insert(String::from("some_collection"), PathBuf::from("/"));
c
});
assert!(entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false).is_ok());
let check_isstr = |entry: &Entry, location, shouldbe| {
let var = entry.get_header().read(location);
assert!(var.is_ok(), "{} is not Ok(_): {:?}", location, var);
let var = var.unwrap();
assert!(var.is_some(), "{} is not Some(_): {:?}", location, var);
let var = var.unwrap().as_str();
assert!(var.is_some(), "{} is not String: {:?}", location, var);
assert_eq!(var.unwrap(), shouldbe, "{} is not == {}", location, shouldbe);
};
check_isstr(&entry, "ref.relpath", "/");
check_isstr(&entry, "ref.hash.Testhasher", "/"); // TestHasher hashes by returning the path itself
check_isstr(&entry, "ref.collection", "some_collection");
}
#[test]
fn test_makeref_remref() {
setup_logging();
let store = get_store();
let mut entry = store.retrieve(PathBuf::from("test_makeref_remref")).unwrap();
let file = PathBuf::from("/"); // has to exist
let collection_name = "some_collection";
let config = Config({
let mut c = BTreeMap::new();
c.insert(String::from("some_collection"), PathBuf::from("/"));
c
});
assert!(entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false).is_ok());
assert!(entry.as_ref_with_hasher::<TestHasher>().is_ref().unwrap());
let res = entry.as_ref_with_hasher_mut::<TestHasher>().remove_ref();
assert!(res.is_ok(), "Expected to be ok: {:?}", res);
assert!(!entry.as_ref_with_hasher::<TestHasher>().is_ref().unwrap());
}
} }

128
lib/entry/libimagentryref/src/refstore.rs
View file

@ -1,128 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::path::Path;
use std::path::PathBuf;
use libimagstore::store::FileLockEntry;
use libimagstore::store::Store;
use libimagstore::storeid::StoreId;
use reference::Ref;
use failure::Fallible as Result;
use failure::Error;
/// A UniqueRefPathGenerator generates unique Pathes
///
/// It is basically a functor which generates a StoreId from a &Path.
/// For more information have a look at the documentation of RefStore.
pub trait UniqueRefPathGenerator {
/// The collection the `StoreId` should be created for
fn collection() -> &'static str {
"ref"
}
/// A function which should generate a unique string for a Path
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String>;
/// Postprocess the generated `StoreId` object
fn postprocess_storeid(sid: StoreId) -> Result<StoreId> {
Ok(sid)
}
}
/// A extensions for the `Store` to handle `Ref` objects
///
/// The RefStore handles refs using a `UniqueRefPathGenerator`. The `UniqueRefPathGenerator`, as it
/// name suggests, generates unique `StoreId`s for a `&Path`. It is a functor `&Path -> StoreId`.
///
/// It provides three functions which are called in the following sequence:
///
/// * The `UniqueRefPathGenerator::collection()` function is used for get the collection a `StoreId`
/// should be in (The first element of the `StoreId` path)
/// * The `UniqueRefPathGenerator::unique_hash()` gets the `&Path` which it then should generate a
/// unique String for. How this is done does not matter. It can hash the Path itself, read the
/// file and hash that or something else. It should be reproduceable, though.
/// * These two parts are joined and put into a `StoreId` which the
/// `UniqueRefPathGenerator::postprocess_storeid()` function is then allowed to postprocess (for
/// example add more parts to the StoreId). The default implementation does nothing.
///
/// The StoreId which is generated is then used to carry out the actual action (reading, creating
/// ...).
/// If a entry is created, header information is set (that it is a ref, the hash which was just
/// generated and the path of the referenced file)
///
/// # Details
///
/// The `UniqueRefPathGenerator` is passed as type parameter to enforce some semantics:
///
/// * The used `UniqueRefPathGenerator` is defined by the implementation rather than by the runtime
/// of the program or some environment. Of course this is only a small hurdle to enforce this, but
/// a hint.
/// * The `UniqueRefPathGenerator` is a functor which does not carry state.
///
pub trait RefStore<'a> {
fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H) -> Result<Option<FileLockEntry<'a>>>;
fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>>;
fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>>;
}
impl<'a> RefStore<'a> for Store {
fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H)
-> Result<Option<FileLockEntry<'a>>>
{
let sid = StoreId::new(PathBuf::from(format!("{}/{}", RPG::collection(), hash.as_ref())))
.map_err(Error::from)?;
debug!("Getting: {:?}", sid);
self.get(sid)
.map_err(Error::from)
}
fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
-> Result<FileLockEntry<'a>>
{
let hash = RPG::unique_hash(&path)?;
let pathbuf = PathBuf::from(format!("{}/{}", RPG::collection(), hash));
let sid = StoreId::new(pathbuf.clone())?;
debug!("Creating: {:?}", sid);
self.create(sid)
.map_err(Error::from)
.and_then(|mut fle| {
fle.make_ref(hash, path)?;
Ok(fle)
})
}
fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
-> Result<FileLockEntry<'a>>
{
match self.get_ref::<RPG, String>(RPG::unique_hash(path.as_ref())?)? {
Some(r) => Ok(r),
None => self.create_ref::<RPG, A>(path),
}
}
}