From db7121ccba11b956c53bd28b246e830446e2c13f Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Tue, 23 Oct 2018 15:39:23 +0200 Subject: [PATCH] Rewrite library * Remove old code * Rewrite with tests This implements a fassade pattern for ref library With the fassade, we can specify the hasher in a rather easy way, which is not possible with default generics for traits. The "default" part in "default generic type" is not properly implemented yet (as visible in the tests), as I don't know how to realize this. For simplicity, the `hasher` module exports a `default` module with a `DefaultHasher` type, which resolves to the `Sha1Hasher`. Signed-off-by: Matthias Beyer --- doc/src/05100-lib-entryref.md | 2 +- lib/entry/libimagentryref/Cargo.toml | 28 +- .../libimagentryref/src/generators/mod.rs | 276 ---------- .../src/{generators/base.rs => hasher.rs} | 50 +- lib/entry/libimagentryref/src/lib.rs | 42 +- lib/entry/libimagentryref/src/reference.rs | 506 +++++++++++++++--- lib/entry/libimagentryref/src/refstore.rs | 128 ----- 7 files changed, 484 insertions(+), 548 deletions(-) delete mode 100644 lib/entry/libimagentryref/src/generators/mod.rs rename lib/entry/libimagentryref/src/{generators/base.rs => hasher.rs} (58%) delete mode 100644 lib/entry/libimagentryref/src/refstore.rs diff --git a/doc/src/05100-lib-entryref.md b/doc/src/05100-lib-entryref.md index 156af102..46409205 100644 --- a/doc/src/05100-lib-entryref.md +++ b/doc/src/05100-lib-entryref.md @@ -42,7 +42,7 @@ libimagentryref does store the following data: ```toml [ref] filehash.sha1 = "" -relpath = "/Psy_trance_2018_yearmix.mp3" +relpath = "Psy_trance_2018_yearmix.mp3" collection = "music" ``` diff --git a/lib/entry/libimagentryref/Cargo.toml b/lib/entry/libimagentryref/Cargo.toml index e82249b9..60692acd 100644 --- a/lib/entry/libimagentryref/Cargo.toml +++ b/lib/entry/libimagentryref/Cargo.toml @@ -20,27 +20,19 @@ is-it-maintained-open-issues = { repository = "matthiasbeyer/imag" } maintenance = { status = "actively-developed" } [dependencies] -itertools = "0.7" -log = "0.4.0" -toml = "0.4" -toml-query = "0.8" -failure = "0.1" -sha-1 = { version = "0.7", optional = true } -sha2 = { version = "0.7", optional = true } -sha3 = { version = "0.7", optional = true } -hex = { version = "0.3", optional = true } +itertools = "0.7" +log = "0.4.0" +failure = "0.1" +sha-1 = "0.8" +toml = "0.4" +toml-query = "0.8" +serde = "1" +serde_derive = "1" libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" } libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" } libimagentryutil = { version = "0.10.0", path = "../../../lib/entry/libimagentryutil" } -[features] -default = [] -generators = [] -generators-sha1 = ["sha-1", "hex"] -generators-sha224 = ["sha2", "hex"] -generators-sha256 = ["sha2", "hex"] -generators-sha384 = ["sha2", "hex"] -generators-sha512 = ["sha2", "hex"] -generators-sha3 = ["sha3", "hex"] +[dev-dependencies] +env_logger = "0.5" diff --git a/lib/entry/libimagentryref/src/generators/mod.rs b/lib/entry/libimagentryref/src/generators/mod.rs deleted file mode 100644 index 27d0fa9b..00000000 --- a/lib/entry/libimagentryref/src/generators/mod.rs +++ /dev/null @@ -1,276 +0,0 @@ -// -// imag - the personal information management suite for the commandline -// Copyright (C) 2015-2019 Matthias Beyer and contributors -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; version -// 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// - -//! Default generators -//! -//! This module provides a number of default `UniqueRefPathGenerator`s -//! which can be used for generating references. -//! -//! These generators are _NOT_ domain specific. So there won't be a "UniqueMailRefPathGenerator" in -//! here, for example. -//! -//! All these generators use "ref" as collection name. -//! They can be overridden using the `make_unique_ref_path_generator!()` convenience macro. -//! -//! # Note -//! -//! You must enable the appropriate crate feature to use any of the provided generators. With the -//! `generators` feature, you only get the convenience macro `make_unique_ref_path_generator!()`. -//! - -/// A convenience macro for wrapping a generator in a new one, reusing the functionality from the -/// underlying generator -/// -/// The UniqueRefPathGenerator must be in scope. -/// -/// The macro creates a new struct `$name` over `$underlying` and changes the collection name to -/// `$collectionname`. -/// If passed, the new implementation is used (defaults to the implementation from the underlying -/// generator). -/// If passed, the new postprocessing is used (defaults to not changing the StoreId) -/// -#[macro_export] -macro_rules! make_unique_ref_path_generator { - ( - $name:ident - over $underlying:ty - => with collection name $collectionname:expr - ) => { - struct $name; - - impl $crate::refstore::UniqueRefPathGenerator for $name { - - fn collection() -> &'static str { - $collectionname - } - - fn unique_hash>(path: A) -> Result { - $underlying::unique_hash(path) - } - - fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId) - -> Result<::libimagstore::storeid::StoreId> - { - Ok(sid) - } - } - }; - - ( - $name:ident - over $underlying:ty - => with collection name $collectionname:expr - => $impl:expr - ) => { - struct $name; - - impl $crate::refstore::UniqueRefPathGenerator for $name { - type Error = $errtype; - - fn collection() -> &'static str { - $collectionname - } - - fn unique_hash>(path: A) -> Result { - debug!("Making unique hash for path: {:?}", path.as_ref()); - $impl(path) - } - - fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId) - -> Result<::libimagstore::storeid::StoreId> - { - Ok(sid) - } - } - }; - - ( - pub $name:ident - over $underlying:ty - => with collection name $collectionname:expr - => $impl:expr - ) => { - make_unique_ref_path_generator!( - pub $name - over $underlying - => with collection name $collectionname - => $impl => |sid| { Ok(sid) } - ); - }; - - ( - pub $name:ident - over $underlying:ty - => with collection name $collectionname:expr - => $impl:expr - => $postproc:expr - ) => { - pub struct $name; - - impl $crate::refstore::UniqueRefPathGenerator for $name { - fn collection() -> &'static str { - $collectionname - } - - fn unique_hash>(path: A) -> ::failure::Fallible { - debug!("Making unique hash for path: {:?}", path.as_ref()); - $impl(path) - } - - fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId) - -> ::failure::Fallible<::libimagstore::storeid::StoreId> - { - $postproc(sid) - } - } - }; -} - - -#[cfg(any( - feature = "generators-sha1", - feature = "generators-sha224", - feature = "generators-sha256", - feature = "generators-sha384", - feature = "generators-sha512", - feature = "generators-sha3", - ))] -mod base; - -/// Helper macro for generating implementations for the various Sha algorithms -macro_rules! make_sha_mod { - { - $modname:ident, - $hashname:ident, - $hashingimpl:expr - } => { - pub mod $modname { - use std::path::Path; - use std::fs::OpenOptions; - use std::io::Read; - - use hex; - make_unique_ref_path_generator! ( - pub $hashname - over generators::base::Base - => with collection name "ref" - => |path| { - OpenOptions::new() - .read(true) - .write(false) - .create(false) - .open(path) - .map_err(::failure::Error::from) - .and_then(|mut file| { - let mut buffer = String::new(); - let _ = file.read_to_string(&mut buffer)?; - $hashingimpl(buffer) - }) - } - ); - - impl $hashname { - - /// Function which can be used by a wrapping UniqueRefPathGenerator to hash only N bytes. - pub fn hash_n_bytes>(path: A, n: usize) -> ::failure::Fallible { - debug!("Opening '{}' for hashing", path.as_ref().display()); - OpenOptions::new() - .read(true) - .write(false) - .create(false) - .open(path) - .map_err(::failure::Error::from) - .and_then(|mut file| { - let mut buffer = vec![0; n]; - debug!("Allocated {} bytes", buffer.capacity()); - - match file.read_exact(&mut buffer) { - Ok(_) => { /* yay */ Ok(()) }, - Err(e) => if e.kind() == ::std::io::ErrorKind::UnexpectedEof { - debug!("Ignoring unexpected EOF before {} bytes were read", n); - Ok(()) - } else { - Err(e) - } - }?; - - let buffer = String::from_utf8(buffer)?; - $hashingimpl(buffer) - }) - } - - } - - } - } -} - -#[cfg(feature = "generators-sha1")] -make_sha_mod! { - sha1, Sha1, |buffer: String| { - use sha1::{Sha1, Digest}; - - trace!("Hashing: '{:?}'", buffer); - let res = hex::encode(Sha1::digest(buffer.as_bytes())); - trace!("Hash => '{:?}'", res); - - Ok(res) - } -} - -#[cfg(feature = "generators-sha224")] -make_sha_mod! { - sha224, Sha224, |buffer: String| { - use sha2::{Sha224, Digest}; - Ok(hex::encode(Sha224::digest(buffer.as_bytes()))) - } -} - -#[cfg(feature = "generators-sha256")] -make_sha_mod! { - sha256, Sha256, |buffer: String| { - use sha2::{Sha256, Digest}; - Ok(hex::encode(Sha256::digest(buffer.as_bytes()))) - } -} - -#[cfg(feature = "generators-sha384")] -make_sha_mod! { - sha384, Sha384, |buffer: String| { - use sha2::{Sha384, Digest}; - Ok(hex::encode(Sha384::digest(buffer.as_bytes()))) - } -} - -#[cfg(feature = "generators-sha512")] -make_sha_mod! { - sha512, Sha512, |buffer: String| { - use sha2::{Sha512, Digest}; - Ok(hex::encode(Sha512::digest(buffer.as_bytes()))) - } -} - -#[cfg(feature = "generators-sha3")] -make_sha_mod! { - sha3, Sha3, |buffer: String| { - use sha3::{Sha3_256, Digest}; - Ok(hex::encode(Sha3_256::digest(buffer.as_bytes()))) - } -} - diff --git a/lib/entry/libimagentryref/src/generators/base.rs b/lib/entry/libimagentryref/src/hasher.rs similarity index 58% rename from lib/entry/libimagentryref/src/generators/base.rs rename to lib/entry/libimagentryref/src/hasher.rs index 1ccbb760..aec7eebc 100644 --- a/lib/entry/libimagentryref/src/generators/base.rs +++ b/lib/entry/libimagentryref/src/hasher.rs @@ -19,29 +19,37 @@ use std::path::Path; -use libimagstore::storeid::StoreId; - -use refstore::UniqueRefPathGenerator; - use failure::Fallible as Result; -/// A base UniqueRefPathGenerator which must be overridden by the actual UniqueRefPathGenerator -/// which is provided by this crate -#[allow(dead_code)] -pub struct Base; +pub trait Hasher { + const NAME: &'static str; + + /// hash the file at path `path` + fn hash>(path: P) -> Result; +} + +pub mod default { + pub use super::sha1::Sha1Hasher as DefaultHasher; +} + +pub mod sha1 { + use std::path::Path; + + use failure::Fallible as Result; + use sha1::{Sha1, Digest}; + + use hasher::Hasher; + + pub struct Sha1Hasher; + + impl Hasher for Sha1Hasher { + const NAME : &'static str = "sha1"; + + fn hash>(path: P) -> Result { + let digest = Sha1::digest(::std::fs::read_to_string(path)?.as_bytes()); + Ok(format!("{:x}", digest)) // TODO: Ugh... + } + } -impl UniqueRefPathGenerator for Base { - fn collection() -> &'static str { - "ref" - } - - fn unique_hash>(_path: A) -> Result { - // This has to be overridden - panic!("Not overridden base functionality. This is a BUG!") - } - - fn postprocess_storeid(sid: StoreId) -> Result { - Ok(sid) // default implementation - } } diff --git a/lib/entry/libimagentryref/src/lib.rs b/lib/entry/libimagentryref/src/lib.rs index da9bda3d..4d8e02e8 100644 --- a/lib/entry/libimagentryref/src/lib.rs +++ b/lib/entry/libimagentryref/src/lib.rs @@ -41,41 +41,17 @@ extern crate itertools; extern crate toml; extern crate toml_query; - -#[macro_use] extern crate libimagstore; -extern crate libimagerror; -#[macro_use] extern crate libimagentryutil; -extern crate failure; - -module_entry_path_mod!("ref"); - -pub mod reference; -pub mod refstore; - -#[cfg(feature = "generators-sha1")] +#[macro_use] extern crate serde_derive; extern crate sha1; -#[cfg(any( - feature = "generators-sha224", - feature = "generators-sha256", - feature = "generators-sha384", - feature = "generators-sha512", -))] -extern crate sha2; +extern crate libimagstore; +extern crate libimagerror; +#[macro_use] extern crate libimagentryutil; +#[macro_use] extern crate failure; -#[cfg(feature = "generators-sha3")] -extern crate sha3; +#[cfg(test)] +extern crate env_logger; -#[cfg(any( - feature = "generators-sha1", - feature = "generators-sha224", - feature = "generators-sha256", - feature = "generators-sha384", - feature = "generators-sha512", - feature = "generators-sha3", -))] -extern crate hex; - -#[cfg(feature = "generators")] -pub mod generators; +pub mod hasher; +pub mod reference; diff --git a/lib/entry/libimagentryref/src/reference.rs b/lib/entry/libimagentryref/src/reference.rs index d09464c5..96a13530 100644 --- a/lib/entry/libimagentryref/src/reference.rs +++ b/lib/entry/libimagentryref/src/reference.rs @@ -17,15 +17,13 @@ // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // -//! The Ref object is a helper over the link functionality, so one is able to create references to -//! files outside of the imag store. - use std::path::Path; use std::path::PathBuf; +use std::collections::BTreeMap; +use std::ops::Deref; use libimagentryutil::isa::Is; use libimagentryutil::isa::IsKindHeaderPathProvider; -use libimagstore::store::Entry; use libimagerror::errors::ErrorMsg as EM; use toml::Value; @@ -34,8 +32,106 @@ use toml_query::delete::TomlValueDeleteExt; use toml_query::insert::TomlValueInsertExt; use failure::Fallible as Result; use failure::Error; +use failure::err_msg; +use failure::ResultExt; + +use hasher::Hasher; + +/// A configuration of "collection name" -> "collection path" mappings +/// +/// Should be deserializeable from the configuration file right away, because we expect a +/// configuration like this in the config file: +/// +/// ```toml +/// [ref.collections] +/// music = "/home/alice/music" +/// documents = "/home/alice/doc" +/// ``` +/// +/// for example. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Config(BTreeMap); + +impl Config { + pub fn new(map: BTreeMap) -> Self { + Config(map) + } +} + +impl Deref for Config { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +provide_kindflag_path!(pub IsRef, "ref.is_ref"); + +/// Fassade module +/// +/// This module is necessary to build a generic fassade around the "entry with a (default) hasher to +/// represent the entry as a ref". +/// +/// The module is for code-structuring only, all types in the module are exported publicly in the +/// supermodule. +pub mod fassade { + use std::marker::PhantomData; + + use libimagstore::store::Entry; + use libimagentryutil::isa::Is; + + use failure::Fallible as Result; + use failure::Error; + + use hasher::sha1::Sha1Hasher; + use hasher::Hasher; + use super::IsRef; + + pub trait RefFassade { + fn is_ref(&self) -> Result; + fn as_ref_with_hasher(&self) -> RefWithHasher; + fn as_ref_with_hasher_mut(&mut self) -> MutRefWithHasher; + } + + impl RefFassade for Entry { + /// Check whether the underlying object is actually a ref + fn is_ref(&self) -> Result { + self.is::().map_err(Error::from) + } + + fn as_ref_with_hasher(&self) -> RefWithHasher { + RefWithHasher::new(self) + } + + fn as_ref_with_hasher_mut(&mut self) -> MutRefWithHasher { + MutRefWithHasher::new(self) + } + + } + + pub struct RefWithHasher<'a, H: Hasher = Sha1Hasher>(pub(crate) &'a Entry, PhantomData); + + impl<'a, H> RefWithHasher<'a, H> + where H: Hasher + { + fn new(entry: &'a Entry) -> Self { + RefWithHasher(entry, PhantomData) + } + } + + pub struct MutRefWithHasher<'a, H: Hasher = Sha1Hasher>(pub(crate) &'a mut Entry, PhantomData); + + impl<'a, H> MutRefWithHasher<'a, H> + where H: Hasher + { + fn new(entry: &'a mut Entry) -> Self { + MutRefWithHasher(entry, PhantomData) + } + } +} +pub use self::fassade::*; -use refstore::UniqueRefPathGenerator; pub trait Ref { @@ -43,76 +139,41 @@ pub trait Ref { fn is_ref(&self) -> Result; /// Get the stored hash. - /// - /// Does not need a `UniqueRefPathGenerator` as it reads the hash stored in the header - fn get_hash(&self) -> Result<&str>; - - /// Make this object a ref - fn make_ref>(&mut self, hash: String, path: P) -> Result<()>; - - /// Get the referenced path. - /// - /// Does not need a `UniqueRefPathGenerator` as it reads the path stored in the header. fn get_path(&self) -> Result; + /// Get the stored hash. + fn get_hash(&self) -> Result<&str>; + /// Check whether the referenced file still matches its hash - fn hash_valid(&self) -> Result; - - fn remove_ref(&mut self) -> Result<()>; - - /// Alias for `r.fs_link_exists() && r.deref().is_file()` - fn is_ref_to_file(&self) -> Result { - self.get_path().map(|p| p.is_file()) - } - - /// Alias for `r.fs_link_exists() && r.deref().is_dir()` - fn is_ref_to_dir(&self) -> Result { - self.get_path().map(|p| p.is_dir()) - } - - /// Alias for `!Ref::fs_link_exists()` - fn is_dangling(&self) -> Result { - self.get_path().map(|p| !p.exists()) - } - + fn hash_valid(&self, config: &Config) -> Result; } -provide_kindflag_path!(pub IsRef, "ref.is_ref"); - -impl Ref for Entry { +impl<'a, H: Hasher> Ref for RefWithHasher<'a, H> { /// Check whether the underlying object is actually a ref fn is_ref(&self) -> Result { - self.is::().map_err(Error::from) + self.0.is::().map_err(Error::from) } fn get_hash(&self) -> Result<&str> { - self.get_header() - .read("ref.hash") + let header_path = format!("ref.hash.{}", H::NAME); + self.0 + .get_header() + .read(&header_path) .map_err(Error::from)? - .ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.hash"))) + .ok_or_else(|| { + Error::from(EM::EntryHeaderFieldMissing("ref.hash.")) + }) .and_then(|v| { - v.as_str().ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash", "string"))) + v.as_str().ok_or_else(|| { + Error::from(EM::EntryHeaderTypeError2("ref.hash.", "string")) + }) }) } - fn make_ref>(&mut self, hash: String, path: P) -> Result<()> { - let path_str : String = path - .as_ref() - .to_str() - .map(String::from) - .ok_or_else(|| EM::UTF8Error)?; - - let _ = self.set_isflag::()?; - let hdr = self.get_header_mut(); - hdr.insert("ref.path", Value::String(String::from(path_str)))?; - hdr.insert("ref.hash", Value::String(hash))?; - - Ok(()) - } - fn get_path(&self) -> Result { - self.get_header() + self.0 + .get_header() .read("ref.path") .map_err(Error::from)? .ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.path"))) @@ -124,20 +185,323 @@ impl Ref for Entry { .map(PathBuf::from) } - fn hash_valid(&self) -> Result { - self.get_path() - .map(PathBuf::from) - .map_err(Error::from) - .and_then(|pb| RPG::unique_hash(pb)) - .and_then(|h| Ok(h == self.get_hash()?)) - } + fn hash_valid(&self, config: &Config) -> Result { + let ref_header = self.0 + .get_header() + .read("ref")? + .ok_or_else(|| err_msg("Header missing at 'ref'"))?; - fn remove_ref(&mut self) -> Result<()> { - let hdr = self.get_header_mut(); - let _ = hdr.delete("ref.hash")?; - let _ = hdr.delete("ref.path")?; - let _ = hdr.delete("ref")?; - Ok(()) + let collection_name = ref_header + .read("collection") + .map_err(Error::from)? + .ok_or_else(|| err_msg("Header missing at 'ref.collection'"))? + .as_str() + .ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash.", "string")))?; + + let path = ref_header + .read("path") + .map_err(Error::from)? + .ok_or_else(|| err_msg("Header missing at 'ref.path'"))? + .as_str() + .map(PathBuf::from) + .ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash.", "string")))?; + + + let file_path = get_file_path(config, collection_name.as_ref(), &path)?; + + ref_header + .read(H::NAME) + .map_err(Error::from)? + .ok_or_else(|| format_err!("Header missing at 'ref.{}'", H::NAME)) + .and_then(|v| { + v.as_str().ok_or_else(|| { + Error::from(EM::EntryHeaderTypeError2("ref.hash.", "string")) + }) + }) + .and_then(|hash| H::hash(file_path).map(|h| h == hash)) + } + +} + +pub trait MutRef { + fn remove_ref(&mut self) -> Result<()>; + + /// Make a ref out of a normal (non-ref) entry. + /// + /// If the entry is already a ref, this fails if `force` is false + fn make_ref(&mut self, path: P, collection_name: Coll, config: &Config, force: bool) + -> Result<()> + where P: AsRef, + Coll: AsRef; +} + + +impl<'a, H> MutRef for MutRefWithHasher<'a, H> + where H: Hasher +{ + + fn remove_ref(&mut self) -> Result<()> { + debug!("Removing 'ref' header section"); + { + let header = self.0.get_header_mut(); + trace!("header = {:?}", header); + + let _ = header.delete("ref.relpath").context("Removing ref.relpath")?; + + if let Some(hash_tbl) = header.read_mut("ref.hash")? { + match hash_tbl { + Value::Table(ref mut tbl) => *tbl = BTreeMap::new(), + _ => { + // should not happen + } + } + } + + let _ = header.delete("ref.hash").context("Removing ref.hash")?; + let _ = header.delete("ref.collection").context("Removing ref.collection")?; + } + + debug!("Removing 'ref' header marker"); + self.0.remove_isflag::().context("Removing ref")?; + + let _ = self.0 + .get_header_mut() + .delete("ref") + .context("Removing ref")?; + + trace!("header = {:?}", self.0.get_header()); + Ok(()) + } + + /// Make a ref out of a normal (non-ref) entry. + /// + /// `path` is the path to refer to, + /// + /// # Warning + /// + /// If the entry is already a ref, this fails if `force` is false + /// + fn make_ref(&mut self, path: P, collection_name: Coll, config: &Config, force: bool) + -> Result<()> + where P: AsRef, + Coll: AsRef + { + trace!("Making ref out of {:?}", self.0); + trace!("Making ref with collection name {:?}", collection_name.as_ref()); + trace!("Making ref with config {:?}", config); + trace!("Making ref forced = {}", force); + + if self.0.get_header().read("ref.is_ref")?.is_some() && !force { + debug!("Entry is already a Ref!"); + let _ = Err(err_msg("Entry is already a reference")).context("Making ref out of entry")?; + } + + let file_path = get_file_path(config, collection_name.as_ref(), &path)?; + + if !file_path.exists() { + let msg = format_err!("File '{:?}' does not exist", file_path); + let _ = Err(msg).context("Making ref out of entry")?; + } + + debug!("Entry hashing..."); + let _ = H::hash(&file_path) + .and_then(|hash| make_header_section(hash, H::NAME, path, collection_name)) + .and_then(|h| self.0.get_header_mut().insert("ref", h).map_err(Error::from)) + .and_then(|_| self.0.set_isflag::()) + .context("Making ref out of entry")?; + + debug!("Setting is-ref flag"); + self.0 + .set_isflag::() + .context("Setting ref-flag") + .map_err(Error::from) + .map(|_| ()) + } + +} + +/// Create a new header section for a "ref". +/// +/// # Warning +/// +/// The `relpath` _must_ be relative to the configured path for that collection. +pub(crate) fn make_header_section(hash: String, hashname: H, relpath: P, collection: C) + -> Result + where P: AsRef, + C: AsRef, + H: AsRef, +{ + let mut header_section = Value::Table(BTreeMap::new()); + { + let relpath = relpath + .as_ref() + .to_str() + .map(String::from) + .ok_or_else(|| { + let msg = format_err!("UTF Error in '{:?}'", relpath.as_ref()); + Error::from(msg) + })?; + + let _ = header_section.insert("relpath", Value::String(relpath))?; + } + + { + let mut hash_table = Value::Table(BTreeMap::new()); + let _ = hash_table.insert(hashname.as_ref(), Value::String(hash))?; + let _ = header_section.insert("hash", hash_table)?; + } + + let _ = header_section.insert("collection", Value::String(String::from(collection.as_ref()))); + + Ok(header_section) +} + +fn get_file_path

(config: &Config, collection_name: &str, path: P) -> Result + where P: AsRef +{ + config + .get(collection_name) + .map(PathBuf::clone) + .ok_or_else(|| { + format_err!("Configuration missing for collection: '{}'", collection_name) + }) + .context("Making ref out of entry") + .map_err(Error::from) + .map(|p| { + let filepath = p.join(&path); + trace!("Found filepath: {:?}", filepath.display()); + filepath + }) +} + + +#[cfg(test)] +mod test { + use std::path::PathBuf; + + use libimagstore::store::Store; + use libimagstore::store::Entry; + + use super::*; + use hasher::Hasher; + + fn setup_logging() { + let _ = ::env_logger::try_init(); + } + + pub fn get_store() -> Store { + Store::new_inmemory(PathBuf::from("/"), &None).unwrap() + } + + struct TestHasher; + impl Hasher for TestHasher { + const NAME: &'static str = "Testhasher"; + + fn hash>(path: P) -> Result { + path.as_ref() + .to_str() + .map(String::from) + .ok_or_else(|| Error::from(err_msg("Failed to create test hash"))) + } + } + + + #[test] + fn test_isref() { + setup_logging(); + let store = get_store(); + let entry = store.retrieve(PathBuf::from("test_isref")).unwrap(); + + assert!(!entry.is_ref().unwrap()); + } + + #[test] + fn test_makeref() { + setup_logging(); + let store = get_store(); + let mut entry = store.retrieve(PathBuf::from("test_makeref")).unwrap(); + let file = PathBuf::from("/"); // has to exist + let collection_name = "some_collection"; + let config = Config({ + let mut c = BTreeMap::new(); + c.insert(String::from("some_collection"), PathBuf::from("/tmp")); + c + }); + + let r = entry.as_ref_with_hasher_mut::().make_ref(file, collection_name, &config, false); + assert!(r.is_ok()); + } + + #[test] + fn test_makeref_isref() { + setup_logging(); + let store = get_store(); + let mut entry = store.retrieve(PathBuf::from("test_makeref_isref")).unwrap(); + let file = PathBuf::from("/"); // has to exists + let collection_name = "some_collection"; + let config = Config({ + let mut c = BTreeMap::new(); + c.insert(String::from("some_collection"), PathBuf::from("/tmp")); + c + }); + + let res = entry.as_ref_with_hasher_mut::().make_ref(file, collection_name, &config, false); + assert!(res.is_ok(), "Expected to be ok: {:?}", res); + + assert!(entry.as_ref_with_hasher::().is_ref().unwrap()); + } + + #[test] + fn test_makeref_is_ref_with_testhash() { + setup_logging(); + let store = get_store(); + let mut entry = store.retrieve(PathBuf::from("test_makeref_is_ref_with_testhash")).unwrap(); + let file = PathBuf::from("/"); // has to exist + let collection_name = "some_collection"; + let config = Config({ + let mut c = BTreeMap::new(); + c.insert(String::from("some_collection"), PathBuf::from("/")); + c + }); + + assert!(entry.as_ref_with_hasher_mut::().make_ref(file, collection_name, &config, false).is_ok()); + + let check_isstr = |entry: &Entry, location, shouldbe| { + let var = entry.get_header().read(location); + + assert!(var.is_ok(), "{} is not Ok(_): {:?}", location, var); + let var = var.unwrap(); + + assert!(var.is_some(), "{} is not Some(_): {:?}", location, var); + let var = var.unwrap().as_str(); + + assert!(var.is_some(), "{} is not String: {:?}", location, var); + assert_eq!(var.unwrap(), shouldbe, "{} is not == {}", location, shouldbe); + }; + + check_isstr(&entry, "ref.relpath", "/"); + check_isstr(&entry, "ref.hash.Testhasher", "/"); // TestHasher hashes by returning the path itself + check_isstr(&entry, "ref.collection", "some_collection"); + } + + #[test] + fn test_makeref_remref() { + setup_logging(); + let store = get_store(); + let mut entry = store.retrieve(PathBuf::from("test_makeref_remref")).unwrap(); + let file = PathBuf::from("/"); // has to exist + let collection_name = "some_collection"; + let config = Config({ + let mut c = BTreeMap::new(); + c.insert(String::from("some_collection"), PathBuf::from("/")); + c + }); + + assert!(entry.as_ref_with_hasher_mut::().make_ref(file, collection_name, &config, false).is_ok()); + assert!(entry.as_ref_with_hasher::().is_ref().unwrap()); + let res = entry.as_ref_with_hasher_mut::().remove_ref(); + assert!(res.is_ok(), "Expected to be ok: {:?}", res); + assert!(!entry.as_ref_with_hasher::().is_ref().unwrap()); } } diff --git a/lib/entry/libimagentryref/src/refstore.rs b/lib/entry/libimagentryref/src/refstore.rs deleted file mode 100644 index c378d6fd..00000000 --- a/lib/entry/libimagentryref/src/refstore.rs +++ /dev/null @@ -1,128 +0,0 @@ -// -// imag - the personal information management suite for the commandline -// Copyright (C) 2015-2019 Matthias Beyer and contributors -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; version -// 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// - -use std::path::Path; -use std::path::PathBuf; - -use libimagstore::store::FileLockEntry; -use libimagstore::store::Store; -use libimagstore::storeid::StoreId; - -use reference::Ref; - -use failure::Fallible as Result; -use failure::Error; - -/// A UniqueRefPathGenerator generates unique Pathes -/// -/// It is basically a functor which generates a StoreId from a &Path. -/// For more information have a look at the documentation of RefStore. -pub trait UniqueRefPathGenerator { - /// The collection the `StoreId` should be created for - fn collection() -> &'static str { - "ref" - } - - /// A function which should generate a unique string for a Path - fn unique_hash>(path: A) -> Result; - - /// Postprocess the generated `StoreId` object - fn postprocess_storeid(sid: StoreId) -> Result { - Ok(sid) - } -} - -/// A extensions for the `Store` to handle `Ref` objects -/// -/// The RefStore handles refs using a `UniqueRefPathGenerator`. The `UniqueRefPathGenerator`, as it -/// name suggests, generates unique `StoreId`s for a `&Path`. It is a functor `&Path -> StoreId`. -/// -/// It provides three functions which are called in the following sequence: -/// -/// * The `UniqueRefPathGenerator::collection()` function is used for get the collection a `StoreId` -/// should be in (The first element of the `StoreId` path) -/// * The `UniqueRefPathGenerator::unique_hash()` gets the `&Path` which it then should generate a -/// unique String for. How this is done does not matter. It can hash the Path itself, read the -/// file and hash that or something else. It should be reproduceable, though. -/// * These two parts are joined and put into a `StoreId` which the -/// `UniqueRefPathGenerator::postprocess_storeid()` function is then allowed to postprocess (for -/// example add more parts to the StoreId). The default implementation does nothing. -/// -/// The StoreId which is generated is then used to carry out the actual action (reading, creating -/// ...). -/// If a entry is created, header information is set (that it is a ref, the hash which was just -/// generated and the path of the referenced file) -/// -/// # Details -/// -/// The `UniqueRefPathGenerator` is passed as type parameter to enforce some semantics: -/// -/// * The used `UniqueRefPathGenerator` is defined by the implementation rather than by the runtime -/// of the program or some environment. Of course this is only a small hurdle to enforce this, but -/// a hint. -/// * The `UniqueRefPathGenerator` is a functor which does not carry state. -/// -pub trait RefStore<'a> { - - fn get_ref>(&'a self, hash: H) -> Result>>; - fn create_ref>(&'a self, path: A) -> Result>; - fn retrieve_ref>(&'a self, path: A) -> Result>; - -} - -impl<'a> RefStore<'a> for Store { - - fn get_ref>(&'a self, hash: H) - -> Result>> - { - let sid = StoreId::new(PathBuf::from(format!("{}/{}", RPG::collection(), hash.as_ref()))) - .map_err(Error::from)?; - - debug!("Getting: {:?}", sid); - self.get(sid) - .map_err(Error::from) - } - - fn create_ref>(&'a self, path: A) - -> Result> - { - let hash = RPG::unique_hash(&path)?; - let pathbuf = PathBuf::from(format!("{}/{}", RPG::collection(), hash)); - let sid = StoreId::new(pathbuf.clone())?; - - debug!("Creating: {:?}", sid); - self.create(sid) - .map_err(Error::from) - .and_then(|mut fle| { - fle.make_ref(hash, path)?; - Ok(fle) - }) - } - - fn retrieve_ref>(&'a self, path: A) - -> Result> - { - match self.get_ref::(RPG::unique_hash(path.as_ref())?)? { - Some(r) => Ok(r), - None => self.create_ref::(path), - } - } - -} -