matthiasbeyer/imag
1
0
Fork 0
Code Releases Activity

Merge pull request #1285 from matthiasbeyer/libimagentryref/refactor

Browse source 
libimagentryref: Rewrite
This commit is contained in:
Matthias Beyer 2018-02-19 14:18:17 +01:00 committed by GitHub
commit 2c0c8347e9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 707 additions and 1319 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
bin/core/imag-ref/Cargo.toml
View file

@ -24,6 +24,7 @@ maintenance = { status = "actively-developed" }
[dependencies] [dependencies]
log = "0.4.0" log = "0.4.0"
libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }
libimagrt = { version = "0.7.0", path = "../../../lib/core/libimagrt" } libimagrt = { version = "0.7.0", path = "../../../lib/core/libimagrt" }
libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" } libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" }
libimagentryref = { version = "0.7.0", path = "../../../lib/entry/libimagentryref" } libimagentryref = { version = "0.7.0", path = "../../../lib/entry/libimagentryref" }

103
bin/core/imag-ref/src/main.rs
View file

@ -35,6 +35,7 @@
#[macro_use] extern crate log; #[macro_use] extern crate log;
extern crate clap; extern crate clap;
extern crate libimagstore;
#[macro_use] extern crate libimagrt; #[macro_use] extern crate libimagrt;
extern crate libimagentryref; extern crate libimagentryref;
extern crate libimagerror; extern crate libimagerror;
@ -48,12 +49,11 @@ use ui::build_ui;
use std::path::PathBuf; use std::path::PathBuf;
use std::process::exit; use std::process::exit;
use libimagentryref::refstore::RefStore;
use libimagentryref::flags::RefFlags;
use libimagerror::trace::trace_error;
use libimagerror::trace::MapErrTrace; use libimagerror::trace::MapErrTrace;
use libimagrt::setup::generate_runtime_setup; use libimagrt::setup::generate_runtime_setup;
use libimagrt::runtime::Runtime; use libimagrt::runtime::Runtime;
use libimagstore::storeid::IntoStoreId;
use libimagentryref::reference::Ref;
fn main() { fn main() {
let version = make_imag_version!(); let version = make_imag_version!();
@ -66,9 +66,8 @@ fn main() {
.map(|name| { .map(|name| {
debug!("Call: {}", name); debug!("Call: {}", name);
match name { match name {
"add" => add(&rt), "deref" => deref(&rt),
"remove" => remove(&rt), "remove" => remove(&rt),
"list" => list(&rt),
_ => { _ => {
debug!("Unknown command"); // More error handling debug!("Unknown command"); // More error handling
}, },
@ -76,83 +75,57 @@ fn main() {
}); });
} }
fn add(rt: &Runtime) { fn deref(rt: &Runtime) {
let cmd = rt.cli().subcommand_matches("add").unwrap(); let cmd = rt.cli().subcommand_matches("deref").unwrap();
let path = cmd.value_of("path").map(PathBuf::from).unwrap(); // saved by clap let id = cmd.value_of("ID")
.map(String::from)
.map(PathBuf::from)
.unwrap() // saved by clap
.into_storeid()
.map_err_trace_exit_unwrap(1);
let flags = RefFlags::default() match rt.store().get(id.clone()).map_err_trace_exit_unwrap(1) {
.with_content_hashing(cmd.is_present("track-content")) Some(entry) => entry
.with_permission_tracking(cmd.is_present("track-permissions")); .get_path()
.map_err_trace_exit_unwrap(1)
match RefStore::create(rt.store(), path, flags) { .to_str()
Ok(r) => { .ok_or_else(|| {
debug!("Reference created: {:?}", r); error!("Could not transform path into string!");
exit(1)
})
.map(|s| info!("{}", s))
.ok(), // safe here because we exited already in the error case
None => {
error!("No entry for id '{}' found", id);
exit(1)
}, },
Err(e) => { };
trace_error(&e);
warn!("Failed to create reference");
},
}
} }
fn remove(rt: &Runtime) { fn remove(rt: &Runtime) {
use libimaginteraction::ask::ask_bool; use libimaginteraction::ask::ask_bool;
let cmd = rt.cli().subcommand_matches("remove").unwrap(); let cmd = rt.cli().subcommand_matches("remove").unwrap();
let hash = cmd.value_of("hash").map(String::from).unwrap(); // saved by clap
let yes = cmd.is_present("yes"); let yes = cmd.is_present("yes");
let id = cmd.value_of("ID")
.map(String::from)
.map(PathBuf::from)
.unwrap() // saved by clap
.into_storeid()
.map_err_trace_exit_unwrap(1);
match rt.store().find_storeid_by_partial_hash(&hash).map_err_trace_exit_unwrap(1) { match rt.store().get(id.clone()).map_err_trace_exit_unwrap(1) {
Some(sid) => { Some(mut entry) => {
if yes || ask_bool(&format!("Delete Ref with hash '{}'", hash)[..], None) { if yes || ask_bool(&format!("Delete ref from entry '{}'", id), None) {
debug!("Found for hash '{}' -> {:?}", hash, sid); let _ = entry.remove_ref().map_err_trace_exit_unwrap(1);
rt.store().delete(sid).map_err_trace_exit_unwrap(1)
} else { } else {
info!("Aborted"); info!("Aborted");
} }
}, },
None => { None => {
error!("Not id for hash '{}' found", hash); error!("No entry for id '{}' found", id);
exit(1) exit(1)
}, },
}; };
}
fn list(rt: &Runtime) {
use std::process::exit;
use libimagentrylist::lister::Lister;
use libimagentryref::lister::RefLister;
let cmd = rt.cli().subcommand_matches("list").unwrap();
let do_check_dead = cmd.is_present("check-dead");
let do_check_changed = cmd.is_present("check-changed");
let do_check_changed_content = cmd.is_present("check-changed-content");
let do_check_changed_permiss = cmd.is_present("check-changed-permissions");
let iter = match rt.store().retrieve_for_module("ref") {
Ok(iter) => iter.filter_map(|id| {
match rt.store().get(id) {
Ok(r) => Some(r),
Err(e) => {
trace_error(&e);
None
},
}
}),
Err(e) => {
trace_error(&e);
exit(1);
}
};
RefLister::new()
.check_dead(do_check_dead)
.check_changed(do_check_changed)
.check_changed_content(do_check_changed_content)
.check_changed_permiss(do_check_changed_permiss)
.list(iter.filter_map(Into::into))
.ok();
} }

60
bin/core/imag-ref/src/ui.rs
View file

@ -19,73 +19,33 @@
use clap::{Arg, App, SubCommand}; use clap::{Arg, App, SubCommand};
use libimagutil::cli_validators::is_existing_path;
pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> { pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> {
app app
.subcommand(SubCommand::with_name("add") .subcommand(SubCommand::with_name("deref")
.about("Add a reference to a file outside of the store") .about("'Dereference' a ref. This prints the Path of the referenced file")
.version("0.1") .version("0.1")
.arg(Arg::with_name("path") .arg(Arg::with_name("ID")
.index(1) .index(1)
.takes_value(true) .takes_value(true)
.required(true) .required(true)
.help("The path of the file") .help("The id of the store entry to dereference")
.validator(is_existing_path) .value_name("ID"))
.value_name("PATH"))
.arg(Arg::with_name("track-content")
.long("content-hash")
.short("C")
.takes_value(false)
.required(false)
.help("Hash the content for the reference"))
.arg(Arg::with_name("track-permissions")
.long("permission-tracking")
.short("P")
.takes_value(false)
.required(false)
.help("Rememeber the permissions of the referenced file"))
) )
.subcommand(SubCommand::with_name("remove") .subcommand(SubCommand::with_name("remove")
.about("Remove a reference") .about("Remove a reference from an entry")
.version("0.1") .version("0.1")
.arg(Arg::with_name("hash") .arg(Arg::with_name("ID")
.index(1) .index(1)
.takes_value(true) .takes_value(true)
.required(true) .required(true)
.help("Remove the reference with this hash") .multiple(true)
.value_name("HASH")) .help("Remove the reference from this store entry")
.value_name("ENTRIES"))
.arg(Arg::with_name("yes") .arg(Arg::with_name("yes")
.long("yes") .long("yes")
.short("y") .short("y")
.help("Don't ask whether this really should be done")) .help("Don't ask whether this really should be done"))
) )
.subcommand(SubCommand::with_name("list")
.about("List references in the store")
.version("0.1")
.arg(Arg::with_name("check-dead")
.long("check-dead")
.short("d")
.help("Check each reference whether it is dead"))
.arg(Arg::with_name("check-changed")
.long("check-changed")
.short("c")
.help("Check whether a reference had changed (content or permissions)"))
.arg(Arg::with_name("check-changed-content")
.long("check-changed-content")
.short("C")
.help("Check whether the content of the referenced file changed"))
.arg(Arg::with_name("check-changed-permissions")
.long("check-changed-perms")
.short("P")
.help("Check whether the permissions of the referenced file changed"))
)
} }

2
bin/domain/imag-contact/Cargo.toml
View file

@ -28,7 +28,7 @@ toml-query = "0.6"
handlebars = "0.29" handlebars = "0.29"
vobject = "0.4" vobject = "0.4"
walkdir = "1" walkdir = "1"
uuid = { version = "0.5", features = ["v4"] } uuid = { version = "0.6", features = ["v4"] }
libimagrt = { version = "0.7.0", path = "../../../lib/core/libimagrt" } libimagrt = { version = "0.7.0", path = "../../../lib/core/libimagrt" }
libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" } libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }

8
bin/domain/imag-contact/src/create.rs
View file

@ -31,13 +31,13 @@ use toml::Value;
use uuid::Uuid; use uuid::Uuid;
use libimagcontact::error::ContactError as CE; use libimagcontact::error::ContactError as CE;
use libimagcontact::store::UniqueContactPathGenerator;
use libimagrt::runtime::Runtime; use libimagrt::runtime::Runtime;
use libimagerror::str::ErrFromStr; use libimagerror::str::ErrFromStr;
use libimagerror::trace::MapErrTrace; use libimagerror::trace::MapErrTrace;
use libimagerror::trace::trace_error; use libimagerror::trace::trace_error;
use libimagutil::warn_result::WarnResult; use libimagutil::warn_result::WarnResult;
use libimagentryref::refstore::RefStore; use libimagentryref::refstore::RefStore;
use libimagentryref::flags::RefFlags;
const TEMPLATE : &'static str = include_str!("../static/new-contact-template.toml"); const TEMPLATE : &'static str = include_str!("../static/new-contact-template.toml");
@ -144,11 +144,7 @@ pub fn create(rt: &Runtime) {
if let Some(location) = location { if let Some(location) = location {
if !scmd.is_present("dont-track") { if !scmd.is_present("dont-track") {
let flags = RefFlags::default() RefStore::create_ref::<UniqueContactPathGenerator, _>(rt.store(), location)
.with_content_hashing(true)
.with_permission_tracking(false);
RefStore::create(rt.store(), location, flags)
.map_err_trace_exit_unwrap(1); .map_err_trace_exit_unwrap(1);
info!("Created entry in store"); info!("Created entry in store");

5
bin/domain/imag-contact/src/main.rs
View file

@ -67,6 +67,7 @@ use libimagerror::trace::MapErrTrace;
use libimagerror::io::ToExitCode; use libimagerror::io::ToExitCode;
use libimagerror::exit::ExitUnwrap; use libimagerror::exit::ExitUnwrap;
use libimagcontact::store::ContactStore; use libimagcontact::store::ContactStore;
use libimagcontact::store::UniqueContactPathGenerator;
use libimagcontact::error::ContactError as CE; use libimagcontact::error::ContactError as CE;
use libimagcontact::contact::Contact; use libimagcontact::contact::Contact;
use libimagstore::iter::get::StoreIdGetIteratorExtension; use libimagstore::iter::get::StoreIdGetIteratorExtension;
@ -130,7 +131,7 @@ fn list(rt: &Runtime) {
}) })
.enumerate() .enumerate()
.map(|(i, (fle, vcard))| { .map(|(i, (fle, vcard))| {
let hash = fle.get_path_hash().map_err_trace_exit_unwrap(1); let hash = String::from(fle.get_hash().map_err_trace_exit_unwrap(1));
let vcard = vcard.unwrap_or_else(|e| { let vcard = vcard.unwrap_or_else(|e| {
error!("Element is not a VCARD object: {:?}", e); error!("Element is not a VCARD object: {:?}", e);
exit(1) exit(1)
@ -190,7 +191,7 @@ fn show(rt: &Runtime) {
let hash = scmd.value_of("hash").map(String::from).unwrap(); // safed by clap let hash = scmd.value_of("hash").map(String::from).unwrap(); // safed by clap
let contact_data = rt.store() let contact_data = rt.store()
.get_by_hash(hash.clone()) .get_ref::<UniqueContactPathGenerator, _>(hash.clone())
.map_err_trace_exit_unwrap(1) .map_err_trace_exit_unwrap(1)
.ok_or(CE::from(format!("No entry for hash {}", hash))) .ok_or(CE::from(format!("No entry for hash {}", hash)))
.map_err_trace_exit_unwrap(1) .map_err_trace_exit_unwrap(1)

70
doc/src/05100-lib-entryref.md
View file

@ -3,27 +3,24 @@
This library crate contains functionality to generate _references_ within the This library crate contains functionality to generate _references_ within the
imag store. imag store.
It can be used to create references to other files on the filesystem (reachable A reference is a "pointer" to a file or directory on the filesystem and outside
via a filesystem path). It differs from `libimagentrylink`/external linking as the store.
It differs from `libimagentrylink`/external linking as
it is designed exclusively for filesystem references, not for URLs. it is designed exclusively for filesystem references, not for URLs.
A reference can have several properties, for example can a reference track the A reference is created with a unique identifier, like a hash. The implementation
content of a filesystem path by hashing the content with a hashsum (SHA1) and how this hash is calculated can be defined by the user of `libimagentryref`.
one can check whether a file was changed by that.
As files can get big (think of `debian.iso`) _partial hashing_ is supported
(think of "hash the first 2048 bytes of a file).
The library contains functionality to re-find a moved file automatically by
checking the content hash which was stored before.
Permission changes can be tracked as well.
So this library helps to resemble something like a _symlink_. So this library helps to resemble something like a _symlink_.
### Usage
Users have to implement the `UniqueRefPathGenerator` trait which should
implement a hashing functionality for pathes.
### Limits ### Limits
Please understand that this is _not_ intended to be a version control system or This is _not_ intended to be a version control system or something like that.
something like that.
We also can not use _real symlinks_ as we need imag-store-objects to be able to We also can not use _real symlinks_ as we need imag-store-objects to be able to
link stuff. link stuff.
@ -31,39 +28,22 @@ link stuff.
This library offers functionality to refer to content outside of the store. This library offers functionality to refer to content outside of the store.
It can be used to refer to _nearly static stuff_ pretty easily - think of a It can be used to refer to _nearly static stuff_ pretty easily - think of a
Maildir - you add new mails by fetching them, but you mostly do not remove mails Maildir - you add new mails by fetching them, but you mostly do not remove
and if you do you end up with a "null pointer" in the store, which can then be mails.
handled properly. If mails get moved, they can be re-found via their hash, because Maildir objects
hardly change. Or because the hash implementation which is used to refer to them
As this library supports custom hashes (you don't have to hash the full file, hashes only the `Message-Id` and that does not change.
you can also parse the file and hash only _some_ content) this is pretty
flexible.
For example if you want to implement a imag module which tracks a certain kind
of files which constantly change... but the first 5 lines do never change
after the file is created - you can write a custom hasher that only uses the
first 5 lines for the hash.
### Internals
Internally, in the store, the file gets created under
`/ref/<hash of the path to the file to refer to>`.
If the content of the file is hashed, we can still re-find the file via the
content hash (which is stored in the header of the store entry).
The reference object can, after the path was re-found, be updated.
### Long-term TODO ### Long-term TODO
Things which have to be done here or are not yet properly tested: Not implemented yet:
- [ ] Testing of different Hashers - [ ] Re-finding of files via their hash.
- [ ] Testing of re-finding of objects, including: This must be implemented with several things in mind
- [ ] Can a moved file automatically be found by content hash? * The user of the library should be able to provide a way how the
- [ ] Does a store-reference get updated automatically if it was moved, filesystem is searched. Basically a Functor which yields pathes to
including links (as in `libimaglink`)? check based on the original path of the missing file.
- [ ] If the content of a file changes, does the content hash get updated This enables implementations which do only search a certain subset
automatically? of pathes, or does depth-first-search rather than
breadth-first-search.
("automatically" is a strechable term here, as these things have to be triggered
by the user anyways)

8
lib/domain/libimagcontact/Cargo.toml
View file

@ -25,9 +25,15 @@ log = "0.3"
toml = "0.4" toml = "0.4"
toml-query = "0.4" toml-query = "0.4"
vobject = "0.4" vobject = "0.4"
uuid = { version = "0.6", features = ["v4"] }
libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" } libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" } libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" }
libimagentryref = { version = "0.7.0", path = "../../../lib/entry/libimagentryref/" }
libimagentryutil = { version = "0.7.0", path = "../../../lib/entry/libimagentryutil/" } libimagentryutil = { version = "0.7.0", path = "../../../lib/entry/libimagentryutil/" }
[dependencies.libimagentryref]
version = "0.7.0"
path = "../../../lib/entry/libimagentryref/"
default-features = false
features = ["generators", "generators-sha1"]

2
lib/domain/libimagcontact/src/contact.rs
View file

@ -54,7 +54,7 @@ impl Contact for Entry {
fn get_contact_data(&self) -> Result<ContactData> { fn get_contact_data(&self) -> Result<ContactData> {
let component = self let component = self
.fs_file() .get_path()
.map_err(From::from) .map_err(From::from)
.and_then(util::read_to_string) .and_then(util::read_to_string)
.and_then(util::parse)?; .and_then(util::parse)?;

1
lib/domain/libimagcontact/src/error.rs
View file

@ -34,6 +34,7 @@ error_chain! {
foreign_links { foreign_links {
Io(::std::io::Error); Io(::std::io::Error);
TomlQueryError(::toml_query::error::Error); TomlQueryError(::toml_query::error::Error);
UuidError(::uuid::ParseError);
} }
errors { errors {

1
lib/domain/libimagcontact/src/lib.rs
View file

@ -38,6 +38,7 @@
extern crate vobject; extern crate vobject;
extern crate toml; extern crate toml;
extern crate toml_query; extern crate toml_query;
extern crate uuid;
#[macro_use] extern crate libimagstore; #[macro_use] extern crate libimagstore;
extern crate libimagerror; extern crate libimagerror;

45
lib/domain/libimagcontact/src/store.rs
View file

@ -17,22 +17,54 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// //
use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use std::result::Result as RResult;
use std::ffi::OsStr;
use vobject::parse_component; use vobject::parse_component;
use uuid::Uuid;
use libimagstore::store::Store; use libimagstore::store::Store;
use libimagstore::store::FileLockEntry; use libimagstore::store::FileLockEntry;
use libimagstore::storeid::StoreIdIterator; use libimagstore::storeid::StoreIdIterator;
use libimagentryref::refstore::RefStore; use libimagentryref::refstore::RefStore;
use libimagentryref::flags::RefFlags; use libimagentryref::refstore::UniqueRefPathGenerator;
use libimagentryref::generators::sha1::Sha1;
use libimagentryutil::isa::Is; use libimagentryutil::isa::Is;
use contact::IsContact; use contact::IsContact;
use error::ContactError as CE;
use error::Result; use error::Result;
use util; use util;
pub trait ContactStore<'a> : RefStore { pub struct UniqueContactPathGenerator;
impl UniqueRefPathGenerator for UniqueContactPathGenerator {
type Error = CE;
/// The collection the `StoreId` should be created for
fn collection() -> &'static str {
"contact"
}
/// A function which should generate a unique string for a Path
fn unique_hash<A: AsRef<Path>>(path: A) -> RResult<String, Self::Error> {
debug!("Generating unique hash for path: {:?}", path.as_ref());
if let Some(p) = path.as_ref().file_name().and_then(OsStr::to_str).map(String::from) {
debug!("Found UUID string: '{}'", p);
Uuid::parse_str(&p)
.map_err(CE::from)
.map(|u| format!("{}", u.hyphenated())) // FIXME I don't know how to do in not-ugly
} else { // else, we sha1 the (complete) content
debug!("Couldn't find UUID string, using SHA1 of contents");
Sha1::unique_hash(path).map_err(CE::from)
}
}
}
pub trait ContactStore<'a> : RefStore<'a> {
// creating // creating
@ -42,7 +74,7 @@ pub trait ContactStore<'a> : RefStore {
/// ///
/// Needs the `p` argument as we're finally creating a reference by path, the buffer is only for /// Needs the `p` argument as we're finally creating a reference by path, the buffer is only for
/// collecting metadata. /// collecting metadata.
fn create_from_buf(&'a self, p: &PathBuf, buf: &String) -> Result<FileLockEntry<'a>>; fn create_from_buf<P: AsRef<Path>>(&'a self, p: P, buf: &String) -> Result<FileLockEntry<'a>>;
// getting // getting
@ -63,12 +95,11 @@ impl<'a> ContactStore<'a> for Store {
} }
/// Create contact ref from buffer /// Create contact ref from buffer
fn create_from_buf(&'a self, p: &PathBuf, buf: &String) -> Result<FileLockEntry<'a>> { fn create_from_buf<P: AsRef<Path>>(&'a self, p: P, buf: &String) -> Result<FileLockEntry<'a>> {
let component = parse_component(&buf)?; let component = parse_component(&buf)?;
debug!("Parsed: {:?}", component); debug!("Parsed: {:?}", component);
let flags = RefFlags::default().with_content_hashing(true).with_permission_tracking(false); RefStore::create_ref::<UniqueContactPathGenerator, P>(self, p)
RefStore::create(self, p.clone(), flags)
.map_err(From::from) .map_err(From::from)
.and_then(|mut entry| { .and_then(|mut entry| {
entry.set_isflag::<IsContact>() entry.set_isflag::<IsContact>()
@ -78,7 +109,7 @@ impl<'a> ContactStore<'a> for Store {
} }
fn all_contacts(&'a self) -> Result<StoreIdIterator> { fn all_contacts(&'a self) -> Result<StoreIdIterator> {
self.all_references().map_err(From::from) unimplemented!()
} }
} }

4
lib/domain/libimagmail/src/error.rs
View file

@ -26,6 +26,10 @@ error_chain! {
RefError(::libimagentryref::error::RefError, ::libimagentryref::error::RefErrorKind); RefError(::libimagentryref::error::RefError, ::libimagentryref::error::RefErrorKind);
} }
foreign_links {
IoError(::std::io::Error);
}
errors { errors {
RefCreationError { RefCreationError {

81
lib/domain/libimagmail/src/hasher.rs
View file

@ -1,81 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::io::Read;
use std::path::PathBuf;
use email::MimeMessage;
use libimagentryref::hasher::Hasher;
use libimagentryref::hasher::DefaultHasher;
use libimagentryref::error::RefErrorKind as REK;
use libimagentryref::error::ResultExt;
use libimagentryref::error::Result as RResult;
pub struct MailHasher {
defaulthasher: DefaultHasher,
}
impl MailHasher {
pub fn new() -> MailHasher {
MailHasher { defaulthasher: DefaultHasher::new() }
}
}
impl Hasher for MailHasher {
fn hash_name(&self) -> &'static str {
"default_mail_hasher"
}
fn create_hash<R: Read>(&mut self, pb: &PathBuf, c: &mut R) -> RResult<String> {
use filters::filter::Filter;
use email::Header;
let mut s = String::new();
c.read_to_string(&mut s)?;
MimeMessage::parse(&s)
.chain_err(|| REK::RefHashingError)
.and_then(|mail| {
let has_key = |hdr: &Header, exp: &str| hdr.name == exp;
let subject_filter = |hdr: &Header| has_key(hdr, "Subject");
let from_filter = |hdr: &Header| has_key(hdr, "From");
let to_filter = |hdr: &Header| has_key(hdr, "To");
let filter = subject_filter.or(from_filter).or(to_filter);
let mut v : Vec<String> = vec![];
for hdr in mail.headers.iter().filter(|item| filter.filter(item)) {
let s = hdr
.get_value()
.chain_err(|| REK::RefHashingError)?;
v.push(s);
}
let s : String = v.join("");
self.defaulthasher.create_hash(pb, &mut s.as_bytes())
})
}
}

1
lib/domain/libimagmail/src/lib.rs
View file

@ -45,7 +45,6 @@ extern crate libimagstore;
extern crate libimagentryref; extern crate libimagentryref;
pub mod error; pub mod error;
pub mod hasher;
pub mod iter; pub mod iter;
pub mod mail; pub mod mail;

75
lib/domain/libimagmail/src/mail.rs
View file

@ -18,22 +18,75 @@
// //
use std::path::Path; use std::path::Path;
use std::path::PathBuf;
use std::fs::File; use std::fs::File;
use std::io::Read; use std::io::Read;
use std::fs::OpenOptions;
use std::result::Result as RResult;
use libimagstore::store::Store; use libimagstore::store::Store;
use libimagstore::storeid::StoreId;
use libimagstore::store::FileLockEntry; use libimagstore::store::FileLockEntry;
use libimagentryref::reference::Ref; use libimagentryref::reference::Ref;
use libimagentryref::flags::RefFlags;
use libimagentryref::refstore::RefStore; use libimagentryref::refstore::RefStore;
use libimagentryref::refstore::UniqueRefPathGenerator;
use email::MimeMessage; use email::MimeMessage;
use email::results::ParsingResult as EmailParsingResult; use email::results::ParsingResult as EmailParsingResult;
use hasher::MailHasher;
use error::Result; use error::Result;
use error::{ResultExt, MailErrorKind as MEK}; use error::{ResultExt, MailError as ME, MailErrorKind as MEK};
struct UniqueMailRefGenerator;
impl UniqueRefPathGenerator for UniqueMailRefGenerator {
type Error = ME;
/// The collection the `StoreId` should be created for
fn collection() -> &'static str {
"mail"
}
/// A function which should generate a unique string for a Path
fn unique_hash<A: AsRef<Path>>(path: A) -> RResult<String, Self::Error> {
use filters::filter::Filter;
use email::Header;
let mut s = String::new();
let _ = OpenOptions::new()
.read(true)
.write(false)
.create(false)
.open(path)?
.read_to_string(&mut s)?;
MimeMessage::parse(&s)
.chain_err(|| MEK::RefCreationError)
.and_then(|mail| {
let has_key = |hdr: &Header, exp: &str| hdr.name == exp;
let subject_filter = |hdr: &Header| has_key(hdr, "Subject");
let from_filter = |hdr: &Header| has_key(hdr, "From");
let to_filter = |hdr: &Header| has_key(hdr, "To");
let filter = subject_filter.or(from_filter).or(to_filter);
let mut v : Vec<String> = vec![];
for hdr in mail.headers.iter().filter(|item| filter.filter(item)) {
let s = hdr
.get_value()
.chain_err(|| MEK::RefCreationError)?;
v.push(s);
}
let s : String = v.join("");
Ok(s)
})
}
/// Postprocess the generated `StoreId` object
fn postprocess_storeid(sid: StoreId) -> RResult<StoreId, Self::Error> {
Ok(sid)
}
}
struct Buffer(String); struct Buffer(String);
@ -56,15 +109,10 @@ impl<'a> Mail<'a> {
/// Imports a mail from the Path passed /// Imports a mail from the Path passed
pub fn import_from_path<P: AsRef<Path>>(store: &Store, p: P) -> Result<Mail> { pub fn import_from_path<P: AsRef<Path>>(store: &Store, p: P) -> Result<Mail> {
debug!("Importing Mail from path"); debug!("Importing Mail from path");
let h = MailHasher::new(); store.retrieve_ref::<UniqueMailRefGenerator, P>(p)
let f = RefFlags::default().with_content_hashing(true).with_permission_tracking(false);
let p = PathBuf::from(p.as_ref());
store.create_with_hasher(p, f, h)
.chain_err(|| MEK::RefCreationError)
.and_then(|reference| { .and_then(|reference| {
debug!("Build reference file: {:?}", reference); debug!("Build reference file: {:?}", reference);
reference.fs_file() reference.get_path()
.chain_err(|| MEK::RefHandlingError) .chain_err(|| MEK::RefHandlingError)
.and_then(|path| File::open(path).chain_err(|| MEK::IOError)) .and_then(|path| File::open(path).chain_err(|| MEK::IOError))
.and_then(|mut file| { .and_then(|mut file| {
@ -81,19 +129,18 @@ impl<'a> Mail<'a> {
/// Opens a mail by the passed hash /// Opens a mail by the passed hash
pub fn open<S: AsRef<str>>(store: &Store, hash: S) -> Result<Option<Mail>> { pub fn open<S: AsRef<str>>(store: &Store, hash: S) -> Result<Option<Mail>> {
debug!("Opening Mail by Hash"); debug!("Opening Mail by Hash");
store.get_by_hash(String::from(hash.as_ref())) store.get_ref::<UniqueMailRefGenerator, S>(hash)
.chain_err(|| MEK::FetchByHashError) .chain_err(|| MEK::FetchByHashError)
.chain_err(|| MEK::FetchError) .chain_err(|| MEK::FetchError)
.and_then(|o| match o { .and_then(|o| match o {
Some(r) => Mail::from_fle(r).map(Some), Some(r) => Mail::from_fle(r).map(Some),
None => Ok(None), None => Ok(None),
}) })
} }
/// Implement me as TryFrom as soon as it is stable /// Implement me as TryFrom as soon as it is stable
pub fn from_fle(fle: FileLockEntry<'a>) -> Result<Mail<'a>> { pub fn from_fle(fle: FileLockEntry<'a>) -> Result<Mail<'a>> {
fle.fs_file() fle.get_path()
.chain_err(|| MEK::RefHandlingError) .chain_err(|| MEK::RefHandlingError)
.and_then(|path| File::open(path).chain_err(|| MEK::IOError)) .and_then(|path| File::open(path).chain_err(|| MEK::IOError))
.and_then(|mut file| { .and_then(|mut file| {

7
lib/entry/libimagentrymarkdown/Cargo.toml
View file

@ -29,6 +29,11 @@ env_logger = "0.5"
libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" } libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" } libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" }
libimagentrylink = { version = "0.7.0", path = "../../../lib/entry/libimagentrylink/" } libimagentrylink = { version = "0.7.0", path = "../../../lib/entry/libimagentrylink/" }
libimagentryref = { version = "0.7.0", path = "../../../lib/entry/libimagentryref/" }
libimagutil = { version = "0.7.0", path = "../../../lib/etc/libimagutil/" } libimagutil = { version = "0.7.0", path = "../../../lib/etc/libimagutil/" }
[dependencies.libimagentryref]
version = "0.7.0"
path = "../../../lib/entry/libimagentryref/"
default-features = false
features = [ "generators", "generators-sha512" ]

30
lib/entry/libimagentrymarkdown/src/processor.rs
View file

@ -17,6 +17,9 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// //
use std::path::Path;
use std::result::Result as RResult;
use error::MarkdownError as ME; use error::MarkdownError as ME;
use error::MarkdownErrorKind as MEK; use error::MarkdownErrorKind as MEK;
use error::*; use error::*;
@ -25,7 +28,8 @@ use link::extract_links;
use libimagentrylink::external::ExternalLinker; use libimagentrylink::external::ExternalLinker;
use libimagentrylink::internal::InternalLinker; use libimagentrylink::internal::InternalLinker;
use libimagentryref::refstore::RefStore; use libimagentryref::refstore::RefStore;
use libimagentryref::flags::RefFlags; use libimagentryref::refstore::UniqueRefPathGenerator;
use libimagentryref::generators::sha512::Sha512;
use libimagstore::store::Entry; use libimagstore::store::Entry;
use libimagstore::store::Store; use libimagstore::store::Store;
use libimagstore::storeid::StoreId; use libimagstore::storeid::StoreId;
@ -34,6 +38,25 @@ use std::path::PathBuf;
use url::Url; use url::Url;
pub struct UniqueMarkdownRefGenerator;
impl UniqueRefPathGenerator for UniqueMarkdownRefGenerator {
type Error = ME;
fn collection() -> &'static str {
"ref" // we can only use this collection, as we don't know about context
}
fn unique_hash<A: AsRef<Path>>(path: A) -> RResult<String, Self::Error> {
Sha512::unique_hash(path).map_err(ME::from)
}
fn postprocess_storeid(sid: StoreId) -> RResult<StoreId, Self::Error> {
Ok(sid) // don't do anything
}
}
/// A link Processor which collects the links from a Markdown and passes them on to /// A link Processor which collects the links from a Markdown and passes them on to
/// `libimagentrylink` functionality /// `libimagentrylink` functionality
/// ///
@ -136,15 +159,12 @@ impl LinkProcessor {
continue continue
} }
let flags = RefFlags::default()
.with_content_hashing(false)
.with_permission_tracking(false);
trace!("URL = {:?}", url); trace!("URL = {:?}", url);
trace!("URL.path() = {:?}", url.path()); trace!("URL.path() = {:?}", url.path());
trace!("URL.host_str() = {:?}", url.host_str()); trace!("URL.host_str() = {:?}", url.host_str());
let path = url.host_str().unwrap_or_else(|| url.path()); let path = url.host_str().unwrap_or_else(|| url.path());
let path = PathBuf::from(path); let path = PathBuf::from(path);
let mut target = RefStore::create(store, path, flags)?; let mut target = store.create_ref::<UniqueMarkdownRefGenerator, PathBuf>(path)?;
entry.add_internal_link(&mut target)?; entry.add_internal_link(&mut target)?;
}, },

17
lib/entry/libimagentryref/Cargo.toml
View file

@ -22,13 +22,26 @@ maintenance = { status = "actively-developed" }
[dependencies] [dependencies]
itertools = "0.7" itertools = "0.7"
log = "0.4.0" log = "0.4.0"
rust-crypto = "0.2"
toml = "0.4" toml = "0.4"
toml-query = "0.6" toml-query = "0.6"
error-chain = "0.11" error-chain = "0.11"
walkdir = "1"
libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" } libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" } libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" }
libimagentrylist = { version = "0.7.0", path = "../../../lib/entry/libimagentrylist" } libimagentrylist = { version = "0.7.0", path = "../../../lib/entry/libimagentrylist" }
libimagentryutil = { version = "0.7.0", path = "../../../lib/entry/libimagentryutil" } libimagentryutil = { version = "0.7.0", path = "../../../lib/entry/libimagentryutil" }
[dependencies.rust-crypto]
version = "0.2"
optional = true
[features]
default = []
generators = []
generators-sha1 = ["rust-crypto"]
generators-sha224 = ["rust-crypto"]
generators-sha256 = ["rust-crypto"]
generators-sha384 = ["rust-crypto"]
generators-sha512 = ["rust-crypto"]
generators-sha3 = ["rust-crypto"]

70
lib/entry/libimagentryref/src/error.rs
View file

@ -23,7 +23,6 @@ error_chain! {
} }
links { links {
ListError(::libimagentrylist::error::ListError, ::libimagentrylist::error::ListErrorKind);
StoreError(::libimagstore::error::StoreError, ::libimagstore::error::StoreErrorKind); StoreError(::libimagstore::error::StoreError, ::libimagstore::error::StoreErrorKind);
TomlQueryError(::toml_query::error::Error, ::toml_query::error::ErrorKind); TomlQueryError(::toml_query::error::Error, ::toml_query::error::ErrorKind);
EntryUtilError(::libimagentryutil::error::EntryUtilError, ::libimagentryutil::error::EntryUtilErrorKind); EntryUtilError(::libimagentryutil::error::EntryUtilError, ::libimagentryutil::error::EntryUtilErrorKind);
@ -34,23 +33,17 @@ error_chain! {
Utf8Error(::std::string::FromUtf8Error); Utf8Error(::std::string::FromUtf8Error);
TomlDeError(::toml::de::Error); TomlDeError(::toml::de::Error);
TomlSerError(::toml::ser::Error); TomlSerError(::toml::ser::Error);
WalkDirError(::walkdir::Error);
} }
errors { errors {
UTF8Error { HeaderTypeError(field: &'static str, expectedtype: &'static str) {
description("UTF8 Error")
display("UTF8 Error")
}
HeaderTypeError {
description("Header type error") description("Header type error")
display("Header type error") display("Header type error: '{}' should be {}", field, expectedtype)
} }
HeaderFieldMissingError { HeaderFieldMissingError(field: &'static str) {
description("Header field missing error") description("Header field missing error")
display("Header field missing error") display("Header field missing: {}", field)
} }
HeaderFieldWriteError { HeaderFieldWriteError {
@ -73,61 +66,6 @@ error_chain! {
display("Path cannot be converted because of UTF8 Error") display("Path cannot be converted because of UTF8 Error")
} }
PathHashingError {
description("Path cannot be hashed")
display("Path cannot be hashed")
}
PathCanonicalizationError {
description("Path cannot be canonicalized")
display("Path cannot be canonicalized")
}
TypeConversionError {
description("Couldn't convert types")
display("Couldn't convert types")
}
RefToDisplayError {
description("Cannot convert Ref to string to show it to user")
display("Cannot convert Ref to string to show it to user")
}
RefNotInStore {
description("Ref/StoreId does not exist in store")
display("Ref/StoreId does not exist in store")
}
RefTargetDoesNotExist {
description("Ref Target does not exist")
display("Ref Target does not exist")
}
RefTargetPermissionError {
description("Ref Target permissions insufficient for referencing")
display("Ref Target permissions insufficient for referencing")
}
RefTargetCannotBeHashed {
description("Ref Target cannot be hashed (is it a directory?)")
display("Ref Target cannot be hashed (is it a directory?)")
}
RefTargetFileCannotBeOpened {
description("Ref Target File cannot be open()ed")
display("Ref Target File cannot be open()ed")
}
RefTargetCannotReadPermissions {
description("Ref Target: Cannot read permissions")
display("Ref Target: Cannot read permissions")
}
RefHashingError {
description("Error while hashing")
display("Error while hashing")
}
} }
} }

100
lib/entry/libimagentryref/src/flags.rs
View file

@ -1,100 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::collections::BTreeMap;
use toml::Value;
use error::RefError as RE;
use error::RefErrorKind as REK;
use error::Result;
pub struct RefFlags {
content_hashing: bool,
permission_tracking: bool,
}
impl RefFlags {
/// Read the RefFlags from a TOML document
///
/// Assumes that the whole TOML tree is passed. So this looks up `ref.flags` to get the flags.
/// It assumes that this is a Map with Key = <name of the setting> and Value = boolean.
pub fn read(v: &Value) -> Result<RefFlags> {
fn get_field(v: &Value, key: &str) -> Result<bool> {
use toml_query::read::TomlValueReadTypeExt;
v.read_bool(key)?.ok_or(RE::from_kind(REK::HeaderFieldMissingError))
}
Ok(RefFlags {
content_hashing: get_field(v, "ref.flags.content_hashing")?,
permission_tracking: get_field(v, "ref.flags.permission_tracking")?,
})
}
/// Alias for `RefFlags::content_hashing()`
pub fn is_often_moving(self, b: bool) -> RefFlags {
self.with_content_hashing(b)
}
pub fn with_content_hashing(mut self, b: bool) -> RefFlags {
self.content_hashing = b;
self
}
pub fn with_permission_tracking(mut self, b: bool) -> RefFlags {
self.permission_tracking = b;
self
}
pub fn get_content_hashing(&self) -> bool {
self.content_hashing
}
pub fn get_permission_tracking(&self) -> bool {
self.permission_tracking
}
}
impl Into<Value> for RefFlags {
/// Build a TOML::Value from this RefFlags object.
///
/// Returns a Map which should be set in `ref.flags` in the header.
fn into(self) -> Value {
let mut btm = BTreeMap::new();
btm.insert(String::from("content_hashing"), Value::Boolean(self.content_hashing));
btm.insert(String::from("permission_tracking"), Value::Boolean(self.permission_tracking));
return Value::Table(btm)
}
}
impl Default for RefFlags {
fn default() -> RefFlags {
RefFlags {
content_hashing: false,
permission_tracking: false,
}
}
}

30
lib/entry/libimagentryref/src/hashers/mod.rs → lib/entry/libimagentryref/src/generators/base.rs
View file

@ -17,4 +17,32 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// //
pub mod nbytes; use std::path::Path;
use libimagstore::storeid::StoreId;
use error::RefError;
use refstore::UniqueRefPathGenerator;
/// A base UniqueRefPathGenerator which must be overridden by the actual UniqueRefPathGenerator
/// which is provided by this crate
#[allow(dead_code)]
pub struct Base;
impl UniqueRefPathGenerator for Base {
type Error = RefError;
fn collection() -> &'static str {
"ref"
}
fn unique_hash<A: AsRef<Path>>(_path: A) -> Result<String, Self::Error> {
// This has to be overridden
panic!("Not overridden base functionality. This is a BUG!")
}
fn postprocess_storeid(sid: StoreId) -> Result<StoreId, Self::Error> {
Ok(sid) // default implementation
}
}

274
lib/entry/libimagentryref/src/generators/mod.rs Normal file
View file

@ -0,0 +1,274 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
//! Default generators
//!
//! This module provides a number of default `UniqueRefPathGenerator`s
//! which can be used for generating references.
//!
//! These generators are _NOT_ domain specific. So there won't be a "UniqueMailRefPathGenerator" in
//! here, for example.
//!
//! All these generators use "ref" as collection name.
//! They can be overridden using the `make_unique_ref_path_generator!()` convenience macro.
//!
//! # Note
//!
//! You must enable the appropriate crate feature to use any of the provided generators. With the
//! `generators` feature, you only get the convenience macro `make_unique_ref_path_generator!()`.
//!
/// A convenience macro for wrapping a generator in a new one, reusing the functionality from the
/// underlying generator
///
/// The UniqueRefPathGenerator must be in scope.
///
/// The macro creates a new struct `$name` over `$underlying` and changes the collection name to
/// `$collectionname`.
/// If passed, the new implementation is used (defaults to the implementation from the underlying
/// generator).
/// If passed, the new postprocessing is used (defaults to not changing the StoreId)
///
#[macro_export]
macro_rules! make_unique_ref_path_generator {
(
$name:ident
over $underlying:ty
=> with error $errtype:ty
=> with collection name $collectionname:expr
) => {
struct $name;
impl $crate::refstore::UniqueRefPathGenerator for $name {
type Error = $errtype;
fn collection() -> &'static str {
$collectionname
}
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String, Self::Error> {
$underlying::unique_hash(path)
}
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
-> Result<::libimagstore::storeid::StoreId, Self::Error>
{
Ok(sid)
}
}
};
(
$name:ident
over $underlying:ty
=> with error $errtype:ty
=> with collection name $collectionname:expr
=> $impl:expr
) => {
struct $name;
impl $crate::refstore::UniqueRefPathGenerator for $name {
type Error = $errtype;
fn collection() -> &'static str {
$collectionname
}
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String, Self::Error> {
$impl(path)
}
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
-> Result<::libimagstore::storeid::StoreId, Self::Error>
{
Ok(sid)
}
}
};
(
pub $name:ident
over $underlying:ty
=> with error $errtype:ty
=> with collection name $collectionname:expr
=> $impl:expr
) => {
make_unique_ref_path_generator!(
pub $name
over $underlying
=> with error $errtype
=> with collection name $collectionname
=> $impl => |sid| { Ok(sid) }
);
};
(
pub $name:ident
over $underlying:ty
=> with error $errtype:ty
=> with collection name $collectionname:expr
=> $impl:expr
=> $postproc:expr
) => {
pub struct $name;
impl $crate::refstore::UniqueRefPathGenerator for $name {
type Error = $errtype;
fn collection() -> &'static str {
$collectionname
}
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String, Self::Error> {
$impl(path)
}
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
-> Result<::libimagstore::storeid::StoreId, Self::Error>
{
$postproc(sid)
}
}
};
}
#[cfg(any(
feature = "generators-sha1",
feature = "generators-sha224",
feature = "generators-sha256",
feature = "generators-sha384",
feature = "generators-sha512",
feature = "generators-sha3",
))]
mod base;
/// Helper macro for generating implementations for the various Sha algorithms
macro_rules! make_sha_mod {
{
$modname:ident,
$hashname:ident,
$hashingimpl:expr
} => {
pub mod $modname {
use std::path::Path;
use std::fs::OpenOptions;
use std::io::Read;
use error::RefError as RE;
use crypto::digest::Digest;
make_unique_ref_path_generator! (
pub $hashname
over generators::base::Base
=> with error RE
=> with collection name "ref"
=> |path| {
OpenOptions::new()
.read(true)
.write(false)
.create(false)
.open(path)
.map_err(RE::from)
.and_then(|mut file| {
let mut buffer = String::new();
let _ = file.read_to_string(&mut buffer)?;
$hashingimpl(buffer)
})
}
);
impl $hashname {
/// Function which can be used by a wrapping UniqueRefPathGenerator to hash only N bytes.
pub fn hash_n_bytes<A: AsRef<Path>>(path: A, n: usize) -> Result<String, RE> {
OpenOptions::new()
.read(true)
.write(false)
.create(false)
.open(path)
.map_err(RE::from)
.and_then(|mut file| {
let mut buffer = Vec::with_capacity(n);
let _ = file.read_exact(&mut buffer)?;
let buffer = String::from_utf8(buffer)?;
$hashingimpl(buffer)
})
}
}
}
}
}
#[cfg(feature = "generators-sha1")]
make_sha_mod! {
sha1, Sha1, |buffer: String| {
let mut hasher = ::crypto::sha1::Sha1::new();
hasher.input_str(&buffer);
Ok(String::from(hasher.result_str()))
}
}
#[cfg(feature = "generators-sha224")]
make_sha_mod! {
sha224, Sha224, |buffer: String| {
let mut hasher = ::crypto::sha2::Sha224::new();
hasher.input_str(&buffer);
Ok(String::from(hasher.result_str()))
}
}
#[cfg(feature = "generators-sha256")]
make_sha_mod! {
sha256, Sha256, |buffer: String| {
let mut hasher = ::crypto::sha2::Sha256::new();
hasher.input_str(&buffer);
Ok(String::from(hasher.result_str()))
}
}
#[cfg(feature = "generators-sha384")]
make_sha_mod! {
sha384, Sha384, |buffer: String| {
let mut hasher = ::crypto::sha2::Sha384::new();
hasher.input_str(&buffer);
Ok(String::from(hasher.result_str()))
}
}
#[cfg(feature = "generators-sha512")]
make_sha_mod! {
sha512, Sha512, |buffer: String| {
let mut hasher = ::crypto::sha2::Sha512::new();
hasher.input_str(&buffer);
Ok(String::from(hasher.result_str()))
}
}
#[cfg(feature = "generators-sha3")]
make_sha_mod! {
sha3, Sha3, |buffer: String| {
let mut hasher = ::crypto::sha3::Sha3::sha3_256();
hasher.input_str(&buffer);
Ok(String::from(hasher.result_str()))
}
}

64
lib/entry/libimagentryref/src/hasher.rs
View file

@ -1,64 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::path::PathBuf;
use std::io::Read;
use crypto::sha1::Sha1;
use crypto::digest::Digest;
use error::Result;
/// The Hasher trait is used to implement custom hashing functions for the ref library.
/// This means that one can define how the hash of a reference is constructed from the content of
/// the file to ref to.
pub trait Hasher {
fn hash_name(&self) -> &'static str;
fn create_hash<R: Read>(&mut self, pb: &PathBuf, contents: &mut R) -> Result<String>;
}
pub struct DefaultHasher {
hasher: Sha1,
}
impl DefaultHasher {
pub fn new() -> DefaultHasher {
DefaultHasher { hasher: Sha1::new() }
}
}
impl Hasher for DefaultHasher {
fn hash_name(&self) -> &'static str {
"default"
}
fn create_hash<R: Read>(&mut self, _: &PathBuf, c: &mut R) -> Result<String> {
let mut s = String::new();
c.read_to_string(&mut s)?;
self.hasher.input_str(&s[..]);
Ok(self.hasher.result_str())
}
}

66
lib/entry/libimagentryref/src/hashers/nbytes.rs
View file

@ -1,66 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::io::Read;
use std::path::PathBuf;
use std::result::Result as RResult;
use crypto::sha1::Sha1;
use crypto::digest::Digest;
use hasher::Hasher;
use error::Result;
use error::RefError as RE;
pub struct NBytesHasher {
hasher: Sha1,
n: usize,
}
impl NBytesHasher {
pub fn new(n: usize) -> NBytesHasher {
NBytesHasher {
hasher: Sha1::new(),
n: n,
}
}
}
impl Hasher for NBytesHasher {
fn hash_name(&self) -> &'static str {
"n-bytes-hasher"
}
fn create_hash<R: Read>(&mut self, _: &PathBuf, contents: &mut R) -> Result<String> {
let s : String = contents
.bytes()
.take(self.n)
.collect::<RResult<Vec<u8>, _>>()
.map_err(From::from)
.and_then(|v| String::from_utf8(v).map_err(RE::from))?;
self.hasher.input_str(&s[..]);
Ok(self.hasher.result_str())
}
}

21
lib/entry/libimagentryref/src/lib.rs
View file

@ -36,11 +36,9 @@
)] )]
#[macro_use] extern crate log; #[macro_use] extern crate log;
extern crate crypto;
extern crate itertools; extern crate itertools;
extern crate toml; extern crate toml;
extern crate toml_query; extern crate toml_query;
extern crate walkdir;
#[macro_use] extern crate libimagstore; #[macro_use] extern crate libimagstore;
extern crate libimagerror; extern crate libimagerror;
@ -51,10 +49,19 @@ extern crate libimagentrylist;
module_entry_path_mod!("ref"); module_entry_path_mod!("ref");
pub mod error; pub mod error;
pub mod flags;
pub mod hasher;
pub mod hashers;
pub mod lister;
pub mod reference; pub mod reference;
pub mod refstore; pub mod refstore;
mod util;
#[cfg(any(
feature = "generators-sha1",
feature = "generators-sha224",
feature = "generators-sha256",
feature = "generators-sha384",
feature = "generators-sha512",
feature = "generators-sha3",
))]
extern crate crypto;
#[cfg(feature = "generators")]
pub mod generators;

161
lib/entry/libimagentryref/src/lister.rs
View file

@ -1,161 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::default::Default;
use std::io::stdout;
use std::io::Write;
use std::ops::Deref;
use libimagentrylist::lister::Lister;
use libimagentrylist::error::Result;
use libimagerror::trace::trace_error;
use libimagstore::store::FileLockEntry;
use libimagentrylist::error::ListErrorKind as LEK;
use libimagentrylist::error as lerror;
use reference::Ref;
pub struct RefLister {
check_dead: bool,
check_changed: bool,
check_changed_content: bool,
check_changed_permiss: bool,
}
impl RefLister {
pub fn new() -> RefLister {
RefLister::default()
}
pub fn check_dead(mut self, b: bool) -> RefLister {
self.check_dead = b;
self
}
pub fn check_changed(mut self, b: bool) -> RefLister {
self.check_changed = b;
self
}
pub fn check_changed_content(mut self, b: bool) -> RefLister {
self.check_changed_content = b;
self
}
pub fn check_changed_permiss(mut self, b: bool) -> RefLister {
self.check_changed_permiss = b;
self
}
}
impl Default for RefLister {
fn default() -> RefLister {
RefLister {
check_dead: false,
check_changed: false,
check_changed_content: false,
check_changed_permiss: false,
}
}
}
impl Lister for RefLister {
fn list<'b, I: Iterator<Item = FileLockEntry<'b>>>(&self, entries: I) -> Result<()> {
debug!("Called list()");
let (r, n) = entries.fold((Ok(()), 0), |(accu, i), entry| {
debug!("fold({:?}, {:?})", accu, entry);
let r = accu.and_then(|_| {
debug!("Listing Entry: {:?}", entry);
{
let is_dead = if self.check_dead {
if lerror::ResultExt::chain_err(entry.fs_link_exists(), || LEK::FormatError)? {
"dead"
} else {
"alive"
}
} else {
"not checked"
};
let is_changed = if self.check_changed {
if check_changed(entry.deref()) { "changed" } else { "unchanged" }
} else {
"not checked"
};
let is_changed_content = if self.check_changed_content {
if check_changed_content(entry.deref()) { "changed" } else { "unchanged" }
} else {
"not checked"
};
let is_changed_permiss = if self.check_changed_permiss {
if check_changed_permiss(entry.deref()) { "changed" } else { "unchanged" }
} else {
"not checked"
};
Ok(format!("{} | {} | {} | {} | {} | {}",
is_dead,
is_changed,
is_changed_content,
is_changed_permiss,
entry.get_path_hash().unwrap_or_else(|_| String::from("Cannot get hash")),
entry.get_location()))
}
.and_then(|s| {
lerror::ResultExt::chain_err(write!(stdout(), "{}\n", s), || LEK::FormatError)
})
})
.map(|_| ());
(r, i + 1)
});
debug!("Iterated over {} entries", n);
r
}
}
fn check_changed<R: Ref>(r: &R) -> bool {
check_changed_content(r) && check_changed_permiss(r)
}
fn check_changed_content<R: Ref>(r: &R) -> bool {
r.get_current_hash()
.and_then(|hash| r.get_stored_hash().map(|stored| (hash, stored)))
.map(|(hash, stored)| hash == stored)
.unwrap_or_else(|e| {
warn!("Could not check whether the ref changed on the FS");
trace_error(&e);
// We continue here and tell the callee that this reference is unchanged
false
})
}
fn check_changed_permiss<R: Ref>(_: &R) -> bool {
warn!("Permission changes tracking not supported yet.");
false
}

318
lib/entry/libimagentryref/src/reference.rs
View file

@ -21,101 +21,55 @@
//! files outside of the imag store. //! files outside of the imag store.
use std::path::PathBuf; use std::path::PathBuf;
use std::fs::File; use std::result::Result as RResult;
use std::fs::Permissions;
use libimagstore::store::Entry;
use libimagentryutil::isa::Is; use libimagentryutil::isa::Is;
use libimagentryutil::isa::IsKindHeaderPathProvider; use libimagentryutil::isa::IsKindHeaderPathProvider;
use libimagstore::store::Entry;
use toml::Value; use toml_query::read::TomlValueReadExt;
use toml_query::read::TomlValueReadTypeExt; use toml_query::delete::TomlValueDeleteExt;
use toml_query::set::TomlValueSetExt;
use error::RefErrorKind as REK; use refstore::UniqueRefPathGenerator;
use error::RefError as RE;
use error::ResultExt;
use error::Result; use error::Result;
use hasher::*; use error::RefError as RE;
use error::RefErrorKind as REK;
pub trait Ref { pub trait Ref {
/// Check whether the underlying object is actually a ref /// Check whether the underlying object is actually a ref
fn is_ref(&self) -> Result<bool>; fn is_ref(&self) -> Result<bool>;
/// Get the hash from the path of the ref /// Get the stored hash.
fn get_path_hash(&self) -> Result<String>; ///
/// Does not need a `UniqueRefPathGenerator` as it reads the hash stored in the header
fn get_hash(&self) -> Result<&str>;
/// Get the hash of the link target which is stored in the ref object /// Get the referenced path.
fn get_stored_hash(&self) -> Result<String>; ///
/// Does not need a `UniqueRefPathGenerator` as it reads the path stored in the header.
fn get_path(&self) -> Result<PathBuf>;
/// Get the hahs of the link target which is stored in the ref object, which is hashed with a /// Check whether the referenced file still matches its hash
/// custom Hasher instance. fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> RResult<bool, RPG::Error>;
fn get_stored_hash_with_hasher<H: Hasher>(&self, h: &H) -> Result<String>;
/// Get the hash of the link target by reading the link target and hashing the contents fn remove_ref(&mut self) -> Result<()>;
fn get_current_hash(&self) -> Result<String>;
/// Get the hash of the link target by reading the link target and hashing the contents with the
/// custom hasher
fn get_current_hash_with_hasher<H: Hasher>(&self, h: H) -> Result<String>;
/// check whether the pointer the Ref represents still points to a file which exists
fn fs_link_exists(&self) -> Result<bool>;
/// Alias for `r.fs_link_exists() && r.deref().is_file()` /// Alias for `r.fs_link_exists() && r.deref().is_file()`
fn is_ref_to_file(&self) -> Result<bool>; fn is_ref_to_file(&self) -> Result<bool> {
self.get_path().map(|p| p.is_file())
}
/// Alias for `r.fs_link_exists() && r.deref().is_dir()` /// Alias for `r.fs_link_exists() && r.deref().is_dir()`
fn is_ref_to_dir(&self) -> Result<bool>; fn is_ref_to_dir(&self) -> Result<bool> {
self.get_path().map(|p| p.is_dir())
}
/// Alias for `!Ref::fs_link_exists()` /// Alias for `!Ref::fs_link_exists()`
fn is_dangling(&self) -> Result<bool>; fn is_dangling(&self) -> Result<bool> {
self.get_path().map(|p| !p.exists())
}
/// check whether the pointer the Ref represents is valid
/// This includes:
/// - Hashsum of the file is still the same as stored in the Ref
/// - file permissions are still valid
fn fs_link_valid(&self) -> Result<bool>;
/// Check whether the file permissions of the referenced file are equal to the stored
/// permissions
fn fs_link_valid_permissions(&self) -> Result<bool>;
/// Check whether the Hashsum of the referenced file is equal to the stored hashsum
fn fs_link_valid_hash(&self) -> Result<bool>;
/// Update the Ref by re-checking the file from FS
/// This errors if the file is not present or cannot be read()
fn update_ref(&mut self) -> Result<()>;
/// Update the Ref by re-checking the file from FS using the passed Hasher instance
/// This errors if the file is not present or cannot be read()
fn update_ref_with_hasher<H: Hasher>(&mut self, h: &H) -> Result<()>;
/// Get the path of the file which is reffered to by this Ref
fn fs_file(&self) -> Result<PathBuf>;
/// Re-find a referenced file
///
/// This function tries to re-find a ref by searching all directories in `search_roots` recursively
/// for a file which matches the hash of the Ref.
///
/// If `search_roots` is `None`, it starts at the filesystem root `/`.
///
/// If the target cannot be found, this yields a RefTargetDoesNotExist error kind.
///
/// # Warning
///
/// This option causes heavy I/O as it recursively searches the Filesystem.
fn refind(&self, search_roots: Option<Vec<PathBuf>>) -> Result<PathBuf>;
/// See documentation of `Ref::refind()`
fn refind_with_hasher<H: Hasher>(&self, search_roots: Option<Vec<PathBuf>>, h: H)
-> Result<PathBuf>;
/// Get the permissions of the file which are present
fn get_current_permissions(&self) -> Result<Permissions>;
} }
provide_kindflag_path!(pub IsRef, "ref.is_ref"); provide_kindflag_path!(pub IsRef, "ref.is_ref");
@ -127,210 +81,38 @@ impl Ref for Entry {
self.is::<IsRef>().map_err(From::from) self.is::<IsRef>().map_err(From::from)
} }
/// Get the hash from the path of the ref fn get_hash(&self) -> Result<&str> {
fn get_path_hash(&self) -> Result<String> {
self.get_location()
.clone()
.into_pathbuf()
.map_err(From::from)
.and_then(|pb| {
pb.file_name()
.and_then(|osstr| osstr.to_str())
.and_then(|s| s.split("~").next())
.map(String::from)
.ok_or(String::from("String splitting error"))
.map_err(From::from)
})
}
/// Get the hash of the link target which is stored in the ref object
fn get_stored_hash(&self) -> Result<String> {
self.get_stored_hash_with_hasher(&DefaultHasher::new())
}
/// Get the hahs of the link target which is stored in the ref object, which is hashed with a
/// custom Hasher instance.
fn get_stored_hash_with_hasher<H: Hasher>(&self, h: &H) -> Result<String> {
self.get_header() self.get_header()
.read_string(&format!("ref.content_hash.{}", h.hash_name())[..])? .read("ref.hash")
.ok_or(RE::from_kind(REK::HeaderFieldMissingError)) .map_err(RE::from)?
.ok_or_else(|| REK::HeaderFieldMissingError("ref.hash").into())
.and_then(|v| v.as_str().ok_or_else(|| REK::HeaderTypeError("ref.hash", "string").into()))
} }
/// Get the hash of the link target by reading the link target and hashing the contents fn get_path(&self) -> Result<PathBuf> {
fn get_current_hash(&self) -> Result<String> {
self.get_current_hash_with_hasher(DefaultHasher::new())
}
/// Get the hash of the link target by reading the link target and hashing the contents with the
/// custom hasher
fn get_current_hash_with_hasher<H: Hasher>(&self, mut h: H) -> Result<String> {
self.fs_file()
.and_then(|pb| File::open(pb.clone()).map(|f| (pb, f)).map_err(From::from))
.and_then(|(path, mut file)| h.create_hash(&path, &mut file))
}
/// check whether the pointer the Ref represents still points to a file which exists
fn fs_link_exists(&self) -> Result<bool> {
self.fs_file().map(|pathbuf| pathbuf.exists())
}
/// Alias for `r.fs_link_exists() && r.deref().is_file()`
fn is_ref_to_file(&self) -> Result<bool> {
self.fs_file().map(|pathbuf| pathbuf.is_file())
}
/// Alias for `r.fs_link_exists() && r.deref().is_dir()`
fn is_ref_to_dir(&self) -> Result<bool> {
self.fs_file().map(|pathbuf| pathbuf.is_dir())
}
/// Alias for `!Ref::fs_link_exists()`
fn is_dangling(&self) -> Result<bool> {
self.fs_link_exists().map(|b| !b)
}
/// check whether the pointer the Ref represents is valid
/// This includes:
/// - Hashsum of the file is still the same as stored in the Ref
/// - file permissions are still valid
fn fs_link_valid(&self) -> Result<bool> {
match (self.fs_link_valid_permissions(), self.fs_link_valid_hash()) {
(Ok(true) , Ok(true)) => Ok(true),
(Ok(_) , Ok(_)) => Ok(false),
(Err(e) , _) => Err(e),
(_ , Err(e)) => Err(e),
}
}
/// Check whether the file permissions of the referenced file are equal to the stored
/// permissions
fn fs_link_valid_permissions(&self) -> Result<bool> {
self
.get_header()
.read_bool("ref.permissions.ro")
.chain_err(|| REK::HeaderFieldReadError)?
.ok_or(RE::from_kind(REK::HeaderFieldMissingError))
.and_then(|ro| self.get_current_permissions().map(|perm| ro == perm.readonly()))
.chain_err(|| REK::RefTargetCannotReadPermissions)
}
/// Check whether the Hashsum of the referenced file is equal to the stored hashsum
fn fs_link_valid_hash(&self) -> Result<bool> {
let stored_hash = self.get_stored_hash()?;
let current_hash = self.get_current_hash()?;
Ok(stored_hash == current_hash)
}
/// Update the Ref by re-checking the file from FS
/// This errors if the file is not present or cannot be read()
fn update_ref(&mut self) -> Result<()> {
self.update_ref_with_hasher(&DefaultHasher::new())
}
/// Update the Ref by re-checking the file from FS using the passed Hasher instance
/// This errors if the file is not present or cannot be read()
fn update_ref_with_hasher<H: Hasher>(&mut self, h: &H) -> Result<()> {
let current_hash = self.get_current_hash()?; // uses the default hasher
let current_perm = self.get_current_permissions()?;
self
.get_header_mut()
.set("ref.permissions.ro", Value::Boolean(current_perm.readonly()))
?;
self
.get_header_mut()
.set(&format!("ref.content_hash.{}", h.hash_name())[..], Value::String(current_hash))
?;
Ok(())
}
/// Get the path of the file which is reffered to by this Ref
fn fs_file(&self) -> Result<PathBuf> {
self.get_header() self.get_header()
.read_string("ref.path")? .read("ref.path")
.ok_or(RE::from_kind(REK::HeaderFieldMissingError)) .map_err(RE::from)?
.ok_or_else(|| REK::HeaderFieldMissingError("ref.path").into())
.and_then(|v| v.as_str().ok_or_else(|| REK::HeaderTypeError("ref.path", "string").into()))
.map(PathBuf::from) .map(PathBuf::from)
} }
/// Re-find a referenced file fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> RResult<bool, RPG::Error> {
/// self.get_path()
/// This function tries to re-find a ref by searching all directories in `search_roots` recursively .map(PathBuf::from)
/// for a file which matches the hash of the Ref. .map_err(RE::from)
/// .map_err(RPG::Error::from)
/// If `search_roots` is `None`, it starts at the filesystem root `/`. .and_then(|pb| RPG::unique_hash(pb))
/// .and_then(|h| Ok(h == self.get_hash()?))
/// If the target cannot be found, this yields a RefTargetDoesNotExist error kind.
///
/// # Warning
///
/// This option causes heavy I/O as it recursively searches the Filesystem.
fn refind(&self, search_roots: Option<Vec<PathBuf>>) -> Result<PathBuf> {
self.refind_with_hasher(search_roots, DefaultHasher::new())
} }
/// See documentation of `Ref::refind()` fn remove_ref(&mut self) -> Result<()> {
fn refind_with_hasher<H: Hasher>(&self, search_roots: Option<Vec<PathBuf>>, mut h: H) let hdr = self.get_header_mut();
-> Result<PathBuf> let _ = hdr.delete("ref.hash")?;
{ let _ = hdr.delete("ref.path")?;
use itertools::Itertools; let _ = hdr.delete("ref")?;
use walkdir::WalkDir; Ok(())
self.get_stored_hash()
.and_then(|stored_hash| {
search_roots
.unwrap_or(vec![PathBuf::from("/")])
.into_iter()
.map(|root| {
WalkDir::new(root)
.follow_links(false)
.into_iter()
.map(|entry| {
entry
.map_err(From::from)
.and_then(|entry| {
let pb = PathBuf::from(entry.path());
File::open(entry.path())
.map(|f| (pb, f))
.map_err(From::from)
})
.and_then(|(p, mut f)| {
h.create_hash(&p, &mut f)
.map(|h| (p, h))
.map_err(From::from)
})
.map(|(path, hash)| {
if hash == stored_hash {
Some(path)
} else {
None
}
})
})
.filter_map(Result::ok)
.filter_map(|e| e)
.next()
})
.flatten()
.next()
.ok_or(RE::from_kind(REK::RefTargetDoesNotExist))
})
}
/// Get the permissions of the file which are present
fn get_current_permissions(&self) -> Result<Permissions> {
self.fs_file()
.and_then(|pb| {
File::open(pb)
.chain_err(|| REK::HeaderFieldReadError)
})
.and_then(|file| {
file
.metadata()
.map(|md| md.permissions())
.chain_err(|| REK::RefTargetCannotReadPermissions)
})
} }
} }

329
lib/entry/libimagentryref/src/refstore.rs
View file

@ -17,280 +17,125 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// //
use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use std::collections::BTreeMap;
use std::fs::File;
use libimagstore::store::FileLockEntry; use libimagstore::store::FileLockEntry;
use libimagstore::storeid::IntoStoreId;
use libimagstore::storeid::StoreId;
use libimagstore::storeid::StoreIdIterator;
use libimagstore::store::Store; use libimagstore::store::Store;
use libimagstore::storeid::StoreId;
use libimagentryutil::isa::Is; use libimagentryutil::isa::Is;
use toml_query::insert::TomlValueInsertExt;
use toml::Value; use toml::Value;
use error::RefErrorKind as REK;
use error::RefError as RE; use error::RefError as RE;
use error::ResultExt; use error::RefErrorKind as REK;
use error::Result;
use flags::RefFlags;
use reference::IsRef; use reference::IsRef;
use hasher::*;
use module_path::ModuleEntryPath;
use util::*;
pub trait RefStore { /// A UniqueRefPathGenerator generates unique Pathes
/// Check whether there is a reference to the file at `pb`
fn exists(&self, pb: PathBuf) -> Result<bool>;
/// Get a Ref object from the store by hash.
/// ///
/// Returns None if the hash cannot be found. /// It is basically a functor which generates a StoreId from a &Path.
fn get_by_hash<'a>(&'a self, hash: String) -> Result<Option<FileLockEntry<'a>>>; /// For more information have a look at the documentation of RefStore.
pub trait UniqueRefPathGenerator {
type Error: From<RE>;
/// Find a store id by partial ref (also see documentation for /// The collection the `StoreId` should be created for
/// `RefStore::get_by_partitial_hash()`. fn collection() -> &'static str {
fn find_storeid_by_partial_hash(&self, hash: &String) -> Result<Option<StoreId>>; "ref"
}
/// Get a Ref object from the store by (eventually partial) hash. /// A function which should generate a unique string for a Path
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String, Self::Error>;
/// Postprocess the generated `StoreId` object
fn postprocess_storeid(sid: StoreId) -> Result<StoreId, Self::Error> {
Ok(sid)
}
}
/// A extensions for the `Store` to handle `Ref` objects
/// ///
/// If the hash is complete, `RefStore::get_by_hash()` should be used as it is cheaper. /// The RefStore handles refs using a `UniqueRefPathGenerator`. The `UniqueRefPathGenerator`, as it
/// If the hash comes from user input and thus might be abbreviated, this function can be used. /// name suggests, generates unique `StoreId`s for a `&Path`. It is a functor `&Path -> StoreId`.
fn get_by_partitial_hash<'a>(&'a self, hash: &String) -> Result<Option<FileLockEntry<'a>>>;
/// Delete a ref by hash
/// ///
/// If the returned Result contains an error, the ref might not be deleted. /// It provides three functions which are called in the following sequence:
fn delete_by_hash(&self, hash: String) -> Result<()>;
/// Create a Ref object which refers to `pb`
fn create<'a>(&'a self, pb: PathBuf, flags: RefFlags) -> Result<FileLockEntry<'a>>;
fn create_with_hasher<'a, H: Hasher>(&'a self, pb: PathBuf, flags: RefFlags, h: H)
-> Result<FileLockEntry<'a>>;
/// Get all reference objects
fn all_references(&self) -> Result<StoreIdIterator>;
}
impl RefStore for Store {
/// Check whether there is a reference to the file at `pb`
fn exists(&self, pb: PathBuf) -> Result<bool> {
pb.canonicalize()
.chain_err(|| REK::PathCanonicalizationError)
.and_then(|c| hash_path(&c))
.chain_err(|| REK::PathHashingError)
.and_then(|hash| {
self.retrieve_for_module("ref").map(|iter| (hash, iter)).map_err(From::from)
})
.and_then(|(hash, possible_refs)| {
// This is kind of a manual Iterator::filter() call what we do here, but with the
// actual ::filter method we cannot return the error in a nice way, so we do it
// manually here. If you can come up with a better version of this, feel free to
// take this note as a todo.
for r in possible_refs {
let contains_hash = r.to_str()
.chain_err(|| REK::TypeConversionError)
.map(|s| s.contains(&hash[..]))
?;
if !contains_hash {
continue;
}
match self.get(r.clone())? {
Some(fle) => {
if read_reference(&fle).map(|path| path == pb).unwrap_or(false) {
return Ok(true)
}
},
None => {
let e = format!("Failed to get from store: {}", r);
return Err(e).map_err(From::from)
},
}
}
Ok(false)
})
}
/// Get a Ref object from the store by hash.
/// ///
/// Returns None if the hash cannot be found. /// * The `UniqueRefPathGenerator::collection()` function is used for get the collection a `StoreId`
fn get_by_hash<'a>(&'a self, hash: String) -> Result<Option<FileLockEntry<'a>>> { /// should be in (The first element of the `StoreId` path)
ModuleEntryPath::new(hash) /// * The `UniqueRefPathGenerator::unique_hash()` gets the `&Path` which it then should generate a
.into_storeid() /// unique String for. How this is done does not matter. It can hash the Path itself, read the
.and_then(|id| self.get(id)) /// file and hash that or something else. It should be reproduceable, though.
.map_err(From::from) /// * These two parts are joined and put into a `StoreId` which the
} /// `UniqueRefPathGenerator::postprocess_storeid()` function is then allowed to postprocess (for
/// example add more parts to the StoreId). The default implementation does nothing.
fn find_storeid_by_partial_hash(&self, hash: &String) -> Result<Option<StoreId>> {
debug!("Trying to find '{}' in store...", hash);
for id in self.retrieve_for_module("ref")? {
let components_have_hash = id
.components()
.any(|c| c.as_os_str().to_str().map(|s| s.contains(hash)).unwrap_or(false));
if components_have_hash {
debug!("Found hash '{}' in {:?}", hash, id);
return Ok(Some(id))
}
}
Ok(None)
}
/// Get a Ref object from the store by (eventually partial) hash.
/// ///
/// If the hash is complete, `RefStore::get_by_hash()` should be used as it is cheaper. /// The StoreId which is generated is then used to carry out the actual action (reading, creating
/// If the hash comes from user input and thus might be abbreviated, this function can be used. /// ...).
fn get_by_partitial_hash<'a>(&'a self, hash: &String) -> Result<Option<FileLockEntry<'a>>> { /// If a entry is created, header information is set (that it is a ref, the hash which was just
match self.find_storeid_by_partial_hash(hash)? { /// generated and the path of the referenced file)
Some(id) => self.get(id).map_err(From::from),
None => Ok(None),
}
}
/// Delete a ref by hash
/// ///
/// If the returned Result contains an error, the ref might not be deleted. /// # Details
fn delete_by_hash(&self, hash: String) -> Result<()> { ///
ModuleEntryPath::new(hash) /// The `UniqueRefPathGenerator` is passed as type parameter to enforce some semantics:
.into_storeid() ///
.and_then(|id| self.delete(id)) /// * The used `UniqueRefPathGenerator` is defined by the implementation rather than by the runtime
.map_err(From::from) /// of the program or some environment. Of course this is only a small hurdle to enforce this, but
/// a hint.
/// * The `UniqueRefPathGenerator` is a functor which does not carry state.
///
pub trait RefStore<'a> {
fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H) -> Result<Option<FileLockEntry<'a>>, RPG::Error>;
fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>, RPG::Error>;
fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>, RPG::Error>;
} }
/// Create a Ref object which refers to `pb` impl<'a> RefStore<'a> for Store {
fn create<'a>(&'a self, pb: PathBuf, flags: RefFlags) -> Result<FileLockEntry<'a>> {
self.create_with_hasher(pb, flags, DefaultHasher::new())
}
fn create_with_hasher<'a, H: Hasher>(&'a self, pb: PathBuf, flags: RefFlags, mut h: H) fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H)
-> Result<FileLockEntry<'a>> -> Result<Option<FileLockEntry<'a>>, RPG::Error>
{ {
use toml_query::insert::TomlValueInsertExt; let sid = StoreId::new_baseless(PathBuf::from(format!("{}/{}", RPG::collection(), hash.as_ref())))
.map_err(RE::from)?;
if !pb.exists() { debug!("Getting: {:?}", sid);
return Err(RE::from_kind(REK::RefTargetDoesNotExist)); self.get(sid)
} .map_err(RE::from)
if flags.get_content_hashing() && pb.is_dir() { .map_err(RPG::Error::from)
return Err(RE::from_kind(REK::RefTargetCannotBeHashed));
} }
let (mut fle, content_hash, permissions, canonical_path) = { // scope to be able to fold fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
File::open(pb.clone()) -> Result<FileLockEntry<'a>, RPG::Error>
.chain_err(|| REK::RefTargetFileCannotBeOpened)
// If we were able to open this file,
// we hash the contents of the file and return (file, hash)
.and_then(|mut file| {
let opt_contenthash = if flags.get_content_hashing() {
Some(h.create_hash(&pb, &mut file)?)
} else {
None
};
Ok((file, opt_contenthash))
})
// and then we get the permissions if we have to
// and return (file, content hash, permissions)
.and_then(|(file, opt_contenthash)| {
let opt_permissions = if flags.get_permission_tracking() {
Some(file.metadata()
.map(|md| md.permissions())
.chain_err(|| REK::RefTargetCannotReadPermissions)?)
} else {
None
};
Ok((opt_contenthash, opt_permissions))
})
// and then we try to canonicalize the PathBuf, because we want to store a
// canonicalized path
// and return (file, content hash, permissions, canonicalized path)
.and_then(|(opt_contenthash, opt_permissions)| {
pb.canonicalize()
.map(|can| (opt_contenthash, opt_permissions, can))
// if PathBuf::canonicalize() failed, build an error from the return value
.chain_err(|| REK::PathCanonicalizationError)
})
// and then we hash the canonicalized path
// and return (file, content hash, permissions, canonicalized path, path hash)
.and_then(|(opt_contenthash, opt_permissions, can)| {
let path_hash = hash_path(&can).chain_err(|| REK::PathHashingError)?;
Ok((opt_contenthash, opt_permissions, can, path_hash))
})
// and then we convert the PathBuf of the canonicalized path to a String to be able
// to save it in the Ref FileLockEntry obj
// and return
// (file, content hash, permissions, canonicalized path as String, path hash)
.and_then(|(opt_conhash, opt_perm, can, path_hash)| {
match can.to_str().map(String::from) {
// UTF convert error in PathBuf::to_str(),
None => Err(RE::from_kind(REK::PathUTF8Error)),
Some(can) => Ok((opt_conhash, opt_perm, can, path_hash))
}
})
// and then we create the FileLockEntry in the Store
// and return (filelockentry, content hash, permissions, canonicalized path)
.and_then(|(opt_conhash, opt_perm, can, path_hash)| {
let fle = self.create(ModuleEntryPath::new(path_hash))?;
Ok((fle, opt_conhash, opt_perm, can))
})?
};
for tpl in [
Some((String::from("ref"), Value::Table(BTreeMap::new()))),
Some((String::from("ref.permissions"), Value::Table(BTreeMap::new()))),
Some((String::from("ref.path"), Value::String(canonical_path))),
Some((String::from("ref.content_hash"), Value::Table(BTreeMap::new()))),
content_hash.map(|hash| {
(format!("ref.content_hash.{}", h.hash_name()), Value::String(hash))
}),
permissions.map(|p| {
(String::from("ref.permissions.ro"), Value::Boolean(p.readonly()))
}),
].into_iter()
{ {
match tpl { let path_str = path.as_ref().to_str().map(String::from).ok_or(REK::PathUTF8Error.into())?;
&Some((ref s, ref v)) => { let hash = RPG::unique_hash(path)?;
match fle.get_header_mut().insert(s, v.clone()) { let pathbuf = PathBuf::from(format!("{}/{}", RPG::collection(), hash));
Ok(Some(_)) => { let sid = StoreId::new_baseless(pathbuf).map_err(RE::from)?;
let e = RE::from_kind(REK::HeaderFieldAlreadyExistsError);
return Err(e).chain_err(|| REK::HeaderFieldWriteError);
},
Ok(None) => {
// Okay, we just inserted a new header value...
},
Err(e) => return Err(e).chain_err(|| REK::HeaderFieldWriteError),
}
}
&None => {
debug!("Not going to insert.");
}
}
}
debug!("Creating: {:?}", sid);
self.create(sid)
.map_err(RE::from)
.and_then(|mut fle| {
let _ = fle.set_isflag::<IsRef>()?; let _ = fle.set_isflag::<IsRef>()?;
{
let hdr = fle.get_header_mut();
hdr.insert("ref.path", Value::String(String::from(path_str)))?;
hdr.insert("ref.hash", Value::String(hash))?;
}
Ok(fle) Ok(fle)
})
.map_err(RPG::Error::from)
} }
fn all_references(&self) -> Result<StoreIdIterator> { fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
self.retrieve_for_module("ref").map_err(From::from) -> Result<FileLockEntry<'a>, RPG::Error>
{
match self.get_ref::<RPG, String>(RPG::unique_hash(path.as_ref())?)? {
Some(r) => Ok(r),
None => self.create_ref::<RPG, A>(path),
} }
} }
}

52
lib/entry/libimagentryref/src/util.rs
View file

@ -1,52 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::path::PathBuf;
use error::RefErrorKind as REK;
use error::RefError as RE;
use error::Result;
use libimagstore::store::Entry;
use toml_query::read::TomlValueReadTypeExt;
/// Creates a Hash from a PathBuf by making the PathBuf absolute and then running a hash
/// algorithm on it
pub fn hash_path(pb: &PathBuf) -> Result<String> {
use crypto::sha1::Sha1;
use crypto::digest::Digest;
pb.to_str()
.ok_or(RE::from_kind(REK::PathUTF8Error))
.map(|s| {
let mut hasher = Sha1::new();
hasher.input_str(s);
hasher.result_str()
})
}
/// Read the reference from a file
pub fn read_reference(refentry: &Entry) -> Result<PathBuf> {
refentry.get_header()
.read_string("ref.path")?
.ok_or(RE::from_kind(REK::HeaderFieldMissingError))
.map(PathBuf::from)
}