matthiasbeyer/imag
1
0
Fork 0
Code Releases Activity

Merge branch 'rewrite-libimagentryref'

Browse source 
This is the merge for the "ref"-infrastructure rewrite. Finally.

The refs are now stored with three bits of information:

    * The "collection" (named basepath)
    * The "relpath" (relative path)
    * The hash (sha1 as of now)

The "collection" is a name which has to be accociated with a path to a
directory (in the config file).
That gives the user the opportunity to have (for example) their music
collection in $HOME/music on one device and in $HOME/media/music on
another.
The "relpath" is the relative path, which results in the path to the
actual file when being joined with the "collection" path.

The hash is a sha1 hash as of now. Re-checking this hash with
libimagentryref is not yet tested.

This merge also removes the "mail" code completely, for the sake of
seperating concerns into branches. The respective commits might be
removed later.

Signed-off-by: Matthias Beyer <mail@beyermatthias.de>
This commit is contained in:
Matthias Beyer 2019-02-17 11:44:54 +01:00
commit c562f35222
25 changed files with 792 additions and 1322 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Cargo.toml
View file

@ -22,7 +22,6 @@ members = [
"bin/domain/imag-diary", "bin/domain/imag-diary",
"bin/domain/imag-habit", "bin/domain/imag-habit",
"bin/domain/imag-log", "bin/domain/imag-log",
"bin/domain/imag-mail",
"bin/domain/imag-notes", "bin/domain/imag-notes",
"bin/domain/imag-timetrack", "bin/domain/imag-timetrack",
"bin/domain/imag-todo", "bin/domain/imag-todo",
@ -35,7 +34,6 @@ members = [
"lib/domain/libimagdiary", "lib/domain/libimagdiary",
"lib/domain/libimaghabit", "lib/domain/libimaghabit",
"lib/domain/libimaglog", "lib/domain/libimaglog",
"lib/domain/libimagmail",
"lib/domain/libimagnotes", "lib/domain/libimagnotes",
"lib/domain/libimagtimetrack", "lib/domain/libimagtimetrack",
"lib/domain/libimagtodo", "lib/domain/libimagtodo",

44
bin/core/imag-ref/src/main.rs
View file

@ -47,15 +47,17 @@ extern crate libimagutil;
mod ui; mod ui;
use ui::build_ui; use ui::build_ui;
use std::path::PathBuf;
use std::process::exit; use std::process::exit;
use std::io::Write;
use libimagerror::trace::MapErrTrace; use libimagerror::trace::MapErrTrace;
use libimagerror::exit::ExitUnwrap; use libimagerror::exit::ExitUnwrap;
use libimagrt::setup::generate_runtime_setup; use libimagrt::setup::generate_runtime_setup;
use libimagrt::runtime::Runtime; use libimagrt::runtime::Runtime;
use libimagstore::storeid::IntoStoreId;
use libimagentryref::reference::Ref; use libimagentryref::reference::Ref;
use libimagentryref::reference::MutRef;
use libimagentryref::reference::RefFassade;
use libimagentryref::hasher::default::DefaultHasher;
fn main() { fn main() {
let version = make_imag_version!(); let version = make_imag_version!();
@ -69,6 +71,7 @@ fn main() {
debug!("Call: {}", name); debug!("Call: {}", name);
match name { match name {
"deref" => deref(&rt), "deref" => deref(&rt),
"create" => create(&rt),
"remove" => remove(&rt), "remove" => remove(&rt),
other => { other => {
debug!("Unknown command"); debug!("Unknown command");
@ -83,16 +86,16 @@ fn main() {
fn deref(rt: &Runtime) { fn deref(rt: &Runtime) {
let cmd = rt.cli().subcommand_matches("deref").unwrap(); let cmd = rt.cli().subcommand_matches("deref").unwrap();
let id = cmd.value_of("ID") let ids = rt.ids::<::ui::PathProvider>().map_err_trace_exit_unwrap();
.map(String::from) let out = rt.stdout();
.map(PathBuf::from) let mut outlock = out.lock();
.unwrap() // saved by clap
.into_storeid()
.map_err_trace_exit_unwrap();
ids.into_iter()
.for_each(|id| {
match rt.store().get(id.clone()).map_err_trace_exit_unwrap() { match rt.store().get(id.clone()).map_err_trace_exit_unwrap() {
Some(entry) => { Some(entry) => {
entry entry
.as_ref_with_hasher::<DefaultHasher>()
.get_path() .get_path()
.map_err_trace_exit_unwrap() .map_err_trace_exit_unwrap()
.to_str() .to_str()
@ -100,7 +103,7 @@ fn deref(rt: &Runtime) {
error!("Could not transform path into string!"); error!("Could not transform path into string!");
exit(1) exit(1)
}) })
.map(|s| info!("{}", s)) .map(|s| writeln!(outlock, "{}", s))
.ok(); // safe here because we exited already in the error case .ok(); // safe here because we exited already in the error case
let _ = rt.report_touched(&id).unwrap_or_exit(); let _ = rt.report_touched(&id).unwrap_or_exit();
@ -109,7 +112,8 @@ fn deref(rt: &Runtime) {
error!("No entry for id '{}' found", id); error!("No entry for id '{}' found", id);
exit(1) exit(1)
}, },
}; }
});
} }
fn remove(rt: &Runtime) { fn remove(rt: &Runtime) {
@ -117,12 +121,7 @@ fn remove(rt: &Runtime) {
let cmd = rt.cli().subcommand_matches("remove").unwrap(); let cmd = rt.cli().subcommand_matches("remove").unwrap();
let yes = cmd.is_present("yes"); let yes = cmd.is_present("yes");
let id = cmd.value_of("ID") let ids = rt.ids::<::ui::PathProvider>().map_err_trace_exit_unwrap();
.map(String::from)
.map(PathBuf::from)
.unwrap() // saved by clap
.into_storeid()
.map_err_trace_exit_unwrap();
let mut input = rt.stdin().unwrap_or_else(|| { let mut input = rt.stdin().unwrap_or_else(|| {
error!("No input stream. Cannot ask for permission"); error!("No input stream. Cannot ask for permission");
@ -131,13 +130,17 @@ fn remove(rt: &Runtime) {
let mut output = rt.stdout(); let mut output = rt.stdout();
ids.into_iter()
.for_each(|id| {
match rt.store().get(id.clone()).map_err_trace_exit_unwrap() { match rt.store().get(id.clone()).map_err_trace_exit_unwrap() {
Some(mut entry) => { Some(mut entry) => {
if yes || if yes ||
ask_bool(&format!("Delete ref from entry '{}'", id), None, &mut input, &mut output) ask_bool(&format!("Delete ref from entry '{}'", id), None, &mut input, &mut output)
.map_err_trace_exit_unwrap() .map_err_trace_exit_unwrap()
{ {
let _ = entry.remove_ref().map_err_trace_exit_unwrap(); let _ = entry.as_ref_with_hasher_mut::<DefaultHasher>()
.remove_ref()
.map_err_trace_exit_unwrap();
} else { } else {
info!("Aborted"); info!("Aborted");
} }
@ -146,6 +149,11 @@ fn remove(rt: &Runtime) {
error!("No entry for id '{}' found", id); error!("No entry for id '{}' found", id);
exit(1) exit(1)
}, },
}; }
});
}
fn create(rt: &Runtime) {
unimplemented!()
} }

108
bin/core/imag-ref/src/ui.rs
View file

@ -17,19 +17,34 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// //
use clap::{Arg, App, SubCommand}; use std::path::PathBuf;
use clap::{Arg, App, ArgMatches, SubCommand};
use libimagstore::storeid::StoreId;
use libimagstore::storeid::IntoStoreId;
use libimagrt::runtime::IdPathProvider;
use libimagerror::trace::MapErrTrace;
pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> { pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> {
app app
.subcommand(SubCommand::with_name("deref") .subcommand(SubCommand::with_name("deref")
.about("'Dereference' a ref. This prints the Path of the referenced file") .about("'Dereference a ref. This prints the Path(es) of the referenced file(s)")
.version("0.1") .version("0.1")
.arg(Arg::with_name("ID") .arg(Arg::with_name("ID")
.index(1) .index(1)
.takes_value(true) .takes_value(true)
.required(true) .required(false)
.help("The id of the store entry to dereference") .multiple(true)
.help("The id of the store entry to dereference.")
.value_name("ID")) .value_name("ID"))
.arg(Arg::with_name("ignore-noref")
.long("ignore-noref")
.takes_value(false)
.required(false)
.multiple(false)
.help("Ignore store entries which are not refs and do not print error message"))
) )
.subcommand(SubCommand::with_name("remove") .subcommand(SubCommand::with_name("remove")
@ -38,14 +53,89 @@ pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> {
.arg(Arg::with_name("ID") .arg(Arg::with_name("ID")
.index(1) .index(1)
.takes_value(true) .takes_value(true)
.required(true) .required(false)
.multiple(true) .multiple(true)
.help("Remove the reference from this store entry") .help("Remove the reference from this store entry")
.value_name("ENTRIES")) .value_name("ENTRIES"))
.arg(Arg::with_name("yes") .arg(Arg::with_name("ignore-noref")
.long("yes") .long("ignore-noref")
.short("y") .takes_value(false)
.help("Don't ask whether this really should be done")) .required(false)
.multiple(false)
.help("Ignore store entries which are not refs and do not print error message"))
)
.subcommand(SubCommand::with_name("create")
.about("Create a reference to a file")
.version("0.1")
.arg(Arg::with_name("ID")
.index(1)
.takes_value(true)
.required(true)
.multiple(false)
.help("Create a reference with that ID in the store. If the store id exists, it will be made into a reference.")
.value_name("ID"))
.arg(Arg::with_name("path")
.index(2)
.takes_value(true)
.required(true)
.multiple(false)
.help("The path to refer to. If there is no basepath configuration in the config file for the path this file is located at, the operation will error.")
.value_name("ID"))
.arg(Arg::with_name("force")
.long("force")
.takes_value(false)
.required(false)
.multiple(false)
.help("Use force to override existing references"))
) )
} }
pub struct PathProvider;
impl IdPathProvider for PathProvider {
fn get_ids(matches: &ArgMatches) -> Vec<StoreId> {
match matches.subcommand() {
("deref", Some(subm)) => {
subm.values_of("ID")
.ok_or_else(|| {
error!("No StoreId found");
::std::process::exit(1)
})
.unwrap()
.into_iter()
.map(PathBuf::from)
.map(|pb| pb.into_storeid())
.collect::<Result<Vec<_>, _>>()
.map_err_trace_exit_unwrap()
},
("remove", Some(subm)) => {
subm.values_of("ID")
.ok_or_else(|| {
error!("No StoreId found");
::std::process::exit(1)
})
.unwrap()
.into_iter()
.map(PathBuf::from)
.map(|pb| pb.into_storeid())
.collect::<Result<Vec<_>, _>>()
.map_err_trace_exit_unwrap()
},
("create", _) => {
error!("Command does not get IDs as input");
::std::process::exit(1)
},
(other, _) => {
error!("Not a known command: {}", other);
::std::process::exit(1)
}
}
}
}

2
bin/core/imag/build.rs
View file

@ -99,7 +99,6 @@ gen_mods_buildui!(
("../../../bin/domain/imag-diary/src/ui.rs" , imagdiary) , ("../../../bin/domain/imag-diary/src/ui.rs" , imagdiary) ,
("../../../bin/domain/imag-habit/src/ui.rs" , imaghabit) , ("../../../bin/domain/imag-habit/src/ui.rs" , imaghabit) ,
("../../../bin/domain/imag-log/src/ui.rs" , imaglog) , ("../../../bin/domain/imag-log/src/ui.rs" , imaglog) ,
("../../../bin/domain/imag-mail/src/ui.rs" , imagmail) ,
("../../../bin/domain/imag-notes/src/ui.rs" , imagnotes) , ("../../../bin/domain/imag-notes/src/ui.rs" , imagnotes) ,
("../../../bin/domain/imag-timetrack/src/ui.rs" , imagtimetrack) , ("../../../bin/domain/imag-timetrack/src/ui.rs" , imagtimetrack) ,
("../../../bin/domain/imag-todo/src/ui.rs" , imagtodo) , ("../../../bin/domain/imag-todo/src/ui.rs" , imagtodo) ,
@ -129,7 +128,6 @@ fn main() {
.subcommand(build_subcommand!("init" , imaginit , version)) .subcommand(build_subcommand!("init" , imaginit , version))
.subcommand(build_subcommand!("link" , imaglink , version)) .subcommand(build_subcommand!("link" , imaglink , version))
.subcommand(build_subcommand!("log" , imaglog , version)) .subcommand(build_subcommand!("log" , imaglog , version))
.subcommand(build_subcommand!("mail" , imagmail , version))
.subcommand(build_subcommand!("mv" , imagmv , version)) .subcommand(build_subcommand!("mv" , imagmv , version))
.subcommand(build_subcommand!("notes" , imagnotes , version)) .subcommand(build_subcommand!("notes" , imagnotes , version))
.subcommand(build_subcommand!("ref" , imagref , version)) .subcommand(build_subcommand!("ref" , imagref , version))

37
bin/domain/imag-mail/Cargo.toml
View file

@ -1,37 +0,0 @@
[package]
name = "imag-mail"
version = "0.10.0"
authors = ["Matthias Beyer <mail@beyermatthias.de>"]
description = "Part of the imag core distribution: imag-mail command"
keywords = ["imag", "PIM", "personal", "information", "management"]
readme = "../../../README.md"
license = "LGPL-2.1"
documentation = "https://imag-pim.org/doc/"
repository = "https://github.com/matthiasbeyer/imag"
homepage = "http://imag-pim.org"
build = "../../../build.rs"
[badges]
travis-ci = { repository = "matthiasbeyer/imag" }
is-it-maintained-issue-resolution = { repository = "matthiasbeyer/imag" }
is-it-maintained-open-issues = { repository = "matthiasbeyer/imag" }
maintenance = { status = "actively-developed" }
[dependencies]
log = "0.4.0"
failure = "0.1"
libimagrt = { version = "0.10.0", path = "../../../lib/core/libimagrt" }
libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
libimagmail = { version = "0.10.0", path = "../../../lib/domain/libimagmail" }
libimagutil = { version = "0.10.0", path = "../../../lib/etc/libimagutil" }
[dependencies.clap]
version = "^2.29"
default-features = false
features = ["color", "suggestions", "wrap_help"]

1
bin/domain/imag-mail/README.md
View file

@ -1 +0,0 @@
../../../doc/src/04020-module-mails.md

175
bin/domain/imag-mail/src/main.rs
View file

@ -1,175 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
#![forbid(unsafe_code)]
#![deny(
non_camel_case_types,
non_snake_case,
path_statements,
trivial_numeric_casts,
unstable_features,
unused_allocation,
unused_import_braces,
unused_imports,
unused_must_use,
unused_mut,
unused_qualifications,
while_true,
)]
extern crate clap;
#[macro_use] extern crate log;
extern crate failure;
#[macro_use] extern crate libimagrt;
extern crate libimagmail;
extern crate libimagerror;
extern crate libimagutil;
use std::io::Write;
use failure::Error;
use failure::err_msg;
use libimagerror::trace::{MapErrTrace, trace_error};
use libimagerror::iter::TraceIterator;
use libimagerror::exit::ExitUnwrap;
use libimagerror::io::ToExitCode;
use libimagmail::mail::Mail;
use libimagrt::runtime::Runtime;
use libimagrt::setup::generate_runtime_setup;
use libimagutil::info_result::*;
mod ui;
use ui::build_ui;
fn main() {
let version = make_imag_version!();
let rt = generate_runtime_setup("imag-mail",
&version,
"Mail collection tool",
build_ui);
rt.cli()
.subcommand_name()
.map(|name| {
debug!("Call {}", name);
match name {
"import-mail" => import_mail(&rt),
"list" => list(&rt),
"mail-store" => mail_store(&rt),
other => {
debug!("Unknown command");
let _ = rt.handle_unknown_subcommand("imag-mail", other, rt.cli())
.map_err_trace_exit_unwrap()
.code()
.map(::std::process::exit);
}
}
});
}
fn import_mail(rt: &Runtime) {
let scmd = rt.cli().subcommand_matches("import-mail").unwrap();
let path = scmd.value_of("path").unwrap(); // enforced by clap
let mail = Mail::import_from_path(rt.store(), path)
.map_info_str("Ok")
.map_err_trace_exit_unwrap();
let _ = rt.report_touched(mail.fle().get_location()).unwrap_or_exit();
}
fn list(rt: &Runtime) {
use failure::ResultExt;
// TODO: Implement lister type in libimagmail for this
fn list_mail(rt: &Runtime, m: Mail) {
let id = match m.get_message_id() {
Ok(Some(f)) => f,
Ok(None) => "<no id>".to_owned(),
Err(e) => {
trace_error(&e);
"<error>".to_owned()
},
};
let from = match m.get_from() {
Ok(Some(f)) => f,
Ok(None) => "<no from>".to_owned(),
Err(e) => {
trace_error(&e);
"<error>".to_owned()
},
};
let to = match m.get_to() {
Ok(Some(f)) => f,
Ok(None) => "<no to>".to_owned(),
Err(e) => {
trace_error(&e);
"<error>".to_owned()
},
};
let subject = match m.get_subject() {
Ok(Some(f)) => f,
Ok(None) => "<no subject>".to_owned(),
Err(e) => {
trace_error(&e);
"<error>".to_owned()
},
};
writeln!(rt.stdout(),
"Mail: {id}\n\tFrom: {from}\n\tTo: {to}\n\t{subj}\n",
from = from,
id = id,
subj = subject,
to = to
).to_exit_code().unwrap_or_exit();
let _ = rt.report_touched(m.fle().get_location()).unwrap_or_exit();
}
let _ = rt.store()
.entries()
.map_err_trace_exit_unwrap()
.trace_unwrap_exit()
.filter(|id| id.is_in_collection(&["mail"]))
.filter_map(|id| {
rt.store()
.get(id)
.context(err_msg("Ref handling error"))
.map_err(Error::from)
.map_err_trace_exit_unwrap()
.map(|fle| Mail::from_fle(fle).map_err_trace().ok())
})
.filter_map(|e| e)
.for_each(|m| list_mail(&rt, m));
}
fn mail_store(rt: &Runtime) {
let _ = rt.cli().subcommand_matches("mail-store").unwrap();
error!("This feature is currently not implemented.");
unimplemented!()
}

74
bin/domain/imag-mail/src/ui.rs
View file

@ -1,74 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use clap::{Arg, App, SubCommand};
pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> {
app
.subcommand(SubCommand::with_name("import-mail")
.about("Import a mail (create a reference to it) (Maildir)")
.version("0.1")
.arg(Arg::with_name("path")
.long("path")
.short("p")
.takes_value(true)
.required(true)
.help("Path to the mail file or a directory which is then searched recursively")
.value_name("PATH"))
)
.subcommand(SubCommand::with_name("list")
.about("List all stored references to mails")
.version("0.1")
// TODO: Thee following four arguments are the same as in imag-ref.
// We should make these importable from libimagentryref.
.arg(Arg::with_name("check-dead")
.long("check-dead")
.short("d")
.help("Check each reference whether it is dead"))
.arg(Arg::with_name("check-changed")
.long("check-changed")
.short("c")
.help("Check whether a reference had changed (content or permissions)"))
.arg(Arg::with_name("check-changed-content")
.long("check-changed-content")
.short("C")
.help("Check whether the content of the referenced file changed"))
.arg(Arg::with_name("check-changed-permissions")
.long("check-changed-perms")
.short("P")
.help("Check whether the permissions of the referenced file changed"))
)
.subcommand(SubCommand::with_name("mail-store")
.about("Operations on (subsets of) all mails")
.version("0.1")
.subcommand(SubCommand::with_name("update-refs")
.about("Create references based on Message-IDs for all loaded mails")
.version("0.1"))
// TODO: We really should be able to filter here.
)
}

102
doc/src/05100-lib-entryref.md
View file

@ -3,52 +3,74 @@
This library crate contains functionality to generate _references_ within the This library crate contains functionality to generate _references_ within the
imag store. imag store.
A reference is a "pointer" to a file or directory on the filesystem and outside ### Problem
the store.
It differs from `libimagentrylink`/external linking as
it is designed exclusively for filesystem references, not for URLs.
A reference is created with a unique identifier, like a hash. The implementation The problem this library solves is the following: A user wants to refer to a
how this hash is calculated can be defined by the user of `libimagentryref`. file which exists on her filesystem from within imag.
But unfortunately, the user has several devices and the filesystem layout (the
way the $HOME is organized) is not the same on every device.
With this library, the user is able to refer to a file, but without specifying
the whole path.
So this library helps to resemble something like a _symlink_. Each device can have a different "base path", files are re-found via their
hashes and file names, assuming that the files are equal on different devices or
have at least the same name.
### Usage
Users have to implement the `UniqueRefPathGenerator` trait which should ### User Story / Usecase
implement a hashing functionality for pathes.
Alice has a music library on her workstation and on her notebook. On her
workstation, the music collection is at `home/alice/music`, on the notebook, it
exists in `/home/al/media/music`.
From within imag, alice wants to create a link to a file
`$music_store/Psy_trance_2018_yearmix.mp3`.
`libimagentryref` helps her, because she can provide a "base path" in the
imag configuration file of each device and then link the file. imag only stores
data about the file and its relative path, but not its abolute path.
When moving the imag store from the workstation to the notebook, the base path
for the music collection is not `/home/alice/music` anymore, but
`/home/al/media/music` and imag can find the file automatically.
### Solution, Details
libimagentryref does store the following data:
```toml
[ref]
filehash.sha1 = "<sha1 hash of the file>"
relpath = "Psy_trance_2018_yearmix.mp3"
collection = "music"
```
The filehash is stored so that libimagentryref can re-find the file whenever it
was moved. The `sha1` key is added to be able to upgrade hashes later to other
hashing algorithms.
`relpath` is the part of the path that when joined with the "base" path from
the configuration results in the full path of the file for the current machine.
The "collection" key hints to the configuration key in the imag config file.
The configuration section for the collections looks like this:
```toml
[ref.basepathes]
music = "/home/alice/music"
documents = "/home/alice/doc"
```
libimagentryref provides functionality to get the file.
libimagentryref also offers functionality to find files _only_ using their
filename (x)or filehash and correct the filehash or filename respectively
(automatically or explicitely).
### Limits ### Limits
This is _not_ intended to be a version control system or something like that. As soon as the file is renamed _and_ modified, this fails.
We also can not use _real symlinks_ as we need imag-store-objects to be able to This does also not cover the use case where the same file has different names on
link stuff. different machines.
### Usecase
This library offers functionality to refer to content outside of the store.
It can be used to refer to _nearly static stuff_ pretty easily - think of a
Maildir - you add new mails by fetching them, but you mostly do not remove
mails.
If mails get moved, they can be re-found via their hash, because Maildir objects
hardly change. Or because the hash implementation which is used to refer to them
hashes only the `Message-Id` and that does not change.
### Long-term TODO
Not implemented yet:
- [ ] Re-finding of files via their hash.
This must be implemented with several things in mind
* The user of the library should be able to provide a way how the
filesystem is searched. Basically a Functor which yields pathes to
check based on the original path of the missing file.
This enables implementations which do only search a certain subset
of pathes, or does depth-first-search rather than
breadth-first-search.
### Known problems
The functionality this library provides fails to work when syncing the imag
store between two devices where the data layout is different on each device.

14
doc/src/05100-lib-mails.md
View file

@ -1,14 +0,0 @@
## libimagmails
The mail library implements everything that is needed for beeing used to
implement a mail reader (MUA).
It therefor providea reading mailboxes, getting related content or mails, saving
attachements to external locations, crafting new mails and responses,...
It also offers, natively, ways to search for mails (which are represented as
imag entries) via tags, categories or even other metadata.
For more information on the domain of the `imag-mail` command, look at the
documentation of the @sec:modules:mails module.

7
imagrc.toml
View file

@ -349,3 +349,10 @@ default = "default"
# if this variable is _true_, imag-git will run git in $IMAG_RTP/store # if this variable is _true_, imag-git will run git in $IMAG_RTP/store
execute_in_store = false execute_in_store = false
[ref]
# configuration for imag-ref and ref using tools.
# The base pathes define the search pathes for libimagentryref
[ref.basepathes]
music = "/home/user/music"

30
lib/domain/libimagmail/Cargo.toml
View file

@ -1,30 +0,0 @@
[package]
name = "libimagmail"
version = "0.10.0"
authors = ["Matthias Beyer <mail@beyermatthias.de>"]
description = "Library for the imag core distribution"
keywords = ["imag", "PIM", "personal", "information", "management"]
readme = "../../../README.md"
license = "LGPL-2.1"
documentation = "https://imag-pim.org/doc/"
repository = "https://github.com/matthiasbeyer/imag"
homepage = "http://imag-pim.org"
[badges]
travis-ci = { repository = "matthiasbeyer/imag" }
is-it-maintained-issue-resolution = { repository = "matthiasbeyer/imag" }
is-it-maintained-open-issues = { repository = "matthiasbeyer/imag" }
maintenance = { status = "actively-developed" }
[dependencies]
log = "0.4.0"
email = "0.0.20"
filters = "0.3"
failure = "0.1"
libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
libimagentryref = { version = "0.10.0", path = "../../../lib/entry/libimagentryref" }

1
lib/domain/libimagmail/README.md
View file

@ -1 +0,0 @@
../../../doc/src/05100-lib-mails.md

55
lib/domain/libimagmail/src/iter.rs
View file

@ -1,55 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
//! Module for the MailIter
//!
//! MailIter is a iterator which takes an Iterator that yields `Ref` and yields itself
//! `Result<Mail>`, where `Err(_)` is returned if the Ref is not a Mail or parsing of the
//! referenced mail file failed.
//!
use mail::Mail;
use failure::Fallible as Result;
use libimagstore::store::FileLockEntry;
use std::marker::PhantomData;
pub struct MailIter<'a, I: Iterator<Item = FileLockEntry<'a>>> {
_marker: PhantomData<I>,
i: I,
}
impl<'a, I: Iterator<Item = FileLockEntry<'a>>> MailIter<'a, I> {
pub fn new(i: I) -> MailIter<'a, I> {
MailIter { _marker: PhantomData, i: i }
}
}
impl<'a, I: Iterator<Item = FileLockEntry<'a>>> Iterator for MailIter<'a, I> {
type Item = Result<Mail<'a>>;
fn next(&mut self) -> Option<Self::Item> {
self.i.next().map(Mail::from_fle)
}
}

51
lib/domain/libimagmail/src/lib.rs
View file

@ -1,51 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
#![forbid(unsafe_code)]
#![recursion_limit="256"]
#![deny(
dead_code,
non_camel_case_types,
non_snake_case,
path_statements,
trivial_numeric_casts,
unstable_features,
unused_allocation,
unused_import_braces,
unused_imports,
unused_must_use,
unused_mut,
unused_qualifications,
while_true,
)]
#[macro_use] extern crate log;
extern crate email;
extern crate filters;
extern crate failure;
extern crate libimagerror;
extern crate libimagstore;
extern crate libimagentryref;
pub mod iter;
pub mod mail;

201
lib/domain/libimagmail/src/mail.rs
View file

@ -1,201 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::path::Path;
use std::fs::File;
use std::io::Read;
use std::fs::OpenOptions;
use libimagstore::store::Store;
use libimagstore::storeid::StoreId;
use libimagstore::store::FileLockEntry;
use libimagentryref::reference::Ref;
use libimagentryref::refstore::RefStore;
use libimagentryref::refstore::UniqueRefPathGenerator;
use libimagerror::errors::ErrorMsg as EM;
use email::MimeMessage;
use email::results::ParsingResult as EmailParsingResult;
use failure::Fallible as Result;
use failure::ResultExt;
use failure::Error;
use failure::err_msg;
struct UniqueMailRefGenerator;
impl UniqueRefPathGenerator for UniqueMailRefGenerator {
/// The collection the `StoreId` should be created for
fn collection() -> &'static str {
"mail"
}
/// A function which should generate a unique string for a Path
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String> {
use filters::filter::Filter;
use email::Header;
let mut s = String::new();
let _ = OpenOptions::new()
.read(true)
.write(false)
.create(false)
.open(path)?
.read_to_string(&mut s)?;
MimeMessage::parse(&s)
.context(err_msg("Error creating ref"))
.map_err(Error::from)
.and_then(|mail| {
let has_key = |hdr: &Header, exp: &str| hdr.name == exp;
let subject_filter = |hdr: &Header| has_key(hdr, "Subject");
let from_filter = |hdr: &Header| has_key(hdr, "From");
let to_filter = |hdr: &Header| has_key(hdr, "To");
let filter = subject_filter.or(from_filter).or(to_filter);
let mut v : Vec<String> = vec![];
for hdr in mail.headers.iter().filter(|item| filter.filter(item)) {
let s = hdr
.get_value()
.context(err_msg("Ref creation error"))?;
v.push(s);
}
let s : String = v.join("");
Ok(s)
})
}
/// Postprocess the generated `StoreId` object
fn postprocess_storeid(sid: StoreId) -> Result<StoreId> {
Ok(sid)
}
}
struct Buffer(String);
impl Buffer {
pub fn parsed(&self) -> EmailParsingResult<MimeMessage> {
MimeMessage::parse(&self.0)
}
}
impl From<String> for Buffer {
fn from(data: String) -> Buffer {
Buffer(data)
}
}
pub struct Mail<'a>(FileLockEntry<'a>, Buffer);
impl<'a> Mail<'a> {
/// Imports a mail from the Path passed
pub fn import_from_path<P: AsRef<Path>>(store: &Store, p: P) -> Result<Mail> {
debug!("Importing Mail from path");
store.retrieve_ref::<UniqueMailRefGenerator, P>(p)
.and_then(|reference| {
debug!("Build reference file: {:?}", reference);
reference.get_path()
.context(err_msg("Ref handling error"))
.map_err(Error::from)
.and_then(|path| File::open(path).context(EM::IO).map_err(Error::from))
.and_then(|mut file| {
let mut s = String::new();
file.read_to_string(&mut s)
.map(|_| s)
.context(EM::IO)
.map_err(Error::from)
})
.map(Buffer::from)
.map(|buffer| Mail(reference, buffer))
})
}
/// Opens a mail by the passed hash
pub fn open<S: AsRef<str>>(store: &Store, hash: S) -> Result<Option<Mail>> {
debug!("Opening Mail by Hash");
store.get_ref::<UniqueMailRefGenerator, S>(hash)
.context(err_msg("Fetch by hash error"))
.context(err_msg("Fetch error"))
.map_err(Error::from)
.and_then(|o| match o {
Some(r) => Mail::from_fle(r).map(Some),
None => Ok(None),
})
}
/// Implement me as TryFrom as soon as it is stable
pub fn from_fle(fle: FileLockEntry<'a>) -> Result<Mail<'a>> {
fle.get_path()
.context(err_msg("Ref handling error"))
.map_err(Error::from)
.and_then(|path| File::open(path).context(EM::IO).map_err(Error::from))
.and_then(|mut file| {
let mut s = String::new();
file.read_to_string(&mut s)
.map(|_| s)
.context(EM::IO)
.map_err(Error::from)
})
.map(Buffer::from)
.map(|buffer| Mail(fle, buffer))
}
pub fn get_field(&self, field: &str) -> Result<Option<String>> {
debug!("Getting field in mail: {:?}", field);
self.1
.parsed()
.context(err_msg("Mail parsing error"))
.map_err(Error::from)
.map(|parsed| {
parsed.headers
.iter()
.filter(|hdr| hdr.name == field)
.nth(0)
.and_then(|field| field.get_value().ok())
})
}
pub fn get_from(&self) -> Result<Option<String>> {
self.get_field("From")
}
pub fn get_to(&self) -> Result<Option<String>> {
self.get_field("To")
}
pub fn get_subject(&self) -> Result<Option<String>> {
self.get_field("Subject")
}
pub fn get_message_id(&self) -> Result<Option<String>> {
self.get_field("Message-ID")
}
pub fn get_in_reply_to(&self) -> Result<Option<String>> {
self.get_field("In-Reply-To")
}
pub fn fle(&self) -> &FileLockEntry<'a> {
&self.0
}
}

8
lib/entry/libimagentrymarkdown/Cargo.toml
View file

@ -25,15 +25,11 @@ hoedown = "6.0.0"
url = "1.5" url = "1.5"
env_logger = "0.5" env_logger = "0.5"
failure = "0.1" failure = "0.1"
sha-1 = "0.8"
libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" } libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" } libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink/" } libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink/" }
libimagentryref = { version = "0.10.0", path = "../../../lib/entry/libimagentryref/" }
libimagutil = { version = "0.10.0", path = "../../../lib/etc/libimagutil/" } libimagutil = { version = "0.10.0", path = "../../../lib/etc/libimagutil/" }
[dependencies.libimagentryref]
version = "0.10.0"
path = "../../../lib/entry/libimagentryref/"
default-features = false
features = [ "generators", "generators-sha512" ]

1
lib/entry/libimagentrymarkdown/src/lib.rs
View file

@ -46,6 +46,7 @@ extern crate libimagentryref;
extern crate libimagutil; extern crate libimagutil;
#[macro_use] extern crate failure; #[macro_use] extern crate failure;
#[macro_use] extern crate log; #[macro_use] extern crate log;
extern crate sha1;
#[cfg(test)] #[cfg(test)]
extern crate env_logger; extern crate env_logger;

105
lib/entry/libimagentrymarkdown/src/processor.rs
View file

@ -17,7 +17,7 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// //
use std::path::Path; use std::collections::BTreeMap;
use failure::Fallible as Result; use failure::Fallible as Result;
use failure::ResultExt; use failure::ResultExt;
@ -26,34 +26,18 @@ use link::extract_links;
use libimagentrylink::external::ExternalLinker; use libimagentrylink::external::ExternalLinker;
use libimagentrylink::internal::InternalLinker; use libimagentrylink::internal::InternalLinker;
use libimagentryref::refstore::RefStore; use libimagentryref::reference::MutRef;
use libimagentryref::refstore::UniqueRefPathGenerator; use libimagentryref::reference::RefFassade;
use libimagentryref::generators::sha512::Sha512; use libimagentryref::hasher::sha1::Sha1Hasher;
use libimagstore::store::Entry; use libimagstore::store::Entry;
use libimagstore::store::Store; use libimagstore::store::Store;
use libimagstore::storeid::StoreId; use libimagstore::storeid::StoreId;
use libimagerror::errors::ErrorMsg;
use std::path::PathBuf; use std::path::PathBuf;
use url::Url; use url::Url;
pub struct UniqueMarkdownRefGenerator;
impl UniqueRefPathGenerator for UniqueMarkdownRefGenerator {
fn collection() -> &'static str {
"ref" // we can only use this collection, as we don't know about context
}
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String> {
Sha512::unique_hash(path).map_err(Error::from)
}
fn postprocess_storeid(sid: StoreId) -> Result<StoreId> {
Ok(sid) // don't do anything
}
}
/// A link Processor which collects the links from a Markdown and passes them on to /// A link Processor which collects the links from a Markdown and passes them on to
/// `libimagentrylink` functionality /// `libimagentrylink` functionality
/// ///
@ -117,11 +101,37 @@ impl LinkProcessor {
/// Process an Entry for its links /// Process an Entry for its links
/// ///
///
/// # Notice
///
/// Whenever a "ref" is created, that means when a URL points to a filesystem path (normally
/// when using `file:///home/user/foobar.file` for example), the _current_ implementation uses
/// libimagentryref to create make the entry into a ref.
///
/// The configuration of the `libimagentryref::reference::Reference::make_ref()` call is as
/// follows:
///
/// * Name of the collection: "root"
/// * Configuration: `{"root": "/"}`
///
/// This implementation might change in the future, so that the configuration and the name of
/// the collection can be passed to the function, or in a way that the user is asked what to do
/// during the runtime of this function.
///
///
/// # Warning /// # Warning
/// ///
/// When `LinkProcessor::create_internal_targets()` was called to set the setting to true, this /// When `LinkProcessor::create_internal_targets()` was called to set the setting to true, this
/// function returns all errors returned by the Store. /// function returns all errors returned by the Store.
/// ///
/// That means:
///
/// * For an internal link, the linked target is created if create_internal_targets() is true,
/// else error
/// * For an external link, if create_internal_targets() is true, libimagentrylink creates the
/// external link entry, else the link is ignored
/// * all other cases do not create elements in the store
///
pub fn process<'a>(&self, entry: &mut Entry, store: &'a Store) -> Result<()> { pub fn process<'a>(&self, entry: &mut Entry, store: &'a Store) -> Result<()> {
let text = entry.to_str()?; let text = entry.to_str()?;
trace!("Processing: {:?}", entry.get_location()); trace!("Processing: {:?}", entry.get_location());
@ -151,18 +161,58 @@ impl LinkProcessor {
entry.add_external_link(store, url)?; entry.add_external_link(store, url)?;
}, },
LinkQualification::RefLink(url) => { LinkQualification::RefLink(url) => {
use sha1::{Sha1, Digest};
if !self.process_refs { if !self.process_refs {
trace!("Not processing refs... continue...");
continue continue
} }
// because we can make one entry only into _one_ ref, but a markdown document
// might contain several "ref" links, we create a new entry for the ref we're
// about to create
//
// We generate the StoreId with the SHA1 hash of the path, which is the best
// option we have
// right now
//
// TODO: Does this make sense? Can we improve this?
let path = url.host_str().unwrap_or_else(|| url.path());
let path = PathBuf::from(path);
let ref_entry_id = {
let digest = Sha1::digest(path.to_str().ok_or(ErrorMsg::UTF8Error)?.as_bytes());
StoreId::new(PathBuf::from(format!("ref/{:x}", digest)))? // TODO: Ugh...
};
let mut ref_entry = store.retrieve(ref_entry_id)?;
let ref_collection_name = "root";
// TODO: Maybe this can be a const?
// TODO: Maybe we need this ot be overrideable? Not sure.
let ref_collection_config = {
let mut map = BTreeMap::new();
map.insert(String::from("root"), PathBuf::from("/"));
::libimagentryref::reference::Config::new(map)
};
trace!("URL = {:?}", url); trace!("URL = {:?}", url);
trace!("URL.path() = {:?}", url.path()); trace!("URL.path() = {:?}", url.path());
trace!("URL.host_str() = {:?}", url.host_str()); trace!("URL.host_str() = {:?}", url.host_str());
let path = url.host_str().unwrap_or_else(|| url.path());
let path = PathBuf::from(path);
let mut target = store.create_ref::<UniqueMarkdownRefGenerator, PathBuf>(path)?;
entry.add_internal_link(&mut target)?; trace!("Processing ref: {:?} -> {path}, collection: {ref_collection_name}, cfg: {cfg:?}",
path = path.display(),
ref_collection_name = ref_collection_name,
cfg = ref_collection_config);
ref_entry.as_ref_with_hasher_mut::<Sha1Hasher>()
.make_ref(path,
ref_collection_name,
&ref_collection_config,
false)?;
trace!("Ready processing, linking new ref entry...");
let _ = entry.add_internal_link(&mut ref_entry)?;
}, },
LinkQualification::Undecidable(e) => { LinkQualification::Undecidable(e) => {
// error // error
@ -188,9 +238,11 @@ enum LinkQualification {
impl LinkQualification { impl LinkQualification {
fn qualify(text: &str) -> LinkQualification { fn qualify(text: &str) -> LinkQualification {
trace!("Qualifying: {}", text);
match Url::parse(text) { match Url::parse(text) {
Ok(url) => { Ok(url) => {
if url.scheme() == "file" { if url.scheme() == "file" {
trace!("Qualifying = RefLink");
return LinkQualification::RefLink(url) return LinkQualification::RefLink(url)
} }
@ -205,6 +257,7 @@ impl LinkQualification {
Err(e) => { Err(e) => {
match e { match e {
::url::ParseError::RelativeUrlWithoutBase => { ::url::ParseError::RelativeUrlWithoutBase => {
trace!("Qualifying = InternalLink");
LinkQualification::InternalLink LinkQualification::InternalLink
}, },
@ -438,7 +491,7 @@ mod tests {
assert!(entries.is_ok()); assert!(entries.is_ok());
let entries : Vec<_> = entries.unwrap().into_storeid_iter().collect(); let entries : Vec<_> = entries.unwrap().into_storeid_iter().collect();
assert_eq!(2, entries.len(), "Expected 2 links, got: {:?}", entries); assert_eq!(2, entries.len(), "Expected 1 entries, got: {:?}", entries);
debug!("{:?}", entries); debug!("{:?}", entries);
} }

20
lib/entry/libimagentryref/Cargo.toml
View file

@ -22,25 +22,17 @@ maintenance = { status = "actively-developed" }
[dependencies] [dependencies]
itertools = "0.7" itertools = "0.7"
log = "0.4.0" log = "0.4.0"
failure = "0.1"
sha-1 = "0.8"
toml = "0.4" toml = "0.4"
toml-query = "0.8" toml-query = "0.8"
failure = "0.1" serde = "1"
sha-1 = { version = "0.7", optional = true } serde_derive = "1"
sha2 = { version = "0.7", optional = true }
sha3 = { version = "0.7", optional = true }
hex = { version = "0.3", optional = true }
libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" } libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" } libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
libimagentryutil = { version = "0.10.0", path = "../../../lib/entry/libimagentryutil" } libimagentryutil = { version = "0.10.0", path = "../../../lib/entry/libimagentryutil" }
[features] [dev-dependencies]
default = [] env_logger = "0.5"
generators = []
generators-sha1 = ["sha-1", "hex"]
generators-sha224 = ["sha2", "hex"]
generators-sha256 = ["sha2", "hex"]
generators-sha384 = ["sha2", "hex"]
generators-sha512 = ["sha2", "hex"]
generators-sha3 = ["sha3", "hex"]

276
lib/entry/libimagentryref/src/generators/mod.rs
View file

@ -1,276 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
//! Default generators
//!
//! This module provides a number of default `UniqueRefPathGenerator`s
//! which can be used for generating references.
//!
//! These generators are _NOT_ domain specific. So there won't be a "UniqueMailRefPathGenerator" in
//! here, for example.
//!
//! All these generators use "ref" as collection name.
//! They can be overridden using the `make_unique_ref_path_generator!()` convenience macro.
//!
//! # Note
//!
//! You must enable the appropriate crate feature to use any of the provided generators. With the
//! `generators` feature, you only get the convenience macro `make_unique_ref_path_generator!()`.
//!
/// A convenience macro for wrapping a generator in a new one, reusing the functionality from the
/// underlying generator
///
/// The UniqueRefPathGenerator must be in scope.
///
/// The macro creates a new struct `$name` over `$underlying` and changes the collection name to
/// `$collectionname`.
/// If passed, the new implementation is used (defaults to the implementation from the underlying
/// generator).
/// If passed, the new postprocessing is used (defaults to not changing the StoreId)
///
#[macro_export]
macro_rules! make_unique_ref_path_generator {
(
$name:ident
over $underlying:ty
=> with collection name $collectionname:expr
) => {
struct $name;
impl $crate::refstore::UniqueRefPathGenerator for $name {
fn collection() -> &'static str {
$collectionname
}
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String> {
$underlying::unique_hash(path)
}
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
-> Result<::libimagstore::storeid::StoreId>
{
Ok(sid)
}
}
};
(
$name:ident
over $underlying:ty
=> with collection name $collectionname:expr
=> $impl:expr
) => {
struct $name;
impl $crate::refstore::UniqueRefPathGenerator for $name {
type Error = $errtype;
fn collection() -> &'static str {
$collectionname
}
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String> {
debug!("Making unique hash for path: {:?}", path.as_ref());
$impl(path)
}
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
-> Result<::libimagstore::storeid::StoreId>
{
Ok(sid)
}
}
};
(
pub $name:ident
over $underlying:ty
=> with collection name $collectionname:expr
=> $impl:expr
) => {
make_unique_ref_path_generator!(
pub $name
over $underlying
=> with collection name $collectionname
=> $impl => |sid| { Ok(sid) }
);
};
(
pub $name:ident
over $underlying:ty
=> with collection name $collectionname:expr
=> $impl:expr
=> $postproc:expr
) => {
pub struct $name;
impl $crate::refstore::UniqueRefPathGenerator for $name {
fn collection() -> &'static str {
$collectionname
}
fn unique_hash<A: AsRef<Path>>(path: A) -> ::failure::Fallible<String> {
debug!("Making unique hash for path: {:?}", path.as_ref());
$impl(path)
}
fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
-> ::failure::Fallible<::libimagstore::storeid::StoreId>
{
$postproc(sid)
}
}
};
}
#[cfg(any(
feature = "generators-sha1",
feature = "generators-sha224",
feature = "generators-sha256",
feature = "generators-sha384",
feature = "generators-sha512",
feature = "generators-sha3",
))]
mod base;
/// Helper macro for generating implementations for the various Sha algorithms
macro_rules! make_sha_mod {
{
$modname:ident,
$hashname:ident,
$hashingimpl:expr
} => {
pub mod $modname {
use std::path::Path;
use std::fs::OpenOptions;
use std::io::Read;
use hex;
make_unique_ref_path_generator! (
pub $hashname
over generators::base::Base
=> with collection name "ref"
=> |path| {
OpenOptions::new()
.read(true)
.write(false)
.create(false)
.open(path)
.map_err(::failure::Error::from)
.and_then(|mut file| {
let mut buffer = String::new();
let _ = file.read_to_string(&mut buffer)?;
$hashingimpl(buffer)
})
}
);
impl $hashname {
/// Function which can be used by a wrapping UniqueRefPathGenerator to hash only N bytes.
pub fn hash_n_bytes<A: AsRef<Path>>(path: A, n: usize) -> ::failure::Fallible<String> {
debug!("Opening '{}' for hashing", path.as_ref().display());
OpenOptions::new()
.read(true)
.write(false)
.create(false)
.open(path)
.map_err(::failure::Error::from)
.and_then(|mut file| {
let mut buffer = vec![0; n];
debug!("Allocated {} bytes", buffer.capacity());
match file.read_exact(&mut buffer) {
Ok(_) => { /* yay */ Ok(()) },
Err(e) => if e.kind() == ::std::io::ErrorKind::UnexpectedEof {
debug!("Ignoring unexpected EOF before {} bytes were read", n);
Ok(())
} else {
Err(e)
}
}?;
let buffer = String::from_utf8(buffer)?;
$hashingimpl(buffer)
})
}
}
}
}
}
#[cfg(feature = "generators-sha1")]
make_sha_mod! {
sha1, Sha1, |buffer: String| {
use sha1::{Sha1, Digest};
trace!("Hashing: '{:?}'", buffer);
let res = hex::encode(Sha1::digest(buffer.as_bytes()));
trace!("Hash => '{:?}'", res);
Ok(res)
}
}
#[cfg(feature = "generators-sha224")]
make_sha_mod! {
sha224, Sha224, |buffer: String| {
use sha2::{Sha224, Digest};
Ok(hex::encode(Sha224::digest(buffer.as_bytes())))
}
}
#[cfg(feature = "generators-sha256")]
make_sha_mod! {
sha256, Sha256, |buffer: String| {
use sha2::{Sha256, Digest};
Ok(hex::encode(Sha256::digest(buffer.as_bytes())))
}
}
#[cfg(feature = "generators-sha384")]
make_sha_mod! {
sha384, Sha384, |buffer: String| {
use sha2::{Sha384, Digest};
Ok(hex::encode(Sha384::digest(buffer.as_bytes())))
}
}
#[cfg(feature = "generators-sha512")]
make_sha_mod! {
sha512, Sha512, |buffer: String| {
use sha2::{Sha512, Digest};
Ok(hex::encode(Sha512::digest(buffer.as_bytes())))
}
}
#[cfg(feature = "generators-sha3")]
make_sha_mod! {
sha3, Sha3, |buffer: String| {
use sha3::{Sha3_256, Digest};
Ok(hex::encode(Sha3_256::digest(buffer.as_bytes())))
}
}

50
lib/entry/libimagentryref/src/generators/base.rs → lib/entry/libimagentryref/src/hasher.rs
View file

@ -19,29 +19,37 @@
use std::path::Path; use std::path::Path;
use libimagstore::storeid::StoreId;
use refstore::UniqueRefPathGenerator;
use failure::Fallible as Result; use failure::Fallible as Result;
/// A base UniqueRefPathGenerator which must be overridden by the actual UniqueRefPathGenerator pub trait Hasher {
/// which is provided by this crate const NAME: &'static str;
#[allow(dead_code)]
pub struct Base; /// hash the file at path `path`
fn hash<P: AsRef<Path>>(path: P) -> Result<String>;
}
pub mod default {
pub use super::sha1::Sha1Hasher as DefaultHasher;
}
pub mod sha1 {
use std::path::Path;
use failure::Fallible as Result;
use sha1::{Sha1, Digest};
use hasher::Hasher;
pub struct Sha1Hasher;
impl Hasher for Sha1Hasher {
const NAME : &'static str = "sha1";
fn hash<P: AsRef<Path>>(path: P) -> Result<String> {
let digest = Sha1::digest(::std::fs::read_to_string(path)?.as_bytes());
Ok(format!("{:x}", digest)) // TODO: Ugh...
}
}
impl UniqueRefPathGenerator for Base {
fn collection() -> &'static str {
"ref"
}
fn unique_hash<A: AsRef<Path>>(_path: A) -> Result<String> {
// This has to be overridden
panic!("Not overridden base functionality. This is a BUG!")
}
fn postprocess_storeid(sid: StoreId) -> Result<StoreId> {
Ok(sid) // default implementation
}
} }

42
lib/entry/libimagentryref/src/lib.rs
View file

@ -41,41 +41,17 @@
extern crate itertools; extern crate itertools;
extern crate toml; extern crate toml;
extern crate toml_query; extern crate toml_query;
#[macro_use] extern crate serde_derive;
#[macro_use] extern crate libimagstore;
extern crate libimagerror;
#[macro_use] extern crate libimagentryutil;
extern crate failure;
module_entry_path_mod!("ref");
pub mod reference;
pub mod refstore;
#[cfg(feature = "generators-sha1")]
extern crate sha1; extern crate sha1;
#[cfg(any( extern crate libimagstore;
feature = "generators-sha224", extern crate libimagerror;
feature = "generators-sha256", #[macro_use] extern crate libimagentryutil;
feature = "generators-sha384", #[macro_use] extern crate failure;
feature = "generators-sha512",
))]
extern crate sha2;
#[cfg(feature = "generators-sha3")] #[cfg(test)]
extern crate sha3; extern crate env_logger;
#[cfg(any( pub mod hasher;
feature = "generators-sha1", pub mod reference;
feature = "generators-sha224",
feature = "generators-sha256",
feature = "generators-sha384",
feature = "generators-sha512",
feature = "generators-sha3",
))]
extern crate hex;
#[cfg(feature = "generators")]
pub mod generators;

506
lib/entry/libimagentryref/src/reference.rs
View file

@ -17,15 +17,13 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// //
//! The Ref object is a helper over the link functionality, so one is able to create references to
//! files outside of the imag store.
use std::path::Path; use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use std::collections::BTreeMap;
use std::ops::Deref;
use libimagentryutil::isa::Is; use libimagentryutil::isa::Is;
use libimagentryutil::isa::IsKindHeaderPathProvider; use libimagentryutil::isa::IsKindHeaderPathProvider;
use libimagstore::store::Entry;
use libimagerror::errors::ErrorMsg as EM; use libimagerror::errors::ErrorMsg as EM;
use toml::Value; use toml::Value;
@ -34,8 +32,106 @@ use toml_query::delete::TomlValueDeleteExt;
use toml_query::insert::TomlValueInsertExt; use toml_query::insert::TomlValueInsertExt;
use failure::Fallible as Result; use failure::Fallible as Result;
use failure::Error; use failure::Error;
use failure::err_msg;
use failure::ResultExt;
use hasher::Hasher;
/// A configuration of "collection name" -> "collection path" mappings
///
/// Should be deserializeable from the configuration file right away, because we expect a
/// configuration like this in the config file:
///
/// ```toml
/// [ref.collections]
/// music = "/home/alice/music"
/// documents = "/home/alice/doc"
/// ```
///
/// for example.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Config(BTreeMap<String, PathBuf>);
impl Config {
pub fn new(map: BTreeMap<String, PathBuf>) -> Self {
Config(map)
}
}
impl Deref for Config {
type Target = BTreeMap<String, PathBuf>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
provide_kindflag_path!(pub IsRef, "ref.is_ref");
/// Fassade module
///
/// This module is necessary to build a generic fassade around the "entry with a (default) hasher to
/// represent the entry as a ref".
///
/// The module is for code-structuring only, all types in the module are exported publicly in the
/// supermodule.
pub mod fassade {
use std::marker::PhantomData;
use libimagstore::store::Entry;
use libimagentryutil::isa::Is;
use failure::Fallible as Result;
use failure::Error;
use hasher::sha1::Sha1Hasher;
use hasher::Hasher;
use super::IsRef;
pub trait RefFassade {
fn is_ref(&self) -> Result<bool>;
fn as_ref_with_hasher<H: Hasher>(&self) -> RefWithHasher<H>;
fn as_ref_with_hasher_mut<H: Hasher>(&mut self) -> MutRefWithHasher<H>;
}
impl RefFassade for Entry {
/// Check whether the underlying object is actually a ref
fn is_ref(&self) -> Result<bool> {
self.is::<IsRef>().map_err(Error::from)
}
fn as_ref_with_hasher<H: Hasher>(&self) -> RefWithHasher<H> {
RefWithHasher::new(self)
}
fn as_ref_with_hasher_mut<H: Hasher>(&mut self) -> MutRefWithHasher<H> {
MutRefWithHasher::new(self)
}
}
pub struct RefWithHasher<'a, H: Hasher = Sha1Hasher>(pub(crate) &'a Entry, PhantomData<H>);
impl<'a, H> RefWithHasher<'a, H>
where H: Hasher
{
fn new(entry: &'a Entry) -> Self {
RefWithHasher(entry, PhantomData)
}
}
pub struct MutRefWithHasher<'a, H: Hasher = Sha1Hasher>(pub(crate) &'a mut Entry, PhantomData<H>);
impl<'a, H> MutRefWithHasher<'a, H>
where H: Hasher
{
fn new(entry: &'a mut Entry) -> Self {
MutRefWithHasher(entry, PhantomData)
}
}
}
pub use self::fassade::*;
use refstore::UniqueRefPathGenerator;
pub trait Ref { pub trait Ref {
@ -43,76 +139,41 @@ pub trait Ref {
fn is_ref(&self) -> Result<bool>; fn is_ref(&self) -> Result<bool>;
/// Get the stored hash. /// Get the stored hash.
///
/// Does not need a `UniqueRefPathGenerator` as it reads the hash stored in the header
fn get_hash(&self) -> Result<&str>;
/// Make this object a ref
fn make_ref<P: AsRef<Path>>(&mut self, hash: String, path: P) -> Result<()>;
/// Get the referenced path.
///
/// Does not need a `UniqueRefPathGenerator` as it reads the path stored in the header.
fn get_path(&self) -> Result<PathBuf>; fn get_path(&self) -> Result<PathBuf>;
/// Get the stored hash.
fn get_hash(&self) -> Result<&str>;
/// Check whether the referenced file still matches its hash /// Check whether the referenced file still matches its hash
fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> Result<bool>; fn hash_valid(&self, config: &Config) -> Result<bool>;
fn remove_ref(&mut self) -> Result<()>;
/// Alias for `r.fs_link_exists() && r.deref().is_file()`
fn is_ref_to_file(&self) -> Result<bool> {
self.get_path().map(|p| p.is_file())
}
/// Alias for `r.fs_link_exists() && r.deref().is_dir()`
fn is_ref_to_dir(&self) -> Result<bool> {
self.get_path().map(|p| p.is_dir())
}
/// Alias for `!Ref::fs_link_exists()`
fn is_dangling(&self) -> Result<bool> {
self.get_path().map(|p| !p.exists())
}
} }
provide_kindflag_path!(pub IsRef, "ref.is_ref"); impl<'a, H: Hasher> Ref for RefWithHasher<'a, H> {
impl Ref for Entry {
/// Check whether the underlying object is actually a ref /// Check whether the underlying object is actually a ref
fn is_ref(&self) -> Result<bool> { fn is_ref(&self) -> Result<bool> {
self.is::<IsRef>().map_err(Error::from) self.0.is::<IsRef>().map_err(Error::from)
} }
fn get_hash(&self) -> Result<&str> { fn get_hash(&self) -> Result<&str> {
self.get_header() let header_path = format!("ref.hash.{}", H::NAME);
.read("ref.hash") self.0
.get_header()
.read(&header_path)
.map_err(Error::from)? .map_err(Error::from)?
.ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.hash"))) .ok_or_else(|| {
Error::from(EM::EntryHeaderFieldMissing("ref.hash.<hash>"))
})
.and_then(|v| { .and_then(|v| {
v.as_str().ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash", "string"))) v.as_str().ok_or_else(|| {
Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string"))
})
}) })
} }
fn make_ref<P: AsRef<Path>>(&mut self, hash: String, path: P) -> Result<()> {
let path_str : String = path
.as_ref()
.to_str()
.map(String::from)
.ok_or_else(|| EM::UTF8Error)?;
let _ = self.set_isflag::<IsRef>()?;
let hdr = self.get_header_mut();
hdr.insert("ref.path", Value::String(String::from(path_str)))?;
hdr.insert("ref.hash", Value::String(hash))?;
Ok(())
}
fn get_path(&self) -> Result<PathBuf> { fn get_path(&self) -> Result<PathBuf> {
self.get_header() self.0
.get_header()
.read("ref.path") .read("ref.path")
.map_err(Error::from)? .map_err(Error::from)?
.ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.path"))) .ok_or_else(|| Error::from(EM::EntryHeaderFieldMissing("ref.path")))
@ -124,20 +185,323 @@ impl Ref for Entry {
.map(PathBuf::from) .map(PathBuf::from)
} }
fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> Result<bool> { fn hash_valid(&self, config: &Config) -> Result<bool> {
self.get_path() let ref_header = self.0
.map(PathBuf::from) .get_header()
.map_err(Error::from) .read("ref")?
.and_then(|pb| RPG::unique_hash(pb)) .ok_or_else(|| err_msg("Header missing at 'ref'"))?;
.and_then(|h| Ok(h == self.get_hash()?))
}
fn remove_ref(&mut self) -> Result<()> { let collection_name = ref_header
let hdr = self.get_header_mut(); .read("collection")
let _ = hdr.delete("ref.hash")?; .map_err(Error::from)?
let _ = hdr.delete("ref.path")?; .ok_or_else(|| err_msg("Header missing at 'ref.collection'"))?
let _ = hdr.delete("ref")?; .as_str()
Ok(()) .ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string")))?;
let path = ref_header
.read("path")
.map_err(Error::from)?
.ok_or_else(|| err_msg("Header missing at 'ref.path'"))?
.as_str()
.map(PathBuf::from)
.ok_or_else(|| Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string")))?;
let file_path = get_file_path(config, collection_name.as_ref(), &path)?;
ref_header
.read(H::NAME)
.map_err(Error::from)?
.ok_or_else(|| format_err!("Header missing at 'ref.{}'", H::NAME))
.and_then(|v| {
v.as_str().ok_or_else(|| {
Error::from(EM::EntryHeaderTypeError2("ref.hash.<hash>", "string"))
})
})
.and_then(|hash| H::hash(file_path).map(|h| h == hash))
}
}
pub trait MutRef {
fn remove_ref(&mut self) -> Result<()>;
/// Make a ref out of a normal (non-ref) entry.
///
/// If the entry is already a ref, this fails if `force` is false
fn make_ref<P, Coll>(&mut self, path: P, collection_name: Coll, config: &Config, force: bool)
-> Result<()>
where P: AsRef<Path>,
Coll: AsRef<str>;
}
impl<'a, H> MutRef for MutRefWithHasher<'a, H>
where H: Hasher
{
fn remove_ref(&mut self) -> Result<()> {
debug!("Removing 'ref' header section");
{
let header = self.0.get_header_mut();
trace!("header = {:?}", header);
let _ = header.delete("ref.relpath").context("Removing ref.relpath")?;
if let Some(hash_tbl) = header.read_mut("ref.hash")? {
match hash_tbl {
Value::Table(ref mut tbl) => *tbl = BTreeMap::new(),
_ => {
// should not happen
}
}
}
let _ = header.delete("ref.hash").context("Removing ref.hash")?;
let _ = header.delete("ref.collection").context("Removing ref.collection")?;
}
debug!("Removing 'ref' header marker");
self.0.remove_isflag::<IsRef>().context("Removing ref")?;
let _ = self.0
.get_header_mut()
.delete("ref")
.context("Removing ref")?;
trace!("header = {:?}", self.0.get_header());
Ok(())
}
/// Make a ref out of a normal (non-ref) entry.
///
/// `path` is the path to refer to,
///
/// # Warning
///
/// If the entry is already a ref, this fails if `force` is false
///
fn make_ref<P, Coll>(&mut self, path: P, collection_name: Coll, config: &Config, force: bool)
-> Result<()>
where P: AsRef<Path>,
Coll: AsRef<str>
{
trace!("Making ref out of {:?}", self.0);
trace!("Making ref with collection name {:?}", collection_name.as_ref());
trace!("Making ref with config {:?}", config);
trace!("Making ref forced = {}", force);
if self.0.get_header().read("ref.is_ref")?.is_some() && !force {
debug!("Entry is already a Ref!");
let _ = Err(err_msg("Entry is already a reference")).context("Making ref out of entry")?;
}
let file_path = get_file_path(config, collection_name.as_ref(), &path)?;
if !file_path.exists() {
let msg = format_err!("File '{:?}' does not exist", file_path);
let _ = Err(msg).context("Making ref out of entry")?;
}
debug!("Entry hashing...");
let _ = H::hash(&file_path)
.and_then(|hash| make_header_section(hash, H::NAME, path, collection_name))
.and_then(|h| self.0.get_header_mut().insert("ref", h).map_err(Error::from))
.and_then(|_| self.0.set_isflag::<IsRef>())
.context("Making ref out of entry")?;
debug!("Setting is-ref flag");
self.0
.set_isflag::<IsRef>()
.context("Setting ref-flag")
.map_err(Error::from)
.map(|_| ())
}
}
/// Create a new header section for a "ref".
///
/// # Warning
///
/// The `relpath` _must_ be relative to the configured path for that collection.
pub(crate) fn make_header_section<P, C, H>(hash: String, hashname: H, relpath: P, collection: C)
-> Result<Value>
where P: AsRef<Path>,
C: AsRef<str>,
H: AsRef<str>,
{
let mut header_section = Value::Table(BTreeMap::new());
{
let relpath = relpath
.as_ref()
.to_str()
.map(String::from)
.ok_or_else(|| {
let msg = format_err!("UTF Error in '{:?}'", relpath.as_ref());
Error::from(msg)
})?;
let _ = header_section.insert("relpath", Value::String(relpath))?;
}
{
let mut hash_table = Value::Table(BTreeMap::new());
let _ = hash_table.insert(hashname.as_ref(), Value::String(hash))?;
let _ = header_section.insert("hash", hash_table)?;
}
let _ = header_section.insert("collection", Value::String(String::from(collection.as_ref())));
Ok(header_section)
}
fn get_file_path<P>(config: &Config, collection_name: &str, path: P) -> Result<PathBuf>
where P: AsRef<Path>
{
config
.get(collection_name)
.map(PathBuf::clone)
.ok_or_else(|| {
format_err!("Configuration missing for collection: '{}'", collection_name)
})
.context("Making ref out of entry")
.map_err(Error::from)
.map(|p| {
let filepath = p.join(&path);
trace!("Found filepath: {:?}", filepath.display());
filepath
})
}
#[cfg(test)]
mod test {
use std::path::PathBuf;
use libimagstore::store::Store;
use libimagstore::store::Entry;
use super::*;
use hasher::Hasher;
fn setup_logging() {
let _ = ::env_logger::try_init();
}
pub fn get_store() -> Store {
Store::new_inmemory(PathBuf::from("/"), &None).unwrap()
}
struct TestHasher;
impl Hasher for TestHasher {
const NAME: &'static str = "Testhasher";
fn hash<P: AsRef<Path>>(path: P) -> Result<String> {
path.as_ref()
.to_str()
.map(String::from)
.ok_or_else(|| Error::from(err_msg("Failed to create test hash")))
}
}
#[test]
fn test_isref() {
setup_logging();
let store = get_store();
let entry = store.retrieve(PathBuf::from("test_isref")).unwrap();
assert!(!entry.is_ref().unwrap());
}
#[test]
fn test_makeref() {
setup_logging();
let store = get_store();
let mut entry = store.retrieve(PathBuf::from("test_makeref")).unwrap();
let file = PathBuf::from("/"); // has to exist
let collection_name = "some_collection";
let config = Config({
let mut c = BTreeMap::new();
c.insert(String::from("some_collection"), PathBuf::from("/tmp"));
c
});
let r = entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false);
assert!(r.is_ok());
}
#[test]
fn test_makeref_isref() {
setup_logging();
let store = get_store();
let mut entry = store.retrieve(PathBuf::from("test_makeref_isref")).unwrap();
let file = PathBuf::from("/"); // has to exists
let collection_name = "some_collection";
let config = Config({
let mut c = BTreeMap::new();
c.insert(String::from("some_collection"), PathBuf::from("/tmp"));
c
});
let res = entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false);
assert!(res.is_ok(), "Expected to be ok: {:?}", res);
assert!(entry.as_ref_with_hasher::<TestHasher>().is_ref().unwrap());
}
#[test]
fn test_makeref_is_ref_with_testhash() {
setup_logging();
let store = get_store();
let mut entry = store.retrieve(PathBuf::from("test_makeref_is_ref_with_testhash")).unwrap();
let file = PathBuf::from("/"); // has to exist
let collection_name = "some_collection";
let config = Config({
let mut c = BTreeMap::new();
c.insert(String::from("some_collection"), PathBuf::from("/"));
c
});
assert!(entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false).is_ok());
let check_isstr = |entry: &Entry, location, shouldbe| {
let var = entry.get_header().read(location);
assert!(var.is_ok(), "{} is not Ok(_): {:?}", location, var);
let var = var.unwrap();
assert!(var.is_some(), "{} is not Some(_): {:?}", location, var);
let var = var.unwrap().as_str();
assert!(var.is_some(), "{} is not String: {:?}", location, var);
assert_eq!(var.unwrap(), shouldbe, "{} is not == {}", location, shouldbe);
};
check_isstr(&entry, "ref.relpath", "/");
check_isstr(&entry, "ref.hash.Testhasher", "/"); // TestHasher hashes by returning the path itself
check_isstr(&entry, "ref.collection", "some_collection");
}
#[test]
fn test_makeref_remref() {
setup_logging();
let store = get_store();
let mut entry = store.retrieve(PathBuf::from("test_makeref_remref")).unwrap();
let file = PathBuf::from("/"); // has to exist
let collection_name = "some_collection";
let config = Config({
let mut c = BTreeMap::new();
c.insert(String::from("some_collection"), PathBuf::from("/"));
c
});
assert!(entry.as_ref_with_hasher_mut::<TestHasher>().make_ref(file, collection_name, &config, false).is_ok());
assert!(entry.as_ref_with_hasher::<TestHasher>().is_ref().unwrap());
let res = entry.as_ref_with_hasher_mut::<TestHasher>().remove_ref();
assert!(res.is_ok(), "Expected to be ok: {:?}", res);
assert!(!entry.as_ref_with_hasher::<TestHasher>().is_ref().unwrap());
} }
} }

128
lib/entry/libimagentryref/src/refstore.rs
View file

@ -1,128 +0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::path::Path;
use std::path::PathBuf;
use libimagstore::store::FileLockEntry;
use libimagstore::store::Store;
use libimagstore::storeid::StoreId;
use reference::Ref;
use failure::Fallible as Result;
use failure::Error;
/// A UniqueRefPathGenerator generates unique Pathes
///
/// It is basically a functor which generates a StoreId from a &Path.
/// For more information have a look at the documentation of RefStore.
pub trait UniqueRefPathGenerator {
/// The collection the `StoreId` should be created for
fn collection() -> &'static str {
"ref"
}
/// A function which should generate a unique string for a Path
fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String>;
/// Postprocess the generated `StoreId` object
fn postprocess_storeid(sid: StoreId) -> Result<StoreId> {
Ok(sid)
}
}
/// A extensions for the `Store` to handle `Ref` objects
///
/// The RefStore handles refs using a `UniqueRefPathGenerator`. The `UniqueRefPathGenerator`, as it
/// name suggests, generates unique `StoreId`s for a `&Path`. It is a functor `&Path -> StoreId`.
///
/// It provides three functions which are called in the following sequence:
///
/// * The `UniqueRefPathGenerator::collection()` function is used for get the collection a `StoreId`
/// should be in (The first element of the `StoreId` path)
/// * The `UniqueRefPathGenerator::unique_hash()` gets the `&Path` which it then should generate a
/// unique String for. How this is done does not matter. It can hash the Path itself, read the
/// file and hash that or something else. It should be reproduceable, though.
/// * These two parts are joined and put into a `StoreId` which the
/// `UniqueRefPathGenerator::postprocess_storeid()` function is then allowed to postprocess (for
/// example add more parts to the StoreId). The default implementation does nothing.
///
/// The StoreId which is generated is then used to carry out the actual action (reading, creating
/// ...).
/// If a entry is created, header information is set (that it is a ref, the hash which was just
/// generated and the path of the referenced file)
///
/// # Details
///
/// The `UniqueRefPathGenerator` is passed as type parameter to enforce some semantics:
///
/// * The used `UniqueRefPathGenerator` is defined by the implementation rather than by the runtime
/// of the program or some environment. Of course this is only a small hurdle to enforce this, but
/// a hint.
/// * The `UniqueRefPathGenerator` is a functor which does not carry state.
///
pub trait RefStore<'a> {
fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H) -> Result<Option<FileLockEntry<'a>>>;
fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>>;
fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>>;
}
impl<'a> RefStore<'a> for Store {
fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H)
-> Result<Option<FileLockEntry<'a>>>
{
let sid = StoreId::new(PathBuf::from(format!("{}/{}", RPG::collection(), hash.as_ref())))
.map_err(Error::from)?;
debug!("Getting: {:?}", sid);
self.get(sid)
.map_err(Error::from)
}
fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
-> Result<FileLockEntry<'a>>
{
let hash = RPG::unique_hash(&path)?;
let pathbuf = PathBuf::from(format!("{}/{}", RPG::collection(), hash));
let sid = StoreId::new(pathbuf.clone())?;
debug!("Creating: {:?}", sid);
self.create(sid)
.map_err(Error::from)
.and_then(|mut fle| {
fle.make_ref(hash, path)?;
Ok(fle)
})
}
fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
-> Result<FileLockEntry<'a>>
{
match self.get_ref::<RPG, String>(RPG::unique_hash(path.as_ref())?)? {
Some(r) => Ok(r),
None => self.create_ref::<RPG, A>(path),
}
}
}