diff --git a/Cargo.toml b/Cargo.toml index 25f73c91..23c9969b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ members = [ "lib/entry/libimagentryfilter", "lib/entry/libimagentrygps", "lib/entry/libimagentrylink", + "lib/entry/libimagentryurl", "lib/entry/libimagentrymarkdown", "lib/entry/libimagentryref", "lib/entry/libimagentrytag", diff --git a/bin/core/imag-annotate/src/main.rs b/bin/core/imag-annotate/src/main.rs index 5904b345..f86d71fa 100644 --- a/bin/core/imag-annotate/src/main.rs +++ b/bin/core/imag-annotate/src/main.rs @@ -66,7 +66,7 @@ use libimagrt::runtime::Runtime; use libimagrt::setup::generate_runtime_setup; use libimagstore::store::FileLockEntry; use libimagstore::iter::get::StoreIdGetIteratorExtension; -use libimagentrylink::internal::InternalLinker; +use libimagentrylink::linker::InternalLinker; mod ui; diff --git a/bin/core/imag-diagnostics/src/main.rs b/bin/core/imag-diagnostics/src/main.rs index 4414c1b8..5aa6844d 100644 --- a/bin/core/imag-diagnostics/src/main.rs +++ b/bin/core/imag-diagnostics/src/main.rs @@ -55,7 +55,7 @@ use libimagerror::io::ToExitCode; use libimagerror::exit::ExitUnwrap; use libimagstore::store::FileLockEntry; use libimagstore::storeid::StoreId; -use libimagentrylink::internal::*; +use libimagentrylink::linker::InternalLinker; use toml::Value; use toml_query::read::TomlValueReadExt; diff --git a/bin/core/imag-link/Cargo.toml b/bin/core/imag-link/Cargo.toml index e2caf2fa..be2fa058 100644 --- a/bin/core/imag-link/Cargo.toml +++ b/bin/core/imag-link/Cargo.toml @@ -31,6 +31,7 @@ libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" libimagrt = { version = "0.10.0", path = "../../../lib/core/libimagrt" } libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" } libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink" } +libimagentryurl = { version = "0.10.0", path = "../../../lib/entry/libimagentryurl" } libimagutil = { version = "0.10.0", path = "../../../lib/etc/libimagutil" } [dependencies.clap] diff --git a/bin/core/imag-link/src/main.rs b/bin/core/imag-link/src/main.rs index 3e12f1c0..df717015 100644 --- a/bin/core/imag-link/src/main.rs +++ b/bin/core/imag-link/src/main.rs @@ -44,6 +44,7 @@ extern crate failure; #[cfg(test)] extern crate env_logger; extern crate libimagentrylink; +extern crate libimagentryurl; #[macro_use] extern crate libimagrt; extern crate libimagstore; extern crate libimagerror; @@ -61,9 +62,9 @@ use std::path::PathBuf; use failure::Error; use failure::err_msg; -use libimagentrylink::external::ExternalLinker; -use libimagentrylink::internal::InternalLinker; -use libimagentrylink::internal::store_check::StoreLinkConsistentExt; +use libimagentryurl::linker::UrlLinker; +use libimagentrylink::linker::InternalLinker; +use libimagentrylink::storecheck::StoreLinkConsistentExt; use libimagerror::trace::{MapErrTrace, trace_error}; use libimagerror::exit::ExitUnwrap; use libimagerror::io::ToExitCode; @@ -157,7 +158,7 @@ fn link_from_to<'a, I>(rt: &'a Runtime, from: &'a str, to: I) }); let iter = from_entry - .add_external_link(rt.store(), url) + .add_url(rt.store(), url) .map_err_trace_exit_unwrap() .into_iter(); @@ -238,7 +239,7 @@ fn remove_linking(rt: &Runtime) { error!("Error parsing URL: {:?}", e); ::std::process::exit(1); }); - from.remove_external_link(rt.store(), url).map_err_trace_exit_unwrap(); + from.remove_url(rt.store(), url).map_err_trace_exit_unwrap(); info!("Ok: {}", id); } else { warn!("Entry not found: {:?}", id); @@ -313,7 +314,7 @@ fn list_linkings(rt: &Runtime) { } if list_externals { - entry.get_external_links(rt.store()) + entry.get_urls(rt.store()) .map_err_trace_exit_unwrap() .enumerate() .for_each(|(i, link)| { diff --git a/bin/core/imag-mv/src/main.rs b/bin/core/imag-mv/src/main.rs index 9f2fde74..9eca7b35 100644 --- a/bin/core/imag-mv/src/main.rs +++ b/bin/core/imag-mv/src/main.rs @@ -56,7 +56,7 @@ use libimagerror::exit::ExitUnwrap; use libimagstore::storeid::StoreId; use libimagstore::store::Store; use libimagstore::store::FileLockEntry; -use libimagentrylink::internal::InternalLinker; +use libimagentrylink::linker::InternalLinker; use libimagstore::iter::get::StoreIdGetIteratorExtension; fn main() { diff --git a/bin/domain/imag-bookmark/src/main.rs b/bin/domain/imag-bookmark/src/main.rs index b2eaa795..f15ceabe 100644 --- a/bin/domain/imag-bookmark/src/main.rs +++ b/bin/domain/imag-bookmark/src/main.rs @@ -61,7 +61,7 @@ use libimagerror::trace::{MapErrTrace, trace_error}; use libimagerror::io::ToExitCode; use libimagerror::exit::ExitUnwrap; use libimagutil::debug_result::DebugResult; -use libimagentrylink::internal::InternalLinker; +use libimagentrylink::linker::InternalLinker; mod ui; diff --git a/bin/domain/imag-wiki/src/main.rs b/bin/domain/imag-wiki/src/main.rs index 1556abbb..765a9e64 100644 --- a/bin/domain/imag-wiki/src/main.rs +++ b/bin/domain/imag-wiki/src/main.rs @@ -252,7 +252,7 @@ fn show(rt: &Runtime, wiki_name: &str) { } fn delete(rt: &Runtime, wiki_name: &str) { - use libimagentrylink::internal::InternalLinker; + use libimagentrylink::linker::InternalLinker; let scmd = rt.cli().subcommand_matches("delete").unwrap(); // safed by clap let name = String::from(scmd.value_of("delete-name").unwrap()); // safe by clap diff --git a/lib/core/libimagrt/Cargo.toml b/lib/core/libimagrt/Cargo.toml index f9b83932..318d7f37 100644 --- a/lib/core/libimagrt/Cargo.toml +++ b/lib/core/libimagrt/Cargo.toml @@ -25,7 +25,6 @@ toml = "0.5" xdg-basedir = "1.0" itertools = "0.7" ansi_term = "0.11" -toml-query = "0.9" atty = "0.2" failure = "0.1" failure_derive = "0.1" @@ -52,6 +51,11 @@ version = "^1.0.5" default-features = false features = ["no_logging"] +[dependencies.toml-query] +version = "0.9" +default-features = false +features = [ "typed" ] + [features] default = [] diff --git a/lib/domain/libimagbookmark/Cargo.toml b/lib/domain/libimagbookmark/Cargo.toml index cd2aaa9e..f351fa5c 100644 --- a/lib/domain/libimagbookmark/Cargo.toml +++ b/lib/domain/libimagbookmark/Cargo.toml @@ -27,4 +27,5 @@ failure = "0.1" libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" } libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" } libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink" } +libimagentryurl = { version = "0.10.0", path = "../../../lib/entry/libimagentryurl" } diff --git a/lib/domain/libimagbookmark/src/collection.rs b/lib/domain/libimagbookmark/src/collection.rs index 19f10f64..3ebf8066 100644 --- a/lib/domain/libimagbookmark/src/collection.rs +++ b/lib/domain/libimagbookmark/src/collection.rs @@ -20,7 +20,7 @@ //! BookmarkCollection module //! //! A BookmarkCollection is nothing more than a simple store entry. One can simply call functions -//! from the libimagentrylink::external::ExternalLinker trait on this to generate external links. +//! from the libimagentryurl::linker::UrlLinker trait on this to generate external links. //! //! The BookmarkCollection type offers helper functions to get all links or such things. @@ -34,10 +34,10 @@ use libimagstore::store::Store; use libimagstore::store::Entry; use libimagstore::store::FileLockEntry; use libimagstore::storeid::StoreId; -use libimagentrylink::external::ExternalLinker; -use libimagentrylink::external::iter::UrlIter; -use libimagentrylink::internal::InternalLinker; -use libimagentrylink::internal::Link as StoreLink; +use libimagentryurl::linker::UrlLinker; +use libimagentryurl::iter::UrlIter; +use libimagentrylink::linker::InternalLinker; +use libimagentrylink::link::Link as StoreLink; use crate::link::Link; @@ -80,7 +80,7 @@ impl<'a> BookmarkCollectionStore<'a> for Store { } -pub trait BookmarkCollection : Sized + InternalLinker + ExternalLinker { +pub trait BookmarkCollection : Sized + InternalLinker + UrlLinker { fn links<'a>(&self, store: &'a Store) -> Result>; fn link_entries(&self) -> Result>; fn add_link(&mut self, store: &Store, l: Link) -> Result>; @@ -91,27 +91,27 @@ pub trait BookmarkCollection : Sized + InternalLinker + ExternalLinker { impl BookmarkCollection for Entry { fn links<'a>(&self, store: &'a Store) -> Result> { - self.get_external_links(store) + self.get_urls(store) } fn link_entries(&self) -> Result> { - use libimagentrylink::external::is_external_link_storeid; + use libimagentryurl::util::is_external_link_storeid; self.get_internal_links().map(|v| v.filter(|id| is_external_link_storeid(id)).collect()) } fn add_link(&mut self, store: &Store, l: Link) -> Result> { use crate::link::IntoUrl; - l.into_url().and_then(|url| self.add_external_link(store, url)) + l.into_url().and_then(|url| self.add_url(store, url)) } fn get_links_matching<'a>(&self, store: &'a Store, r: Regex) -> Result> { use self::iter::IntoLinksMatchingRegexIter; - self.get_external_links(store).map(|iter| iter.matching_regex(r)) + self.get_urls(store).map(|iter| iter.matching_regex(r)) } fn remove_link(&mut self, store: &Store, l: Link) -> Result> { use crate::link::IntoUrl; - l.into_url().and_then(|url| self.remove_external_link(store, url)) + l.into_url().and_then(|url| self.remove_url(store, url)) } } @@ -120,6 +120,9 @@ pub mod iter { use crate::link::Link; use failure::Fallible as Result; use failure::Error; + use regex::Regex; + + use libimagentryurl::iter::UrlIter; pub struct LinkIter(I) where I: Iterator; @@ -144,9 +147,6 @@ pub mod iter { } } - use libimagentrylink::external::iter::UrlIter; - use regex::Regex; - pub struct LinksMatchingRegexIter<'a>(UrlIter<'a>, Regex); impl<'a> LinksMatchingRegexIter<'a> { diff --git a/lib/domain/libimagbookmark/src/lib.rs b/lib/domain/libimagbookmark/src/lib.rs index 6072070c..c8b2c59c 100644 --- a/lib/domain/libimagbookmark/src/lib.rs +++ b/lib/domain/libimagbookmark/src/lib.rs @@ -44,6 +44,7 @@ extern crate regex; #[macro_use] extern crate libimagstore; extern crate libimagerror; extern crate libimagentrylink; +extern crate libimagentryurl; module_entry_path_mod!("bookmark"); diff --git a/lib/domain/libimaghabit/src/habit.rs b/lib/domain/libimaghabit/src/habit.rs index 7041dff7..bd203189 100644 --- a/lib/domain/libimaghabit/src/habit.rs +++ b/lib/domain/libimaghabit/src/habit.rs @@ -33,7 +33,7 @@ use crate::util::IsHabitCheck; use crate::util::get_string_header_from_entry; use crate::instance::IsHabitInstance; -use libimagentrylink::internal::InternalLinker; +use libimagentrylink::linker::InternalLinker; use libimagstore::store::Store; use libimagstore::store::FileLockEntry; use libimagstore::store::Entry; diff --git a/lib/domain/libimagwiki/src/entry.rs b/lib/domain/libimagwiki/src/entry.rs index 2619a943..9ced2299 100644 --- a/lib/domain/libimagwiki/src/entry.rs +++ b/lib/domain/libimagwiki/src/entry.rs @@ -57,7 +57,7 @@ impl WikiEntry for Entry { let processor = LinkProcessor::default() .process_internal_links(true) .create_internal_targets(true) - .process_external_links(true) + .process_urls(true) .process_refs(true); self.autolink_with_processor(store, processor) diff --git a/lib/domain/libimagwiki/src/wiki.rs b/lib/domain/libimagwiki/src/wiki.rs index c61a6962..9cd3d236 100644 --- a/lib/domain/libimagwiki/src/wiki.rs +++ b/lib/domain/libimagwiki/src/wiki.rs @@ -22,7 +22,7 @@ use std::path::PathBuf; use libimagstore::store::Store; use libimagstore::store::FileLockEntry; use libimagstore::iter::Entries; -use libimagentrylink::internal::InternalLinker; +use libimagentrylink::linker::InternalLinker; use failure::Fallible as Result; use failure::Error; diff --git a/lib/entry/libimagentryannotation/src/annotateable.rs b/lib/entry/libimagentryannotation/src/annotateable.rs index 46b8cdbc..f89ca631 100644 --- a/lib/entry/libimagentryannotation/src/annotateable.rs +++ b/lib/entry/libimagentryannotation/src/annotateable.rs @@ -24,7 +24,7 @@ use libimagstore::store::Entry; use libimagstore::store::FileLockEntry; use libimagstore::store::Store; use libimagstore::storeid::StoreIdIterator; -use libimagentrylink::internal::InternalLinker; +use libimagentrylink::linker::InternalLinker; use libimagentryutil::isa::Is; use libimagentryutil::isa::IsKindHeaderPathProvider; diff --git a/lib/entry/libimagentrycategory/src/category.rs b/lib/entry/libimagentrycategory/src/category.rs index fb7268a7..f33a7ed5 100644 --- a/lib/entry/libimagentrycategory/src/category.rs +++ b/lib/entry/libimagentrycategory/src/category.rs @@ -22,7 +22,7 @@ use libimagentryutil::isa::IsKindHeaderPathProvider; use libimagstore::store::Entry; use libimagstore::store::Store; use libimagstore::storeid::StoreIdIterator; -use libimagentrylink::internal::InternalLinker; +use libimagentrylink::linker::InternalLinker; use toml_query::read::TomlValueReadTypeExt; diff --git a/lib/entry/libimagentrycategory/src/entry.rs b/lib/entry/libimagentrycategory/src/entry.rs index 9876add4..a9e39da0 100644 --- a/lib/entry/libimagentrycategory/src/entry.rs +++ b/lib/entry/libimagentrycategory/src/entry.rs @@ -23,7 +23,7 @@ use toml_query::read::TomlValueReadTypeExt; use toml::Value; use libimagstore::store::Entry; -use libimagentrylink::internal::InternalLinker; +use libimagentrylink::linker::InternalLinker; use libimagerror::errors::ErrorMsg as EM; use failure::Fallible as Result; diff --git a/lib/entry/libimagentrycategory/src/store.rs b/lib/entry/libimagentrycategory/src/store.rs index 3ea78648..3f0f7df2 100644 --- a/lib/entry/libimagentrycategory/src/store.rs +++ b/lib/entry/libimagentrycategory/src/store.rs @@ -84,7 +84,7 @@ impl CategoryStore for Store { /// /// Automatically removes all category settings from entries which are linked to this category. fn delete_category(&self, name: &str) -> Result<()> { - use libimagentrylink::internal::InternalLinker; + use libimagentrylink::linker::InternalLinker; use crate::category::Category; trace!("Deleting category: '{}'", name); diff --git a/lib/entry/libimagentrylink/src/external.rs b/lib/entry/libimagentrylink/src/external.rs deleted file mode 100644 index 95398785..00000000 --- a/lib/entry/libimagentrylink/src/external.rs +++ /dev/null @@ -1,476 +0,0 @@ -// -// imag - the personal information management suite for the commandline -// Copyright (C) 2015-2019 Matthias Beyer and contributors -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; version -// 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// - -/// External linking is a complex implementation to be able to serve a clean and easy-to-use -/// interface. -/// -/// Internally, there are no such things as "external links" (plural). Each Entry in the store can -/// only have _one_ external link. -/// -/// This library does the following therefor: It allows you to have several external links with one -/// entry, which are internally one file in the store for each link, linked with "internal -/// linking". -/// -/// This helps us greatly with deduplication of URLs. -/// - -use std::ops::DerefMut; -use std::fmt::Debug; - -use libimagstore::store::Entry; -use libimagstore::store::Store; -use libimagstore::storeid::StoreId; -use libimagutil::debug_result::*; -use libimagerror::errors::ErrorMsg as EM; - -use toml_query::read::TomlValueReadExt; -use toml_query::read::TomlValueReadTypeExt; -use toml_query::insert::TomlValueInsertExt; -use toml::map::Map; -use failure::Error; -use failure::Fallible as Result; -use failure::ResultExt; -use failure::err_msg; - -use crate::internal::InternalLinker; - -use self::iter::*; - -use toml::Value; -use url::Url; -use sha1::{Sha1, Digest}; -use hex; - -pub trait Link { - - fn get_link_uri_from_filelockentry(&self) -> Result>; - - fn get_url(&self) -> Result>; - -} - -impl Link for Entry { - - fn get_link_uri_from_filelockentry(&self) -> Result> { - self.get_header() - .read_string("links.external.content.url") - .context(format_err!("Error reading header 'links.external.content.url' from '{}'", self.get_location())) - .context(EM::EntryHeaderReadError) - .map_err(Error::from) - .and_then(|opt| match opt { - None => Ok(None), - Some(ref s) => { - debug!("Found url, parsing: {:?}", s); - Url::parse(&s[..]) - .map_err(Error::from) - .context(format_err!("Failed to parse URL: '{}'", s)) - .context(err_msg("Invalid URI")) - .map_err(Error::from) - .map(Some) - }, - }) - .context("Failed to get link URI from entry") - .map_err(Error::from) - } - - fn get_url(&self) -> Result> { - match self.get_header().read_string("links.external.url")? { - None => Ok(None), - Some(ref s) => Url::parse(&s[..]) - .context(format_err!("Failed to parse URL: '{}'", s)) - .map(Some) - .map_err(Error::from) - .context(EM::EntryHeaderReadError) - .map_err(Error::from), - } - } - -} - -pub trait ExternalLinker : InternalLinker { - - /// Get the external links from the implementor object - fn get_external_links<'a>(&self, store: &'a Store) -> Result>; - - /// Set the external links for the implementor object - fn set_external_links(&mut self, store: &Store, links: Vec) -> Result>; - - /// Add an external link to the implementor object - fn add_external_link(&mut self, store: &Store, link: Url) -> Result>; - - /// Remove an external link from the implementor object - fn remove_external_link(&mut self, store: &Store, link: Url) -> Result>; - -} - -pub mod iter { - //! Iterator helpers for external linking stuff - //! - //! Contains also helpers to filter iterators for external/internal links - //! - //! - //! # Warning - //! - //! This module uses `internal::Link` as link type, so we operate on _store ids_ here. - //! - //! Not to confuse with `external::Link` which is a real `FileLockEntry` under the hood. - //! - - use libimagutil::debug_result::*; - use libimagstore::store::Store; - - use crate::internal::Link; - use crate::internal::iter::LinkIter; - use failure::Fallible as Result; - - use url::Url; - - /// Helper for building `OnlyExternalIter` and `NoExternalIter` - /// - /// The boolean value defines, how to interpret the `is_external_link_storeid()` return value - /// (here as "pred"): - /// - /// ```ignore - /// pred | bool | xor | take? - /// ---- | ---- | --- | ---- - /// 0 | 0 | 0 | 1 - /// 0 | 1 | 1 | 0 - /// 1 | 0 | 1 | 0 - /// 1 | 1 | 0 | 1 - /// ``` - /// - /// If `bool` says "take if return value is false", we take the element if the `pred` returns - /// false... and so on. - /// - /// As we can see, the operator between these two operants is `!(a ^ b)`. - pub struct ExternalFilterIter(LinkIter, bool); - - impl Iterator for ExternalFilterIter { - type Item = Link; - - fn next(&mut self) -> Option { - use super::is_external_link_storeid; - - while let Some(elem) = self.0.next() { - trace!("Check whether is external: {:?}", elem); - if !(self.1 ^ is_external_link_storeid(&elem)) { - trace!("Is external id: {:?}", elem); - return Some(elem); - } - } - None - } - } - - /// Helper trait to be implemented on `LinkIter` to select or deselect all external links - /// - /// # See also - /// - /// Also see `OnlyExternalIter` and `NoExternalIter` and the helper traits/functions - /// `OnlyInteralLinks`/`only_internal_links()` and `OnlyExternalLinks`/`only_external_links()`. - pub trait SelectExternal { - fn select_external_links(self, b: bool) -> ExternalFilterIter; - } - - impl SelectExternal for LinkIter { - fn select_external_links(self, b: bool) -> ExternalFilterIter { - ExternalFilterIter(self, b) - } - } - - - pub struct OnlyExternalIter(ExternalFilterIter); - - impl OnlyExternalIter { - pub fn new(li: LinkIter) -> OnlyExternalIter { - OnlyExternalIter(ExternalFilterIter(li, true)) - } - - pub fn urls<'a>(self, store: &'a Store) -> UrlIter<'a> { - UrlIter(self, store) - } - } - - impl Iterator for OnlyExternalIter { - type Item = Link; - - fn next(&mut self) -> Option { - self.0.next() - } - } - - pub struct NoExternalIter(ExternalFilterIter); - - impl NoExternalIter { - pub fn new(li: LinkIter) -> NoExternalIter { - NoExternalIter(ExternalFilterIter(li, false)) - } - } - - impl Iterator for NoExternalIter { - type Item = Link; - - fn next(&mut self) -> Option { - self.0.next() - } - } - - pub trait OnlyExternalLinks : Sized { - fn only_external_links(self) -> OnlyExternalIter ; - - fn no_internal_links(self) -> OnlyExternalIter { - self.only_external_links() - } - } - - impl OnlyExternalLinks for LinkIter { - fn only_external_links(self) -> OnlyExternalIter { - OnlyExternalIter::new(self) - } - } - - pub trait OnlyInternalLinks : Sized { - fn only_internal_links(self) -> NoExternalIter; - - fn no_external_links(self) -> NoExternalIter { - self.only_internal_links() - } - } - - impl OnlyInternalLinks for LinkIter { - fn only_internal_links(self) -> NoExternalIter { - NoExternalIter::new(self) - } - } - - pub struct UrlIter<'a>(OnlyExternalIter, &'a Store); - - impl<'a> Iterator for UrlIter<'a> { - type Item = Result; - - fn next(&mut self) -> Option { - use crate::external::Link; - - loop { - let next = self.0 - .next() - .map(|id| { - debug!("Retrieving entry for id: '{:?}'", id); - self.1 - .retrieve(id.clone()) - .map_dbg_err(|_| format!("Retrieving entry for id: '{:?}' failed", id)) - .map_err(From::from) - .and_then(|f| { - debug!("Store::retrieve({:?}) succeeded", id); - debug!("getting external link from file now"); - f.get_link_uri_from_filelockentry() - .map_dbg_str("Error happened while getting link URI from FLE") - .map_dbg_err(|e| format!("URL -> Err = {:?}", e)) - }) - }); - - match next { - Some(Ok(Some(link))) => return Some(Ok(link)), - Some(Ok(None)) => continue, - Some(Err(e)) => return Some(Err(e)), - None => return None - } - } - } - - } - -} - - -/// Check whether the StoreId starts with `/link/external/` -pub fn is_external_link_storeid + Debug>(id: A) -> bool { - debug!("Checking whether this is a 'links/external/': '{:?}'", id); - id.as_ref().is_in_collection(&["links", "external"]) -} - -/// Implement `ExternalLinker` for `Entry`, hiding the fact that there is no such thing as an external -/// link in an entry, but internal links to other entries which serve as external links, as one -/// entry in the store can only have one external link. -impl ExternalLinker for Entry { - - /// Get the external links from the implementor object - fn get_external_links<'a>(&self, store: &'a Store) -> Result> { - // Iterate through all internal links and filter for FileLockEntries which live in - // /link/external/ -> load these files and get the external link from their headers, - // put them into the return vector. - self.get_internal_links() - .map(|iter| { - debug!("Getting external links"); - iter.only_external_links().urls(store) - }) - } - - /// Set the external links for the implementor object - /// - /// # Return Value - /// - /// Returns the StoreIds which were newly created for the new external links, if there are more - /// external links than before. - /// If there are less external links than before, an empty vec![] is returned. - /// - fn set_external_links(&mut self, store: &Store, links: Vec) -> Result> { - // Take all the links, generate a SHA sum out of each one, filter out the already existing - // store entries and store the other URIs in the header of one FileLockEntry each, in - // the path /link/external/ - - debug!("Iterating {} links = {:?}", links.len(), links); - links.into_iter().map(|link| { - let hash = hex::encode(Sha1::digest(&link.as_str().as_bytes())); - let file_id = crate::module_path::new_id(format!("external/{}", hash)) - .map_dbg_err(|_| { - format!("Failed to build StoreId for this hash '{:?}'", hash) - })?; - - debug!("Link = '{:?}'", link); - debug!("Hash = '{:?}'", hash); - debug!("StoreId = '{:?}'", file_id); - - let link_already_exists = store.get(file_id.clone())?.is_some(); - - // retrieve the file from the store, which implicitely creates the entry if it does not - // exist - let mut file = store - .retrieve(file_id.clone()) - .map_dbg_err(|_| { - format!("Failed to create or retrieve an file for this link '{:?}'", link) - })?; - - debug!("Generating header content!"); - { - let hdr = file.deref_mut().get_header_mut(); - - let mut table = match hdr.read("links.external.content")? { - Some(&Value::Table(ref table)) => table.clone(), - Some(_) => { - warn!("There is a value at 'links.external.content' which is not a table."); - warn!("Going to override this value"); - Map::new() - }, - None => Map::new(), - }; - - let v = Value::String(link.into_string()); - - debug!("setting URL = '{:?}", v); - table.insert(String::from("url"), v); - - let _ = hdr.insert("links.external.content", Value::Table(table))?; - debug!("Setting URL worked"); - } - - // then add an internal link to the new file or return an error if this fails - let _ = self.add_internal_link(file.deref_mut())?; - debug!("Error adding internal link"); - - Ok((link_already_exists, file_id)) - }) - .filter_map(|res| match res { - Ok((exists, entry)) => if exists { Some(Ok(entry)) } else { None }, - Err(e) => Some(Err(e)) - }) - .collect() - } - - /// Add an external link to the implementor object - /// - /// # Return Value - /// - /// (See ExternalLinker::set_external_links()) - /// - /// Returns the StoreIds which were newly created for the new external links, if there are more - /// external links than before. - /// If there are less external links than before, an empty vec![] is returned. - /// - fn add_external_link(&mut self, store: &Store, link: Url) -> Result> { - // get external links, add this one, save them - debug!("Getting links"); - self.get_external_links(store) - .and_then(|links| { - let mut links = links.collect::>>()?; - - debug!("Adding link = '{:?}' to links = {:?}", link, links); - links.push(link); - - debug!("Setting {} links = {:?}", links.len(), links); - self.set_external_links(store, links) - }) - } - - /// Remove an external link from the implementor object - /// - /// # Return Value - /// - /// (See ExternalLinker::set_external_links()) - /// - /// Returns the StoreIds which were newly created for the new external links, if there are more - /// external links than before. - /// If there are less external links than before, an empty vec![] is returned. - /// - fn remove_external_link(&mut self, store: &Store, link: Url) -> Result> { - // get external links, remove this one, save them - self.get_external_links(store) - .and_then(|links| { - debug!("Removing link = '{:?}'", link); - let links = links - .filter_map(Result::ok) - .filter(|l| l.as_str() != link.as_str()) - .collect::>(); - self.set_external_links(store, links) - }) - } - -} - -#[cfg(test)] -mod tests { - use super::*; - use std::path::PathBuf; - - use libimagstore::store::Store; - - fn setup_logging() { - let _ = env_logger::try_init(); - } - - pub fn get_store() -> Store { - Store::new_inmemory(PathBuf::from("/"), &None).unwrap() - } - - - #[test] - fn test_simple() { - setup_logging(); - let store = get_store(); - let mut e = store.retrieve(PathBuf::from("base-test_simple")).unwrap(); - let url = Url::parse("http://google.de").unwrap(); - - assert!(e.add_external_link(&store, url.clone()).is_ok()); - - assert_eq!(1, e.get_external_links(&store).unwrap().count()); - assert_eq!(url, e.get_external_links(&store).unwrap().next().unwrap().unwrap()); - } - -} - diff --git a/lib/entry/libimagentrylink/src/iter.rs b/lib/entry/libimagentrylink/src/iter.rs new file mode 100644 index 00000000..5d76d0b6 --- /dev/null +++ b/lib/entry/libimagentrylink/src/iter.rs @@ -0,0 +1,97 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + + +use std::vec::IntoIter; + +use failure::Error; +use failure::ResultExt; +use failure::Fallible as Result; +use toml::Value; +use itertools::Itertools; + +use libimagstore::store::Store; +use libimagstore::store::FileLockEntry; +use libimagerror::errors::ErrorMsg as EM; + +use crate::link::Link; + +pub struct LinkIter(IntoIter); + +impl LinkIter { + + pub fn new(v: Vec) -> LinkIter { + LinkIter(v.into_iter()) + } + + pub fn into_getter(self, store: &Store) -> GetIter { + GetIter(self.0, store) + } + +} + +impl Iterator for LinkIter { + type Item = Link; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +pub trait IntoValues { + fn into_values(self) -> Vec>; +} + +impl> IntoValues for I { + fn into_values(self) -> Vec> { + self.map(|s| s.without_base()) + .unique() + .sorted() + .into_iter() // Cannot sort toml::Value, hence uglyness here + .map(|link| link.to_value().context(EM::ConversionError).map_err(Error::from)) + .collect() + } +} + +/// An Iterator that `Store::get()`s the Entries from the store while consumed +pub struct GetIter<'a>(IntoIter, &'a Store); + +impl<'a> GetIter<'a> { + pub fn new(i: IntoIter, store: &'a Store) -> GetIter<'a> { + GetIter(i, store) + } + + pub fn store(&self) -> &Store { + self.1 + } +} + +impl<'a> Iterator for GetIter<'a> { + type Item = Result>; + + fn next(&mut self) -> Option { + self.0.next().and_then(|id| match self.1.get(id) { + Ok(None) => None, + Ok(Some(x)) => Some(Ok(x)), + Err(e) => Some(Err(e).map_err(From::from)), + }) + } + +} + diff --git a/lib/entry/libimagentrylink/src/lib.rs b/lib/entry/libimagentrylink/src/lib.rs index ce258815..84fcba78 100644 --- a/lib/entry/libimagentrylink/src/lib.rs +++ b/lib/entry/libimagentrylink/src/lib.rs @@ -56,6 +56,8 @@ extern crate libimagutil; module_entry_path_mod!("links"); -pub mod external; -pub mod internal; +pub mod iter; +pub mod linker; +pub mod link; +pub mod storecheck; diff --git a/lib/entry/libimagentrylink/src/link.rs b/lib/entry/libimagentrylink/src/link.rs new file mode 100644 index 00000000..eb594877 --- /dev/null +++ b/lib/entry/libimagentrylink/src/link.rs @@ -0,0 +1,150 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use libimagstore::storeid::StoreId; +use libimagstore::storeid::IntoStoreId; +use libimagstore::store::Store; +use libimagerror::errors::ErrorMsg as EM; + +use toml::Value; +use toml::map::Map; +use failure::ResultExt; +use failure::Fallible as Result; +use failure::Error; + +#[derive(Eq, PartialOrd, Ord, Hash, Debug, Clone)] +pub enum Link { + Id { link: StoreId }, + Annotated { link: StoreId, annotation: String }, +} + +impl Link { + + pub fn exists(&self, store: &Store) -> Result { + match *self { + Link::Id { ref link } => store.exists(link.clone()), + Link::Annotated { ref link, .. } => store.exists(link.clone()), + } + .map_err(From::from) + } + + pub fn to_str(&self) -> Result { + match *self { + Link::Id { ref link } => link.to_str(), + Link::Annotated { ref link, .. } => link.to_str(), + } + .map_err(From::from) + } + + + pub(crate) fn eq_store_id(&self, id: &StoreId) -> bool { + match self { + &Link::Id { link: ref s } => s.eq(id), + &Link::Annotated { link: ref s, .. } => s.eq(id), + } + } + + /// Get the StoreId inside the Link, which is always present + pub fn get_store_id(&self) -> &StoreId { + match self { + &Link::Id { link: ref s } => s, + &Link::Annotated { link: ref s, .. } => s, + } + } + + /// Helper wrapper around Link for StoreId + pub(crate) fn without_base(self) -> Link { + match self { + Link::Id { link: s } => Link::Id { link: s }, + Link::Annotated { link: s, annotation: ann } => + Link::Annotated { link: s, annotation: ann }, + } + } + + pub(crate) fn to_value(&self) -> Result { + match self { + &Link::Id { link: ref s } => + s.to_str() + .map(Value::String) + .context(EM::ConversionError) + .map_err(Error::from), + &Link::Annotated { ref link, annotation: ref anno } => { + link.to_str() + .map(Value::String) + .context(EM::ConversionError) + .map_err(Error::from) + .map(|link| { + let mut tab = Map::new(); + + tab.insert("link".to_owned(), link); + tab.insert("annotation".to_owned(), Value::String(anno.clone())); + Value::Table(tab) + }) + } + } + } + +} + +impl ::std::cmp::PartialEq for Link { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (&Link::Id { link: ref a }, &Link::Id { link: ref b }) => a.eq(&b), + (&Link::Annotated { link: ref a, annotation: ref ann1 }, + &Link::Annotated { link: ref b, annotation: ref ann2 }) => + (a, ann1).eq(&(b, ann2)), + _ => false, + } + } +} + +impl From for Link { + + fn from(s: StoreId) -> Link { + Link::Id { link: s } + } +} + +impl Into for Link { + fn into(self) -> StoreId { + match self { + Link::Id { link } => link, + Link::Annotated { link, .. } => link, + } + } +} + +impl IntoStoreId for Link { + fn into_storeid(self) -> Result { + match self { + Link::Id { link } => Ok(link), + Link::Annotated { link, .. } => Ok(link), + } + } +} + +impl AsRef for Link { + fn as_ref(&self) -> &StoreId { + match self { + &Link::Id { ref link } => &link, + &Link::Annotated { ref link, .. } => &link, + } + } +} + diff --git a/lib/entry/libimagentrylink/src/internal.rs b/lib/entry/libimagentrylink/src/linker.rs similarity index 62% rename from lib/entry/libimagentrylink/src/internal.rs rename to lib/entry/libimagentrylink/src/linker.rs index ca24201f..a94a5707 100644 --- a/lib/entry/libimagentrylink/src/internal.rs +++ b/lib/entry/libimagentrylink/src/linker.rs @@ -18,144 +18,23 @@ // use libimagstore::storeid::StoreId; -use libimagstore::storeid::IntoStoreId; use libimagstore::store::Entry; use libimagstore::store::Store; use libimagerror::errors::ErrorMsg as EM; use toml_query::read::TomlValueReadExt; use toml_query::insert::TomlValueInsertExt; -use toml::map::Map; use failure::ResultExt; use failure::Fallible as Result; use failure::Error; use failure::err_msg; -use self::iter::LinkIter; -use self::iter::IntoValues; +use crate::iter::LinkIter; +use crate::iter::IntoValues; +use crate::link::Link; use toml::Value; -#[derive(Eq, PartialOrd, Ord, Hash, Debug, Clone)] -pub enum Link { - Id { link: StoreId }, - Annotated { link: StoreId, annotation: String }, -} - -impl Link { - - pub fn exists(&self, store: &Store) -> Result { - match *self { - Link::Id { ref link } => store.exists(link.clone()), - Link::Annotated { ref link, .. } => store.exists(link.clone()), - } - .map_err(From::from) - } - - pub fn to_str(&self) -> Result { - match *self { - Link::Id { ref link } => link.to_str(), - Link::Annotated { ref link, .. } => link.to_str(), - } - .map_err(From::from) - } - - - fn eq_store_id(&self, id: &StoreId) -> bool { - match self { - &Link::Id { link: ref s } => s.eq(id), - &Link::Annotated { link: ref s, .. } => s.eq(id), - } - } - - /// Get the StoreId inside the Link, which is always present - pub fn get_store_id(&self) -> &StoreId { - match self { - &Link::Id { link: ref s } => s, - &Link::Annotated { link: ref s, .. } => s, - } - } - - /// Helper wrapper around Link for StoreId - fn without_base(self) -> Link { - match self { - Link::Id { link: s } => Link::Id { link: s }, - Link::Annotated { link: s, annotation: ann } => - Link::Annotated { link: s, annotation: ann }, - } - } - - fn to_value(&self) -> Result { - match self { - &Link::Id { link: ref s } => - s.to_str() - .map(Value::String) - .context(EM::ConversionError) - .map_err(Error::from), - &Link::Annotated { ref link, annotation: ref anno } => { - link.to_str() - .map(Value::String) - .context(EM::ConversionError) - .map_err(Error::from) - .map(|link| { - let mut tab = Map::new(); - - tab.insert("link".to_owned(), link); - tab.insert("annotation".to_owned(), Value::String(anno.clone())); - Value::Table(tab) - }) - } - } - } - -} - -impl ::std::cmp::PartialEq for Link { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (&Link::Id { link: ref a }, &Link::Id { link: ref b }) => a.eq(&b), - (&Link::Annotated { link: ref a, annotation: ref ann1 }, - &Link::Annotated { link: ref b, annotation: ref ann2 }) => - (a, ann1).eq(&(b, ann2)), - _ => false, - } - } -} - -impl From for Link { - - fn from(s: StoreId) -> Link { - Link::Id { link: s } - } -} - -impl Into for Link { - fn into(self) -> StoreId { - match self { - Link::Id { link } => link, - Link::Annotated { link, .. } => link, - } - } -} - -impl IntoStoreId for Link { - fn into_storeid(self) -> Result { - match self { - Link::Id { link } => Ok(link), - Link::Annotated { link, .. } => Ok(link), - } - } -} - -impl AsRef for Link { - fn as_ref(&self) -> &StoreId { - match self { - &Link::Id { ref link } => &link, - &Link::Annotated { ref link, .. } => &link, - } - } -} - pub trait InternalLinker { /// Get the internal links from the implementor object @@ -174,85 +53,6 @@ pub trait InternalLinker { fn add_internal_annotated_link(&mut self, link: &mut Entry, annotation: String) -> Result<()>; } -pub mod iter { - use std::vec::IntoIter; - use super::Link; - - use failure::Error; - use failure::Fallible as Result; - use failure::ResultExt; - use toml::Value; - use itertools::Itertools; - - use libimagstore::store::Store; - use libimagstore::store::FileLockEntry; - use libimagerror::errors::ErrorMsg as EM; - - pub struct LinkIter(IntoIter); - - impl LinkIter { - - pub fn new(v: Vec) -> LinkIter { - LinkIter(v.into_iter()) - } - - pub fn into_getter(self, store: &Store) -> GetIter { - GetIter(self.0, store) - } - - } - - impl Iterator for LinkIter { - type Item = Link; - - fn next(&mut self) -> Option { - self.0.next() - } - } - - pub trait IntoValues { - fn into_values(self) -> Vec>; - } - - impl> IntoValues for I { - fn into_values(self) -> Vec> { - self.map(|s| s.without_base()) - .unique() - .sorted() - .into_iter() // Cannot sort toml::Value, hence uglyness here - .map(|link| link.to_value().context(EM::ConversionError).map_err(Error::from)) - .collect() - } - } - - /// An Iterator that `Store::get()`s the Entries from the store while consumed - pub struct GetIter<'a>(IntoIter, &'a Store); - - impl<'a> GetIter<'a> { - pub fn new(i: IntoIter, store: &'a Store) -> GetIter<'a> { - GetIter(i, store) - } - - pub fn store(&self) -> &Store { - self.1 - } - } - - impl<'a> Iterator for GetIter<'a> { - type Item = Result>; - - fn next(&mut self) -> Option { - self.0.next().and_then(|id| match self.1.get(id) { - Ok(None) => None, - Ok(Some(x)) => Some(Ok(x)), - Err(e) => Some(Err(e).map_err(From::from)), - }) - } - - } - -} - impl InternalLinker for Entry { fn get_internal_links(&self) -> Result { @@ -452,164 +252,6 @@ fn process_rw_result(links: Result>) -> Result { Ok(LinkIter::new(links)) } -pub mod store_check { - use libimagstore::store::Store; - - use failure::ResultExt; - use failure::Fallible as Result; - use failure::Error; - use failure::err_msg; - - pub trait StoreLinkConsistentExt { - fn check_link_consistency(&self) -> Result<()>; - } - - impl StoreLinkConsistentExt for Store { - fn check_link_consistency(&self) -> Result<()> { - use std::collections::HashMap; - - use crate::internal::InternalLinker; - - use libimagstore::storeid::StoreId; - use libimagutil::debug_result::DebugResult; - - // Helper data structure to collect incoming and outgoing links for each StoreId - #[derive(Debug, Default)] - struct Linking { - outgoing: Vec, - incoming: Vec, - } - - // Helper function to aggregate the Link network - // - // This function aggregates a HashMap which maps each StoreId object in the store onto - // a Linking object, which contains a list of StoreIds which this entry links to and a - // list of StoreIds which link to the current one. - // - // The lambda returns an error if something fails - let aggregate_link_network = |store: &Store| -> Result> { - store - .entries()? - .into_get_iter() - .fold(Ok(HashMap::new()), |map, element| { - map.and_then(|mut map| { - debug!("Checking element = {:?}", element); - let entry = element?.ok_or_else(|| err_msg("TODO: Not yet handled"))?; - - debug!("Checking entry = {:?}", entry.get_location()); - - let internal_links = entry - .get_internal_links()? - .into_getter(store); // get the FLEs from the Store - - let mut linking = Linking::default(); - for internal_link in internal_links { - debug!("internal link = {:?}", internal_link); - - linking.outgoing.push(internal_link?.get_location().clone()); - linking.incoming.push(entry.get_location().clone()); - } - - map.insert(entry.get_location().clone(), linking); - Ok(map) - }) - }) - }; - - // Helper to check whethre all StoreIds in the network actually exists - // - // Because why not? - let all_collected_storeids_exist = |network: &HashMap| -> Result<()> { - for (id, _) in network.iter() { - if is_match!(self.get(id.clone()), Ok(Some(_))) { - debug!("Exists in store: {:?}", id); - - if !self.exists(id.clone())? { - warn!("Does exist in store but not on FS: {:?}", id); - return Err(err_msg("Link target does not exist")) - } - } else { - warn!("Does not exist in store: {:?}", id); - return Err(err_msg("Link target does not exist")) - } - } - - Ok(()) - }; - - // Helper function to create a SLCECD::OneDirectionalLink error object - let mk_one_directional_link_err = |src: StoreId, target: StoreId| -> Error { - Error::from(format_err!("Dead link: {} -> {}", - src.local_display_string(), - target.local_display_string())) - }; - - // Helper lambda to check whether the _incoming_ links of each entry actually also - // appear in the _outgoing_ list of the linked entry - let incoming_links_exists_as_outgoing_links = - |src: &StoreId, linking: &Linking, network: &HashMap| -> Result<()> { - for link in linking.incoming.iter() { - // Check whether the links which are _incoming_ on _src_ are outgoing - // in each of the links in the incoming list. - let incoming_consistent = network.get(link) - .map(|l| l.outgoing.contains(src)) - .unwrap_or(false); - - if !incoming_consistent { - return Err(mk_one_directional_link_err(src.clone(), link.clone())) - } - } - - Ok(()) - }; - - // Helper lambda to check whether the _outgoing links of each entry actually also - // appear in the _incoming_ list of the linked entry - let outgoing_links_exist_as_incoming_links = - |src: &StoreId, linking: &Linking, network: &HashMap| -> Result<()> { - for link in linking.outgoing.iter() { - // Check whether the links which are _outgoing_ on _src_ are incoming - // in each of the links in the outgoing list. - let outgoing_consistent = network.get(link) - .map(|l| l.incoming.contains(src)) - .unwrap_or(false); - - if !outgoing_consistent { - return Err(mk_one_directional_link_err(link.clone(), src.clone())) - } - } - - Ok(()) - }; - - aggregate_link_network(&self) - .map_dbg_str("Aggregated") - .map_dbg(|nw| { - let mut s = String::new(); - for (k, v) in nw { - s.push_str(&format!("{}\n in: {:?}\n out: {:?}", k, v.incoming, v.outgoing)); - } - s - }) - .and_then(|nw| { - all_collected_storeids_exist(&nw) - .map(|_| nw) - .context(err_msg("Link handling error")) - .map_err(Error::from) - }) - .and_then(|nw| { - for (id, linking) in nw.iter() { - incoming_links_exists_as_outgoing_links(id, linking, &nw)?; - outgoing_links_exist_as_incoming_links(id, linking, &nw)?; - } - Ok(()) - }) - .map(|_| ()) - } - } - -} - #[cfg(test)] mod test { use std::path::PathBuf; @@ -852,4 +494,3 @@ mod test { } } - diff --git a/lib/entry/libimagentrylink/src/storecheck.rs b/lib/entry/libimagentrylink/src/storecheck.rs new file mode 100644 index 00000000..decf4cdd --- /dev/null +++ b/lib/entry/libimagentrylink/src/storecheck.rs @@ -0,0 +1,173 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use std::collections::HashMap; + +use libimagstore::store::Store; +use libimagstore::storeid::StoreId; +use libimagutil::debug_result::DebugResult; + +use failure::ResultExt; +use failure::Fallible as Result; +use failure::Error; +use failure::err_msg; + +use crate::linker::*; + +pub trait StoreLinkConsistentExt { + fn check_link_consistency(&self) -> Result<()>; +} + +impl StoreLinkConsistentExt for Store { + fn check_link_consistency(&self) -> Result<()> { + // Helper data structure to collect incoming and outgoing links for each StoreId + #[derive(Debug, Default)] + struct Linking { + outgoing: Vec, + incoming: Vec, + } + + // Helper function to aggregate the Link network + // + // This function aggregates a HashMap which maps each StoreId object in the store onto + // a Linking object, which contains a list of StoreIds which this entry links to and a + // list of StoreIds which link to the current one. + // + // The lambda returns an error if something fails + let aggregate_link_network = |store: &Store| -> Result> { + store + .entries()? + .into_get_iter() + .fold(Ok(HashMap::new()), |map, element| { + map.and_then(|mut map| { + debug!("Checking element = {:?}", element); + let entry = element?.ok_or_else(|| err_msg("TODO: Not yet handled"))?; + + debug!("Checking entry = {:?}", entry.get_location()); + + let internal_links = entry + .get_internal_links()? + .into_getter(store); // get the FLEs from the Store + + let mut linking = Linking::default(); + for internal_link in internal_links { + debug!("internal link = {:?}", internal_link); + + linking.outgoing.push(internal_link?.get_location().clone()); + linking.incoming.push(entry.get_location().clone()); + } + + map.insert(entry.get_location().clone(), linking); + Ok(map) + }) + }) + }; + + // Helper to check whethre all StoreIds in the network actually exists + // + // Because why not? + let all_collected_storeids_exist = |network: &HashMap| -> Result<()> { + for (id, _) in network.iter() { + if is_match!(self.get(id.clone()), Ok(Some(_))) { + debug!("Exists in store: {:?}", id); + + if !self.exists(id.clone())? { + warn!("Does exist in store but not on FS: {:?}", id); + return Err(err_msg("Link target does not exist")) + } + } else { + warn!("Does not exist in store: {:?}", id); + return Err(err_msg("Link target does not exist")) + } + } + + Ok(()) + }; + + // Helper function to create a SLCECD::OneDirectionalLink error object + let mk_one_directional_link_err = |src: StoreId, target: StoreId| -> Error { + Error::from(format_err!("Dead link: {} -> {}", + src.local_display_string(), + target.local_display_string())) + }; + + // Helper lambda to check whether the _incoming_ links of each entry actually also + // appear in the _outgoing_ list of the linked entry + let incoming_links_exists_as_outgoing_links = + |src: &StoreId, linking: &Linking, network: &HashMap| -> Result<()> { + for link in linking.incoming.iter() { + // Check whether the links which are _incoming_ on _src_ are outgoing + // in each of the links in the incoming list. + let incoming_consistent = network.get(link) + .map(|l| l.outgoing.contains(src)) + .unwrap_or(false); + + if !incoming_consistent { + return Err(mk_one_directional_link_err(src.clone(), link.clone())) + } + } + + Ok(()) + }; + + // Helper lambda to check whether the _outgoing links of each entry actually also + // appear in the _incoming_ list of the linked entry + let outgoing_links_exist_as_incoming_links = + |src: &StoreId, linking: &Linking, network: &HashMap| -> Result<()> { + for link in linking.outgoing.iter() { + // Check whether the links which are _outgoing_ on _src_ are incoming + // in each of the links in the outgoing list. + let outgoing_consistent = network.get(link) + .map(|l| l.incoming.contains(src)) + .unwrap_or(false); + + if !outgoing_consistent { + return Err(mk_one_directional_link_err(link.clone(), src.clone())) + } + } + + Ok(()) + }; + + aggregate_link_network(&self) + .map_dbg_str("Aggregated") + .map_dbg(|nw| { + let mut s = String::new(); + for (k, v) in nw { + s.push_str(&format!("{}\n in: {:?}\n out: {:?}", k, v.incoming, v.outgoing)); + } + s + }) + .and_then(|nw| { + all_collected_storeids_exist(&nw) + .map(|_| nw) + .context(err_msg("Link handling error")) + .map_err(Error::from) + }) + .and_then(|nw| { + for (id, linking) in nw.iter() { + incoming_links_exists_as_outgoing_links(id, linking, &nw)?; + outgoing_links_exist_as_incoming_links(id, linking, &nw)?; + } + Ok(()) + }) + .map(|_| ()) + } +} + diff --git a/lib/entry/libimagentrymarkdown/Cargo.toml b/lib/entry/libimagentrymarkdown/Cargo.toml index cf082a03..bcd97a5c 100644 --- a/lib/entry/libimagentrymarkdown/Cargo.toml +++ b/lib/entry/libimagentrymarkdown/Cargo.toml @@ -30,6 +30,7 @@ sha-1 = "0.8" libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" } libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" } libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink/" } +libimagentryurl = { version = "0.10.0", path = "../../../lib/entry/libimagentryurl/" } libimagentryref = { version = "0.10.0", path = "../../../lib/entry/libimagentryref/" } libimagutil = { version = "0.10.0", path = "../../../lib/etc/libimagutil/" } diff --git a/lib/entry/libimagentrymarkdown/src/lib.rs b/lib/entry/libimagentrymarkdown/src/lib.rs index 8eedff8d..0a8f0b9b 100644 --- a/lib/entry/libimagentrymarkdown/src/lib.rs +++ b/lib/entry/libimagentrymarkdown/src/lib.rs @@ -42,6 +42,7 @@ extern crate url; extern crate libimagstore; extern crate libimagerror; extern crate libimagentrylink; +extern crate libimagentryurl; extern crate libimagentryref; extern crate libimagutil; #[macro_use] extern crate failure; diff --git a/lib/entry/libimagentrymarkdown/src/processor.rs b/lib/entry/libimagentrymarkdown/src/processor.rs index 87f03ad7..c2626f4d 100644 --- a/lib/entry/libimagentrymarkdown/src/processor.rs +++ b/lib/entry/libimagentrymarkdown/src/processor.rs @@ -24,8 +24,8 @@ use failure::ResultExt; use failure::Error; use crate::link::extract_links; -use libimagentrylink::external::ExternalLinker; -use libimagentrylink::internal::InternalLinker; +use libimagentryurl::linker::UrlLinker; +use libimagentrylink::linker::InternalLinker; use libimagentryref::reference::MutRef; use libimagentryref::reference::RefFassade; use libimagentryref::hasher::sha1::Sha1Hasher; @@ -58,7 +58,7 @@ use url::Url; pub struct LinkProcessor { process_internal_links: bool, create_internal_targets: bool, - process_external_links: bool, + process_urls: bool, process_refs: bool } @@ -86,8 +86,8 @@ impl LinkProcessor { /// Switch external link processing on/off /// /// An external link must start with `https://` or `http://`. - pub fn process_external_links(mut self, b: bool) -> Self { - self.process_external_links = b; + pub fn process_urls(mut self, b: bool) -> Self { + self.process_urls = b; self } @@ -154,11 +154,11 @@ impl LinkProcessor { let _ = entry.add_internal_link(&mut target)?; }, LinkQualification::ExternalLink(url) => { - if !self.process_external_links { + if !self.process_urls { continue } - entry.add_external_link(store, url)?; + entry.add_url(store, url)?; }, LinkQualification::RefLink(url) => { use sha1::{Sha1, Digest}; @@ -273,7 +273,7 @@ impl Default for LinkProcessor { LinkProcessor { process_internal_links: true, create_internal_targets: false, - process_external_links: true, + process_urls: true, process_refs: false } } @@ -286,7 +286,7 @@ mod tests { use std::path::PathBuf; use libimagstore::store::Store; - use libimagentrylink::internal::InternalLinker; + use libimagentrylink::linker::InternalLinker; fn setup_logging() { let _ = ::env_logger::try_init(); @@ -330,7 +330,7 @@ mod tests { let processor = LinkProcessor::default() .process_internal_links(true) .create_internal_targets(false) - .process_external_links(false) + .process_urls(false) .process_refs(false); let result = processor.process(&mut base, &store); @@ -370,7 +370,7 @@ mod tests { let processor = LinkProcessor::default() .process_internal_links(true) .create_internal_targets(false) - .process_external_links(false) + .process_urls(false) .process_refs(false); let result = processor.process(&mut base, &store); @@ -391,7 +391,7 @@ mod tests { let processor = LinkProcessor::default() .process_internal_links(true) .create_internal_targets(true) - .process_external_links(false) + .process_urls(false) .process_refs(false); let result = processor.process(&mut base, &store); @@ -418,7 +418,7 @@ mod tests { } #[test] - fn test_process_one_external_link() { + fn test_process_one_url() { setup_logging(); let store = get_store(); @@ -431,14 +431,14 @@ mod tests { let processor = LinkProcessor::default() .process_internal_links(true) .create_internal_targets(true) - .process_external_links(true) + .process_urls(true) .process_refs(false); let result = processor.process(&mut base, &store); assert!(result.is_ok(), "Should be Ok(()): {:?}", result); // The hash of "http://example.com" processed in the `libimagentrylink` way. - let expected_link = "links/external/9c17e047f58f9220a7008d4f18152fee4d111d14"; + let expected_link = "url/external/9c17e047f58f9220a7008d4f18152fee4d111d14"; { let base_links = base.get_internal_links(); assert!(base_links.is_ok()); @@ -481,7 +481,7 @@ mod tests { let processor = LinkProcessor::default() .process_internal_links(false) .create_internal_targets(false) - .process_external_links(false) + .process_urls(false) .process_refs(true); let result = processor.process(&mut base, &store); @@ -514,7 +514,7 @@ mod tests { let processor = LinkProcessor::default() .process_internal_links(false) .create_internal_targets(false) - .process_external_links(false) + .process_urls(false) .process_refs(true); let result = processor.process(&mut base, &store); @@ -547,7 +547,7 @@ mod tests { let processor = LinkProcessor::default() .process_internal_links(false) .create_internal_targets(false) - .process_external_links(false) + .process_urls(false) .process_refs(false); let result = processor.process(&mut base, &store); @@ -575,7 +575,7 @@ mod tests { let processor = LinkProcessor::default() .process_internal_links(true) .create_internal_targets(true) - .process_external_links(false) + .process_urls(false) .process_refs(false); let result = processor.process(&mut base, &store); @@ -605,7 +605,7 @@ mod tests { let processor = LinkProcessor::default() .process_internal_links(false) .create_internal_targets(false) - .process_external_links(false) + .process_urls(false) .process_refs(false); let result = processor.process(&mut base, &store); diff --git a/lib/entry/libimagentryurl/Cargo.toml b/lib/entry/libimagentryurl/Cargo.toml new file mode 100644 index 00000000..2c8ede95 --- /dev/null +++ b/lib/entry/libimagentryurl/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "libimagentryurl" +version = "0.10.0" +authors = ["Matthias Beyer "] + +description = "Library for the imag core distribution" + +keywords = ["imag", "PIM", "personal", "information", "management"] +readme = "../../../README.md" +license = "LGPL-2.1" + +documentation = "https://imag-pim.org/doc/" +repository = "https://github.com/matthiasbeyer/imag" +homepage = "http://imag-pim.org" + +[badges] +travis-ci = { repository = "matthiasbeyer/imag" } +is-it-maintained-issue-resolution = { repository = "matthiasbeyer/imag" } +is-it-maintained-open-issues = { repository = "matthiasbeyer/imag" } +maintenance = { status = "actively-developed" } + +[dependencies] +itertools = "0.7" +log = "0.4.0" +toml = "0.5" +url = "1.5" +sha-1 = "0.7" +hex = "0.3" +is-match = "0.1" +toml-query = "0.9" +failure = "0.1" +failure_derive = "0.1" + +libimagstore = { version = "0.10.0", path = "../../../lib/core/libimagstore" } +libimagerror = { version = "0.10.0", path = "../../../lib/core/libimagerror" } +libimagutil = { version = "0.10.0", path = "../../../lib/etc/libimagutil" } +libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink" } + +[dev-dependencies] +env_logger = "0.5" + diff --git a/lib/entry/libimagentryurl/src/iter.rs b/lib/entry/libimagentryurl/src/iter.rs new file mode 100644 index 00000000..cfa0ccf5 --- /dev/null +++ b/lib/entry/libimagentryurl/src/iter.rs @@ -0,0 +1,195 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +//! Iterator helpers for external linking stuff +//! +//! Contains also helpers to filter iterators for external/internal links +//! +//! +//! # Warning +//! +//! This module uses `internal::Link` as link type, so we operate on _store ids_ here. +//! +//! Not to confuse with `external::Link` which is a real `FileLockEntry` under the hood. +//! + +use libimagentrylink::link::Link; +use libimagentrylink::iter::LinkIter; +use libimagstore::store::Store; +use libimagutil::debug_result::DebugResult; + +use failure::Fallible as Result; +use url::Url; + +/// Helper for building `OnlyExternalIter` and `NoExternalIter` +/// +/// The boolean value defines, how to interpret the `is_external_link_storeid()` return value +/// (here as "pred"): +/// +/// ```ignore +/// pred | bool | xor | take? +/// ---- | ---- | --- | ---- +/// 0 | 0 | 0 | 1 +/// 0 | 1 | 1 | 0 +/// 1 | 0 | 1 | 0 +/// 1 | 1 | 0 | 1 +/// ``` +/// +/// If `bool` says "take if return value is false", we take the element if the `pred` returns +/// false... and so on. +/// +/// As we can see, the operator between these two operants is `!(a ^ b)`. +pub struct ExternalFilterIter(LinkIter, bool); + +impl Iterator for ExternalFilterIter { + type Item = Link; + + fn next(&mut self) -> Option { + use crate::util::is_external_link_storeid; + + while let Some(elem) = self.0.next() { + trace!("Check whether is external: {:?}", elem); + if !(self.1 ^ is_external_link_storeid(&elem)) { + trace!("Is external id: {:?}", elem); + return Some(elem); + } + } + None + } +} + +/// Helper trait to be implemented on `LinkIter` to select or deselect all external links +/// +/// # See also +/// +/// Also see `OnlyExternalIter` and `NoExternalIter` and the helper traits/functions +/// `OnlyInteralLinks`/`only_internal_links()` and `OnlyExternalLinks`/`only_urls()`. +pub trait SelectExternal { + fn select_urls(self, b: bool) -> ExternalFilterIter; +} + +impl SelectExternal for LinkIter { + fn select_urls(self, b: bool) -> ExternalFilterIter { + ExternalFilterIter(self, b) + } +} + + +pub struct OnlyExternalIter(ExternalFilterIter); + +impl OnlyExternalIter { + pub fn new(li: LinkIter) -> OnlyExternalIter { + OnlyExternalIter(ExternalFilterIter(li, true)) + } + + pub fn urls<'a>(self, store: &'a Store) -> UrlIter<'a> { + UrlIter(self, store) + } +} + +impl Iterator for OnlyExternalIter { + type Item = Link; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +pub struct NoExternalIter(ExternalFilterIter); + +impl NoExternalIter { + pub fn new(li: LinkIter) -> NoExternalIter { + NoExternalIter(ExternalFilterIter(li, false)) + } +} + +impl Iterator for NoExternalIter { + type Item = Link; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +pub trait OnlyExternalLinks : Sized { + fn only_urls(self) -> OnlyExternalIter ; + + fn no_internal_links(self) -> OnlyExternalIter { + self.only_urls() + } +} + +impl OnlyExternalLinks for LinkIter { + fn only_urls(self) -> OnlyExternalIter { + OnlyExternalIter::new(self) + } +} + +pub trait OnlyInternalLinks : Sized { + fn only_internal_links(self) -> NoExternalIter; + + fn no_urls(self) -> NoExternalIter { + self.only_internal_links() + } +} + +impl OnlyInternalLinks for LinkIter { + fn only_internal_links(self) -> NoExternalIter { + NoExternalIter::new(self) + } +} + +pub struct UrlIter<'a>(OnlyExternalIter, &'a Store); + +impl<'a> Iterator for UrlIter<'a> { + type Item = Result; + + fn next(&mut self) -> Option { + use crate::link::Link; + + loop { + let next = self.0 + .next() + .map(|id| { + debug!("Retrieving entry for id: '{:?}'", id); + self.1 + .retrieve(id.clone()) + .map_dbg_err(|_| format!("Retrieving entry for id: '{:?}' failed", id)) + .map_err(From::from) + .and_then(|f| { + debug!("Store::retrieve({:?}) succeeded", id); + debug!("getting external link from file now"); + f.get_link_uri_from_filelockentry() + .map_dbg_str("Error happened while getting link URI from FLE") + .map_dbg_err(|e| format!("URL -> Err = {:?}", e)) + }) + }); + + match next { + Some(Ok(Some(link))) => return Some(Ok(link)), + Some(Ok(None)) => continue, + Some(Err(e)) => return Some(Err(e)), + None => return None + } + } + } + +} + + diff --git a/lib/entry/libimagentryurl/src/lib.rs b/lib/entry/libimagentryurl/src/lib.rs new file mode 100644 index 00000000..7d08ba14 --- /dev/null +++ b/lib/entry/libimagentryurl/src/lib.rs @@ -0,0 +1,74 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +#![forbid(unsafe_code)] + +#![deny( + dead_code, + non_camel_case_types, + non_snake_case, + path_statements, + trivial_numeric_casts, + unstable_features, + unused_allocation, + unused_import_braces, + unused_imports, + unused_must_use, + unused_mut, + unused_qualifications, + while_true, +)] + +//! External linking is a complex implementation to be able to serve a clean and easy-to-use +//! interface. +//! +//! Internally, there are no such things as "external links" (plural). Each Entry in the store can +//! only have _one_ external link. +//! +//! This library does the following therefor: It allows you to have several external links with one +//! entry, which are internally one file in the store for each link, linked with "internal +//! linking". +//! +//! This helps us greatly with deduplication of URLs. +//! + +extern crate itertools; +#[macro_use] extern crate log; +extern crate toml; +extern crate toml_query; +extern crate url; +extern crate sha1; +extern crate hex; +#[macro_use] extern crate failure; + +#[cfg(test)] +extern crate env_logger; + +#[macro_use] extern crate libimagstore; +extern crate libimagerror; +extern crate libimagutil; +extern crate libimagentrylink; + +module_entry_path_mod!("url"); + +pub mod iter; +pub mod link; +pub mod linker; +pub mod util; + diff --git a/lib/entry/libimagentryurl/src/link.rs b/lib/entry/libimagentryurl/src/link.rs new file mode 100644 index 00000000..60f0bf88 --- /dev/null +++ b/lib/entry/libimagentryurl/src/link.rs @@ -0,0 +1,76 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use failure::Error; +use failure::ResultExt; +use failure::Fallible as Result; +use failure::err_msg; +use url::Url; + +use libimagstore::store::Entry; +use libimagerror::errors::ErrorMsg as EM; + +use toml_query::read::TomlValueReadTypeExt; + +pub trait Link { + + fn get_link_uri_from_filelockentry(&self) -> Result>; + + fn get_url(&self) -> Result>; + +} + +impl Link for Entry { + + fn get_link_uri_from_filelockentry(&self) -> Result> { + self.get_header() + .read_string("links.external.content.url") + .context(format_err!("Error reading header 'links.external.content.url' from '{}'", self.get_location())) + .context(EM::EntryHeaderReadError) + .map_err(Error::from) + .and_then(|opt| match opt { + None => Ok(None), + Some(ref s) => { + debug!("Found url, parsing: {:?}", s); + Url::parse(&s[..]) + .map_err(Error::from) + .context(format_err!("Failed to parse URL: '{}'", s)) + .context(err_msg("Invalid URI")) + .map_err(Error::from) + .map(Some) + }, + }) + .context("Failed to get link URI from entry") + .map_err(Error::from) + } + + fn get_url(&self) -> Result> { + match self.get_header().read_string("links.external.url")? { + None => Ok(None), + Some(ref s) => Url::parse(&s[..]) + .context(format_err!("Failed to parse URL: '{}'", s)) + .map(Some) + .map_err(Error::from) + .context(EM::EntryHeaderReadError) + .map_err(Error::from), + } + } + +} + diff --git a/lib/entry/libimagentryurl/src/linker.rs b/lib/entry/libimagentryurl/src/linker.rs new file mode 100644 index 00000000..fe98ade3 --- /dev/null +++ b/lib/entry/libimagentryurl/src/linker.rs @@ -0,0 +1,224 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use std::ops::DerefMut; + +use libimagstore::storeid::StoreId; +use libimagstore::store::Store; +use libimagstore::store::Entry; +use libimagutil::debug_result::DebugResult; +use libimagentrylink::linker::InternalLinker; + +use failure::Fallible as Result; +use toml::Value; +use toml::map::Map; +use toml_query::read::TomlValueReadExt; +use toml_query::insert::TomlValueInsertExt; +use url::Url; +use sha1::{Sha1, Digest}; +use hex; + +use crate::iter::UrlIter; + +pub trait UrlLinker : InternalLinker { + + /// Get the external links from the implementor object + fn get_urls<'a>(&self, store: &'a Store) -> Result>; + + /// Set the external links for the implementor object + fn set_urls(&mut self, store: &Store, links: Vec) -> Result>; + + /// Add an external link to the implementor object + fn add_url(&mut self, store: &Store, link: Url) -> Result>; + + /// Remove an external link from the implementor object + fn remove_url(&mut self, store: &Store, link: Url) -> Result>; + +} + +/// Implement `ExternalLinker` for `Entry`, hiding the fact that there is no such thing as an external +/// link in an entry, but internal links to other entries which serve as external links, as one +/// entry in the store can only have one external link. +impl UrlLinker for Entry { + + /// Get the external links from the implementor object + fn get_urls<'a>(&self, store: &'a Store) -> Result> { + use crate::iter::OnlyExternalLinks; + + // Iterate through all internal links and filter for FileLockEntries which live in + // /link/external/ -> load these files and get the external link from their headers, + // put them into the return vector. + self.get_internal_links() + .map(|iter| { + debug!("Getting external links"); + iter.only_urls().urls(store) + }) + } + + /// Set the external links for the implementor object + /// + /// # Return Value + /// + /// Returns the StoreIds which were newly created for the new external links, if there are more + /// external links than before. + /// If there are less external links than before, an empty vec![] is returned. + /// + fn set_urls(&mut self, store: &Store, links: Vec) -> Result> { + // Take all the links, generate a SHA sum out of each one, filter out the already existing + // store entries and store the other URIs in the header of one FileLockEntry each, in + // the path /link/external/ + + debug!("Iterating {} links = {:?}", links.len(), links); + links.into_iter().map(|link| { + let hash = hex::encode(Sha1::digest(&link.as_str().as_bytes())); + let file_id = crate::module_path::new_id(format!("external/{}", hash)) + .map_dbg_err(|_| { + format!("Failed to build StoreId for this hash '{:?}'", hash) + })?; + + debug!("Link = '{:?}'", link); + debug!("Hash = '{:?}'", hash); + debug!("StoreId = '{:?}'", file_id); + + let link_already_exists = store.get(file_id.clone())?.is_some(); + + // retrieve the file from the store, which implicitely creates the entry if it does not + // exist + let mut file = store + .retrieve(file_id.clone()) + .map_dbg_err(|_| { + format!("Failed to create or retrieve an file for this link '{:?}'", link) + })?; + + debug!("Generating header content!"); + { + let hdr = file.deref_mut().get_header_mut(); + + let mut table = match hdr.read("links.external.content")? { + Some(&Value::Table(ref table)) => table.clone(), + Some(_) => { + warn!("There is a value at 'links.external.content' which is not a table."); + warn!("Going to override this value"); + Map::new() + }, + None => Map::new(), + }; + + let v = Value::String(link.into_string()); + + debug!("setting URL = '{:?}", v); + table.insert(String::from("url"), v); + + let _ = hdr.insert("links.external.content", Value::Table(table))?; + debug!("Setting URL worked"); + } + + // then add an internal link to the new file or return an error if this fails + let _ = self.add_internal_link(file.deref_mut())?; + debug!("Added internal link"); + + Ok((link_already_exists, file_id)) + }) + .filter_map(|res| match res { + Ok((exists, entry)) => if exists { Some(Ok(entry)) } else { None }, + Err(e) => Some(Err(e)) + }) + .collect() + } + + /// Add an external link to the implementor object + /// + /// # Return Value + /// + /// (See ExternalLinker::set_urls()) + /// + /// Returns the StoreIds which were newly created for the new external links, if there are more + /// external links than before. + /// If there are less external links than before, an empty vec![] is returned. + /// + fn add_url(&mut self, store: &Store, link: Url) -> Result> { + // get external links, add this one, save them + debug!("Getting links"); + self.get_urls(store) + .and_then(|links| { + let mut links = links.collect::>>()?; + + debug!("Adding link = '{:?}' to links = {:?}", link, links); + links.push(link); + + debug!("Setting {} links = {:?}", links.len(), links); + self.set_urls(store, links) + }) + } + + /// Remove an external link from the implementor object + /// + /// # Return Value + /// + /// (See ExternalLinker::set_urls()) + /// + /// Returns the StoreIds which were newly created for the new external links, if there are more + /// external links than before. + /// If there are less external links than before, an empty vec![] is returned. + /// + fn remove_url(&mut self, store: &Store, link: Url) -> Result> { + // get external links, remove this one, save them + self.get_urls(store) + .and_then(|links| { + debug!("Removing link = '{:?}'", link); + let links = links + .filter_map(Result::ok) + .filter(|l| l.as_str() != link.as_str()) + .collect::>(); + self.set_urls(store, links) + }) + } + +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + use libimagstore::store::Store; + + fn setup_logging() { + let _ = env_logger::try_init(); + } + + pub fn get_store() -> Store { + Store::new_inmemory(PathBuf::from("/"), &None).unwrap() + } + + + #[test] + fn test_simple() { + setup_logging(); + let store = get_store(); + let mut e = store.retrieve(PathBuf::from("base-test_simple")).unwrap(); + let url = Url::parse("http://google.de").unwrap(); + + assert!(e.add_url(&store, url.clone()).is_ok()); + + assert_eq!(1, e.get_urls(&store).unwrap().count()); + assert_eq!(url, e.get_urls(&store).unwrap().next().unwrap().unwrap()); + } + +} diff --git a/lib/entry/libimagentryurl/src/util.rs b/lib/entry/libimagentryurl/src/util.rs new file mode 100644 index 00000000..48d02fff --- /dev/null +++ b/lib/entry/libimagentryurl/src/util.rs @@ -0,0 +1,28 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use std::fmt::Debug; + +use libimagstore::storeid::StoreId; + +/// Check whether the StoreId starts with `/link/external/` +pub fn is_external_link_storeid + Debug>(id: A) -> bool { + debug!("Checking whether this is a 'url/external/': '{:?}'", id); + id.as_ref().is_in_collection(&["url", "external"]) +} diff --git a/scripts/release.sh b/scripts/release.sh index 09fc299b..ff1598db 100644 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -16,6 +16,7 @@ CRATES=( ./lib/etc/libimaginteraction ./lib/core/libimagrt ./lib/entry/libimagentrylink + ./lib/entry/libimagentryurl ./lib/entry/libimagentrytag ./lib/entry/libimagentryfilter ./lib/entry/libimagentrygps