From 098d25db896dd4ba627f36d172c300dc65033985 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 29 May 2019 18:03:46 +0200 Subject: [PATCH] Move code from external libimagentrylink module to libimagentryurl Signed-off-by: Matthias Beyer --- lib/entry/libimagentrylink/src/lib.rs | 1 - lib/entry/libimagentryurl/src/iter.rs | 195 +++++++++++++ lib/entry/libimagentryurl/src/lib.rs | 79 ++++- lib/entry/libimagentryurl/src/link.rs | 76 +++++ .../src/linker.rs} | 276 +----------------- lib/entry/libimagentryurl/src/util.rs | 28 ++ 6 files changed, 384 insertions(+), 271 deletions(-) create mode 100644 lib/entry/libimagentryurl/src/iter.rs create mode 100644 lib/entry/libimagentryurl/src/link.rs rename lib/entry/{libimagentrylink/src/external.rs => libimagentryurl/src/linker.rs} (50%) create mode 100644 lib/entry/libimagentryurl/src/util.rs diff --git a/lib/entry/libimagentrylink/src/lib.rs b/lib/entry/libimagentrylink/src/lib.rs index ce258815..992d3dd2 100644 --- a/lib/entry/libimagentrylink/src/lib.rs +++ b/lib/entry/libimagentrylink/src/lib.rs @@ -56,6 +56,5 @@ extern crate libimagutil; module_entry_path_mod!("links"); -pub mod external; pub mod internal; diff --git a/lib/entry/libimagentryurl/src/iter.rs b/lib/entry/libimagentryurl/src/iter.rs new file mode 100644 index 00000000..f5ef6ed0 --- /dev/null +++ b/lib/entry/libimagentryurl/src/iter.rs @@ -0,0 +1,195 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +//! Iterator helpers for external linking stuff +//! +//! Contains also helpers to filter iterators for external/internal links +//! +//! +//! # Warning +//! +//! This module uses `internal::Link` as link type, so we operate on _store ids_ here. +//! +//! Not to confuse with `external::Link` which is a real `FileLockEntry` under the hood. +//! + +use libimagentrylink::internal::Link; +use libimagentrylink::internal::iter::LinkIter; +use libimagstore::store::Store; +use libimagutil::debug_result::DebugResult; + +use failure::Fallible as Result; +use url::Url; + +/// Helper for building `OnlyExternalIter` and `NoExternalIter` +/// +/// The boolean value defines, how to interpret the `is_external_link_storeid()` return value +/// (here as "pred"): +/// +/// ```ignore +/// pred | bool | xor | take? +/// ---- | ---- | --- | ---- +/// 0 | 0 | 0 | 1 +/// 0 | 1 | 1 | 0 +/// 1 | 0 | 1 | 0 +/// 1 | 1 | 0 | 1 +/// ``` +/// +/// If `bool` says "take if return value is false", we take the element if the `pred` returns +/// false... and so on. +/// +/// As we can see, the operator between these two operants is `!(a ^ b)`. +pub struct ExternalFilterIter(LinkIter, bool); + +impl Iterator for ExternalFilterIter { + type Item = Link; + + fn next(&mut self) -> Option { + use crate::util::is_external_link_storeid; + + while let Some(elem) = self.0.next() { + trace!("Check whether is external: {:?}", elem); + if !(self.1 ^ is_external_link_storeid(&elem)) { + trace!("Is external id: {:?}", elem); + return Some(elem); + } + } + None + } +} + +/// Helper trait to be implemented on `LinkIter` to select or deselect all external links +/// +/// # See also +/// +/// Also see `OnlyExternalIter` and `NoExternalIter` and the helper traits/functions +/// `OnlyInteralLinks`/`only_internal_links()` and `OnlyExternalLinks`/`only_external_links()`. +pub trait SelectExternal { + fn select_external_links(self, b: bool) -> ExternalFilterIter; +} + +impl SelectExternal for LinkIter { + fn select_external_links(self, b: bool) -> ExternalFilterIter { + ExternalFilterIter(self, b) + } +} + + +pub struct OnlyExternalIter(ExternalFilterIter); + +impl OnlyExternalIter { + pub fn new(li: LinkIter) -> OnlyExternalIter { + OnlyExternalIter(ExternalFilterIter(li, true)) + } + + pub fn urls<'a>(self, store: &'a Store) -> UrlIter<'a> { + UrlIter(self, store) + } +} + +impl Iterator for OnlyExternalIter { + type Item = Link; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +pub struct NoExternalIter(ExternalFilterIter); + +impl NoExternalIter { + pub fn new(li: LinkIter) -> NoExternalIter { + NoExternalIter(ExternalFilterIter(li, false)) + } +} + +impl Iterator for NoExternalIter { + type Item = Link; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +pub trait OnlyExternalLinks : Sized { + fn only_external_links(self) -> OnlyExternalIter ; + + fn no_internal_links(self) -> OnlyExternalIter { + self.only_external_links() + } +} + +impl OnlyExternalLinks for LinkIter { + fn only_external_links(self) -> OnlyExternalIter { + OnlyExternalIter::new(self) + } +} + +pub trait OnlyInternalLinks : Sized { + fn only_internal_links(self) -> NoExternalIter; + + fn no_external_links(self) -> NoExternalIter { + self.only_internal_links() + } +} + +impl OnlyInternalLinks for LinkIter { + fn only_internal_links(self) -> NoExternalIter { + NoExternalIter::new(self) + } +} + +pub struct UrlIter<'a>(OnlyExternalIter, &'a Store); + +impl<'a> Iterator for UrlIter<'a> { + type Item = Result; + + fn next(&mut self) -> Option { + use crate::link::Link; + + loop { + let next = self.0 + .next() + .map(|id| { + debug!("Retrieving entry for id: '{:?}'", id); + self.1 + .retrieve(id.clone()) + .map_dbg_err(|_| format!("Retrieving entry for id: '{:?}' failed", id)) + .map_err(From::from) + .and_then(|f| { + debug!("Store::retrieve({:?}) succeeded", id); + debug!("getting external link from file now"); + f.get_link_uri_from_filelockentry() + .map_dbg_str("Error happened while getting link URI from FLE") + .map_dbg_err(|e| format!("URL -> Err = {:?}", e)) + }) + }); + + match next { + Some(Ok(Some(link))) => return Some(Ok(link)), + Some(Ok(None)) => continue, + Some(Err(e)) => return Some(Err(e)), + None => return None + } + } + } + +} + + diff --git a/lib/entry/libimagentryurl/src/lib.rs b/lib/entry/libimagentryurl/src/lib.rs index 31e1bb20..7d08ba14 100644 --- a/lib/entry/libimagentryurl/src/lib.rs +++ b/lib/entry/libimagentryurl/src/lib.rs @@ -1,7 +1,74 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +#![forbid(unsafe_code)] + +#![deny( + dead_code, + non_camel_case_types, + non_snake_case, + path_statements, + trivial_numeric_casts, + unstable_features, + unused_allocation, + unused_import_braces, + unused_imports, + unused_must_use, + unused_mut, + unused_qualifications, + while_true, +)] + +//! External linking is a complex implementation to be able to serve a clean and easy-to-use +//! interface. +//! +//! Internally, there are no such things as "external links" (plural). Each Entry in the store can +//! only have _one_ external link. +//! +//! This library does the following therefor: It allows you to have several external links with one +//! entry, which are internally one file in the store for each link, linked with "internal +//! linking". +//! +//! This helps us greatly with deduplication of URLs. +//! + +extern crate itertools; +#[macro_use] extern crate log; +extern crate toml; +extern crate toml_query; +extern crate url; +extern crate sha1; +extern crate hex; +#[macro_use] extern crate failure; + #[cfg(test)] -mod tests { - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); - } -} +extern crate env_logger; + +#[macro_use] extern crate libimagstore; +extern crate libimagerror; +extern crate libimagutil; +extern crate libimagentrylink; + +module_entry_path_mod!("url"); + +pub mod iter; +pub mod link; +pub mod linker; +pub mod util; + diff --git a/lib/entry/libimagentryurl/src/link.rs b/lib/entry/libimagentryurl/src/link.rs new file mode 100644 index 00000000..60f0bf88 --- /dev/null +++ b/lib/entry/libimagentryurl/src/link.rs @@ -0,0 +1,76 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use failure::Error; +use failure::ResultExt; +use failure::Fallible as Result; +use failure::err_msg; +use url::Url; + +use libimagstore::store::Entry; +use libimagerror::errors::ErrorMsg as EM; + +use toml_query::read::TomlValueReadTypeExt; + +pub trait Link { + + fn get_link_uri_from_filelockentry(&self) -> Result>; + + fn get_url(&self) -> Result>; + +} + +impl Link for Entry { + + fn get_link_uri_from_filelockentry(&self) -> Result> { + self.get_header() + .read_string("links.external.content.url") + .context(format_err!("Error reading header 'links.external.content.url' from '{}'", self.get_location())) + .context(EM::EntryHeaderReadError) + .map_err(Error::from) + .and_then(|opt| match opt { + None => Ok(None), + Some(ref s) => { + debug!("Found url, parsing: {:?}", s); + Url::parse(&s[..]) + .map_err(Error::from) + .context(format_err!("Failed to parse URL: '{}'", s)) + .context(err_msg("Invalid URI")) + .map_err(Error::from) + .map(Some) + }, + }) + .context("Failed to get link URI from entry") + .map_err(Error::from) + } + + fn get_url(&self) -> Result> { + match self.get_header().read_string("links.external.url")? { + None => Ok(None), + Some(ref s) => Url::parse(&s[..]) + .context(format_err!("Failed to parse URL: '{}'", s)) + .map(Some) + .map_err(Error::from) + .context(EM::EntryHeaderReadError) + .map_err(Error::from), + } + } + +} + diff --git a/lib/entry/libimagentrylink/src/external.rs b/lib/entry/libimagentryurl/src/linker.rs similarity index 50% rename from lib/entry/libimagentrylink/src/external.rs rename to lib/entry/libimagentryurl/src/linker.rs index 95398785..8e409743 100644 --- a/lib/entry/libimagentrylink/src/external.rs +++ b/lib/entry/libimagentryurl/src/linker.rs @@ -17,93 +17,26 @@ // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // -/// External linking is a complex implementation to be able to serve a clean and easy-to-use -/// interface. -/// -/// Internally, there are no such things as "external links" (plural). Each Entry in the store can -/// only have _one_ external link. -/// -/// This library does the following therefor: It allows you to have several external links with one -/// entry, which are internally one file in the store for each link, linked with "internal -/// linking". -/// -/// This helps us greatly with deduplication of URLs. -/// - use std::ops::DerefMut; -use std::fmt::Debug; -use libimagstore::store::Entry; -use libimagstore::store::Store; use libimagstore::storeid::StoreId; -use libimagutil::debug_result::*; -use libimagerror::errors::ErrorMsg as EM; +use libimagstore::store::Store; +use libimagstore::store::Entry; +use libimagutil::debug_result::DebugResult; +use libimagentrylink::internal::InternalLinker; -use toml_query::read::TomlValueReadExt; -use toml_query::read::TomlValueReadTypeExt; -use toml_query::insert::TomlValueInsertExt; -use toml::map::Map; -use failure::Error; use failure::Fallible as Result; -use failure::ResultExt; -use failure::err_msg; - -use crate::internal::InternalLinker; - -use self::iter::*; - use toml::Value; +use toml::map::Map; +use toml_query::read::TomlValueReadExt; +use toml_query::insert::TomlValueInsertExt; use url::Url; use sha1::{Sha1, Digest}; use hex; -pub trait Link { +use crate::iter::UrlIter; - fn get_link_uri_from_filelockentry(&self) -> Result>; - - fn get_url(&self) -> Result>; - -} - -impl Link for Entry { - - fn get_link_uri_from_filelockentry(&self) -> Result> { - self.get_header() - .read_string("links.external.content.url") - .context(format_err!("Error reading header 'links.external.content.url' from '{}'", self.get_location())) - .context(EM::EntryHeaderReadError) - .map_err(Error::from) - .and_then(|opt| match opt { - None => Ok(None), - Some(ref s) => { - debug!("Found url, parsing: {:?}", s); - Url::parse(&s[..]) - .map_err(Error::from) - .context(format_err!("Failed to parse URL: '{}'", s)) - .context(err_msg("Invalid URI")) - .map_err(Error::from) - .map(Some) - }, - }) - .context("Failed to get link URI from entry") - .map_err(Error::from) - } - - fn get_url(&self) -> Result> { - match self.get_header().read_string("links.external.url")? { - None => Ok(None), - Some(ref s) => Url::parse(&s[..]) - .context(format_err!("Failed to parse URL: '{}'", s)) - .map(Some) - .map_err(Error::from) - .context(EM::EntryHeaderReadError) - .map_err(Error::from), - } - } - -} - -pub trait ExternalLinker : InternalLinker { +pub trait UrlLinker : InternalLinker { /// Get the external links from the implementor object fn get_external_links<'a>(&self, store: &'a Store) -> Result>; @@ -119,199 +52,15 @@ pub trait ExternalLinker : InternalLinker { } -pub mod iter { - //! Iterator helpers for external linking stuff - //! - //! Contains also helpers to filter iterators for external/internal links - //! - //! - //! # Warning - //! - //! This module uses `internal::Link` as link type, so we operate on _store ids_ here. - //! - //! Not to confuse with `external::Link` which is a real `FileLockEntry` under the hood. - //! - - use libimagutil::debug_result::*; - use libimagstore::store::Store; - - use crate::internal::Link; - use crate::internal::iter::LinkIter; - use failure::Fallible as Result; - - use url::Url; - - /// Helper for building `OnlyExternalIter` and `NoExternalIter` - /// - /// The boolean value defines, how to interpret the `is_external_link_storeid()` return value - /// (here as "pred"): - /// - /// ```ignore - /// pred | bool | xor | take? - /// ---- | ---- | --- | ---- - /// 0 | 0 | 0 | 1 - /// 0 | 1 | 1 | 0 - /// 1 | 0 | 1 | 0 - /// 1 | 1 | 0 | 1 - /// ``` - /// - /// If `bool` says "take if return value is false", we take the element if the `pred` returns - /// false... and so on. - /// - /// As we can see, the operator between these two operants is `!(a ^ b)`. - pub struct ExternalFilterIter(LinkIter, bool); - - impl Iterator for ExternalFilterIter { - type Item = Link; - - fn next(&mut self) -> Option { - use super::is_external_link_storeid; - - while let Some(elem) = self.0.next() { - trace!("Check whether is external: {:?}", elem); - if !(self.1 ^ is_external_link_storeid(&elem)) { - trace!("Is external id: {:?}", elem); - return Some(elem); - } - } - None - } - } - - /// Helper trait to be implemented on `LinkIter` to select or deselect all external links - /// - /// # See also - /// - /// Also see `OnlyExternalIter` and `NoExternalIter` and the helper traits/functions - /// `OnlyInteralLinks`/`only_internal_links()` and `OnlyExternalLinks`/`only_external_links()`. - pub trait SelectExternal { - fn select_external_links(self, b: bool) -> ExternalFilterIter; - } - - impl SelectExternal for LinkIter { - fn select_external_links(self, b: bool) -> ExternalFilterIter { - ExternalFilterIter(self, b) - } - } - - - pub struct OnlyExternalIter(ExternalFilterIter); - - impl OnlyExternalIter { - pub fn new(li: LinkIter) -> OnlyExternalIter { - OnlyExternalIter(ExternalFilterIter(li, true)) - } - - pub fn urls<'a>(self, store: &'a Store) -> UrlIter<'a> { - UrlIter(self, store) - } - } - - impl Iterator for OnlyExternalIter { - type Item = Link; - - fn next(&mut self) -> Option { - self.0.next() - } - } - - pub struct NoExternalIter(ExternalFilterIter); - - impl NoExternalIter { - pub fn new(li: LinkIter) -> NoExternalIter { - NoExternalIter(ExternalFilterIter(li, false)) - } - } - - impl Iterator for NoExternalIter { - type Item = Link; - - fn next(&mut self) -> Option { - self.0.next() - } - } - - pub trait OnlyExternalLinks : Sized { - fn only_external_links(self) -> OnlyExternalIter ; - - fn no_internal_links(self) -> OnlyExternalIter { - self.only_external_links() - } - } - - impl OnlyExternalLinks for LinkIter { - fn only_external_links(self) -> OnlyExternalIter { - OnlyExternalIter::new(self) - } - } - - pub trait OnlyInternalLinks : Sized { - fn only_internal_links(self) -> NoExternalIter; - - fn no_external_links(self) -> NoExternalIter { - self.only_internal_links() - } - } - - impl OnlyInternalLinks for LinkIter { - fn only_internal_links(self) -> NoExternalIter { - NoExternalIter::new(self) - } - } - - pub struct UrlIter<'a>(OnlyExternalIter, &'a Store); - - impl<'a> Iterator for UrlIter<'a> { - type Item = Result; - - fn next(&mut self) -> Option { - use crate::external::Link; - - loop { - let next = self.0 - .next() - .map(|id| { - debug!("Retrieving entry for id: '{:?}'", id); - self.1 - .retrieve(id.clone()) - .map_dbg_err(|_| format!("Retrieving entry for id: '{:?}' failed", id)) - .map_err(From::from) - .and_then(|f| { - debug!("Store::retrieve({:?}) succeeded", id); - debug!("getting external link from file now"); - f.get_link_uri_from_filelockentry() - .map_dbg_str("Error happened while getting link URI from FLE") - .map_dbg_err(|e| format!("URL -> Err = {:?}", e)) - }) - }); - - match next { - Some(Ok(Some(link))) => return Some(Ok(link)), - Some(Ok(None)) => continue, - Some(Err(e)) => return Some(Err(e)), - None => return None - } - } - } - - } - -} - - -/// Check whether the StoreId starts with `/link/external/` -pub fn is_external_link_storeid + Debug>(id: A) -> bool { - debug!("Checking whether this is a 'links/external/': '{:?}'", id); - id.as_ref().is_in_collection(&["links", "external"]) -} - /// Implement `ExternalLinker` for `Entry`, hiding the fact that there is no such thing as an external /// link in an entry, but internal links to other entries which serve as external links, as one /// entry in the store can only have one external link. -impl ExternalLinker for Entry { +impl UrlLinker for Entry { /// Get the external links from the implementor object fn get_external_links<'a>(&self, store: &'a Store) -> Result> { + use crate::iter::OnlyExternalLinks; + // Iterate through all internal links and filter for FileLockEntries which live in // /link/external/ -> load these files and get the external link from their headers, // put them into the return vector. @@ -473,4 +222,3 @@ mod tests { } } - diff --git a/lib/entry/libimagentryurl/src/util.rs b/lib/entry/libimagentryurl/src/util.rs new file mode 100644 index 00000000..48d02fff --- /dev/null +++ b/lib/entry/libimagentryurl/src/util.rs @@ -0,0 +1,28 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015-2019 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use std::fmt::Debug; + +use libimagstore::storeid::StoreId; + +/// Check whether the StoreId starts with `/link/external/` +pub fn is_external_link_storeid + Debug>(id: A) -> bool { + debug!("Checking whether this is a 'url/external/': '{:?}'", id); + id.as_ref().is_in_collection(&["url", "external"]) +}