matthiasbeyer/imag
1
0
Fork 0
Code Releases Activity

Move code from external libimagentrylink module to libimagentryurl

Browse source 
Signed-off-by: Matthias Beyer <mail@beyermatthias.de>
This commit is contained in:
Matthias Beyer 2019-05-29 18:03:46 +02:00
parent 5cb641044f
commit 098d25db89
6 changed files with 384 additions and 271 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
lib/entry/libimagentrylink/src/lib.rs
View file

@ -56,6 +56,5 @@ extern crate libimagutil;
module_entry_path_mod!("links"); module_entry_path_mod!("links");
pub mod external;
pub mod internal; pub mod internal;

195
lib/entry/libimagentryurl/src/iter.rs Normal file
View file

@ -0,0 +1,195 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
//! Iterator helpers for external linking stuff
//!
//! Contains also helpers to filter iterators for external/internal links
//!
//!
//! # Warning
//!
//! This module uses `internal::Link` as link type, so we operate on _store ids_ here.
//!
//! Not to confuse with `external::Link` which is a real `FileLockEntry` under the hood.
//!
use libimagentrylink::internal::Link;
use libimagentrylink::internal::iter::LinkIter;
use libimagstore::store::Store;
use libimagutil::debug_result::DebugResult;
use failure::Fallible as Result;
use url::Url;
/// Helper for building `OnlyExternalIter` and `NoExternalIter`
///
/// The boolean value defines, how to interpret the `is_external_link_storeid()` return value
/// (here as "pred"):
///
/// ```ignore
/// pred | bool | xor | take?
/// ---- | ---- | --- | ----
/// 0 | 0 | 0 | 1
/// 0 | 1 | 1 | 0
/// 1 | 0 | 1 | 0
/// 1 | 1 | 0 | 1
/// ```
///
/// If `bool` says "take if return value is false", we take the element if the `pred` returns
/// false... and so on.
///
/// As we can see, the operator between these two operants is `!(a ^ b)`.
pub struct ExternalFilterIter(LinkIter, bool);
impl Iterator for ExternalFilterIter {
type Item = Link;
fn next(&mut self) -> Option<Self::Item> {
use crate::util::is_external_link_storeid;
while let Some(elem) = self.0.next() {
trace!("Check whether is external: {:?}", elem);
if !(self.1 ^ is_external_link_storeid(&elem)) {
trace!("Is external id: {:?}", elem);
return Some(elem);
}
}
None
}
}
/// Helper trait to be implemented on `LinkIter` to select or deselect all external links
///
/// # See also
///
/// Also see `OnlyExternalIter` and `NoExternalIter` and the helper traits/functions
/// `OnlyInteralLinks`/`only_internal_links()` and `OnlyExternalLinks`/`only_external_links()`.
pub trait SelectExternal {
fn select_external_links(self, b: bool) -> ExternalFilterIter;
}
impl SelectExternal for LinkIter {
fn select_external_links(self, b: bool) -> ExternalFilterIter {
ExternalFilterIter(self, b)
}
}
pub struct OnlyExternalIter(ExternalFilterIter);
impl OnlyExternalIter {
pub fn new(li: LinkIter) -> OnlyExternalIter {
OnlyExternalIter(ExternalFilterIter(li, true))
}
pub fn urls<'a>(self, store: &'a Store) -> UrlIter<'a> {
UrlIter(self, store)
}
}
impl Iterator for OnlyExternalIter {
type Item = Link;
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
}
pub struct NoExternalIter(ExternalFilterIter);
impl NoExternalIter {
pub fn new(li: LinkIter) -> NoExternalIter {
NoExternalIter(ExternalFilterIter(li, false))
}
}
impl Iterator for NoExternalIter {
type Item = Link;
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
}
pub trait OnlyExternalLinks : Sized {
fn only_external_links(self) -> OnlyExternalIter ;
fn no_internal_links(self) -> OnlyExternalIter {
self.only_external_links()
}
}
impl OnlyExternalLinks for LinkIter {
fn only_external_links(self) -> OnlyExternalIter {
OnlyExternalIter::new(self)
}
}
pub trait OnlyInternalLinks : Sized {
fn only_internal_links(self) -> NoExternalIter;
fn no_external_links(self) -> NoExternalIter {
self.only_internal_links()
}
}
impl OnlyInternalLinks for LinkIter {
fn only_internal_links(self) -> NoExternalIter {
NoExternalIter::new(self)
}
}
pub struct UrlIter<'a>(OnlyExternalIter, &'a Store);
impl<'a> Iterator for UrlIter<'a> {
type Item = Result<Url>;
fn next(&mut self) -> Option<Self::Item> {
use crate::link::Link;
loop {
let next = self.0
.next()
.map(|id| {
debug!("Retrieving entry for id: '{:?}'", id);
self.1
.retrieve(id.clone())
.map_dbg_err(|_| format!("Retrieving entry for id: '{:?}' failed", id))
.map_err(From::from)
.and_then(|f| {
debug!("Store::retrieve({:?}) succeeded", id);
debug!("getting external link from file now");
f.get_link_uri_from_filelockentry()
.map_dbg_str("Error happened while getting link URI from FLE")
.map_dbg_err(|e| format!("URL -> Err = {:?}", e))
})
});
match next {
Some(Ok(Some(link))) => return Some(Ok(link)),
Some(Ok(None)) => continue,
Some(Err(e)) => return Some(Err(e)),
None => return None
}
}
}
}

79
lib/entry/libimagentryurl/src/lib.rs
View file

@ -1,7 +1,74 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
#![forbid(unsafe_code)]
#![deny(
dead_code,
non_camel_case_types,
non_snake_case,
path_statements,
trivial_numeric_casts,
unstable_features,
unused_allocation,
unused_import_braces,
unused_imports,
unused_must_use,
unused_mut,
unused_qualifications,
while_true,
)]
//! External linking is a complex implementation to be able to serve a clean and easy-to-use
//! interface.
//!
//! Internally, there are no such things as "external links" (plural). Each Entry in the store can
//! only have _one_ external link.
//!
//! This library does the following therefor: It allows you to have several external links with one
//! entry, which are internally one file in the store for each link, linked with "internal
//! linking".
//!
//! This helps us greatly with deduplication of URLs.
//!
extern crate itertools;
#[macro_use] extern crate log;
extern crate toml;
extern crate toml_query;
extern crate url;
extern crate sha1;
extern crate hex;
#[macro_use] extern crate failure;
#[cfg(test)] #[cfg(test)]
mod tests { extern crate env_logger;
#[test]
fn it_works() { #[macro_use] extern crate libimagstore;
assert_eq!(2 + 2, 4); extern crate libimagerror;
} extern crate libimagutil;
} extern crate libimagentrylink;
module_entry_path_mod!("url");
pub mod iter;
pub mod link;
pub mod linker;
pub mod util;

76
lib/entry/libimagentryurl/src/link.rs Normal file
View file

@ -0,0 +1,76 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use failure::Error;
use failure::ResultExt;
use failure::Fallible as Result;
use failure::err_msg;
use url::Url;
use libimagstore::store::Entry;
use libimagerror::errors::ErrorMsg as EM;
use toml_query::read::TomlValueReadTypeExt;
pub trait Link {
fn get_link_uri_from_filelockentry(&self) -> Result<Option<Url>>;
fn get_url(&self) -> Result<Option<Url>>;
}
impl Link for Entry {
fn get_link_uri_from_filelockentry(&self) -> Result<Option<Url>> {
self.get_header()
.read_string("links.external.content.url")
.context(format_err!("Error reading header 'links.external.content.url' from '{}'", self.get_location()))
.context(EM::EntryHeaderReadError)
.map_err(Error::from)
.and_then(|opt| match opt {
None => Ok(None),
Some(ref s) => {
debug!("Found url, parsing: {:?}", s);
Url::parse(&s[..])
.map_err(Error::from)
.context(format_err!("Failed to parse URL: '{}'", s))
.context(err_msg("Invalid URI"))
.map_err(Error::from)
.map(Some)
},
})
.context("Failed to get link URI from entry")
.map_err(Error::from)
}
fn get_url(&self) -> Result<Option<Url>> {
match self.get_header().read_string("links.external.url")? {
None => Ok(None),
Some(ref s) => Url::parse(&s[..])
.context(format_err!("Failed to parse URL: '{}'", s))
.map(Some)
.map_err(Error::from)
.context(EM::EntryHeaderReadError)
.map_err(Error::from),
}
}
}

276
lib/entry/libimagentrylink/src/external.rs → lib/entry/libimagentryurl/src/linker.rs
View file

@ -17,93 +17,26 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// //
/// External linking is a complex implementation to be able to serve a clean and easy-to-use
/// interface.
///
/// Internally, there are no such things as "external links" (plural). Each Entry in the store can
/// only have _one_ external link.
///
/// This library does the following therefor: It allows you to have several external links with one
/// entry, which are internally one file in the store for each link, linked with "internal
/// linking".
///
/// This helps us greatly with deduplication of URLs.
///
use std::ops::DerefMut; use std::ops::DerefMut;
use std::fmt::Debug;
use libimagstore::store::Entry;
use libimagstore::store::Store;
use libimagstore::storeid::StoreId; use libimagstore::storeid::StoreId;
use libimagutil::debug_result::*; use libimagstore::store::Store;
use libimagerror::errors::ErrorMsg as EM; use libimagstore::store::Entry;
use libimagutil::debug_result::DebugResult;
use libimagentrylink::internal::InternalLinker;
use toml_query::read::TomlValueReadExt;
use toml_query::read::TomlValueReadTypeExt;
use toml_query::insert::TomlValueInsertExt;
use toml::map::Map;
use failure::Error;
use failure::Fallible as Result; use failure::Fallible as Result;
use failure::ResultExt;
use failure::err_msg;
use crate::internal::InternalLinker;
use self::iter::*;
use toml::Value; use toml::Value;
use toml::map::Map;
use toml_query::read::TomlValueReadExt;
use toml_query::insert::TomlValueInsertExt;
use url::Url; use url::Url;
use sha1::{Sha1, Digest}; use sha1::{Sha1, Digest};
use hex; use hex;
pub trait Link { use crate::iter::UrlIter;
fn get_link_uri_from_filelockentry(&self) -> Result<Option<Url>>; pub trait UrlLinker : InternalLinker {
fn get_url(&self) -> Result<Option<Url>>;
}
impl Link for Entry {
fn get_link_uri_from_filelockentry(&self) -> Result<Option<Url>> {
self.get_header()
.read_string("links.external.content.url")
.context(format_err!("Error reading header 'links.external.content.url' from '{}'", self.get_location()))
.context(EM::EntryHeaderReadError)
.map_err(Error::from)
.and_then(|opt| match opt {
None => Ok(None),
Some(ref s) => {
debug!("Found url, parsing: {:?}", s);
Url::parse(&s[..])
.map_err(Error::from)
.context(format_err!("Failed to parse URL: '{}'", s))
.context(err_msg("Invalid URI"))
.map_err(Error::from)
.map(Some)
},
})
.context("Failed to get link URI from entry")
.map_err(Error::from)
}
fn get_url(&self) -> Result<Option<Url>> {
match self.get_header().read_string("links.external.url")? {
None => Ok(None),
Some(ref s) => Url::parse(&s[..])
.context(format_err!("Failed to parse URL: '{}'", s))
.map(Some)
.map_err(Error::from)
.context(EM::EntryHeaderReadError)
.map_err(Error::from),
}
}
}
pub trait ExternalLinker : InternalLinker {
/// Get the external links from the implementor object /// Get the external links from the implementor object
fn get_external_links<'a>(&self, store: &'a Store) -> Result<UrlIter<'a>>; fn get_external_links<'a>(&self, store: &'a Store) -> Result<UrlIter<'a>>;
@ -119,199 +52,15 @@ pub trait ExternalLinker : InternalLinker {
} }
pub mod iter {
//! Iterator helpers for external linking stuff
//!
//! Contains also helpers to filter iterators for external/internal links
//!
//!
//! # Warning
//!
//! This module uses `internal::Link` as link type, so we operate on _store ids_ here.
//!
//! Not to confuse with `external::Link` which is a real `FileLockEntry` under the hood.
//!
use libimagutil::debug_result::*;
use libimagstore::store::Store;
use crate::internal::Link;
use crate::internal::iter::LinkIter;
use failure::Fallible as Result;
use url::Url;
/// Helper for building `OnlyExternalIter` and `NoExternalIter`
///
/// The boolean value defines, how to interpret the `is_external_link_storeid()` return value
/// (here as "pred"):
///
/// ```ignore
/// pred | bool | xor | take?
/// ---- | ---- | --- | ----
/// 0 | 0 | 0 | 1
/// 0 | 1 | 1 | 0
/// 1 | 0 | 1 | 0
/// 1 | 1 | 0 | 1
/// ```
///
/// If `bool` says "take if return value is false", we take the element if the `pred` returns
/// false... and so on.
///
/// As we can see, the operator between these two operants is `!(a ^ b)`.
pub struct ExternalFilterIter(LinkIter, bool);
impl Iterator for ExternalFilterIter {
type Item = Link;
fn next(&mut self) -> Option<Self::Item> {
use super::is_external_link_storeid;
while let Some(elem) = self.0.next() {
trace!("Check whether is external: {:?}", elem);
if !(self.1 ^ is_external_link_storeid(&elem)) {
trace!("Is external id: {:?}", elem);
return Some(elem);
}
}
None
}
}
/// Helper trait to be implemented on `LinkIter` to select or deselect all external links
///
/// # See also
///
/// Also see `OnlyExternalIter` and `NoExternalIter` and the helper traits/functions
/// `OnlyInteralLinks`/`only_internal_links()` and `OnlyExternalLinks`/`only_external_links()`.
pub trait SelectExternal {
fn select_external_links(self, b: bool) -> ExternalFilterIter;
}
impl SelectExternal for LinkIter {
fn select_external_links(self, b: bool) -> ExternalFilterIter {
ExternalFilterIter(self, b)
}
}
pub struct OnlyExternalIter(ExternalFilterIter);
impl OnlyExternalIter {
pub fn new(li: LinkIter) -> OnlyExternalIter {
OnlyExternalIter(ExternalFilterIter(li, true))
}
pub fn urls<'a>(self, store: &'a Store) -> UrlIter<'a> {
UrlIter(self, store)
}
}
impl Iterator for OnlyExternalIter {
type Item = Link;
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
}
pub struct NoExternalIter(ExternalFilterIter);
impl NoExternalIter {
pub fn new(li: LinkIter) -> NoExternalIter {
NoExternalIter(ExternalFilterIter(li, false))
}
}
impl Iterator for NoExternalIter {
type Item = Link;
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
}
pub trait OnlyExternalLinks : Sized {
fn only_external_links(self) -> OnlyExternalIter ;
fn no_internal_links(self) -> OnlyExternalIter {
self.only_external_links()
}
}
impl OnlyExternalLinks for LinkIter {
fn only_external_links(self) -> OnlyExternalIter {
OnlyExternalIter::new(self)
}
}
pub trait OnlyInternalLinks : Sized {
fn only_internal_links(self) -> NoExternalIter;
fn no_external_links(self) -> NoExternalIter {
self.only_internal_links()
}
}
impl OnlyInternalLinks for LinkIter {
fn only_internal_links(self) -> NoExternalIter {
NoExternalIter::new(self)
}
}
pub struct UrlIter<'a>(OnlyExternalIter, &'a Store);
impl<'a> Iterator for UrlIter<'a> {
type Item = Result<Url>;
fn next(&mut self) -> Option<Self::Item> {
use crate::external::Link;
loop {
let next = self.0
.next()
.map(|id| {
debug!("Retrieving entry for id: '{:?}'", id);
self.1
.retrieve(id.clone())
.map_dbg_err(|_| format!("Retrieving entry for id: '{:?}' failed", id))
.map_err(From::from)
.and_then(|f| {
debug!("Store::retrieve({:?}) succeeded", id);
debug!("getting external link from file now");
f.get_link_uri_from_filelockentry()
.map_dbg_str("Error happened while getting link URI from FLE")
.map_dbg_err(|e| format!("URL -> Err = {:?}", e))
})
});
match next {
Some(Ok(Some(link))) => return Some(Ok(link)),
Some(Ok(None)) => continue,
Some(Err(e)) => return Some(Err(e)),
None => return None
}
}
}
}
}
/// Check whether the StoreId starts with `/link/external/`
pub fn is_external_link_storeid<A: AsRef<StoreId> + Debug>(id: A) -> bool {
debug!("Checking whether this is a 'links/external/': '{:?}'", id);
id.as_ref().is_in_collection(&["links", "external"])
}
/// Implement `ExternalLinker` for `Entry`, hiding the fact that there is no such thing as an external /// Implement `ExternalLinker` for `Entry`, hiding the fact that there is no such thing as an external
/// link in an entry, but internal links to other entries which serve as external links, as one /// link in an entry, but internal links to other entries which serve as external links, as one
/// entry in the store can only have one external link. /// entry in the store can only have one external link.
impl ExternalLinker for Entry { impl UrlLinker for Entry {
/// Get the external links from the implementor object /// Get the external links from the implementor object
fn get_external_links<'a>(&self, store: &'a Store) -> Result<UrlIter<'a>> { fn get_external_links<'a>(&self, store: &'a Store) -> Result<UrlIter<'a>> {
use crate::iter::OnlyExternalLinks;
// Iterate through all internal links and filter for FileLockEntries which live in // Iterate through all internal links and filter for FileLockEntries which live in
// /link/external/<SHA> -> load these files and get the external link from their headers, // /link/external/<SHA> -> load these files and get the external link from their headers,
// put them into the return vector. // put them into the return vector.
@ -473,4 +222,3 @@ mod tests {
} }
} }

28
lib/entry/libimagentryurl/src/util.rs Normal file
View file

@ -0,0 +1,28 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
use std::fmt::Debug;
use libimagstore::storeid::StoreId;
/// Check whether the StoreId starts with `/link/external/`
pub fn is_external_link_storeid<A: AsRef<StoreId> + Debug>(id: A) -> bool {
debug!("Checking whether this is a 'url/external/': '{:?}'", id);
id.as_ref().is_in_collection(&["url", "external"])
}