Merge pull request #326 from matthiasbeyer/libimaglink/external-linking-rewrite

Libimaglink/external linking rewrite

libimaglink/Cargo.toml

@@ -7,7 +7,9 @@ authors = ["Matthias Beyer <mail@beyermatthias.de>"]
 itertools = "0.4.7"
 log = "0.3.4"
 toml = "0.1.27"
+semver = "0.2"
 url = "0.5.5"
+rust-crypto = "0.2.35"
 
 [dependencies.libimagstore]
 path = "../libimagstore"

libimaglink/src/error.rs

@@ -12,6 +12,8 @@ pub enum LinkErrorKind {
     LinkTargetDoesNotExist,
     InternalConversionError,
     InvalidUri,
+    StoreReadError,
+    StoreWriteError,
 }
 
 fn link_error_type_as_str(e: &LinkErrorKind) -> &'static str {
@@ -33,6 +35,12 @@ fn link_error_type_as_str(e: &LinkErrorKind) -> &'static str {
 
         &LinkErrorKind::InvalidUri
             => "URI is not valid",
+
+        &LinkErrorKind::StoreReadError
+            => "Store read error",
+
+        &LinkErrorKind::StoreWriteError
+            => "Store write error",
     }
 }
 

libimaglink/src/external.rs

@@ -1,140 +1,247 @@
+/// External linking is a complex implementation to be able to serve a clean and easy-to-use
+/// interface.
+///
+/// Internally, there are no such things as "external links" (plural). Each Entry in the store can
+/// only have _one_ external link.
+///
+/// This library does the following therefor: It allows you to have several external links with one
+/// entry, which are internally one file in the store for each link, linked with "internal
+/// linking".
+///
+/// This helps us greatly with deduplication of URLs.
+///
+
 use std::convert::Into;
+use std::ops::Deref;
+use std::ops::DerefMut;
+use std::collections::BTreeMap;
 
 use libimagstore::store::Entry;
 use libimagstore::store::EntryHeader;
+use libimagstore::store::FileLockEntry;
+use libimagstore::store::Store;
+use libimagstore::storeid::StoreId;
+use libimagstore::storeid::IntoStoreId;
 
-use error::{LinkError, LinkErrorKind};
+use error::LinkError as LE;
+use error::LinkErrorKind as LEK;
 use result::Result;
+use internal::InternalLinker;
+use module_path::ModuleEntryPath;
 
 use toml::Value;
 use toml::Table;
 use url::Url;
+use crypto::sha1::Sha1;
+use crypto::digest::Digest;
 
-#[derive(PartialOrd, Ord, Eq, PartialEq, Clone, Debug)]
+/// "Link" Type, just an abstraction over FileLockEntry to have some convenience internally.
-pub struct Link {
+struct Link<'a> {
-    link: String
+    link: FileLockEntry<'a>
 }
 
-impl Link {
+impl<'a> Link<'a> {
 
-    pub fn new(s: String) -> Link {
+    pub fn new(fle: FileLockEntry<'a>) -> Link<'a> {
-        Link { link: s }
+        Link { link: fle }
     }
 
-    pub fn is_valid(&self) -> bool {
+    /// For interal use only. Load an Link from a store id, if this is actually a Link
-        Url::parse(&self.link[..]).is_ok()
+    fn retrieve(store: &'a Store, id: StoreId) -> Result<Option<Link<'a>>> {
+        store.retrieve(id)
+            .map(|fle| {
+                if let Some(_) = Link::get_link_uri_from_filelockentry(&fle) {
+                    Some(Link {
+                        link: fle
+                    })
+                } else {
+                    None
+                }
+            })
+            .map_err(|e| LE::new(LEK::StoreReadError, Some(Box::new(e))))
     }
 
+    /// Get a link Url object from a FileLockEntry, ignore errors.
+    fn get_link_uri_from_filelockentry(file: &FileLockEntry<'a>) -> Option<Url> {
+        file.deref()
+            .get_header()
+            .read("imag.content.uri")
+            .ok()
+            .and_then(|opt| {
+                match opt {
+                    Some(Value::String(s)) => Url::parse(&s[..]).ok(),
+                    _ => None
+                }
+            })
     }
 
-#[derive(Eq, PartialEq, Clone, Debug)]
+    pub fn get_url(&self) -> Result<Option<Url>> {
-pub struct Links {
+        let opt = self.link
-    links: Vec<Link>,
+            .deref()
-}
+            .get_header()
+            .read("imag.content.uri");
 
-impl Links {
+        match opt {
-
+            Ok(Some(Value::String(s))) => {
-    pub fn new(s: Vec<Link>) -> Links {
+                Url::parse(&s[..])
-        Links { links: s }
+                     .map(|s| Some(s))
-    }
+                     .map_err(|e| LE::new(LEK::EntryHeaderReadError, Some(Box::new(e))))
-
+            },
-    pub fn add(&mut self, l: Link) {
+            Ok(None) => Ok(None),
-        self.links.push(l);
+            _ => Err(LE::new(LEK::EntryHeaderReadError, None))
-    }
-
-    pub fn remove(&mut self, l: Link) {
-        self.links.retain(|link| l != link.clone());
-    }
-
-    pub fn all_valid(&self) -> bool {
-        self.links.iter().all(|l| l.is_valid())
-    }
-
-}
-
-impl Into<String> for Link {
-
-    fn into(self) -> String {
-        self.link
-    }
-
-}
-
-impl Into<Vec<Link>> for Links {
-
-    fn into(self) -> Vec<Link> {
-        self.links
-    }
-
-}
-
-pub trait ExternalLinker {
-
-    /// get the external link from the implementor object
-    fn get_external_link(&self) -> Result<Option<Link>>;
-
-    /// set the external link for the implementor object and return the current link from the entry,
-    /// if any.
-    fn set_external_link(&mut self, l: Link) -> Result<Option<Link>>;
-}
-
-impl ExternalLinker for EntryHeader {
-
-    fn get_external_link(&self) -> Result<Option<Link>> {
-        let uri = self.read("imag.content.uri");
-
-        if uri.is_err() {
-            let kind = LinkErrorKind::EntryHeaderReadError;
-            let lerr = LinkError::new(kind, Some(Box::new(uri.err().unwrap())));
-            return Err(lerr);
-        }
-        let uri = uri.unwrap();
-
-        match uri {
-            Some(Value::String(s)) => Ok(Some(Link::new(s))),
-            _ => Err(LinkError::new(LinkErrorKind::ExistingLinkTypeWrong, None)),
-        }
-    }
-
-    /// Set an external link in the header
-    ///
-    /// Return the previous set link if there was any
-    fn set_external_link(&mut self, l: Link) -> Result<Option<Link>> {
-        if !l.is_valid() {
-            return Err(LinkError::new(LinkErrorKind::InvalidUri, None));
-        }
-
-        let old_link = self.set("imag.content.uri", Value::String(l.into()));
-
-        if old_link.is_err() {
-            let kind = LinkErrorKind::EntryHeaderWriteError;
-            let lerr = LinkError::new(kind, Some(Box::new(old_link.err().unwrap())));
-            return Err(lerr);
-        }
-        let old_link = old_link.unwrap();
-
-        if old_link.is_none() {
-            return Ok(None);
-        }
-
-        match old_link.unwrap() {
-            Value::String(s) => Ok(Some(Link::new(s))),
-
-            // We don't do anything in this case and be glad we corrected the type error with this set()
-            _ => Ok(None),
         }
     }
 
 }
 
+pub trait ExternalLinker : InternalLinker {
+
+    /// Get the external links from the implementor object
+    fn get_external_links(&self, store: &Store) -> Result<Vec<Url>>;
+
+    /// Set the external links for the implementor object
+    fn set_external_links(&mut self, store: &Store, links: Vec<Url>) -> Result<()>;
+
+    /// Add an external link to the implementor object
+    fn add_external_link(&mut self, store: &Store, link: Url) -> Result<()>;
+
+    /// Remove an external link from the implementor object
+    fn remove_external_link(&mut self, store: &Store, link: Url) -> Result<()>;
+
+}
+
+/// Check whether the StoreId starts with `/link/external/`
+fn is_link_store_id(id: &StoreId) -> bool {
+    debug!("Checking whether this is a /link/external/*: '{:?}'", id);
+    id.starts_with("/link/external/")
+}
+
+fn get_external_link_from_file(entry: &FileLockEntry) -> Result<Url> {
+    Link::get_link_uri_from_filelockentry(entry) // TODO: Do not hide error by using this function
+        .ok_or(LE::new(LEK::StoreReadError, None))
+}
+
+/// Implement ExternalLinker for Entry, hiding the fact that there is no such thing as an external
+/// link in an entry, but internal links to other entries which serve as external links, as one
+/// entry in the store can only have one external link.
 impl ExternalLinker for Entry {
 
-    fn get_external_link(&self) -> Result<Option<Link>> {
+    /// Get the external links from the implementor object
-        self.get_header().get_external_link()
+    fn get_external_links(&self, store: &Store) -> Result<Vec<Url>> {
+        // Iterate through all internal links and filter for FileLockEntries which live in
+        // /link/external/<SHA> -> load these files and get the external link from their headers,
+        // put them into the return vector.
+        self.get_internal_links()
+            .map(|vect| {
+                debug!("Getting external links");
+                vect.into_iter()
+                    .filter(is_link_store_id)
+                    .map(|id| {
+                        debug!("Retrieving entry for id: '{:?}'", id);
+                        match store.retrieve(id.clone()) {
+                            Ok(f) => get_external_link_from_file(&f),
+                            Err(e) => {
+                                debug!("Retrieving entry for id: '{:?}' failed", id);
+                                Err(LE::new(LEK::StoreReadError, Some(Box::new(e))))
+                            }
+                        }
+                    })
+                    .filter_map(|x| x.ok()) // TODO: Do not ignore error here
+                    .collect()
+            })
+            .map_err(|e| LE::new(LEK::StoreReadError, Some(Box::new(e))))
     }
 
-    fn set_external_link(&mut self, l: Link) -> Result<Option<Link>> {
+    /// Set the external links for the implementor object
-        self.get_header_mut().set_external_link(l)
+    fn set_external_links(&mut self, store: &Store, links: Vec<Url>) -> Result<()> {
+        // Take all the links, generate a SHA sum out of each one, filter out the already existing
+        // store entries and store the other URIs in the header of one FileLockEntry each, in
+        // the path /link/external/<SHA of the URL>
+
+        debug!("Iterating {} links = {:?}", links.len(), links);
+        for link in links { // for all links
+            let hash = {
+                let mut s = Sha1::new();
+                s.input_str(&link.serialize()[..]);
+                s.result_str()
+            };
+            let file_id = ModuleEntryPath::new(format!("external/{}", hash)).into_storeid();
+
+            debug!("Link    = '{:?}'", link);
+            debug!("Hash    = '{:?}'", hash);
+            debug!("StoreId = '{:?}'", file_id);
+
+            // retrieve the file from the store, which implicitely creates the entry if it does not
+            // exist
+            let file = store.retrieve(file_id.clone());
+            if file.is_err() {
+                debug!("Failed to create or retrieve an file for this link '{:?}'", link);
+                return Err(LE::new(LEK::StoreWriteError, Some(Box::new(file.err().unwrap()))));
+            }
+            let mut file = file.unwrap();
+
+            debug!("Generating header content!");
+            {
+                let mut hdr = file.deref_mut().get_header_mut();
+
+                let mut table = match hdr.read("imag.content") {
+                    Ok(Some(Value::Table(table))) => table,
+                    Ok(Some(_)) => {
+                        warn!("There is a value at 'imag.content' which is not a table.");
+                        warn!("Going to override this value");
+                        BTreeMap::new()
+                    },
+                    Ok(None) => BTreeMap::new(),
+                    Err(e)   => return Err(LE::new(LEK::StoreWriteError, Some(Box::new(e)))),
+                };
+
+                let v = Value::String(link.serialize());
+
+                debug!("setting URL = '{:?}", v);
+                table.insert(String::from("url"), v);
+
+                if let Err(e) = hdr.set("imag.content", Value::Table(table)) {
+                    return Err(LE::new(LEK::StoreWriteError, Some(Box::new(e))));
+                } else {
+                    debug!("Setting URL worked");
+                }
+            }
+
+            // then add an internal link to the new file or return an error if this fails
+            if let Err(e) = self.add_internal_link(file.deref_mut()) {
+                debug!("Error adding internal link");
+                return Err(LE::new(LEK::StoreWriteError, Some(Box::new(e))));
+            }
+        }
+        debug!("Ready iterating");
+        Ok(())
+    }
+
+    /// Add an external link to the implementor object
+    fn add_external_link(&mut self, store: &Store, link: Url) -> Result<()> {
+        // get external links, add this one, save them
+        debug!("Getting links");
+        self.get_external_links(store)
+            .and_then(|mut links| {
+                debug!("Adding link = '{:?}' to links = {:?}", link, links);
+                links.push(link);
+                debug!("Setting {} links = {:?}", links.len(), links);
+                self.set_external_links(store, links)
+            })
+    }
+
+    /// Remove an external link from the implementor object
+    fn remove_external_link(&mut self, store: &Store, link: Url) -> Result<()> {
+        // get external links, remove this one, save them
+        self.get_external_links(store)
+            .and_then(|mut links| {
+                debug!("Removing link = '{:?}' from links = {:?}", link, links);
+                let links = links.into_iter()
+                    .filter(|l| l.serialize() != link.serialize())
+                    .collect();
+                self.set_external_links(store, links)
+            })
     }
 
 }
+

libimaglink/src/lib.rs

@@ -1,9 +1,13 @@
 extern crate itertools;
 #[macro_use] extern crate log;
 extern crate toml;
+extern crate semver;
 extern crate url;
+extern crate crypto;
 
-extern crate libimagstore;
+#[macro_use] extern crate libimagstore;
+
+module_entry_path_mod!("links", "0.1.0");
 
 pub mod error;
 pub mod external;