Merge branch 'storage' into master

src/storage/backend.rs

@@ -6,8 +6,11 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::vec::Vec;
 use std::fs::File as FSFile;
+use std::fs::create_dir_all;
+use std::fs::remove_file;
 use std::io::Read;
 use std::io::Write;
+use std::vec::IntoIter;
 
 use glob::glob;
 use glob::Paths;
@@ -19,18 +22,34 @@ use storage::parser::{FileHeaderParser, Parser, ParserError};
 use module::Module;
 use runtime::Runtime;
 
-pub type BackendOperationResult = Result<(), StorageBackendError>;
+pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;
 
 pub struct StorageBackend {
     basepath: String,
+    storepath: String,
 }
 
 impl StorageBackend {
 
-    pub fn new(basepath: String) -> StorageBackend {
+    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
-        StorageBackend {
+        let storepath = rt.get_rtp() + "/store/";
-            basepath: basepath,
+        debug!("Trying to create {}", storepath);
-        }
+        create_dir_all(&storepath).and_then(|_| {
+            debug!("Creating succeeded, constructing backend instance");
+            Ok(StorageBackend {
+                basepath: rt.get_rtp(),
+                storepath: storepath.clone(),
+            })
+        }).or_else(|e| {
+            debug!("Creating failed, constructing error instance");
+            let mut serr = StorageBackendError::build(
+                "create_dir_all()",
+                "Could not create store directories",
+                Some(storepath)
+            );
+            serr.caused_by = Some(Box::new(e));
+            Err(serr)
+        })
     }
 
     fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
@@ -40,9 +59,8 @@ impl StorageBackend {
         StorageBackend::new(path)
     }
 
-    fn get_file_ids(&self) -> Option<Vec<FileID>> {
+    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
-        debug!("Getting files from {}", self.basepath);
+        let list = glob(&self.prefix_of_files_for_module(m)[..]);
-        let list = glob(&self.basepath[..]);
 
         if let Ok(globlist) = list {
             let mut v = vec![];
@@ -61,69 +79,107 @@ impl StorageBackend {
         }
     }
 
+    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
+    {
+        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
+            let v = globlist.filter_map(Result::ok)
+                            .map(|pbuf| from_pathbuf(&pbuf))
+                            .collect::<Vec<FileID>>()
+                            .into_iter();
+            Ok(v)
+        }).ok()
+    }
+
+    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
+        -> Option<IntoIter<File<'a>>>
+        where HP: FileHeaderParser
+    {
+        self.iter_ids(m).and_then(|ids| {
+            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
+                    .collect::<Vec<File>>()
+                    .into_iter())
+        })
+    }
+
     /*
      * Write a file to disk.
      *
      * The file is moved to this function as the file won't be edited afterwards
      */
-    pub fn put_file<'a, HP>(&self, f: File, p: &Parser<HP>) ->
+    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
-        Result<BackendOperationResult, ParserError>
+        where HP: FileHeaderParser
-        where HP: FileHeaderParser<'a>
     {
-        let written = p.write(f.contents());
+        let written = write_with_parser(&f, p);
-        if let Ok(string) = written {
+        if written.is_err() { return Err(written.err().unwrap()); }
-            let path = self.build_filepath(&f);
+        let string = written.unwrap();
-            debug!("Writing file: {}", path);
+
-            debug!("    contents: {}", string);
+        let path = self.build_filepath(&f);
-            Ok(Ok(()))
+        debug!("Writing file: {}", path);
-        } else {
+        debug!("      string: {}", string);
-            debug!("Error parsing : {:?}", f.contents());
+
-            Err(written.err().unwrap())
+        FSFile::create(&path).map(|mut file| {
-        }
+            debug!("Created file at '{}'", path);
+            file.write_all(&string.clone().into_bytes())
+                .map_err(|ioerr| {
+                    debug!("Could not write file");
+                    let mut err = StorageBackendError::build(
+                            "File::write_all()",
+                            "Could not write out File contents",
+                            None
+                        );
+                    err.caused_by = Some(Box::new(ioerr));
+                    err
+                })
+        }).map_err(|writeerr| {
+            debug!("Could not create file at '{}'", path);
+            let mut err = StorageBackendError::build(
+                "File::create()",
+                "Creating file on disk failed",
+                None
+            );
+            err.caused_by = Some(Box::new(writeerr));
+            err
+        }).and(Ok(()))
     }
 
     /*
      * Update a file. We have the UUID and can find the file on FS with it and
      * then replace its contents with the contents of the passed file object
      */
-    pub fn update_file<'a, HP>(&self, f: File, p: &Parser<HP>)
+    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
-        -> Result<BackendOperationResult, ParserError>
+        where HP: FileHeaderParser
-        where HP: FileHeaderParser<'a>
     {
-        let contents = p.write(f.contents());
+        let contents = write_with_parser(&f, p);
-
+        if contents.is_err() { return Err(contents.err().unwrap()); }
-        if contents.is_err() {
+        let string = contents.unwrap();
-            debug!("Error parsing contents: {:?}", f.contents());
-            return Err(contents.err().unwrap());
-        }
-
-        let content = contents.unwrap();
-        debug!("Success parsing content : {}", content);
 
         let path = self.build_filepath(&f);
-        debug!("Trying to write to file at {}", path);
+        debug!("Writing file: {}", path);
-        if let Err(_) = FSFile::open(&path) {
+        debug!("      string: {}", string);
-            debug!("Error opening {}", path);
-            return Ok(Err(StorageBackendError::new(
-                String::from("File::open()"),
-                format!("Tried to open '{}'", path),
-                String::from("Tried to update contents of this file, though file doesn't exist"),
-                None)))
-        }
 
-        if let Ok(mut file) = FSFile::create(&path) {
+        FSFile::open(&path).map(|mut file| {
-            if let Err(writeerr) = file.write_all(&content.clone().into_bytes()) {
+            debug!("Open file at '{}'", path);
-                debug!("Error writing to {}", path);
+            file.write_all(&string.clone().into_bytes())
-                return Ok(Err(StorageBackendError::new(
+                .map_err(|ioerr| {
-                    String::from("File::write()"),
+                    debug!("Could not write file");
-                    format!("Tried to write '{}'", path),
+                    let mut err = StorageBackendError::build(
-                    String::from("Tried to write contents of this file, though operation did not succeed"),
+                            "File::write()",
-                    Some(content))))
+                            "Tried to write contents of this file, though operation did not succeed",
-            }
+                            Some(string)
-        }
+                        );
-
+                    err.caused_by = Some(Box::new(ioerr));
-        debug!("Successfully written to file.");
+                    err
-        Ok(Ok(()))
+                })
+        }).map_err(|writeerr| {
+            debug!("Could not write file at '{}'", path);
+            let mut err = StorageBackendError::build(
+                "File::open()",
+                "Tried to update contents of this file, though file doesn't exist",
+                None
+            );
+            err.caused_by = Some(Box::new(writeerr));
+            err
+        }).and(Ok(()))
     }
 
     /*
@@ -133,29 +189,57 @@ impl StorageBackend {
      * TODO: Needs refactoring, as there might be an error when reading from
      * disk OR the id just does not exist.
      */
-    pub fn get_file_by_id<'a, HP>(&self, id: FileID, p: &Parser<HP>) -> Option<File>
+    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
-        where HP: FileHeaderParser<'a>
+        where HP: FileHeaderParser
     {
         debug!("Searching for file with id '{}'", id);
-        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(id.clone())) {
+        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
             let mut s = String::new();
             fs.read_to_string(&mut s);
             debug!("Success reading file with id '{}'", id);
             debug!("Parsing to internal structure now");
-            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(id, h, d))).ok()
+            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
         } else {
             debug!("No file with id '{}'", id);
             None
         }
     }
 
-    fn build_filepath(&self, f: &File) -> String {
+    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
-        self.build_filepath_with_id(f.id())
+        if checked {
+            error!("Checked remove not implemented yet. I will crash now");
+            unimplemented!()
+        }
+
+        debug!("Doing unchecked remove");
+        info!("Going to remove file: {}", file);
+
+        let fp = self.build_filepath(&file);
+        remove_file(fp).map_err(|e| {
+            let mut serr = StorageBackendError::build(
+                "remove_file()",
+                "File removal failed",
+                Some(format!("{}", file))
+            );
+            serr.caused_by = Some(Box::new(e));
+            serr
+        })
     }
 
-    fn build_filepath_with_id(&self, id: FileID) -> String {
+    fn build_filepath(&self, f: &File) -> String {
-        debug!("Building filepath for id '{}'", id);
+        self.build_filepath_with_id(f.owner(), f.id())
-        self.basepath.clone() + &id[..]
+    }
+
+    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
+        debug!("Building filepath with id");
+        debug!("  basepath: '{}'", self.basepath);
+        debug!(" storepath: '{}'", self.storepath);
+        debug!("  id      : '{}'", id);
+        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
+    }
+
+    fn prefix_of_files_for_module(&self, m: &Module) -> String {
+        self.storepath.clone() + m.name()
     }
 
 }
@@ -164,23 +248,35 @@ impl StorageBackend {
 pub struct StorageBackendError {
     pub action: String,             // The file system action in words
     pub desc: String,               // A short description
-    pub explanation: String,        // A long, user friendly description
+    pub data_dump: Option<String>,  // Data dump, if any
-    pub data_dump: Option<String>   // Data dump, if any
+    pub caused_by: Option<Box<Error>>,  // caused from this error
 }
 
 impl StorageBackendError {
     fn new(action: String,
            desc  : String,
-           explan: String,
            data  : Option<String>) -> StorageBackendError
     {
         StorageBackendError {
             action:         action,
             desc:           desc,
-            explanation:    explan,
             data_dump:      data,
+            caused_by:      None,
         }
     }
+
+    fn build(action: &'static str,
+             desc:   &'static str,
+           data  : Option<String>) -> StorageBackendError
+    {
+        StorageBackendError {
+            action:         String::from(action),
+            desc:           String::from(desc),
+            dataDump:       data,
+            caused_by:      None,
+        }
+    }
+
 }
 
 impl Error for StorageBackendError {
@@ -190,15 +286,30 @@ impl Error for StorageBackendError {
     }
 
     fn cause(&self) -> Option<&Error> {
-        None
+        self.caused_by.as_ref().map(|e| &**e)
     }
 
 }
 
 impl Display for StorageBackendError {
     fn fmt(&self, f: &mut Formatter) -> FMTResult {
-        write!(f, "StorageBackendError[{}]: {}\n\n{}",
+        write!(f, "StorageBackendError[{}]: {}",
-               self.action, self.desc, self.explanation)
+               self.action, self.desc)
     }
 }
 
+
+fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
+    where HP: FileHeaderParser
+{
+    p.write(f.contents())
+        .or_else(|err| {
+            let mut serr = StorageBackendError::build(
+                "Parser::write()",
+                "Cannot translate internal representation of file contents into on-disk representation",
+                None
+            );
+            serr.caused_by = Some(Box::new(err));
+            Err(serr)
+        })
+}

src/storage/file.rs

@@ -2,9 +2,12 @@ use std::error::Error;
 use std::fmt::{Debug, Display, Formatter};
 use std::fmt;
 
-use super::parser::FileHeaderParser;
+use module::Module;
+use super::parser::{FileHeaderParser, Parser, ParserError};
 use storage::file_id::*;
 
+use regex::Regex;
+
 #[derive(Debug)]
 pub enum FileHeaderSpec {
     Null,
@@ -27,7 +30,7 @@ pub enum FileHeaderData {
     UInteger(u64),
     Float(f64),
     Text(String),
-    Key { name: String, value: Box<FileHeaderData> },
+    Key { name: &'static str, value: Box<FileHeaderData> },
     Map { keys: Vec<FileHeaderData> },
     Array { values: Box<Vec<FileHeaderData>> },
 }
@@ -56,6 +59,28 @@ impl Display for FileHeaderSpec {
 
 }
 
+impl FileHeaderData {
+
+    pub fn matches_with(&self, r: &Regex) -> bool {
+        match self {
+            &FileHeaderData::Text(ref t) => r.is_match(&t[..]),
+            &FileHeaderData::Key{name: ref n, value: ref val} => {
+                r.is_match(n) || val.matches_with(r)
+            },
+
+            &FileHeaderData::Map{keys: ref dks} => {
+                dks.iter().any(|x| x.matches_with(r))
+            },
+
+            &FileHeaderData::Array{values: ref vs} => {
+                vs.iter().any(|x| x.matches_with(r))
+            }
+
+            _ => false,
+        }
+    }
+}
+
 pub struct MatchError<'a> {
     summary: String,
     expected: &'a FileHeaderSpec,
@@ -176,16 +201,18 @@ pub fn match_header_spec<'a>(spec: &'a FileHeaderSpec, data: &'a FileHeaderData)
  * Internal abstract view on a file. Does not exist on the FS and is just kept
  * internally until it is written to disk.
  */
-pub struct File {
+pub struct File<'a> {
-    header  : FileHeaderData,
+    owning_module   : &'a Module,
-    data    : String,
+    header          : FileHeaderData,
-    id      : FileID,
+    data            : String,
+    id              : FileID,
 }
 
-impl File {
+impl<'a> File<'a> {
 
-    pub fn new() -> File {
+    pub fn new(module: &'a Module) -> File<'a> {
         let f = File {
+            owning_module: module,
             header: FileHeaderData::Null,
             data: String::from(""),
             id: File::get_new_file_id(),
@@ -194,8 +221,9 @@ impl File {
         f
     }
 
-    pub fn from_parser_result(id: FileID, header: FileHeaderData, data: String) -> File {
+    pub fn from_parser_result(module: &Module, id: FileID, header: FileHeaderData, data: String) -> File {
         let f = File {
+            owning_module: module,
             header: header,
             data: data,
             id: id,
@@ -204,8 +232,9 @@ impl File {
         f
     }
 
-    pub fn new_with_header(h: FileHeaderData) -> File {
+    pub fn new_with_header(module: &Module, h: FileHeaderData) -> File {
         let f = File {
+            owning_module: module,
             header: h,
             data: String::from(""),
             id: File::get_new_file_id(),
@@ -214,8 +243,9 @@ impl File {
         f
     }
 
-    pub fn new_with_data(d: String) -> File {
+    pub fn new_with_data(module: &Module, d: String) -> File {
         let f = File {
+            owning_module: module,
             header: FileHeaderData::Null,
             data: d,
             id: File::get_new_file_id(),
@@ -224,8 +254,9 @@ impl File {
         f
     }
 
-    pub fn new_with_content(h: FileHeaderData, d: String) -> File {
+    pub fn new_with_content(module: &Module, h: FileHeaderData, d: String) -> File {
         let f = File {
+            owning_module: module,
             header: h,
             data: d,
             id: File::get_new_file_id(),
@@ -234,25 +265,66 @@ impl File {
         f
     }
 
+    pub fn header(&self) -> FileHeaderData {
+        self.header.clone()
+    }
+
+    pub fn data(&self) -> String {
+        self.data.clone()
+    }
+
     pub fn contents(&self) -> (FileHeaderData, String) {
-        (self.header.clone(), self.data.clone())
+        (self.header(), self.data())
     }
 
     pub fn id(&self) -> FileID {
         self.id.clone()
     }
 
+    pub fn owner(&self) -> &Module {
+        self.owning_module
+    }
+
+    pub fn matches_with(&self, r: &Regex) -> bool {
+        r.is_match(&self.data[..]) || self.header.matches_with(r)
+    }
+
     fn get_new_file_id() -> FileID {
         use uuid::Uuid;
         Uuid::new_v4().to_hyphenated_string()
     }
 }
 
-impl Debug for File {
+impl<'a> Display for File<'a> {
 
-    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
-        write!(f, "File[{:?}] header: '{:?}', data: '{:?}')",
+        write!(fmt,
-            self.id, self.header, self.data)
+"[File] Owner : '{:?}'
+        FileID: '{:?}'
+        Header: '{:?}'
+        Data  : '{:?}'",
+               self.owning_module,
+               self.header,
+               self.data,
+               self.id);
+        Ok(())
     }
+
 }
 
+impl<'a> Debug for File<'a> {
+
+    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
+        write!(fmt,
+"[File] Owner : '{:?}'
+        FileID: '{:?}'
+        Header: '{:?}'
+        Data  : '{:?}'",
+               self.owning_module,
+               self.header,
+               self.data,
+               self.id);
+        Ok(())
+    }
+
+}

src/storage/json/parser.rs

@@ -11,18 +11,22 @@ use super::super::parser::{FileHeaderParser, ParserError};
 use super::super::file::{FileHeaderSpec, FileHeaderData};
 
 
-struct JsonHeaderParser<'a> {
+pub struct JsonHeaderParser {
-    spec: &'a FileHeaderSpec,
+    spec: Option<FileHeaderSpec>,
 }
 
-impl<'a> FileHeaderParser<'a> for JsonHeaderParser<'a> {
+impl JsonHeaderParser {
 
-    fn new(spec: &'a FileHeaderSpec) -> JsonHeaderParser<'a> {
+    pub fn new(spec: Option<FileHeaderSpec>) -> JsonHeaderParser {
         JsonHeaderParser {
             spec: spec
         }
     }
 
+}
+
+impl FileHeaderParser for JsonHeaderParser {
+
     fn read(&self, string: Option<String>)
         -> Result<FileHeaderData, ParserError>
     {
@@ -41,9 +45,16 @@ impl<'a> FileHeaderParser<'a> for JsonHeaderParser<'a> {
     }
 
     fn write(&self, data: &FileHeaderData) -> Result<String, ParserError> {
-        let mut ser = Serializer::pretty(stdout());
+        let mut s = Vec::<u8>::new();
-        data.serialize(&mut ser);
+        {
-        Ok(String::from(""))
+            let mut ser = Serializer::pretty(&mut s);
+            data.serialize(&mut ser);
+        }
+
+        String::from_utf8(s).or(
+            Err(ParserError::short("Cannot parse utf8 bytes",
+                                   String::from("<not printable>"),
+                                   0)))
     }
 
 }
@@ -63,11 +74,12 @@ fn visit_json(v: &Value) -> FileHeaderData {
             }
         },
         &Value::Object(ref btree)    => {
+            let btree = btree.clone();
             FileHeaderData::Map{
-                keys: btree.clone().iter().map(|(k, v)|
+                keys: btree.into_iter().map(|(k, v)|
                     FileHeaderData::Key {
-                        name: k.clone(),
+                        name: k,
-                        value: Box::new(visit_json(v)),
+                        value: Box::new(visit_json(&v)),
                     }
                 ).collect()
             }
@@ -91,12 +103,21 @@ impl Serialize for FileHeaderData {
             &FileHeaderData::Float(ref f)           => f.serialize(ser),
             &FileHeaderData::Text(ref s)            => (&s[..]).serialize(ser),
             &FileHeaderData::Array{values: ref vs}  => vs.serialize(ser),
-            &FileHeaderData::Map{keys: ref ks}      => ks.serialize(ser),
+            &FileHeaderData::Map{keys: ref ks}      => {
-            &FileHeaderData::Key{name: ref n, value: ref v}   => {
                 let mut hm = HashMap::new();
-                hm.insert(n, v);
+
+                for key in ks {
+                    if let &FileHeaderData::Key{name: ref n, value: ref v} = key {
+                        hm.insert(n, v);
+                    } else {
+                        panic!("Not a key: {:?}", key);
+                    }
+                }
+
                 hm.serialize(ser)
-            }
+            },
+            &FileHeaderData::Key{name: ref n, value: ref v} => unreachable!(),
+
         }
     }
 

src/storage/parser.rs

@@ -73,9 +73,7 @@ impl Display for ParserError {
 
 }
 
-
+pub trait FileHeaderParser : Sized {
-pub trait FileHeaderParser<'a> : Sized {
-    fn new(spec: &'a FileHeaderSpec) -> Self;
     fn read(&self, string: Option<String>) -> Result<FileHeaderData, ParserError>;
     fn write(&self, data: &FileHeaderData) -> Result<String, ParserError>;
 }
@@ -87,11 +85,11 @@ pub struct Parser<HP>
     headerp : HP,
 }
 
-impl<'a, HP> Parser<HP> where
+impl<HP> Parser<HP> where
-    HP: FileHeaderParser<'a>,
+    HP: FileHeaderParser,
 {
 
-    fn new(headerp: HP) -> Parser<HP> {
+    pub fn new(headerp: HP) -> Parser<HP> {
         Parser {
             headerp: headerp,
         }