diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 2b8ab577..077cf8b4 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -6,8 +6,11 @@ use std::path::Path; use std::path::PathBuf; use std::vec::Vec; use std::fs::File as FSFile; +use std::fs::create_dir_all; +use std::fs::remove_file; use std::io::Read; use std::io::Write; +use std::vec::IntoIter; use glob::glob; use glob::Paths; @@ -19,18 +22,34 @@ use storage::parser::{FileHeaderParser, Parser, ParserError}; use module::Module; use runtime::Runtime; -pub type BackendOperationResult = Result<(), StorageBackendError>; +pub type BackendOperationResult = Result; pub struct StorageBackend { basepath: String, + storepath: String, } impl StorageBackend { - pub fn new(basepath: String) -> StorageBackend { - StorageBackend { - basepath: basepath, - } + pub fn new(rt: &Runtime) -> BackendOperationResult { + let storepath = rt.get_rtp() + "/store/"; + debug!("Trying to create {}", storepath); + create_dir_all(&storepath).and_then(|_| { + debug!("Creating succeeded, constructing backend instance"); + Ok(StorageBackend { + basepath: rt.get_rtp(), + storepath: storepath.clone(), + }) + }).or_else(|e| { + debug!("Creating failed, constructing error instance"); + let mut serr = StorageBackendError::build( + "create_dir_all()", + "Could not create store directories", + Some(storepath) + ); + serr.caused_by = Some(Box::new(e)); + Err(serr) + }) } fn build(rt: &Runtime, m: &M) -> StorageBackend { @@ -40,9 +59,8 @@ impl StorageBackend { StorageBackend::new(path) } - fn get_file_ids(&self) -> Option> { - debug!("Getting files from {}", self.basepath); - let list = glob(&self.basepath[..]); + fn get_file_ids(&self, m: &Module) -> Option> { + let list = glob(&self.prefix_of_files_for_module(m)[..]); if let Ok(globlist) = list { let mut v = vec![]; @@ -61,69 +79,107 @@ impl StorageBackend { } } + pub fn iter_ids(&self, m: &Module) -> Option> + { + glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| { + let v = globlist.filter_map(Result::ok) + .map(|pbuf| from_pathbuf(&pbuf)) + .collect::>() + .into_iter(); + Ok(v) + }).ok() + } + + pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser) + -> Option>> + where HP: FileHeaderParser + { + self.iter_ids(m).and_then(|ids| { + Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p)) + .collect::>() + .into_iter()) + }) + } + /* * Write a file to disk. * * The file is moved to this function as the file won't be edited afterwards */ - pub fn put_file<'a, HP>(&self, f: File, p: &Parser) -> - Result - where HP: FileHeaderParser<'a> + pub fn put_file(&self, f: File, p: &Parser) -> BackendOperationResult + where HP: FileHeaderParser { - let written = p.write(f.contents()); - if let Ok(string) = written { - let path = self.build_filepath(&f); - debug!("Writing file: {}", path); - debug!(" contents: {}", string); - Ok(Ok(())) - } else { - debug!("Error parsing : {:?}", f.contents()); - Err(written.err().unwrap()) - } + let written = write_with_parser(&f, p); + if written.is_err() { return Err(written.err().unwrap()); } + let string = written.unwrap(); + + let path = self.build_filepath(&f); + debug!("Writing file: {}", path); + debug!(" string: {}", string); + + FSFile::create(&path).map(|mut file| { + debug!("Created file at '{}'", path); + file.write_all(&string.clone().into_bytes()) + .map_err(|ioerr| { + debug!("Could not write file"); + let mut err = StorageBackendError::build( + "File::write_all()", + "Could not write out File contents", + None + ); + err.caused_by = Some(Box::new(ioerr)); + err + }) + }).map_err(|writeerr| { + debug!("Could not create file at '{}'", path); + let mut err = StorageBackendError::build( + "File::create()", + "Creating file on disk failed", + None + ); + err.caused_by = Some(Box::new(writeerr)); + err + }).and(Ok(())) } /* * Update a file. We have the UUID and can find the file on FS with it and * then replace its contents with the contents of the passed file object */ - pub fn update_file<'a, HP>(&self, f: File, p: &Parser) - -> Result - where HP: FileHeaderParser<'a> + pub fn update_file(&self, f: File, p: &Parser) -> BackendOperationResult + where HP: FileHeaderParser { - let contents = p.write(f.contents()); - - if contents.is_err() { - debug!("Error parsing contents: {:?}", f.contents()); - return Err(contents.err().unwrap()); - } - - let content = contents.unwrap(); - debug!("Success parsing content : {}", content); + let contents = write_with_parser(&f, p); + if contents.is_err() { return Err(contents.err().unwrap()); } + let string = contents.unwrap(); let path = self.build_filepath(&f); - debug!("Trying to write to file at {}", path); - if let Err(_) = FSFile::open(&path) { - debug!("Error opening {}", path); - return Ok(Err(StorageBackendError::new( - String::from("File::open()"), - format!("Tried to open '{}'", path), - String::from("Tried to update contents of this file, though file doesn't exist"), - None))) - } + debug!("Writing file: {}", path); + debug!(" string: {}", string); - if let Ok(mut file) = FSFile::create(&path) { - if let Err(writeerr) = file.write_all(&content.clone().into_bytes()) { - debug!("Error writing to {}", path); - return Ok(Err(StorageBackendError::new( - String::from("File::write()"), - format!("Tried to write '{}'", path), - String::from("Tried to write contents of this file, though operation did not succeed"), - Some(content)))) - } - } - - debug!("Successfully written to file."); - Ok(Ok(())) + FSFile::open(&path).map(|mut file| { + debug!("Open file at '{}'", path); + file.write_all(&string.clone().into_bytes()) + .map_err(|ioerr| { + debug!("Could not write file"); + let mut err = StorageBackendError::build( + "File::write()", + "Tried to write contents of this file, though operation did not succeed", + Some(string) + ); + err.caused_by = Some(Box::new(ioerr)); + err + }) + }).map_err(|writeerr| { + debug!("Could not write file at '{}'", path); + let mut err = StorageBackendError::build( + "File::open()", + "Tried to update contents of this file, though file doesn't exist", + None + ); + err.caused_by = Some(Box::new(writeerr)); + err + }).and(Ok(())) } /* @@ -133,29 +189,57 @@ impl StorageBackend { * TODO: Needs refactoring, as there might be an error when reading from * disk OR the id just does not exist. */ - pub fn get_file_by_id<'a, HP>(&self, id: FileID, p: &Parser) -> Option - where HP: FileHeaderParser<'a> + pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser) -> Option> + where HP: FileHeaderParser { debug!("Searching for file with id '{}'", id); - if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(id.clone())) { + if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) { let mut s = String::new(); fs.read_to_string(&mut s); debug!("Success reading file with id '{}'", id); debug!("Parsing to internal structure now"); - p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(id, h, d))).ok() + p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok() } else { debug!("No file with id '{}'", id); None } } - fn build_filepath(&self, f: &File) -> String { - self.build_filepath_with_id(f.id()) + pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult { + if checked { + error!("Checked remove not implemented yet. I will crash now"); + unimplemented!() + } + + debug!("Doing unchecked remove"); + info!("Going to remove file: {}", file); + + let fp = self.build_filepath(&file); + remove_file(fp).map_err(|e| { + let mut serr = StorageBackendError::build( + "remove_file()", + "File removal failed", + Some(format!("{}", file)) + ); + serr.caused_by = Some(Box::new(e)); + serr + }) } - fn build_filepath_with_id(&self, id: FileID) -> String { - debug!("Building filepath for id '{}'", id); - self.basepath.clone() + &id[..] + fn build_filepath(&self, f: &File) -> String { + self.build_filepath_with_id(f.owner(), f.id()) + } + + fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String { + debug!("Building filepath with id"); + debug!(" basepath: '{}'", self.basepath); + debug!(" storepath: '{}'", self.storepath); + debug!(" id : '{}'", id); + self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag" + } + + fn prefix_of_files_for_module(&self, m: &Module) -> String { + self.storepath.clone() + m.name() } } @@ -164,23 +248,35 @@ impl StorageBackend { pub struct StorageBackendError { pub action: String, // The file system action in words pub desc: String, // A short description - pub explanation: String, // A long, user friendly description - pub data_dump: Option // Data dump, if any + pub data_dump: Option, // Data dump, if any + pub caused_by: Option>, // caused from this error } impl StorageBackendError { fn new(action: String, desc : String, - explan: String, data : Option) -> StorageBackendError { StorageBackendError { action: action, desc: desc, - explanation: explan, data_dump: data, + caused_by: None, } } + + fn build(action: &'static str, + desc: &'static str, + data : Option) -> StorageBackendError + { + StorageBackendError { + action: String::from(action), + desc: String::from(desc), + dataDump: data, + caused_by: None, + } + } + } impl Error for StorageBackendError { @@ -190,15 +286,30 @@ impl Error for StorageBackendError { } fn cause(&self) -> Option<&Error> { - None + self.caused_by.as_ref().map(|e| &**e) } } impl Display for StorageBackendError { fn fmt(&self, f: &mut Formatter) -> FMTResult { - write!(f, "StorageBackendError[{}]: {}\n\n{}", - self.action, self.desc, self.explanation) + write!(f, "StorageBackendError[{}]: {}", + self.action, self.desc) } } + +fn write_with_parser<'a, HP>(f: &File, p: &Parser) -> Result + where HP: FileHeaderParser +{ + p.write(f.contents()) + .or_else(|err| { + let mut serr = StorageBackendError::build( + "Parser::write()", + "Cannot translate internal representation of file contents into on-disk representation", + None + ); + serr.caused_by = Some(Box::new(err)); + Err(serr) + }) +} diff --git a/src/storage/file.rs b/src/storage/file.rs index 800cccbc..75bc3b8e 100644 --- a/src/storage/file.rs +++ b/src/storage/file.rs @@ -2,9 +2,12 @@ use std::error::Error; use std::fmt::{Debug, Display, Formatter}; use std::fmt; -use super::parser::FileHeaderParser; +use module::Module; +use super::parser::{FileHeaderParser, Parser, ParserError}; use storage::file_id::*; +use regex::Regex; + #[derive(Debug)] pub enum FileHeaderSpec { Null, @@ -27,7 +30,7 @@ pub enum FileHeaderData { UInteger(u64), Float(f64), Text(String), - Key { name: String, value: Box }, + Key { name: &'static str, value: Box }, Map { keys: Vec }, Array { values: Box> }, } @@ -56,6 +59,28 @@ impl Display for FileHeaderSpec { } +impl FileHeaderData { + + pub fn matches_with(&self, r: &Regex) -> bool { + match self { + &FileHeaderData::Text(ref t) => r.is_match(&t[..]), + &FileHeaderData::Key{name: ref n, value: ref val} => { + r.is_match(n) || val.matches_with(r) + }, + + &FileHeaderData::Map{keys: ref dks} => { + dks.iter().any(|x| x.matches_with(r)) + }, + + &FileHeaderData::Array{values: ref vs} => { + vs.iter().any(|x| x.matches_with(r)) + } + + _ => false, + } + } +} + pub struct MatchError<'a> { summary: String, expected: &'a FileHeaderSpec, @@ -176,16 +201,18 @@ pub fn match_header_spec<'a>(spec: &'a FileHeaderSpec, data: &'a FileHeaderData) * Internal abstract view on a file. Does not exist on the FS and is just kept * internally until it is written to disk. */ -pub struct File { - header : FileHeaderData, - data : String, - id : FileID, +pub struct File<'a> { + owning_module : &'a Module, + header : FileHeaderData, + data : String, + id : FileID, } -impl File { +impl<'a> File<'a> { - pub fn new() -> File { + pub fn new(module: &'a Module) -> File<'a> { let f = File { + owning_module: module, header: FileHeaderData::Null, data: String::from(""), id: File::get_new_file_id(), @@ -194,8 +221,9 @@ impl File { f } - pub fn from_parser_result(id: FileID, header: FileHeaderData, data: String) -> File { + pub fn from_parser_result(module: &Module, id: FileID, header: FileHeaderData, data: String) -> File { let f = File { + owning_module: module, header: header, data: data, id: id, @@ -204,8 +232,9 @@ impl File { f } - pub fn new_with_header(h: FileHeaderData) -> File { + pub fn new_with_header(module: &Module, h: FileHeaderData) -> File { let f = File { + owning_module: module, header: h, data: String::from(""), id: File::get_new_file_id(), @@ -214,8 +243,9 @@ impl File { f } - pub fn new_with_data(d: String) -> File { + pub fn new_with_data(module: &Module, d: String) -> File { let f = File { + owning_module: module, header: FileHeaderData::Null, data: d, id: File::get_new_file_id(), @@ -224,8 +254,9 @@ impl File { f } - pub fn new_with_content(h: FileHeaderData, d: String) -> File { + pub fn new_with_content(module: &Module, h: FileHeaderData, d: String) -> File { let f = File { + owning_module: module, header: h, data: d, id: File::get_new_file_id(), @@ -234,25 +265,66 @@ impl File { f } + pub fn header(&self) -> FileHeaderData { + self.header.clone() + } + + pub fn data(&self) -> String { + self.data.clone() + } + pub fn contents(&self) -> (FileHeaderData, String) { - (self.header.clone(), self.data.clone()) + (self.header(), self.data()) } pub fn id(&self) -> FileID { self.id.clone() } + pub fn owner(&self) -> &Module { + self.owning_module + } + + pub fn matches_with(&self, r: &Regex) -> bool { + r.is_match(&self.data[..]) || self.header.matches_with(r) + } + fn get_new_file_id() -> FileID { use uuid::Uuid; Uuid::new_v4().to_hyphenated_string() } } -impl Debug for File { +impl<'a> Display for File<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { - write!(f, "File[{:?}] header: '{:?}', data: '{:?}')", - self.id, self.header, self.data) + fn fmt(&self, fmt: &mut Formatter) -> fmt::Result { + write!(fmt, +"[File] Owner : '{:?}' + FileID: '{:?}' + Header: '{:?}' + Data : '{:?}'", + self.owning_module, + self.header, + self.data, + self.id); + Ok(()) } + } +impl<'a> Debug for File<'a> { + + fn fmt(&self, fmt: &mut Formatter) -> fmt::Result { + write!(fmt, +"[File] Owner : '{:?}' + FileID: '{:?}' + Header: '{:?}' + Data : '{:?}'", + self.owning_module, + self.header, + self.data, + self.id); + Ok(()) + } + +} diff --git a/src/storage/json/parser.rs b/src/storage/json/parser.rs index e0981a9d..ee230620 100644 --- a/src/storage/json/parser.rs +++ b/src/storage/json/parser.rs @@ -11,18 +11,22 @@ use super::super::parser::{FileHeaderParser, ParserError}; use super::super::file::{FileHeaderSpec, FileHeaderData}; -struct JsonHeaderParser<'a> { - spec: &'a FileHeaderSpec, +pub struct JsonHeaderParser { + spec: Option, } -impl<'a> FileHeaderParser<'a> for JsonHeaderParser<'a> { +impl JsonHeaderParser { - fn new(spec: &'a FileHeaderSpec) -> JsonHeaderParser<'a> { + pub fn new(spec: Option) -> JsonHeaderParser { JsonHeaderParser { spec: spec } } +} + +impl FileHeaderParser for JsonHeaderParser { + fn read(&self, string: Option) -> Result { @@ -41,9 +45,16 @@ impl<'a> FileHeaderParser<'a> for JsonHeaderParser<'a> { } fn write(&self, data: &FileHeaderData) -> Result { - let mut ser = Serializer::pretty(stdout()); - data.serialize(&mut ser); - Ok(String::from("")) + let mut s = Vec::::new(); + { + let mut ser = Serializer::pretty(&mut s); + data.serialize(&mut ser); + } + + String::from_utf8(s).or( + Err(ParserError::short("Cannot parse utf8 bytes", + String::from(""), + 0))) } } @@ -63,11 +74,12 @@ fn visit_json(v: &Value) -> FileHeaderData { } }, &Value::Object(ref btree) => { + let btree = btree.clone(); FileHeaderData::Map{ - keys: btree.clone().iter().map(|(k, v)| + keys: btree.into_iter().map(|(k, v)| FileHeaderData::Key { - name: k.clone(), - value: Box::new(visit_json(v)), + name: k, + value: Box::new(visit_json(&v)), } ).collect() } @@ -91,12 +103,21 @@ impl Serialize for FileHeaderData { &FileHeaderData::Float(ref f) => f.serialize(ser), &FileHeaderData::Text(ref s) => (&s[..]).serialize(ser), &FileHeaderData::Array{values: ref vs} => vs.serialize(ser), - &FileHeaderData::Map{keys: ref ks} => ks.serialize(ser), - &FileHeaderData::Key{name: ref n, value: ref v} => { + &FileHeaderData::Map{keys: ref ks} => { let mut hm = HashMap::new(); - hm.insert(n, v); + + for key in ks { + if let &FileHeaderData::Key{name: ref n, value: ref v} = key { + hm.insert(n, v); + } else { + panic!("Not a key: {:?}", key); + } + } + hm.serialize(ser) - } + }, + &FileHeaderData::Key{name: ref n, value: ref v} => unreachable!(), + } } diff --git a/src/storage/parser.rs b/src/storage/parser.rs index dc0d28c9..c7df264c 100644 --- a/src/storage/parser.rs +++ b/src/storage/parser.rs @@ -73,9 +73,7 @@ impl Display for ParserError { } - -pub trait FileHeaderParser<'a> : Sized { - fn new(spec: &'a FileHeaderSpec) -> Self; +pub trait FileHeaderParser : Sized { fn read(&self, string: Option) -> Result; fn write(&self, data: &FileHeaderData) -> Result; } @@ -87,11 +85,11 @@ pub struct Parser headerp : HP, } -impl<'a, HP> Parser where - HP: FileHeaderParser<'a>, +impl Parser where + HP: FileHeaderParser, { - fn new(headerp: HP) -> Parser { + pub fn new(headerp: HP) -> Parser { Parser { headerp: headerp, }