diff --git a/src/main.rs b/src/main.rs index 99494a4b..20acbda0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,7 @@ use module::Module; use module::ModuleError; use module::CommandEnv; use module::bm::BMModule; -use storage::backend::StorageBackend; +use storage::StorageBackend; mod cli; mod configuration; diff --git a/src/module/bm/commands.rs b/src/module/bm/commands.rs index 09d52c44..2ca11cc5 100644 --- a/src/module/bm/commands.rs +++ b/src/module/bm/commands.rs @@ -6,7 +6,7 @@ use regex::Regex; use module::{CommandEnv, CommandResult, Module, ModuleError}; use module::bm::header::{build_header, get_tags_from_header}; use runtime::Runtime; -use storage::backend::StorageBackendError; +use storage::StorageBackendError; use storage::file::File; use storage::json::parser::JsonHeaderParser; use storage::parser::Parser; diff --git a/src/module/mod.rs b/src/module/mod.rs index a3cd4eaf..7bd75fed 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -7,7 +7,7 @@ use std::result::Result; use clap::ArgMatches; use runtime::Runtime; -use storage::backend::StorageBackend; +use storage::StorageBackend; pub mod bm; pub mod helpers; diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 28a75232..e69de29b 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -1,319 +0,0 @@ -use std::error::Error; -use std::fmt::{Display, Formatter}; -use std::fmt::Result as FMTResult; -use std::fs::File as FSFile; -use std::fs::{create_dir_all, remove_file}; -use std::io::{Read, Write}; -use std::vec::{Vec, IntoIter}; - -use glob::glob; -use glob::Paths; - -use module::Module; -use runtime::Runtime; -use storage::file::File; -use storage::file_id::*; -use storage::parser::{FileHeaderParser, Parser}; - -pub type BackendOperationResult = Result; - -pub struct StorageBackend { - basepath: String, - storepath: String, -} - -impl StorageBackend { - - pub fn new(rt: &Runtime) -> BackendOperationResult { - use self::StorageBackendError as SBE; - - let storepath = rt.get_rtp() + "/store/"; - debug!("Trying to create {}", storepath); - create_dir_all(&storepath).and_then(|_| { - debug!("Creating succeeded, constructing backend instance"); - Ok(StorageBackend { - basepath: rt.get_rtp(), - storepath: storepath.clone(), - }) - }).or_else(|e| { - debug!("Creating failed, constructing error instance"); - Err(SBE::new("create_dir_all()", "Could not create store directories", - Some(storepath), Some(Box::new(e)))) - }) - } - - pub fn iter_ids(&self, m: &Module) -> Result, StorageBackendError> - { - use self::StorageBackendError as SBE; - - let globstr = self.prefix_of_files_for_module(m) + "*.imag"; - debug!("Globstring = {}", globstr); - glob(&globstr[..]) - .and_then(|globlist| { - debug!("Iterating over globlist"); - Ok(globlist_to_file_id_vec(globlist).into_iter()) - }) - .map_err(|e| { - debug!("glob() returned error: {:?}", e); - SBE::new("iter_ids()", "Cannot iter on file ids", None, None) - }) - } - - pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser) - -> Result>, StorageBackendError> - where HP: FileHeaderParser - { - use self::StorageBackendError as SBE; - - self.iter_ids(m) - .and_then(|ids| { - debug!("Iterating ids and building files from them"); - debug!(" number of ids = {}", ids.len()); - Ok(self.filter_map_ids_to_files(m, p, ids).into_iter()) - }) - .map_err(|e| { - debug!("StorageBackend::iter_ids() returned error = {:?}", e); - SBE::new("iter_files()", "Cannot iter on files", None, - Some(Box::new(e))) - }) - } - - /* - * Write a file to disk. - * - * The file is moved to this function as the file won't be edited afterwards - */ - pub fn put_file(&self, f: File, p: &Parser) -> BackendOperationResult - where HP: FileHeaderParser - { - use self::StorageBackendError as SBE; - - let written = write_with_parser(&f, p); - if written.is_err() { return Err(written.err().unwrap()); } - let string = written.unwrap(); - - let path = self.build_filepath(&f); - debug!("Writing file: {}", path); - debug!(" string: {}", string); - - FSFile::create(&path).map(|mut file| { - debug!("Created file at '{}'", path); - file.write_all(&string.clone().into_bytes()) - .map_err(|ioerr| { - debug!("Could not write file"); - SBE::new("File::write_all()", - "Could not write out File contents", - None, Some(Box::new(ioerr))) - }) - }).map_err(|writeerr| { - debug!("Could not create file at '{}'", path); - SBE::new("File::create()", "Creating file on disk failed", None, - Some(Box::new(writeerr))) - }).and(Ok(())) - } - - /* - * Update a file. We have the UUID and can find the file on FS with it and - * then replace its contents with the contents of the passed file object - */ - pub fn update_file(&self, f: File, p: &Parser) -> BackendOperationResult - where HP: FileHeaderParser - { - use self::StorageBackendError as SBE; - - let contents = write_with_parser(&f, p); - if contents.is_err() { return Err(contents.err().unwrap()); } - let string = contents.unwrap(); - - let path = self.build_filepath(&f); - debug!("Writing file: {}", path); - debug!(" string: {}", string); - - FSFile::open(&path).map(|mut file| { - debug!("Open file at '{}'", path); - file.write_all(&string.clone().into_bytes()) - .map_err(|ioerr| { - debug!("Could not write file"); - SBE::new("File::write()", - "Tried to write contents of this file, though operation did not succeed", - Some(string), Some(Box::new(ioerr))) - }) - }).map_err(|writeerr| { - debug!("Could not write file at '{}'", path); - SBE::new("File::open()", - "Tried to update contents of this file, though file doesn't exist", - None, Some(Box::new(writeerr))) - }).and(Ok(())) - } - - /* - * Find a file by its ID and return it if found. Return nothing if not - * found, of course. - * - * TODO: Needs refactoring, as there might be an error when reading from - * disk OR the id just does not exist. - */ - pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser) -> Option> - where HP: FileHeaderParser - { - debug!("Searching for file with id '{}'", id); - - if id.get_type() == FileIDType::NONE { - // We don't know the hash type, so we glob() around a bit. - debug!("Having FileIDType::NONE, so we glob() for the raw ID"); - - let id_str = id.get_id().unwrap_or(String::from("INVALID")); - let globstr = self.prefix_of_files_for_module(m) + "*" + &id_str[..] + ".imag"; - debug!("Globbing with globstr = '{}'", globstr); - glob(&globstr[..]).map(|globlist| { - let idvec = globlist_to_file_id_vec(globlist).into_iter(); - let mut vec = self.filter_map_ids_to_files(m, p, idvec); - vec.reverse(); - vec.pop() - }).unwrap_or({ - debug!("No glob matches, actually. We can't do anything at this point"); - None - }) - } else { - // The (hash)type is already in the FileID object, so we can just - // build a path from the information we already have - debug!("We know FileIDType, so we build the path directly now"); - let filepath = self.build_filepath_with_id(m, id.clone()); - if let Ok(mut fs) = FSFile::open(filepath) { - let mut s = String::new(); - fs.read_to_string(&mut s); - - debug!("Success opening file with id '{}'", id); - debug!("Parsing to internal structure now"); - p.read(s).and_then(|(h, d)| { - Ok(File::from_parser_result(m, id.clone(), h, d)) - }).ok() - } else { - debug!("No file with id '{}'", id); - None - } - } - } - - pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult { - use self::StorageBackendError as SBE; - - if checked { - error!("Checked remove not implemented yet. I will crash now"); - unimplemented!() - } - - debug!("Doing unchecked remove"); - info!("Going to remove file: {}", file); - - let fp = self.build_filepath(&file); - remove_file(fp).map_err(|e| { - SBE::new("remove_file()", "File removal failed", - Some(format!("{}", file)), Some(Box::new(e))) - }) - } - - fn build_filepath(&self, f: &File) -> String { - self.build_filepath_with_id(f.owner(), f.id()) - } - - fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String { - let idstr : String = id.clone().into(); - let idtype : FileIDType = id.into(); - let typestr : String = idtype.into(); - - debug!("Building filepath with id"); - debug!(" basepath: '{}'", self.basepath); - debug!(" storepath: '{}'", self.storepath); - debug!(" id: '{}'", idstr); - debug!(" type: '{}'", typestr); - - self.prefix_of_files_for_module(owner) + - "-" + &typestr[..] + - "-" + &idstr[..] + - ".imag" - } - - fn prefix_of_files_for_module(&self, m: &Module) -> String { - self.storepath.clone() + m.name() - } - - fn filter_map_ids_to_files<'a, HP>(&self, - m: &'a Module, - p: &Parser, - ids: IntoIter) - -> Vec> - where HP: FileHeaderParser - { - ids.filter_map(|id| self.get_file_by_id(m, &id, p)) - .collect::>() - } - -} - -#[derive(Debug)] -pub struct StorageBackendError { - pub action: String, // The file system action in words - pub desc: String, // A short description - pub data_dump: Option, // Data dump, if any - pub caused_by: Option>, // caused from this error -} - -impl StorageBackendError { - - fn new(action: S, - desc: S, - data: Option, - cause: Option>) - -> StorageBackendError - where S: Into - { - StorageBackendError { - action: action.into(), - desc: desc.into(), - data_dump: data, - caused_by: None, - } - } - -} - -impl Error for StorageBackendError { - - fn description(&self) -> &str { - &self.desc[..] - } - - fn cause(&self) -> Option<&Error> { - self.caused_by.as_ref().map(|e| &**e) - } - -} - -impl<'a> Display for StorageBackendError { - fn fmt(&self, f: &mut Formatter) -> FMTResult { - write!(f, "StorageBackendError[{}]: {}", - self.action, self.desc) - } -} - - -fn write_with_parser<'a, HP>(f: &File, p: &Parser) -> Result - where HP: FileHeaderParser -{ - use self::StorageBackendError as SBE; - - p.write(f.contents()) - .or_else(|err| { - Err(SBE::new("Parser::write()", - "Cannot translate internal representation of file contents into on-disk representation", - None, Some(Box::new(err)))) - }) -} - -fn globlist_to_file_id_vec(globlist: Paths) -> Vec { - globlist.filter_map(Result::ok) - .map(|pbuf| FileID::from(&pbuf)) - .collect::>() -} - diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 861d813a..8bb13fe4 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,13 +1,326 @@ -pub use std::path::Path; -pub use std::fs::File; -pub use std::error::Error; - -pub use runtime::Runtime; +use std::error::Error; +use std::fmt::Result as FMTResult; +use std::fmt::{Display, Formatter}; +use std::fs::File as FSFile; +use std::fs::{create_dir_all, remove_file}; +use std::io::{Read, Write}; +use std::path::Path; +use std::vec::{Vec, IntoIter}; pub mod file; pub mod file_id; pub mod parser; pub mod backend; - pub mod json; +use glob::glob; +use glob::Paths; + +use module::Module; +use runtime::Runtime; +use storage::file::File; +use storage::file_id::*; +use storage::parser::{FileHeaderParser, Parser}; + +pub type BackendOperationResult = Result; + +pub struct StorageBackend { + basepath: String, + storepath: String, +} + +impl StorageBackend { + + pub fn new(rt: &Runtime) -> BackendOperationResult { + use self::StorageBackendError as SBE; + + let storepath = rt.get_rtp() + "/store/"; + debug!("Trying to create {}", storepath); + create_dir_all(&storepath).and_then(|_| { + debug!("Creating succeeded, constructing backend instance"); + Ok(StorageBackend { + basepath: rt.get_rtp(), + storepath: storepath.clone(), + }) + }).or_else(|e| { + debug!("Creating failed, constructing error instance"); + Err(SBE::new("create_dir_all()", "Could not create store directories", + Some(storepath), Some(Box::new(e)))) + }) + } + + pub fn iter_ids(&self, m: &Module) -> Result, StorageBackendError> + { + use self::StorageBackendError as SBE; + + let globstr = self.prefix_of_files_for_module(m) + "*.imag"; + debug!("Globstring = {}", globstr); + glob(&globstr[..]) + .and_then(|globlist| { + debug!("Iterating over globlist"); + Ok(globlist_to_file_id_vec(globlist).into_iter()) + }) + .map_err(|e| { + debug!("glob() returned error: {:?}", e); + SBE::new("iter_ids()", "Cannot iter on file ids", None, None) + }) + } + + pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser) + -> Result>, StorageBackendError> + where HP: FileHeaderParser + { + use self::StorageBackendError as SBE; + + self.iter_ids(m) + .and_then(|ids| { + debug!("Iterating ids and building files from them"); + debug!(" number of ids = {}", ids.len()); + Ok(self.filter_map_ids_to_files(m, p, ids).into_iter()) + }) + .map_err(|e| { + debug!("StorageBackend::iter_ids() returned error = {:?}", e); + SBE::new("iter_files()", "Cannot iter on files", None, + Some(Box::new(e))) + }) + } + + /* + * Write a file to disk. + * + * The file is moved to this function as the file won't be edited afterwards + */ + pub fn put_file(&self, f: File, p: &Parser) -> BackendOperationResult + where HP: FileHeaderParser + { + use self::StorageBackendError as SBE; + + let written = write_with_parser(&f, p); + if written.is_err() { return Err(written.err().unwrap()); } + let string = written.unwrap(); + + let path = self.build_filepath(&f); + debug!("Writing file: {}", path); + debug!(" string: {}", string); + + FSFile::create(&path).map(|mut file| { + debug!("Created file at '{}'", path); + file.write_all(&string.clone().into_bytes()) + .map_err(|ioerr| { + debug!("Could not write file"); + SBE::new("File::write_all()", + "Could not write out File contents", + None, Some(Box::new(ioerr))) + }) + }).map_err(|writeerr| { + debug!("Could not create file at '{}'", path); + SBE::new("File::create()", "Creating file on disk failed", None, + Some(Box::new(writeerr))) + }).and(Ok(())) + } + + /* + * Update a file. We have the UUID and can find the file on FS with it and + * then replace its contents with the contents of the passed file object + */ + pub fn update_file(&self, f: File, p: &Parser) -> BackendOperationResult + where HP: FileHeaderParser + { + use self::StorageBackendError as SBE; + + let contents = write_with_parser(&f, p); + if contents.is_err() { return Err(contents.err().unwrap()); } + let string = contents.unwrap(); + + let path = self.build_filepath(&f); + debug!("Writing file: {}", path); + debug!(" string: {}", string); + + FSFile::open(&path).map(|mut file| { + debug!("Open file at '{}'", path); + file.write_all(&string.clone().into_bytes()) + .map_err(|ioerr| { + debug!("Could not write file"); + SBE::new("File::write()", + "Tried to write contents of this file, though operation did not succeed", + Some(string), Some(Box::new(ioerr))) + }) + }).map_err(|writeerr| { + debug!("Could not write file at '{}'", path); + SBE::new("File::open()", + "Tried to update contents of this file, though file doesn't exist", + None, Some(Box::new(writeerr))) + }).and(Ok(())) + } + + /* + * Find a file by its ID and return it if found. Return nothing if not + * found, of course. + * + * TODO: Needs refactoring, as there might be an error when reading from + * disk OR the id just does not exist. + */ + pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser) -> Option> + where HP: FileHeaderParser + { + debug!("Searching for file with id '{}'", id); + + if id.get_type() == FileIDType::NONE { + // We don't know the hash type, so we glob() around a bit. + debug!("Having FileIDType::NONE, so we glob() for the raw ID"); + + let id_str = id.get_id().unwrap_or(String::from("INVALID")); + let globstr = self.prefix_of_files_for_module(m) + "*" + &id_str[..] + ".imag"; + debug!("Globbing with globstr = '{}'", globstr); + glob(&globstr[..]).map(|globlist| { + let idvec = globlist_to_file_id_vec(globlist).into_iter(); + let mut vec = self.filter_map_ids_to_files(m, p, idvec); + vec.reverse(); + vec.pop() + }).unwrap_or({ + debug!("No glob matches, actually. We can't do anything at this point"); + None + }) + } else { + // The (hash)type is already in the FileID object, so we can just + // build a path from the information we already have + debug!("We know FileIDType, so we build the path directly now"); + let filepath = self.build_filepath_with_id(m, id.clone()); + if let Ok(mut fs) = FSFile::open(filepath) { + let mut s = String::new(); + fs.read_to_string(&mut s); + + debug!("Success opening file with id '{}'", id); + debug!("Parsing to internal structure now"); + p.read(s).and_then(|(h, d)| { + Ok(File::from_parser_result(m, id.clone(), h, d)) + }).ok() + } else { + debug!("No file with id '{}'", id); + None + } + } + } + + pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult { + use self::StorageBackendError as SBE; + + if checked { + error!("Checked remove not implemented yet. I will crash now"); + unimplemented!() + } + + debug!("Doing unchecked remove"); + info!("Going to remove file: {}", file); + + let fp = self.build_filepath(&file); + remove_file(fp).map_err(|e| { + SBE::new("remove_file()", "File removal failed", + Some(format!("{}", file)), Some(Box::new(e))) + }) + } + + fn build_filepath(&self, f: &File) -> String { + self.build_filepath_with_id(f.owner(), f.id()) + } + + fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String { + let idstr : String = id.clone().into(); + let idtype : FileIDType = id.into(); + let typestr : String = idtype.into(); + + debug!("Building filepath with id"); + debug!(" basepath: '{}'", self.basepath); + debug!(" storepath: '{}'", self.storepath); + debug!(" id: '{}'", idstr); + debug!(" type: '{}'", typestr); + + self.prefix_of_files_for_module(owner) + + "-" + &typestr[..] + + "-" + &idstr[..] + + ".imag" + } + + fn prefix_of_files_for_module(&self, m: &Module) -> String { + self.storepath.clone() + m.name() + } + + fn filter_map_ids_to_files<'a, HP>(&self, + m: &'a Module, + p: &Parser, + ids: IntoIter) + -> Vec> + where HP: FileHeaderParser + { + ids.filter_map(|id| self.get_file_by_id(m, &id, p)) + .collect::>() + } + +} + +#[derive(Debug)] +pub struct StorageBackendError { + pub action: String, // The file system action in words + pub desc: String, // A short description + pub data_dump: Option, // Data dump, if any + pub caused_by: Option>, // caused from this error +} + +impl StorageBackendError { + + fn new(action: S, + desc: S, + data: Option, + cause: Option>) + -> StorageBackendError + where S: Into + { + StorageBackendError { + action: action.into(), + desc: desc.into(), + data_dump: data, + caused_by: None, + } + } + +} + +impl Error for StorageBackendError { + + fn description(&self) -> &str { + &self.desc[..] + } + + fn cause(&self) -> Option<&Error> { + self.caused_by.as_ref().map(|e| &**e) + } + +} + +impl<'a> Display for StorageBackendError { + fn fmt(&self, f: &mut Formatter) -> FMTResult { + write!(f, "StorageBackendError[{}]: {}", + self.action, self.desc) + } +} + + +fn write_with_parser<'a, HP>(f: &File, p: &Parser) -> Result + where HP: FileHeaderParser +{ + use self::StorageBackendError as SBE; + + p.write(f.contents()) + .or_else(|err| { + Err(SBE::new("Parser::write()", + "Cannot translate internal representation of file contents into on-disk representation", + None, Some(Box::new(err)))) + }) +} + +fn globlist_to_file_id_vec(globlist: Paths) -> Vec { + globlist.filter_map(Result::ok) + .map(|pbuf| FileID::from(&pbuf)) + .collect::>() +} +