matthiasbeyer/imag
1
0
Fork 0
Code Releases Activity

Merge branch 'storage' into master

Browse source
This commit is contained in:
Matthias Beyer 2015-12-02 12:07:40 +01:00
commit 78aeddb42e
4 changed files with 310 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

253
src/storage/backend.rs
View file

@ -6,8 +6,11 @@ use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;
use glob::glob;
use glob::Paths;
@ -19,18 +22,34 @@ use storage::parser::{FileHeaderParser, Parser, ParserError};
use module::Module;
use runtime::Runtime;
pub type BackendOperationResult = Result<(), StorageBackendError>;
pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;
pub struct StorageBackend {
basepath: String,
storepath: String,
}
impl StorageBackend {
pub fn new(basepath: String) -> StorageBackend {
StorageBackend {
basepath: basepath,
}
pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
let storepath = rt.get_rtp() + "/store/";
debug!("Trying to create {}", storepath);
create_dir_all(&storepath).and_then(|_| {
debug!("Creating succeeded, constructing backend instance");
Ok(StorageBackend {
basepath: rt.get_rtp(),
storepath: storepath.clone(),
})
}).or_else(|e| {
debug!("Creating failed, constructing error instance");
let mut serr = StorageBackendError::build(
"create_dir_all()",
"Could not create store directories",
Some(storepath)
);
serr.caused_by = Some(Box::new(e));
Err(serr)
})
}
fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
@ -40,9 +59,8 @@ impl StorageBackend {
StorageBackend::new(path)
}
fn get_file_ids(&self) -> Option<Vec<FileID>> {
debug!("Getting files from {}", self.basepath);
let list = glob(&self.basepath[..]);
fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
let list = glob(&self.prefix_of_files_for_module(m)[..]);
if let Ok(globlist) = list {
let mut v = vec![];
@ -61,69 +79,107 @@ impl StorageBackend {
}
}
pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
{
glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
let v = globlist.filter_map(Result::ok)
.map(|pbuf| from_pathbuf(&pbuf))
.collect::<Vec<FileID>>()
.into_iter();
Ok(v)
}).ok()
}
pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
-> Option<IntoIter<File<'a>>>
where HP: FileHeaderParser
{
self.iter_ids(m).and_then(|ids| {
Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
.collect::<Vec<File>>()
.into_iter())
})
}
/*
* Write a file to disk.
*
* The file is moved to this function as the file won't be edited afterwards
*/
pub fn put_file<'a, HP>(&self, f: File, p: &Parser<HP>) ->
Result<BackendOperationResult, ParserError>
where HP: FileHeaderParser<'a>
pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
where HP: FileHeaderParser
{
let written = p.write(f.contents());
if let Ok(string) = written {
let path = self.build_filepath(&f);
debug!("Writing file: {}", path);
debug!(" contents: {}", string);
Ok(Ok(()))
} else {
debug!("Error parsing : {:?}", f.contents());
Err(written.err().unwrap())
}
let written = write_with_parser(&f, p);
if written.is_err() { return Err(written.err().unwrap()); }
let string = written.unwrap();
let path = self.build_filepath(&f);
debug!("Writing file: {}", path);
debug!(" string: {}", string);
FSFile::create(&path).map(|mut file| {
debug!("Created file at '{}'", path);
file.write_all(&string.clone().into_bytes())
.map_err(|ioerr| {
debug!("Could not write file");
let mut err = StorageBackendError::build(
"File::write_all()",
"Could not write out File contents",
None
);
err.caused_by = Some(Box::new(ioerr));
err
})
}).map_err(|writeerr| {
debug!("Could not create file at '{}'", path);
let mut err = StorageBackendError::build(
"File::create()",
"Creating file on disk failed",
None
);
err.caused_by = Some(Box::new(writeerr));
err
}).and(Ok(()))
}
/*
* Update a file. We have the UUID and can find the file on FS with it and
* then replace its contents with the contents of the passed file object
*/
pub fn update_file<'a, HP>(&self, f: File, p: &Parser<HP>)
-> Result<BackendOperationResult, ParserError>
where HP: FileHeaderParser<'a>
pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
where HP: FileHeaderParser
{
let contents = p.write(f.contents());
if contents.is_err() {
debug!("Error parsing contents: {:?}", f.contents());
return Err(contents.err().unwrap());
}
let content = contents.unwrap();
debug!("Success parsing content : {}", content);
let contents = write_with_parser(&f, p);
if contents.is_err() { return Err(contents.err().unwrap()); }
let string = contents.unwrap();
let path = self.build_filepath(&f);
debug!("Trying to write to file at {}", path);
if let Err(_) = FSFile::open(&path) {
debug!("Error opening {}", path);
return Ok(Err(StorageBackendError::new(
String::from("File::open()"),
format!("Tried to open '{}'", path),
String::from("Tried to update contents of this file, though file doesn't exist"),
None)))
}
debug!("Writing file: {}", path);
debug!(" string: {}", string);
if let Ok(mut file) = FSFile::create(&path) {
if let Err(writeerr) = file.write_all(&content.clone().into_bytes()) {
debug!("Error writing to {}", path);
return Ok(Err(StorageBackendError::new(
String::from("File::write()"),
format!("Tried to write '{}'", path),
String::from("Tried to write contents of this file, though operation did not succeed"),
Some(content))))
}
}
debug!("Successfully written to file.");
Ok(Ok(()))
FSFile::open(&path).map(|mut file| {
debug!("Open file at '{}'", path);
file.write_all(&string.clone().into_bytes())
.map_err(|ioerr| {
debug!("Could not write file");
let mut err = StorageBackendError::build(
"File::write()",
"Tried to write contents of this file, though operation did not succeed",
Some(string)
);
err.caused_by = Some(Box::new(ioerr));
err
})
}).map_err(|writeerr| {
debug!("Could not write file at '{}'", path);
let mut err = StorageBackendError::build(
"File::open()",
"Tried to update contents of this file, though file doesn't exist",
None
);
err.caused_by = Some(Box::new(writeerr));
err
}).and(Ok(()))
}
/*
@ -133,29 +189,57 @@ impl StorageBackend {
* TODO: Needs refactoring, as there might be an error when reading from
* disk OR the id just does not exist.
*/
pub fn get_file_by_id<'a, HP>(&self, id: FileID, p: &Parser<HP>) -> Option<File>
where HP: FileHeaderParser<'a>
pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
where HP: FileHeaderParser
{
debug!("Searching for file with id '{}'", id);
if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(id.clone())) {
if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
let mut s = String::new();
fs.read_to_string(&mut s);
debug!("Success reading file with id '{}'", id);
debug!("Parsing to internal structure now");
p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(id, h, d))).ok()
p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
} else {
debug!("No file with id '{}'", id);
None
}
}
fn build_filepath(&self, f: &File) -> String {
self.build_filepath_with_id(f.id())
pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
if checked {
error!("Checked remove not implemented yet. I will crash now");
unimplemented!()
}
debug!("Doing unchecked remove");
info!("Going to remove file: {}", file);
let fp = self.build_filepath(&file);
remove_file(fp).map_err(|e| {
let mut serr = StorageBackendError::build(
"remove_file()",
"File removal failed",
Some(format!("{}", file))
);
serr.caused_by = Some(Box::new(e));
serr
})
}
fn build_filepath_with_id(&self, id: FileID) -> String {
debug!("Building filepath for id '{}'", id);
self.basepath.clone() + &id[..]
fn build_filepath(&self, f: &File) -> String {
self.build_filepath_with_id(f.owner(), f.id())
}
fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
debug!("Building filepath with id");
debug!(" basepath: '{}'", self.basepath);
debug!(" storepath: '{}'", self.storepath);
debug!(" id : '{}'", id);
self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
}
fn prefix_of_files_for_module(&self, m: &Module) -> String {
self.storepath.clone() + m.name()
}
}
@ -164,23 +248,35 @@ impl StorageBackend {
pub struct StorageBackendError {
pub action: String, // The file system action in words
pub desc: String, // A short description
pub explanation: String, // A long, user friendly description
pub data_dump: Option<String> // Data dump, if any
pub data_dump: Option<String>, // Data dump, if any
pub caused_by: Option<Box<Error>>, // caused from this error
}
impl StorageBackendError {
fn new(action: String,
desc : String,
explan: String,
data : Option<String>) -> StorageBackendError
{
StorageBackendError {
action: action,
desc: desc,
explanation: explan,
data_dump: data,
caused_by: None,
}
}
fn build(action: &'static str,
desc: &'static str,
data : Option<String>) -> StorageBackendError
{
StorageBackendError {
action: String::from(action),
desc: String::from(desc),
dataDump: data,
caused_by: None,
}
}
}
impl Error for StorageBackendError {
@ -190,15 +286,30 @@ impl Error for StorageBackendError {
}
fn cause(&self) -> Option<&Error> {
None
self.caused_by.as_ref().map(|e| &**e)
}
}
impl Display for StorageBackendError {
fn fmt(&self, f: &mut Formatter) -> FMTResult {
write!(f, "StorageBackendError[{}]: {}\n\n{}",
self.action, self.desc, self.explanation)
write!(f, "StorageBackendError[{}]: {}",
self.action, self.desc)
}
}
fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
where HP: FileHeaderParser
{
p.write(f.contents())
.or_else(|err| {
let mut serr = StorageBackendError::build(
"Parser::write()",
"Cannot translate internal representation of file contents into on-disk representation",
None
);
serr.caused_by = Some(Box::new(err));
Err(serr)
})
}

106
src/storage/file.rs
View file

@ -2,9 +2,12 @@ use std::error::Error;
use std::fmt::{Debug, Display, Formatter};
use std::fmt;
use super::parser::FileHeaderParser;
use module::Module;
use super::parser::{FileHeaderParser, Parser, ParserError};
use storage::file_id::*;
use regex::Regex;
#[derive(Debug)]
pub enum FileHeaderSpec {
Null,
@ -27,7 +30,7 @@ pub enum FileHeaderData {
UInteger(u64),
Float(f64),
Text(String),
Key { name: String, value: Box<FileHeaderData> },
Key { name: &'static str, value: Box<FileHeaderData> },
Map { keys: Vec<FileHeaderData> },
Array { values: Box<Vec<FileHeaderData>> },
}
@ -56,6 +59,28 @@ impl Display for FileHeaderSpec {
}
impl FileHeaderData {
pub fn matches_with(&self, r: &Regex) -> bool {
match self {
&FileHeaderData::Text(ref t) => r.is_match(&t[..]),
&FileHeaderData::Key{name: ref n, value: ref val} => {
r.is_match(n) || val.matches_with(r)
},
&FileHeaderData::Map{keys: ref dks} => {
dks.iter().any(|x| x.matches_with(r))
},
&FileHeaderData::Array{values: ref vs} => {
vs.iter().any(|x| x.matches_with(r))
}
_ => false,
}
}
}
pub struct MatchError<'a> {
summary: String,
expected: &'a FileHeaderSpec,
@ -176,16 +201,18 @@ pub fn match_header_spec<'a>(spec: &'a FileHeaderSpec, data: &'a FileHeaderData)
* Internal abstract view on a file. Does not exist on the FS and is just kept
* internally until it is written to disk.
*/
pub struct File {
header : FileHeaderData,
data : String,
id : FileID,
pub struct File<'a> {
owning_module : &'a Module,
header : FileHeaderData,
data : String,
id : FileID,
}
impl File {
impl<'a> File<'a> {
pub fn new() -> File {
pub fn new(module: &'a Module) -> File<'a> {
let f = File {
owning_module: module,
header: FileHeaderData::Null,
data: String::from(""),
id: File::get_new_file_id(),
@ -194,8 +221,9 @@ impl File {
f
}
pub fn from_parser_result(id: FileID, header: FileHeaderData, data: String) -> File {
pub fn from_parser_result(module: &Module, id: FileID, header: FileHeaderData, data: String) -> File {
let f = File {
owning_module: module,
header: header,
data: data,
id: id,
@ -204,8 +232,9 @@ impl File {
f
}
pub fn new_with_header(h: FileHeaderData) -> File {
pub fn new_with_header(module: &Module, h: FileHeaderData) -> File {
let f = File {
owning_module: module,
header: h,
data: String::from(""),
id: File::get_new_file_id(),
@ -214,8 +243,9 @@ impl File {
f
}
pub fn new_with_data(d: String) -> File {
pub fn new_with_data(module: &Module, d: String) -> File {
let f = File {
owning_module: module,
header: FileHeaderData::Null,
data: d,
id: File::get_new_file_id(),
@ -224,8 +254,9 @@ impl File {
f
}
pub fn new_with_content(h: FileHeaderData, d: String) -> File {
pub fn new_with_content(module: &Module, h: FileHeaderData, d: String) -> File {
let f = File {
owning_module: module,
header: h,
data: d,
id: File::get_new_file_id(),
@ -234,25 +265,66 @@ impl File {
f
}
pub fn header(&self) -> FileHeaderData {
self.header.clone()
}
pub fn data(&self) -> String {
self.data.clone()
}
pub fn contents(&self) -> (FileHeaderData, String) {
(self.header.clone(), self.data.clone())
(self.header(), self.data())
}
pub fn id(&self) -> FileID {
self.id.clone()
}
pub fn owner(&self) -> &Module {
self.owning_module
}
pub fn matches_with(&self, r: &Regex) -> bool {
r.is_match(&self.data[..]) || self.header.matches_with(r)
}
fn get_new_file_id() -> FileID {
use uuid::Uuid;
Uuid::new_v4().to_hyphenated_string()
}
}
impl Debug for File {
impl<'a> Display for File<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
write!(f, "File[{:?}] header: '{:?}', data: '{:?}')",
self.id, self.header, self.data)
fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
write!(fmt,
"[File] Owner : '{:?}'
FileID: '{:?}'
Header: '{:?}'
Data : '{:?}'",
self.owning_module,
self.header,
self.data,
self.id);
Ok(())
}
}
impl<'a> Debug for File<'a> {
fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
write!(fmt,
"[File] Owner : '{:?}'
FileID: '{:?}'
Header: '{:?}'
Data : '{:?}'",
self.owning_module,
self.header,
self.data,
self.id);
Ok(())
}
}

49
src/storage/json/parser.rs
View file

@ -11,18 +11,22 @@ use super::super::parser::{FileHeaderParser, ParserError};
use super::super::file::{FileHeaderSpec, FileHeaderData};
struct JsonHeaderParser<'a> {
spec: &'a FileHeaderSpec,
pub struct JsonHeaderParser {
spec: Option<FileHeaderSpec>,
}
impl<'a> FileHeaderParser<'a> for JsonHeaderParser<'a> {
impl JsonHeaderParser {
fn new(spec: &'a FileHeaderSpec) -> JsonHeaderParser<'a> {
pub fn new(spec: Option<FileHeaderSpec>) -> JsonHeaderParser {
JsonHeaderParser {
spec: spec
}
}
}
impl FileHeaderParser for JsonHeaderParser {
fn read(&self, string: Option<String>)
-> Result<FileHeaderData, ParserError>
{
@ -41,9 +45,16 @@ impl<'a> FileHeaderParser<'a> for JsonHeaderParser<'a> {
}
fn write(&self, data: &FileHeaderData) -> Result<String, ParserError> {
let mut ser = Serializer::pretty(stdout());
data.serialize(&mut ser);
Ok(String::from(""))
let mut s = Vec::<u8>::new();
{
let mut ser = Serializer::pretty(&mut s);
data.serialize(&mut ser);
}
String::from_utf8(s).or(
Err(ParserError::short("Cannot parse utf8 bytes",
String::from("<not printable>"),
0)))
}
}
@ -63,11 +74,12 @@ fn visit_json(v: &Value) -> FileHeaderData {
}
},
&Value::Object(ref btree) => {
let btree = btree.clone();
FileHeaderData::Map{
keys: btree.clone().iter().map(|(k, v)|
keys: btree.into_iter().map(|(k, v)|
FileHeaderData::Key {
name: k.clone(),
value: Box::new(visit_json(v)),
name: k,
value: Box::new(visit_json(&v)),
}
).collect()
}
@ -91,12 +103,21 @@ impl Serialize for FileHeaderData {
&FileHeaderData::Float(ref f) => f.serialize(ser),
&FileHeaderData::Text(ref s) => (&s[..]).serialize(ser),
&FileHeaderData::Array{values: ref vs} => vs.serialize(ser),
&FileHeaderData::Map{keys: ref ks} => ks.serialize(ser),
&FileHeaderData::Key{name: ref n, value: ref v} => {
&FileHeaderData::Map{keys: ref ks} => {
let mut hm = HashMap::new();
hm.insert(n, v);
for key in ks {
if let &FileHeaderData::Key{name: ref n, value: ref v} = key {
hm.insert(n, v);
} else {
panic!("Not a key: {:?}", key);
}
}
hm.serialize(ser)
}
},
&FileHeaderData::Key{name: ref n, value: ref v} => unreachable!(),
}
}

10
src/storage/parser.rs
View file

@ -73,9 +73,7 @@ impl Display for ParserError {
}
pub trait FileHeaderParser<'a> : Sized {
fn new(spec: &'a FileHeaderSpec) -> Self;
pub trait FileHeaderParser : Sized {
fn read(&self, string: Option<String>) -> Result<FileHeaderData, ParserError>;
fn write(&self, data: &FileHeaderData) -> Result<String, ParserError>;
}
@ -87,11 +85,11 @@ pub struct Parser<HP>
headerp : HP,
}
impl<'a, HP> Parser<HP> where
HP: FileHeaderParser<'a>,
impl<HP> Parser<HP> where
HP: FileHeaderParser,
{
fn new(headerp: HP) -> Parser<HP> {
pub fn new(headerp: HP) -> Parser<HP> {
Parser {
headerp: headerp,
}