From 2a468059aa1f4afffc21ac28a6481fc783fa6b16 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:28:21 +0100 Subject: [PATCH 01/30] parser: Remove livetimes --- src/storage/parser.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/storage/parser.rs b/src/storage/parser.rs index dc0d28c9..c7df264c 100644 --- a/src/storage/parser.rs +++ b/src/storage/parser.rs @@ -73,9 +73,7 @@ impl Display for ParserError { } - -pub trait FileHeaderParser<'a> : Sized { - fn new(spec: &'a FileHeaderSpec) -> Self; +pub trait FileHeaderParser : Sized { fn read(&self, string: Option) -> Result; fn write(&self, data: &FileHeaderData) -> Result; } @@ -87,11 +85,11 @@ pub struct Parser headerp : HP, } -impl<'a, HP> Parser where - HP: FileHeaderParser<'a>, +impl Parser where + HP: FileHeaderParser, { - fn new(headerp: HP) -> Parser { + pub fn new(headerp: HP) -> Parser { Parser { headerp: headerp, } From 8f8badebb394dfcd33a9093d35cf79345d4e2edf Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:30:14 +0100 Subject: [PATCH 02/30] Fix: FileHeaderData::Key::name must be a &'static str --- src/storage/file.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/file.rs b/src/storage/file.rs index 800cccbc..d6fa0337 100644 --- a/src/storage/file.rs +++ b/src/storage/file.rs @@ -27,7 +27,7 @@ pub enum FileHeaderData { UInteger(u64), Float(f64), Text(String), - Key { name: String, value: Box }, + Key { name: &'static str, value: Box }, Map { keys: Vec }, Array { values: Box> }, } From 58bd36599f2fece8843a631a4df684571b7a5c69 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:30:50 +0100 Subject: [PATCH 03/30] Implement FileHeaderData::matches_with(&Regex) --- src/storage/file.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/storage/file.rs b/src/storage/file.rs index d6fa0337..7a8ab900 100644 --- a/src/storage/file.rs +++ b/src/storage/file.rs @@ -5,6 +5,8 @@ use std::fmt; use super::parser::FileHeaderParser; use storage::file_id::*; +use regex::Regex; + #[derive(Debug)] pub enum FileHeaderSpec { Null, @@ -56,6 +58,28 @@ impl Display for FileHeaderSpec { } +impl FileHeaderData { + + pub fn matches_with(&self, r: &Regex) -> bool { + match self { + &FileHeaderData::Text(ref t) => r.is_match(&t[..]), + &FileHeaderData::Key{name: ref n, value: ref val} => { + r.is_match(n) || val.matches_with(r) + }, + + &FileHeaderData::Map{keys: ref dks} => { + dks.iter().any(|x| x.matches_with(r)) + }, + + &FileHeaderData::Array{values: ref vs} => { + vs.iter().any(|x| x.matches_with(r)) + } + + _ => false, + } + } +} + pub struct MatchError<'a> { summary: String, expected: &'a FileHeaderSpec, From d89977a09e852b35689759337ca177a51f4785dd Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:32:14 +0100 Subject: [PATCH 04/30] Implement File::matches_with(&Regex) --- src/storage/file.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/storage/file.rs b/src/storage/file.rs index 7a8ab900..7b8043cc 100644 --- a/src/storage/file.rs +++ b/src/storage/file.rs @@ -266,6 +266,10 @@ impl File { self.id.clone() } + pub fn matches_with(&self, r: &Regex) -> bool { + r.is_match(&self.data[..]) || self.header.matches_with(r) + } + fn get_new_file_id() -> FileID { use uuid::Uuid; Uuid::new_v4().to_hyphenated_string() From 90f08cc1f21b23d9e4b0b2615c57fb3f8bb53518 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:32:59 +0100 Subject: [PATCH 05/30] Implement File::{header(),data()} as header/data getters --- src/storage/file.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/storage/file.rs b/src/storage/file.rs index 7b8043cc..96555595 100644 --- a/src/storage/file.rs +++ b/src/storage/file.rs @@ -258,6 +258,14 @@ impl File { f } + pub fn header(&self) -> FileHeaderData { + self.header.clone() + } + + pub fn data(&self) -> String { + self.data.clone() + } + pub fn contents(&self) -> (FileHeaderData, String) { (self.header.clone(), self.data.clone()) } From 09940f32286a0b89787bab1d5eceea4da710adee Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:33:58 +0100 Subject: [PATCH 06/30] Each file must have an owning module --- src/storage/file.rs | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/storage/file.rs b/src/storage/file.rs index 96555595..a6d93389 100644 --- a/src/storage/file.rs +++ b/src/storage/file.rs @@ -2,7 +2,8 @@ use std::error::Error; use std::fmt::{Debug, Display, Formatter}; use std::fmt; -use super::parser::FileHeaderParser; +use module::Module; +use super::parser::{FileHeaderParser, Parser, ParserError}; use storage::file_id::*; use regex::Regex; @@ -200,16 +201,18 @@ pub fn match_header_spec<'a>(spec: &'a FileHeaderSpec, data: &'a FileHeaderData) * Internal abstract view on a file. Does not exist on the FS and is just kept * internally until it is written to disk. */ -pub struct File { - header : FileHeaderData, - data : String, - id : FileID, +pub struct File<'a> { + owning_module : &'a Module, + header : FileHeaderData, + data : String, + id : FileID, } -impl File { +impl<'a> File<'a> { - pub fn new() -> File { + pub fn new(module: &'a Module) -> File<'a> { let f = File { + owning_module: module, header: FileHeaderData::Null, data: String::from(""), id: File::get_new_file_id(), @@ -218,8 +221,9 @@ impl File { f } - pub fn from_parser_result(id: FileID, header: FileHeaderData, data: String) -> File { + pub fn from_parser_result(module: &Module, id: FileID, header: FileHeaderData, data: String) -> File { let f = File { + owning_module: module, header: header, data: data, id: id, @@ -228,8 +232,9 @@ impl File { f } - pub fn new_with_header(h: FileHeaderData) -> File { + pub fn new_with_header(module: &Module, h: FileHeaderData) -> File { let f = File { + owning_module: module, header: h, data: String::from(""), id: File::get_new_file_id(), @@ -238,8 +243,9 @@ impl File { f } - pub fn new_with_data(d: String) -> File { + pub fn new_with_data(module: &Module, d: String) -> File { let f = File { + owning_module: module, header: FileHeaderData::Null, data: d, id: File::get_new_file_id(), @@ -248,8 +254,9 @@ impl File { f } - pub fn new_with_content(h: FileHeaderData, d: String) -> File { + pub fn new_with_content(module: &Module, h: FileHeaderData, d: String) -> File { let f = File { + owning_module: module, header: h, data: d, id: File::get_new_file_id(), @@ -274,6 +281,10 @@ impl File { self.id.clone() } + pub fn owner(&self) -> &Module { + self.owning_module + } + pub fn matches_with(&self, r: &Regex) -> bool { r.is_match(&self.data[..]) || self.header.matches_with(r) } From 6eb7bee35581dd7ff93844bce385317aef2cdd8c Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:34:23 +0100 Subject: [PATCH 07/30] Reimplement File::contents() with File::{header(),data()} --- src/storage/file.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/file.rs b/src/storage/file.rs index a6d93389..5f3a7eb0 100644 --- a/src/storage/file.rs +++ b/src/storage/file.rs @@ -274,7 +274,7 @@ impl<'a> File<'a> { } pub fn contents(&self) -> (FileHeaderData, String) { - (self.header.clone(), self.data.clone()) + (self.header(), self.data()) } pub fn id(&self) -> FileID { From 535952c2a50e50a36254a9a1ac0266b1278384e8 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:34:51 +0100 Subject: [PATCH 08/30] Implement Display for File --- src/storage/file.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/storage/file.rs b/src/storage/file.rs index 5f3a7eb0..1f9026a5 100644 --- a/src/storage/file.rs +++ b/src/storage/file.rs @@ -303,3 +303,20 @@ impl Debug for File { } } +impl<'a> Display for File<'a> { + + fn fmt(&self, fmt: &mut Formatter) -> fmt::Result { + write!(fmt, +"[File] Owner : '{:?}' + FileID: '{:?}' + Header: '{:?}' + Data : '{:?}'", + self.owning_module, + self.header, + self.data, + self.id); + Ok(()) + } + +} + From 9493c79104fb2888edd0a9ca62eea83d01548a14 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:35:01 +0100 Subject: [PATCH 09/30] Reimplement Debug for File --- src/storage/file.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/storage/file.rs b/src/storage/file.rs index 1f9026a5..75bc3b8e 100644 --- a/src/storage/file.rs +++ b/src/storage/file.rs @@ -295,14 +295,6 @@ impl<'a> File<'a> { } } -impl Debug for File { - - fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { - write!(f, "File[{:?}] header: '{:?}', data: '{:?}')", - self.id, self.header, self.data) - } -} - impl<'a> Display for File<'a> { fn fmt(&self, fmt: &mut Formatter) -> fmt::Result { @@ -320,3 +312,19 @@ impl<'a> Display for File<'a> { } +impl<'a> Debug for File<'a> { + + fn fmt(&self, fmt: &mut Formatter) -> fmt::Result { + write!(fmt, +"[File] Owner : '{:?}' + FileID: '{:?}' + Header: '{:?}' + Data : '{:?}'", + self.owning_module, + self.header, + self.data, + self.id); + Ok(()) + } + +} From c73cb88df43121e270ee7845e27b93c18e924230 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:39:04 +0100 Subject: [PATCH 10/30] Remove StorageBackendError::explanation member --- src/storage/backend.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 2b8ab577..e93d7037 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -164,20 +164,17 @@ impl StorageBackend { pub struct StorageBackendError { pub action: String, // The file system action in words pub desc: String, // A short description - pub explanation: String, // A long, user friendly description pub data_dump: Option // Data dump, if any } impl StorageBackendError { fn new(action: String, desc : String, - explan: String, data : Option) -> StorageBackendError { StorageBackendError { action: action, desc: desc, - explanation: explan, data_dump: data, } } @@ -197,8 +194,8 @@ impl Error for StorageBackendError { impl Display for StorageBackendError { fn fmt(&self, f: &mut Formatter) -> FMTResult { - write!(f, "StorageBackendError[{}]: {}\n\n{}", - self.action, self.desc, self.explanation) + write!(f, "StorageBackendError[{}]: {}", + self.action, self.desc) } } From 8ff5dcc9e8e02a259fb5c977054dea3f41ea6027 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:39:50 +0100 Subject: [PATCH 11/30] Add StorageBackendError::caused_by member --- src/storage/backend.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index e93d7037..882207e0 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -164,7 +164,8 @@ impl StorageBackend { pub struct StorageBackendError { pub action: String, // The file system action in words pub desc: String, // A short description - pub data_dump: Option // Data dump, if any + pub data_dump: Option, // Data dump, if any + pub caused_by: Option>, // caused from this error } impl StorageBackendError { @@ -176,6 +177,7 @@ impl StorageBackendError { action: action, desc: desc, data_dump: data, + caused_by: None, } } } @@ -187,7 +189,7 @@ impl Error for StorageBackendError { } fn cause(&self) -> Option<&Error> { - None + self.caused_by.as_ref().map(|e| &**e) } } From d22b991da0a6f0a14195f1495dcc2f082bbbe336 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:40:32 +0100 Subject: [PATCH 12/30] Implement StorageBackendError::build() helper --- src/storage/backend.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 882207e0..0b7dbe72 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -180,6 +180,19 @@ impl StorageBackendError { caused_by: None, } } + + fn build(action: &'static str, + desc: &'static str, + data : Option) -> StorageBackendError + { + StorageBackendError { + action: String::from(action), + desc: String::from(desc), + dataDump: data, + caused_by: None, + } + } + } impl Error for StorageBackendError { From 1b6711fe109bc3a11cccdaa72d8be58c9187a2b4 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:41:49 +0100 Subject: [PATCH 13/30] Add StorageBackend::iter_ids() --- src/storage/backend.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 0b7dbe72..7c413b24 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -8,6 +8,7 @@ use std::vec::Vec; use std::fs::File as FSFile; use std::io::Read; use std::io::Write; +use std::vec::IntoIter; use glob::glob; use glob::Paths; @@ -61,6 +62,17 @@ impl StorageBackend { } } + pub fn iter_ids(&self, m: &Module) -> Option> + { + glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| { + let v = globlist.filter_map(Result::ok) + .map(|pbuf| from_pathbuf(&pbuf)) + .collect::>() + .into_iter(); + Ok(v) + }).ok() + } + /* * Write a file to disk. * From d141a3670f9b52e75b7f67ef4b2e4ecd6baddc4e Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:42:06 +0100 Subject: [PATCH 14/30] Add StorageBackend::iter_files() --- src/storage/backend.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 7c413b24..b2a3c499 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -73,6 +73,17 @@ impl StorageBackend { }).ok() } + pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser) + -> Option>> + where HP: FileHeaderParser + { + self.iter_ids(m).and_then(|ids| { + Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p)) + .collect::>() + .into_iter()) + }) + } + /* * Write a file to disk. * From 8bd782c1a7ddb645c64fc60fac40b85b7abb47a4 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:43:11 +0100 Subject: [PATCH 15/30] Reimplement StorageBackend::put_file() --- src/storage/backend.rs | 46 ++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index b2a3c499..d806efac 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -89,20 +89,40 @@ impl StorageBackend { * * The file is moved to this function as the file won't be edited afterwards */ - pub fn put_file<'a, HP>(&self, f: File, p: &Parser) -> - Result - where HP: FileHeaderParser<'a> + pub fn put_file(&self, f: File, p: &Parser) -> BackendOperationResult + where HP: FileHeaderParser { - let written = p.write(f.contents()); - if let Ok(string) = written { - let path = self.build_filepath(&f); - debug!("Writing file: {}", path); - debug!(" contents: {}", string); - Ok(Ok(())) - } else { - debug!("Error parsing : {:?}", f.contents()); - Err(written.err().unwrap()) - } + let written = write_with_parser(&f, p); + if written.is_err() { return Err(written.err().unwrap()); } + let string = written.unwrap(); + + let path = self.build_filepath(&f); + debug!("Writing file: {}", path); + debug!(" string: {}", string); + + FSFile::create(&path).map(|mut file| { + debug!("Created file at '{}'", path); + file.write_all(&string.clone().into_bytes()) + .map_err(|ioerr| { + debug!("Could not write file"); + let mut err = StorageBackendError::build( + "File::write_all()", + "Could not write out File contents", + None + ); + err.caused_by = Some(Box::new(ioerr)); + err + }) + }).map_err(|writeerr| { + debug!("Could not create file at '{}'", path); + let mut err = StorageBackendError::build( + "File::create()", + "Creating file on disk failed", + None + ); + err.caused_by = Some(Box::new(writeerr)); + err + }).and(Ok(())) } /* From 25b26b03a66d5eb673f65d04121113f7d3ab80fc Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:44:10 +0100 Subject: [PATCH 16/30] Reimplement StorageBackend::update_file() --- src/storage/backend.rs | 64 ++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index d806efac..12873509 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -129,44 +129,40 @@ impl StorageBackend { * Update a file. We have the UUID and can find the file on FS with it and * then replace its contents with the contents of the passed file object */ - pub fn update_file<'a, HP>(&self, f: File, p: &Parser) - -> Result - where HP: FileHeaderParser<'a> + pub fn update_file(&self, f: File, p: &Parser) -> BackendOperationResult + where HP: FileHeaderParser { - let contents = p.write(f.contents()); - - if contents.is_err() { - debug!("Error parsing contents: {:?}", f.contents()); - return Err(contents.err().unwrap()); - } - - let content = contents.unwrap(); - debug!("Success parsing content : {}", content); + let contents = write_with_parser(&f, p); + if contents.is_err() { return Err(contents.err().unwrap()); } + let string = contents.unwrap(); let path = self.build_filepath(&f); - debug!("Trying to write to file at {}", path); - if let Err(_) = FSFile::open(&path) { - debug!("Error opening {}", path); - return Ok(Err(StorageBackendError::new( - String::from("File::open()"), - format!("Tried to open '{}'", path), - String::from("Tried to update contents of this file, though file doesn't exist"), - None))) - } + debug!("Writing file: {}", path); + debug!(" string: {}", string); - if let Ok(mut file) = FSFile::create(&path) { - if let Err(writeerr) = file.write_all(&content.clone().into_bytes()) { - debug!("Error writing to {}", path); - return Ok(Err(StorageBackendError::new( - String::from("File::write()"), - format!("Tried to write '{}'", path), - String::from("Tried to write contents of this file, though operation did not succeed"), - Some(content)))) - } - } - - debug!("Successfully written to file."); - Ok(Ok(())) + FSFile::open(&path).map(|mut file| { + debug!("Open file at '{}'", path); + file.write_all(&string.clone().into_bytes()) + .map_err(|ioerr| { + debug!("Could not write file"); + let mut err = StorageBackendError::build( + "File::write()", + "Tried to write contents of this file, though operation did not succeed", + Some(string) + ); + err.caused_by = Some(Box::new(ioerr)); + err + }) + }).map_err(|writeerr| { + debug!("Could not write file at '{}'", path); + let mut err = StorageBackendError::build( + "File::open()", + "Tried to update contents of this file, though file doesn't exist", + None + ); + err.caused_by = Some(Box::new(writeerr)); + err + }).and(Ok(())) } /* From c23a1e22ab8222ee1c1d4bf110f6c27677c730a3 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:44:31 +0100 Subject: [PATCH 17/30] Implement helper write_with_parser() --- src/storage/backend.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 12873509..e2cbd1c6 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -253,3 +253,18 @@ impl Display for StorageBackendError { } } + +fn write_with_parser<'a, HP>(f: &File, p: &Parser) -> Result + where HP: FileHeaderParser +{ + p.write(f.contents()) + .or_else(|err| { + let mut serr = StorageBackendError::build( + "Parser::write()", + "Cannot translate internal representation of file contents into on-disk representation", + None + ); + serr.caused_by = Some(Box::new(err)); + Err(serr) + }) +} From 0363075f9c46c62ce48c9a5edad20132a1c958a3 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:45:15 +0100 Subject: [PATCH 18/30] Implement StorageBackend::remove_file() --- src/storage/backend.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index e2cbd1c6..4ca0c005 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -188,6 +188,27 @@ impl StorageBackend { } } + pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult { + if checked { + error!("Checked remove not implemented yet. I will crash now"); + unimplemented!() + } + + debug!("Doing unchecked remove"); + info!("Going to remove file: {}", file); + + let fp = self.build_filepath(&file); + remove_file(fp).map_err(|e| { + let mut serr = StorageBackendError::build( + "remove_file()", + "File removal failed", + Some(format!("{}", file)) + ); + serr.caused_by = Some(Box::new(e)); + serr + }) + } + fn build_filepath(&self, f: &File) -> String { self.build_filepath_with_id(f.id()) } From c81800d985ead113671f2cdb47b34ba9a98c5d3d Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:46:07 +0100 Subject: [PATCH 19/30] Implement helpers: build_filepath_with_id(), prefix_of_files_for_module() --- src/storage/backend.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 4ca0c005..2971ba29 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -210,12 +210,19 @@ impl StorageBackend { } fn build_filepath(&self, f: &File) -> String { - self.build_filepath_with_id(f.id()) + self.build_filepath_with_id(f.owner(), f.id()) } - fn build_filepath_with_id(&self, id: FileID) -> String { - debug!("Building filepath for id '{}'", id); - self.basepath.clone() + &id[..] + fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String { + debug!("Building filepath with id"); + debug!(" basepath: '{}'", self.basepath); + debug!(" storepath: '{}'", self.storepath); + debug!(" id : '{}'", id); + self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag" + } + + fn prefix_of_files_for_module(&self, m: &Module) -> String { + self.storepath.clone() + m.name() } } From d35e89f27c543982599773e3ad9a10614a8654a0 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:47:37 +0100 Subject: [PATCH 20/30] Fixup: get_file_by_id() gets &Module as well, so it can use build_filepath_with_id() --- src/storage/backend.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 2971ba29..acaed6b4 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -172,11 +172,11 @@ impl StorageBackend { * TODO: Needs refactoring, as there might be an error when reading from * disk OR the id just does not exist. */ - pub fn get_file_by_id<'a, HP>(&self, id: FileID, p: &Parser) -> Option - where HP: FileHeaderParser<'a> + pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser) -> Option> + where HP: FileHeaderParser { debug!("Searching for file with id '{}'", id); - if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(id.clone())) { + if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) { let mut s = String::new(); fs.read_to_string(&mut s); debug!("Success reading file with id '{}'", id); From 9a0cc75c6fbd6e3d042d9e7e9bd3cf51610fdb8a Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:48:07 +0100 Subject: [PATCH 21/30] BackendOperationResult: Add optional type parameter --- src/storage/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index acaed6b4..8c069e38 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -20,7 +20,7 @@ use storage::parser::{FileHeaderParser, Parser, ParserError}; use module::Module; use runtime::Runtime; -pub type BackendOperationResult = Result<(), StorageBackendError>; +pub type BackendOperationResult = Result; pub struct StorageBackend { basepath: String, From 0796d0ab36a4060b480f87502696edd6e389a9f0 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:48:49 +0100 Subject: [PATCH 22/30] Fixup: File::from_parser_result() gets module now as well --- src/storage/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 8c069e38..f3e584e4 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -181,7 +181,7 @@ impl StorageBackend { fs.read_to_string(&mut s); debug!("Success reading file with id '{}'", id); debug!("Parsing to internal structure now"); - p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(id, h, d))).ok() + p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok() } else { debug!("No file with id '{}'", id); None From 1e5c50bc0bc8a60115b668c31bafd7d137fe1c80 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:49:11 +0100 Subject: [PATCH 23/30] Fixup: Import remove_file() --- src/storage/backend.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index f3e584e4..832f6fec 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::path::PathBuf; use std::vec::Vec; use std::fs::File as FSFile; +use std::fs::remove_file; use std::io::Read; use std::io::Write; use std::vec::IntoIter; From 0e4a56fe681de7fc06c4d26fd9f4c4cc3b3024bc Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:49:53 +0100 Subject: [PATCH 24/30] Reimplement StorageBackend::new(), create directory if not existing, store storepath in extra variable --- src/storage/backend.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 832f6fec..0a123c83 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::path::PathBuf; use std::vec::Vec; use std::fs::File as FSFile; +use std::fs::create_dir_all; use std::fs::remove_file; use std::io::Read; use std::io::Write; @@ -25,14 +26,30 @@ pub type BackendOperationResult = Result; pub struct StorageBackend { basepath: String, + storepath: String, } impl StorageBackend { - pub fn new(basepath: String) -> StorageBackend { - StorageBackend { - basepath: basepath, - } + pub fn new(rt: &Runtime) -> BackendOperationResult { + let storepath = rt.get_rtp() + "/store/"; + debug!("Trying to create {}", storepath); + create_dir_all(&storepath).and_then(|_| { + debug!("Creating succeeded, constructing backend instance"); + Ok(StorageBackend { + basepath: rt.get_rtp(), + storepath: storepath.clone(), + }) + }).or_else(|e| { + debug!("Creating failed, constructing error instance"); + let mut serr = StorageBackendError::build( + "create_dir_all()", + "Could not create store directories", + Some(storepath) + ); + serr.caused_by = Some(Box::new(e)); + Err(serr) + }) } fn build(rt: &Runtime, m: &M) -> StorageBackend { From cf196b2492473d6ec03dfc9a420e1af38d7fa183 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:50:24 +0100 Subject: [PATCH 25/30] Fixup: get_file_ids() - use prefix_of_files_for_module() --- src/storage/backend.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/storage/backend.rs b/src/storage/backend.rs index 0a123c83..077cf8b4 100644 --- a/src/storage/backend.rs +++ b/src/storage/backend.rs @@ -59,9 +59,8 @@ impl StorageBackend { StorageBackend::new(path) } - fn get_file_ids(&self) -> Option> { - debug!("Getting files from {}", self.basepath); - let list = glob(&self.basepath[..]); + fn get_file_ids(&self, m: &Module) -> Option> { + let list = glob(&self.prefix_of_files_for_module(m)[..]); if let Ok(globlist) = list { let mut v = vec![]; From df18071d9a02c575c3ec5fe345621c6216cad788 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:51:50 +0100 Subject: [PATCH 26/30] Move JsonHeaderParser::new() out of trait implementation --- src/storage/json/parser.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/storage/json/parser.rs b/src/storage/json/parser.rs index e0981a9d..fc663303 100644 --- a/src/storage/json/parser.rs +++ b/src/storage/json/parser.rs @@ -15,7 +15,7 @@ struct JsonHeaderParser<'a> { spec: &'a FileHeaderSpec, } -impl<'a> FileHeaderParser<'a> for JsonHeaderParser<'a> { +impl JsonHeaderParser { fn new(spec: &'a FileHeaderSpec) -> JsonHeaderParser<'a> { JsonHeaderParser { @@ -23,6 +23,10 @@ impl<'a> FileHeaderParser<'a> for JsonHeaderParser<'a> { } } +} + +impl FileHeaderParser for JsonHeaderParser { + fn read(&self, string: Option) -> Result { From 34cf5076f9b9108a8f31c5f15ad69e921e85b00a Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:52:36 +0100 Subject: [PATCH 27/30] Reimplement JsonHeaderParser::write() --- src/storage/json/parser.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/storage/json/parser.rs b/src/storage/json/parser.rs index fc663303..b2ea1e1d 100644 --- a/src/storage/json/parser.rs +++ b/src/storage/json/parser.rs @@ -45,9 +45,16 @@ impl FileHeaderParser for JsonHeaderParser { } fn write(&self, data: &FileHeaderData) -> Result { - let mut ser = Serializer::pretty(stdout()); - data.serialize(&mut ser); - Ok(String::from("")) + let mut s = Vec::::new(); + { + let mut ser = Serializer::pretty(&mut s); + data.serialize(&mut ser); + } + + String::from_utf8(s).or( + Err(ParserError::short("Cannot parse utf8 bytes", + String::from(""), + 0))) } } From 500fb281a078a05103c8c50a6f33309c2f6cf020 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:53:11 +0100 Subject: [PATCH 28/30] JsonHeaderParser: Make spec optional --- src/storage/json/parser.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/storage/json/parser.rs b/src/storage/json/parser.rs index b2ea1e1d..c813e9c5 100644 --- a/src/storage/json/parser.rs +++ b/src/storage/json/parser.rs @@ -11,13 +11,13 @@ use super::super::parser::{FileHeaderParser, ParserError}; use super::super::file::{FileHeaderSpec, FileHeaderData}; -struct JsonHeaderParser<'a> { - spec: &'a FileHeaderSpec, +pub struct JsonHeaderParser { + spec: Option, } impl JsonHeaderParser { - fn new(spec: &'a FileHeaderSpec) -> JsonHeaderParser<'a> { + pub fn new(spec: Option) -> JsonHeaderParser { JsonHeaderParser { spec: spec } From 7549db4ffc27feb15b55f009d02299135ac1b3c7 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:54:10 +0100 Subject: [PATCH 29/30] Fix JsonHeaderParser::visit_json() --- src/storage/json/parser.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/storage/json/parser.rs b/src/storage/json/parser.rs index c813e9c5..815e77be 100644 --- a/src/storage/json/parser.rs +++ b/src/storage/json/parser.rs @@ -74,11 +74,12 @@ fn visit_json(v: &Value) -> FileHeaderData { } }, &Value::Object(ref btree) => { + let btree = btree.clone(); FileHeaderData::Map{ - keys: btree.clone().iter().map(|(k, v)| + keys: btree.into_iter().map(|(k, v)| FileHeaderData::Key { - name: k.clone(), - value: Box::new(visit_json(v)), + name: k, + value: Box::new(visit_json(&v)), } ).collect() } From 2bab3d48e60903d0b9317594401b7bdaadaaddee Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 2 Dec 2015 11:54:37 +0100 Subject: [PATCH 30/30] Fix JsonHeaderParser::serialize() --- src/storage/json/parser.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/storage/json/parser.rs b/src/storage/json/parser.rs index 815e77be..ee230620 100644 --- a/src/storage/json/parser.rs +++ b/src/storage/json/parser.rs @@ -103,12 +103,21 @@ impl Serialize for FileHeaderData { &FileHeaderData::Float(ref f) => f.serialize(ser), &FileHeaderData::Text(ref s) => (&s[..]).serialize(ser), &FileHeaderData::Array{values: ref vs} => vs.serialize(ser), - &FileHeaderData::Map{keys: ref ks} => ks.serialize(ser), - &FileHeaderData::Key{name: ref n, value: ref v} => { + &FileHeaderData::Map{keys: ref ks} => { let mut hm = HashMap::new(); - hm.insert(n, v); + + for key in ks { + if let &FileHeaderData::Key{name: ref n, value: ref v} = key { + hm.insert(n, v); + } else { + panic!("Not a key: {:?}", key); + } + } + hm.serialize(ser) - } + }, + &FileHeaderData::Key{name: ref n, value: ref v} => unreachable!(), + } }