From 146e2a1140ac08df39e64c534637c564748325b7 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Mon, 12 Jun 2017 16:18:40 +0200 Subject: [PATCH 01/13] Implement backend code for in-memory IO backend This commit implements a backend which reads from a Read when created and writes to a Write when dropped. This way, one can initialize stores which are build from commandline feeded JSON or TOML (currently JSON is implemented/about to be implemented). --- libimagstore/src/file_abstraction.rs | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/libimagstore/src/file_abstraction.rs b/libimagstore/src/file_abstraction.rs index 81db7ca7..69d309c7 100644 --- a/libimagstore/src/file_abstraction.rs +++ b/libimagstore/src/file_abstraction.rs @@ -375,6 +375,79 @@ mod inmemory { } +mod stdio { + use std::cell::RefCell; + use std::collections::HashMap; + use std::fmt::Debug; + use std::fmt::Error as FmtError; + use std::fmt::Formatter; + use std::io::Cursor; + use std::io::{Read, Write}; + use std::path::PathBuf; + use std::sync::Arc; + use std::sync::Mutex; + + use error::StoreError as SE; + use super::FileAbstraction; + use super::FileAbstractionInstance; + use super::InMemoryFileAbstraction; + + // Because this is not exported in super::inmemory; + type Backend = Arc>>>>>; + + pub struct StdIoFileAbstraction { + mem: InMemoryFileAbstraction, + stdin: Box, + stdout: Box, + } + + impl Debug for StdIoFileAbstraction { + fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> { + write!(f, "StdIoFileAbstraction({:?}", self.mem) + } + } + + impl StdIoFileAbstraction { + + pub fn new(in_stream: Box, out_stream: Box) -> StdIoFileAbstraction { + StdIoFileAbstraction { + mem: InMemoryFileAbstraction::new(), + stdin: in_stream, + stdout: out_stream + } + } + + pub fn backend(&self) -> &Backend { + &self.mem.backend() + } + + } + + impl FileAbstraction for StdIoFileAbstraction { + + fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { + self.mem.remove_file(path) + } + + fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { + self.mem.copy(from, to) + } + + fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { + self.mem.rename(from, to) + } + + fn create_dir_all(&self, pb: &PathBuf) -> Result<(), SE> { + self.mem.create_dir_all(pb) + } + + fn new_instance(&self, p: PathBuf) -> Box { + self.mem.new_instance(p) + } + } + +} + #[cfg(test)] mod test { use super::FileAbstractionInstance; From 447f2610ef4b9b312f8c73dec9fdc19fed61036c Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 18:50:28 +0200 Subject: [PATCH 02/13] Make StdIo backend abstract over IO->"fs"->IO mapper --- libimagstore/src/file_abstraction.rs | 73 +++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 12 deletions(-) diff --git a/libimagstore/src/file_abstraction.rs b/libimagstore/src/file_abstraction.rs index 69d309c7..ad4697d8 100644 --- a/libimagstore/src/file_abstraction.rs +++ b/libimagstore/src/file_abstraction.rs @@ -387,6 +387,10 @@ mod stdio { use std::sync::Arc; use std::sync::Mutex; + use libimagerror::into::IntoError; + use libimagerror::trace::*; + + use error::StoreErrorKind as SEK; use error::StoreError as SE; use super::FileAbstraction; use super::FileAbstractionInstance; @@ -395,26 +399,55 @@ mod stdio { // Because this is not exported in super::inmemory; type Backend = Arc>>>>>; - pub struct StdIoFileAbstraction { - mem: InMemoryFileAbstraction, - stdin: Box, - stdout: Box, + mod mapper { + use std::collections::HashMap; + use std::io::Cursor; + use std::io::{Read, Write}; + use std::path::PathBuf; + use store::Result; + + pub trait Mapper { + fn read_to_fs(&self, Box, &mut HashMap>>) -> Result<()>; + fn fs_to_write(&self, &mut HashMap>>, &mut Write) -> Result<()>; + } } - impl Debug for StdIoFileAbstraction { + use self::mapper::Mapper; + + pub struct StdIoFileAbstraction { + mapper: M, + mem: InMemoryFileAbstraction, + out: Box, + } + + impl Debug for StdIoFileAbstraction + where M: Mapper + { fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> { write!(f, "StdIoFileAbstraction({:?}", self.mem) } } - impl StdIoFileAbstraction { + impl StdIoFileAbstraction + where M: Mapper + { - pub fn new(in_stream: Box, out_stream: Box) -> StdIoFileAbstraction { - StdIoFileAbstraction { - mem: InMemoryFileAbstraction::new(), - stdin: in_stream, - stdout: out_stream + pub fn new(in_stream: Box, out_stream: Box, mapper: M) -> Result, SE> { + let mem = InMemoryFileAbstraction::new(); + + { + let fill_res = match mem.backend().lock() { + Err(_) => Err(SEK::LockError.into_error()), + Ok(mut mtx) => mapper.read_to_fs(in_stream, mtx.get_mut()) + }; + let _ = try!(fill_res); } + + Ok(StdIoFileAbstraction { + mapper: mapper, + mem: mem, + out: out_stream, + }) } pub fn backend(&self) -> &Backend { @@ -423,7 +456,23 @@ mod stdio { } - impl FileAbstraction for StdIoFileAbstraction { + impl Drop for StdIoFileAbstraction + where M: Mapper + { + fn drop(&mut self) { + let fill_res = match self.mem.backend().lock() { + Err(_) => Err(SEK::LockError.into_error()), + Ok(mut mtx) => self.mapper.fs_to_write(mtx.get_mut(), &mut *self.out) + }; + + // We can do nothing but end this here with a trace. + // As this drop gets called when imag almost exits, there is no point in exit()ing here + // again. + let _ = fill_res.map_err_trace(); + } + } + + impl FileAbstraction for StdIoFileAbstraction { fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { self.mem.remove_file(path) From 4abd9dd7cfae1999a681047531c73d419d837390 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 18:52:48 +0200 Subject: [PATCH 03/13] Split into module-files --- libimagstore/src/file_abstraction.rs | 519 ------------------ libimagstore/src/file_abstraction/fs.rs | 142 +++++ libimagstore/src/file_abstraction/inmemory.rs | 166 ++++++ libimagstore/src/file_abstraction/mod.rs | 126 +++++ .../src/file_abstraction/stdio/mapper/mod.rs | 30 + .../src/file_abstraction/stdio/mod.rs | 127 +++++ 6 files changed, 591 insertions(+), 519 deletions(-) delete mode 100644 libimagstore/src/file_abstraction.rs create mode 100644 libimagstore/src/file_abstraction/fs.rs create mode 100644 libimagstore/src/file_abstraction/inmemory.rs create mode 100644 libimagstore/src/file_abstraction/mod.rs create mode 100644 libimagstore/src/file_abstraction/stdio/mapper/mod.rs create mode 100644 libimagstore/src/file_abstraction/stdio/mod.rs diff --git a/libimagstore/src/file_abstraction.rs b/libimagstore/src/file_abstraction.rs deleted file mode 100644 index ad4697d8..00000000 --- a/libimagstore/src/file_abstraction.rs +++ /dev/null @@ -1,519 +0,0 @@ -// -// imag - the personal information management suite for the commandline -// Copyright (C) 2015, 2016 Matthias Beyer and contributors -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; version -// 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// - -//! The filesystem abstraction code -//! -//! # Problem -//! -//! First, we had a compiletime backend for the store. This means that the actual filesystem -//! operations were compiled into the store either as real filesystem operations (in a normal debug -//! or release build) but as a in-memory variant in the 'test' case. -//! So tests did not hit the filesystem when running. -//! This gave us us the possibility to run tests concurrently with multiple -//! stores that did not interfere with eachother. -//! -//! This approach worked perfectly well until we started to test not the -//! store itself but crates that depend on the store implementation. -//! When running tests in a crate that depends on the store, the store -//! itself was compiled with the filesystem-hitting-backend. -//! This was problematic, as tests could not be implemented without hitting -//! the filesystem. -//! -//! Hence we implemented this. -//! -//! # Implementation -//! -//! The filesystem is abstracted via a trait `FileAbstraction` which -//! contains the essential functions for working with the filesystem. -//! -//! Two implementations are provided in the code: -//! -//! * FSFileAbstraction -//! * InMemoryFileAbstraction -//! -//! whereas the first actually works with the filesystem and the latter -//! works with an in-memory HashMap that is used as filesystem. -//! -//! Further, the trait `FileAbstractionInstance` was introduced for -//! functions which are executed on actual instances of content from the -//! filesystem, which was previousely tied into the general abstraction -//! mechanism. -//! -//! So, the `FileAbstraction` trait is for working with the filesystem, the -//! `FileAbstractionInstance` trait is for working with instances of content -//! from the filesystem (speak: actual Files). -//! -//! In case of the `FSFileAbstractionInstance`, which is the implementation -//! of the `FileAbstractionInstance` for the actual filesystem-hitting code, -//! the underlying resource is managed like with the old code before. -//! The `InMemoryFileAbstractionInstance` implementation is corrosponding to -//! the `InMemoryFileAbstraction` implementation - for the in-memory -//! "filesystem". -//! -//! The implementation of the `get_file_content()` function had to be -//! changed to return a `String` rather than a `&mut Read` because of -//! lifetime issues. -//! This change is store-internally and the API of the store itself was not -//! affected. -//! - -use std::path::PathBuf; -use std::fmt::Debug; - -use error::StoreError as SE; - -pub use self::fs::FSFileAbstraction; -pub use self::fs::FSFileAbstractionInstance; -pub use self::inmemory::InMemoryFileAbstraction; -pub use self::inmemory::InMemoryFileAbstractionInstance; - -/// An abstraction trait over filesystem actions -pub trait FileAbstraction : Debug { - fn remove_file(&self, path: &PathBuf) -> Result<(), SE>; - fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE>; - fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE>; - fn create_dir_all(&self, _: &PathBuf) -> Result<(), SE>; - - fn new_instance(&self, p: PathBuf) -> Box; -} - -/// An abstraction trait over actions on files -pub trait FileAbstractionInstance : Debug { - fn get_file_content(&mut self) -> Result; - fn write_file_content(&mut self, buf: &[u8]) -> Result<(), SE>; -} - -mod fs { - use std::fs::{File, OpenOptions, create_dir_all, remove_file, copy, rename}; - use std::io::{Seek, SeekFrom, Read}; - use std::path::{Path, PathBuf}; - - use error::{MapErrInto, StoreError as SE, StoreErrorKind as SEK}; - - use super::FileAbstraction; - use super::FileAbstractionInstance; - - #[derive(Debug)] - pub enum FSFileAbstractionInstance { - Absent(PathBuf), - File(File, PathBuf) - } - - impl FileAbstractionInstance for FSFileAbstractionInstance { - - /** - * Get the content behind this file - */ - fn get_file_content(&mut self) -> Result { - debug!("Getting lazy file: {:?}", self); - let (file, path) = match *self { - FSFileAbstractionInstance::File(ref mut f, _) => return { - // We seek to the beginning of the file since we expect each - // access to the file to be in a different context - try!(f.seek(SeekFrom::Start(0)) - .map_err_into(SEK::FileNotSeeked)); - - let mut s = String::new(); - f.read_to_string(&mut s) - .map_err_into(SEK::IoError) - .map(|_| s) - }, - FSFileAbstractionInstance::Absent(ref p) => - (try!(open_file(p).map_err_into(SEK::FileNotFound)), p.clone()), - }; - *self = FSFileAbstractionInstance::File(file, path); - if let FSFileAbstractionInstance::File(ref mut f, _) = *self { - let mut s = String::new(); - f.read_to_string(&mut s) - .map_err_into(SEK::IoError) - .map(|_| s) - } else { - unreachable!() - } - } - - /** - * Write the content of this file - */ - fn write_file_content(&mut self, buf: &[u8]) -> Result<(), SE> { - use std::io::Write; - let (file, path) = match *self { - FSFileAbstractionInstance::File(ref mut f, _) => return { - // We seek to the beginning of the file since we expect each - // access to the file to be in a different context - try!(f.seek(SeekFrom::Start(0)) - .map_err_into(SEK::FileNotCreated)); - f.write_all(buf).map_err_into(SEK::FileNotWritten) - }, - FSFileAbstractionInstance::Absent(ref p) => - (try!(create_file(p).map_err_into(SEK::FileNotCreated)), p.clone()), - }; - *self = FSFileAbstractionInstance::File(file, path); - if let FSFileAbstractionInstance::File(ref mut f, _) = *self { - return f.write_all(buf).map_err_into(SEK::FileNotWritten); - } - unreachable!(); - } - - } - - /// `FSFileAbstraction` state type - /// - /// A lazy file is either absent, but a path to it is available, or it is present. - #[derive(Debug)] - pub struct FSFileAbstraction { - } - - impl FSFileAbstraction { - pub fn new() -> FSFileAbstraction { - FSFileAbstraction { } - } - } - - impl FileAbstraction for FSFileAbstraction { - - fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { - remove_file(path).map_err_into(SEK::FileNotRemoved) - } - - fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - copy(from, to).map_err_into(SEK::FileNotCopied).map(|_| ()) - } - - fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - rename(from, to).map_err_into(SEK::FileNotRenamed) - } - - fn create_dir_all(&self, path: &PathBuf) -> Result<(), SE> { - create_dir_all(path).map_err_into(SEK::DirNotCreated) - } - - fn new_instance(&self, p: PathBuf) -> Box { - Box::new(FSFileAbstractionInstance::Absent(p)) - } - } - - fn open_file>(p: A) -> ::std::io::Result { - OpenOptions::new().write(true).read(true).open(p) - } - - fn create_file>(p: A) -> ::std::io::Result { - if let Some(parent) = p.as_ref().parent() { - debug!("Implicitely creating directory: {:?}", parent); - if let Err(e) = create_dir_all(parent) { - return Err(e); - } - } - OpenOptions::new().write(true).read(true).create(true).open(p) - } - -} - -mod inmemory { - use error::StoreError as SE; - use error::StoreErrorKind as SEK; - use std::io::Read; - use std::io::Cursor; - use std::path::PathBuf; - use std::collections::HashMap; - use std::sync::Mutex; - use std::cell::RefCell; - use std::sync::Arc; - - use libimagerror::into::IntoError; - - use super::FileAbstraction; - use super::FileAbstractionInstance; - use error::MapErrInto; - - type Backend = Arc>>>>>; - - /// `FileAbstraction` type, this is the Test version! - /// - /// A lazy file is either absent, but a path to it is available, or it is present. - #[derive(Debug)] - pub struct InMemoryFileAbstractionInstance { - fs_abstraction: Backend, - absent_path: PathBuf, - } - - impl InMemoryFileAbstractionInstance { - - pub fn new(fs: Backend, pb: PathBuf) -> InMemoryFileAbstractionInstance { - InMemoryFileAbstractionInstance { - fs_abstraction: fs, - absent_path: pb - } - } - - } - - impl FileAbstractionInstance for InMemoryFileAbstractionInstance { - - /** - * Get the mutable file behind a InMemoryFileAbstraction object - */ - fn get_file_content(&mut self) -> Result { - debug!("Getting lazy file: {:?}", self); - - let p = self.absent_path.clone(); - match self.fs_abstraction.lock() { - Ok(mut mtx) => { - mtx.get_mut() - .get_mut(&p) - .ok_or(SEK::FileNotFound.into_error()) - .and_then(|t| { - let mut s = String::new(); - t.read_to_string(&mut s) - .map_err_into(SEK::IoError) - .map(|_| s) - }) - } - - Err(_) => Err(SEK::LockError.into_error()) - } - } - - fn write_file_content(&mut self, buf: &[u8]) -> Result<(), SE> { - match *self { - InMemoryFileAbstractionInstance { ref absent_path, .. } => { - let mut mtx = self.fs_abstraction.lock().expect("Locking Mutex failed"); - let mut backend = mtx.get_mut(); - - if let Some(ref mut cur) = backend.get_mut(absent_path) { - let mut vec = cur.get_mut(); - vec.clear(); - vec.extend_from_slice(buf); - return Ok(()); - } - let vec = Vec::from(buf); - backend.insert(absent_path.clone(), Cursor::new(vec)); - return Ok(()); - }, - }; - } - } - - #[derive(Debug)] - pub struct InMemoryFileAbstraction { - virtual_filesystem: Backend, - } - - impl InMemoryFileAbstraction { - - pub fn new() -> InMemoryFileAbstraction { - InMemoryFileAbstraction { - virtual_filesystem: Arc::new(Mutex::new(RefCell::new(HashMap::new()))), - } - } - - pub fn backend(&self) -> &Backend { - &self.virtual_filesystem - } - - } - - impl FileAbstraction for InMemoryFileAbstraction { - - fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { - debug!("Removing: {:?}", path); - self.backend() - .lock() - .expect("Locking Mutex failed") - .get_mut() - .remove(path) - .map(|_| ()) - .ok_or(SEK::FileNotFound.into_error()) - } - - fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - debug!("Copying : {:?} -> {:?}", from, to); - let mut mtx = self.backend().lock().expect("Locking Mutex failed"); - let mut backend = mtx.get_mut(); - - let a = try!(backend.get(from).cloned().ok_or(SEK::FileNotFound.into_error())); - backend.insert(to.clone(), a); - debug!("Copying: {:?} -> {:?} worked", from, to); - Ok(()) - } - - fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - debug!("Renaming: {:?} -> {:?}", from, to); - let mut mtx = self.backend().lock().expect("Locking Mutex failed"); - let mut backend = mtx.get_mut(); - - let a = try!(backend.get(from).cloned().ok_or(SEK::FileNotFound.into_error())); - backend.insert(to.clone(), a); - debug!("Renaming: {:?} -> {:?} worked", from, to); - Ok(()) - } - - fn create_dir_all(&self, _: &PathBuf) -> Result<(), SE> { - Ok(()) - } - - fn new_instance(&self, p: PathBuf) -> Box { - Box::new(InMemoryFileAbstractionInstance::new(self.backend().clone(), p)) - } - } - -} - -mod stdio { - use std::cell::RefCell; - use std::collections::HashMap; - use std::fmt::Debug; - use std::fmt::Error as FmtError; - use std::fmt::Formatter; - use std::io::Cursor; - use std::io::{Read, Write}; - use std::path::PathBuf; - use std::sync::Arc; - use std::sync::Mutex; - - use libimagerror::into::IntoError; - use libimagerror::trace::*; - - use error::StoreErrorKind as SEK; - use error::StoreError as SE; - use super::FileAbstraction; - use super::FileAbstractionInstance; - use super::InMemoryFileAbstraction; - - // Because this is not exported in super::inmemory; - type Backend = Arc>>>>>; - - mod mapper { - use std::collections::HashMap; - use std::io::Cursor; - use std::io::{Read, Write}; - use std::path::PathBuf; - use store::Result; - - pub trait Mapper { - fn read_to_fs(&self, Box, &mut HashMap>>) -> Result<()>; - fn fs_to_write(&self, &mut HashMap>>, &mut Write) -> Result<()>; - } - } - - use self::mapper::Mapper; - - pub struct StdIoFileAbstraction { - mapper: M, - mem: InMemoryFileAbstraction, - out: Box, - } - - impl Debug for StdIoFileAbstraction - where M: Mapper - { - fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> { - write!(f, "StdIoFileAbstraction({:?}", self.mem) - } - } - - impl StdIoFileAbstraction - where M: Mapper - { - - pub fn new(in_stream: Box, out_stream: Box, mapper: M) -> Result, SE> { - let mem = InMemoryFileAbstraction::new(); - - { - let fill_res = match mem.backend().lock() { - Err(_) => Err(SEK::LockError.into_error()), - Ok(mut mtx) => mapper.read_to_fs(in_stream, mtx.get_mut()) - }; - let _ = try!(fill_res); - } - - Ok(StdIoFileAbstraction { - mapper: mapper, - mem: mem, - out: out_stream, - }) - } - - pub fn backend(&self) -> &Backend { - &self.mem.backend() - } - - } - - impl Drop for StdIoFileAbstraction - where M: Mapper - { - fn drop(&mut self) { - let fill_res = match self.mem.backend().lock() { - Err(_) => Err(SEK::LockError.into_error()), - Ok(mut mtx) => self.mapper.fs_to_write(mtx.get_mut(), &mut *self.out) - }; - - // We can do nothing but end this here with a trace. - // As this drop gets called when imag almost exits, there is no point in exit()ing here - // again. - let _ = fill_res.map_err_trace(); - } - } - - impl FileAbstraction for StdIoFileAbstraction { - - fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { - self.mem.remove_file(path) - } - - fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - self.mem.copy(from, to) - } - - fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - self.mem.rename(from, to) - } - - fn create_dir_all(&self, pb: &PathBuf) -> Result<(), SE> { - self.mem.create_dir_all(pb) - } - - fn new_instance(&self, p: PathBuf) -> Box { - self.mem.new_instance(p) - } - } - -} - -#[cfg(test)] -mod test { - use super::FileAbstractionInstance; - use super::inmemory::InMemoryFileAbstraction; - use super::inmemory::InMemoryFileAbstractionInstance; - use std::path::PathBuf; - - #[test] - fn lazy_file() { - let fs = InMemoryFileAbstraction::new(); - - let mut path = PathBuf::from("/tests"); - path.set_file_name("test1"); - let mut lf = InMemoryFileAbstractionInstance::new(fs.backend().clone(), path); - lf.write_file_content(b"Hello World").unwrap(); - let bah = lf.get_file_content().unwrap(); - assert_eq!(bah, "Hello World"); - } - -} diff --git a/libimagstore/src/file_abstraction/fs.rs b/libimagstore/src/file_abstraction/fs.rs new file mode 100644 index 00000000..4210a2e5 --- /dev/null +++ b/libimagstore/src/file_abstraction/fs.rs @@ -0,0 +1,142 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015, 2016 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use std::fs::{File, OpenOptions, create_dir_all, remove_file, copy, rename}; +use std::io::{Seek, SeekFrom, Read}; +use std::path::{Path, PathBuf}; + +use error::{MapErrInto, StoreError as SE, StoreErrorKind as SEK}; + +use super::FileAbstraction; +use super::FileAbstractionInstance; + +#[derive(Debug)] +pub enum FSFileAbstractionInstance { + Absent(PathBuf), + File(File, PathBuf) +} + +impl FileAbstractionInstance for FSFileAbstractionInstance { + + /** + * Get the content behind this file + */ + fn get_file_content(&mut self) -> Result { + debug!("Getting lazy file: {:?}", self); + let (file, path) = match *self { + FSFileAbstractionInstance::File(ref mut f, _) => return { + // We seek to the beginning of the file since we expect each + // access to the file to be in a different context + try!(f.seek(SeekFrom::Start(0)) + .map_err_into(SEK::FileNotSeeked)); + + let mut s = String::new(); + f.read_to_string(&mut s) + .map_err_into(SEK::IoError) + .map(|_| s) + }, + FSFileAbstractionInstance::Absent(ref p) => + (try!(open_file(p).map_err_into(SEK::FileNotFound)), p.clone()), + }; + *self = FSFileAbstractionInstance::File(file, path); + if let FSFileAbstractionInstance::File(ref mut f, _) = *self { + let mut s = String::new(); + f.read_to_string(&mut s) + .map_err_into(SEK::IoError) + .map(|_| s) + } else { + unreachable!() + } + } + + /** + * Write the content of this file + */ + fn write_file_content(&mut self, buf: &[u8]) -> Result<(), SE> { + use std::io::Write; + let (file, path) = match *self { + FSFileAbstractionInstance::File(ref mut f, _) => return { + // We seek to the beginning of the file since we expect each + // access to the file to be in a different context + try!(f.seek(SeekFrom::Start(0)) + .map_err_into(SEK::FileNotCreated)); + f.write_all(buf).map_err_into(SEK::FileNotWritten) + }, + FSFileAbstractionInstance::Absent(ref p) => + (try!(create_file(p).map_err_into(SEK::FileNotCreated)), p.clone()), + }; + *self = FSFileAbstractionInstance::File(file, path); + if let FSFileAbstractionInstance::File(ref mut f, _) = *self { + return f.write_all(buf).map_err_into(SEK::FileNotWritten); + } + unreachable!(); + } + +} + +/// `FSFileAbstraction` state type +/// +/// A lazy file is either absent, but a path to it is available, or it is present. +#[derive(Debug)] +pub struct FSFileAbstraction { +} + +impl FSFileAbstraction { + pub fn new() -> FSFileAbstraction { + FSFileAbstraction { } + } +} + +impl FileAbstraction for FSFileAbstraction { + + fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { + remove_file(path).map_err_into(SEK::FileNotRemoved) + } + + fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { + copy(from, to).map_err_into(SEK::FileNotCopied).map(|_| ()) + } + + fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { + rename(from, to).map_err_into(SEK::FileNotRenamed) + } + + fn create_dir_all(&self, path: &PathBuf) -> Result<(), SE> { + create_dir_all(path).map_err_into(SEK::DirNotCreated) + } + + fn new_instance(&self, p: PathBuf) -> Box { + Box::new(FSFileAbstractionInstance::Absent(p)) + } +} + +fn open_file>(p: A) -> ::std::io::Result { + OpenOptions::new().write(true).read(true).open(p) +} + +fn create_file>(p: A) -> ::std::io::Result { + if let Some(parent) = p.as_ref().parent() { + debug!("Implicitely creating directory: {:?}", parent); + if let Err(e) = create_dir_all(parent) { + return Err(e); + } + } + OpenOptions::new().write(true).read(true).create(true).open(p) +} + diff --git a/libimagstore/src/file_abstraction/inmemory.rs b/libimagstore/src/file_abstraction/inmemory.rs new file mode 100644 index 00000000..c93e3679 --- /dev/null +++ b/libimagstore/src/file_abstraction/inmemory.rs @@ -0,0 +1,166 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015, 2016 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use error::StoreError as SE; +use error::StoreErrorKind as SEK; +use std::io::Read; +use std::io::Cursor; +use std::path::PathBuf; +use std::collections::HashMap; +use std::sync::Mutex; +use std::cell::RefCell; +use std::sync::Arc; + +use libimagerror::into::IntoError; + +use super::FileAbstraction; +use super::FileAbstractionInstance; +use error::MapErrInto; + +type Backend = Arc>>>>>; + +/// `FileAbstraction` type, this is the Test version! +/// +/// A lazy file is either absent, but a path to it is available, or it is present. +#[derive(Debug)] +pub struct InMemoryFileAbstractionInstance { + fs_abstraction: Backend, + absent_path: PathBuf, +} + +impl InMemoryFileAbstractionInstance { + + pub fn new(fs: Backend, pb: PathBuf) -> InMemoryFileAbstractionInstance { + InMemoryFileAbstractionInstance { + fs_abstraction: fs, + absent_path: pb + } + } + +} + +impl FileAbstractionInstance for InMemoryFileAbstractionInstance { + + /** + * Get the mutable file behind a InMemoryFileAbstraction object + */ + fn get_file_content(&mut self) -> Result { + debug!("Getting lazy file: {:?}", self); + + let p = self.absent_path.clone(); + match self.fs_abstraction.lock() { + Ok(mut mtx) => { + mtx.get_mut() + .get_mut(&p) + .ok_or(SEK::FileNotFound.into_error()) + .and_then(|t| { + let mut s = String::new(); + t.read_to_string(&mut s) + .map_err_into(SEK::IoError) + .map(|_| s) + }) + } + + Err(_) => Err(SEK::LockError.into_error()) + } + } + + fn write_file_content(&mut self, buf: &[u8]) -> Result<(), SE> { + match *self { + InMemoryFileAbstractionInstance { ref absent_path, .. } => { + let mut mtx = self.fs_abstraction.lock().expect("Locking Mutex failed"); + let mut backend = mtx.get_mut(); + + if let Some(ref mut cur) = backend.get_mut(absent_path) { + let mut vec = cur.get_mut(); + vec.clear(); + vec.extend_from_slice(buf); + return Ok(()); + } + let vec = Vec::from(buf); + backend.insert(absent_path.clone(), Cursor::new(vec)); + return Ok(()); + }, + }; + } +} + +#[derive(Debug)] +pub struct InMemoryFileAbstraction { + virtual_filesystem: Backend, +} + +impl InMemoryFileAbstraction { + + pub fn new() -> InMemoryFileAbstraction { + InMemoryFileAbstraction { + virtual_filesystem: Arc::new(Mutex::new(RefCell::new(HashMap::new()))), + } + } + + pub fn backend(&self) -> &Backend { + &self.virtual_filesystem + } + +} + +impl FileAbstraction for InMemoryFileAbstraction { + + fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { + debug!("Removing: {:?}", path); + self.backend() + .lock() + .expect("Locking Mutex failed") + .get_mut() + .remove(path) + .map(|_| ()) + .ok_or(SEK::FileNotFound.into_error()) + } + + fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { + debug!("Copying : {:?} -> {:?}", from, to); + let mut mtx = self.backend().lock().expect("Locking Mutex failed"); + let mut backend = mtx.get_mut(); + + let a = try!(backend.get(from).cloned().ok_or(SEK::FileNotFound.into_error())); + backend.insert(to.clone(), a); + debug!("Copying: {:?} -> {:?} worked", from, to); + Ok(()) + } + + fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { + debug!("Renaming: {:?} -> {:?}", from, to); + let mut mtx = self.backend().lock().expect("Locking Mutex failed"); + let mut backend = mtx.get_mut(); + + let a = try!(backend.get(from).cloned().ok_or(SEK::FileNotFound.into_error())); + backend.insert(to.clone(), a); + debug!("Renaming: {:?} -> {:?} worked", from, to); + Ok(()) + } + + fn create_dir_all(&self, _: &PathBuf) -> Result<(), SE> { + Ok(()) + } + + fn new_instance(&self, p: PathBuf) -> Box { + Box::new(InMemoryFileAbstractionInstance::new(self.backend().clone(), p)) + } +} + diff --git a/libimagstore/src/file_abstraction/mod.rs b/libimagstore/src/file_abstraction/mod.rs new file mode 100644 index 00000000..6b98c83d --- /dev/null +++ b/libimagstore/src/file_abstraction/mod.rs @@ -0,0 +1,126 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015, 2016 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +//! The filesystem abstraction code +//! +//! # Problem +//! +//! First, we had a compiletime backend for the store. This means that the actual filesystem +//! operations were compiled into the store either as real filesystem operations (in a normal debug +//! or release build) but as a in-memory variant in the 'test' case. +//! So tests did not hit the filesystem when running. +//! This gave us us the possibility to run tests concurrently with multiple +//! stores that did not interfere with eachother. +//! +//! This approach worked perfectly well until we started to test not the +//! store itself but crates that depend on the store implementation. +//! When running tests in a crate that depends on the store, the store +//! itself was compiled with the filesystem-hitting-backend. +//! This was problematic, as tests could not be implemented without hitting +//! the filesystem. +//! +//! Hence we implemented this. +//! +//! # Implementation +//! +//! The filesystem is abstracted via a trait `FileAbstraction` which +//! contains the essential functions for working with the filesystem. +//! +//! Two implementations are provided in the code: +//! +//! * FSFileAbstraction +//! * InMemoryFileAbstraction +//! +//! whereas the first actually works with the filesystem and the latter +//! works with an in-memory HashMap that is used as filesystem. +//! +//! Further, the trait `FileAbstractionInstance` was introduced for +//! functions which are executed on actual instances of content from the +//! filesystem, which was previousely tied into the general abstraction +//! mechanism. +//! +//! So, the `FileAbstraction` trait is for working with the filesystem, the +//! `FileAbstractionInstance` trait is for working with instances of content +//! from the filesystem (speak: actual Files). +//! +//! In case of the `FSFileAbstractionInstance`, which is the implementation +//! of the `FileAbstractionInstance` for the actual filesystem-hitting code, +//! the underlying resource is managed like with the old code before. +//! The `InMemoryFileAbstractionInstance` implementation is corrosponding to +//! the `InMemoryFileAbstraction` implementation - for the in-memory +//! "filesystem". +//! +//! The implementation of the `get_file_content()` function had to be +//! changed to return a `String` rather than a `&mut Read` because of +//! lifetime issues. +//! This change is store-internally and the API of the store itself was not +//! affected. +//! + +use std::path::PathBuf; +use std::fmt::Debug; + +use error::StoreError as SE; + +pub use self::fs::FSFileAbstraction; +pub use self::fs::FSFileAbstractionInstance; +pub use self::inmemory::InMemoryFileAbstraction; +pub use self::inmemory::InMemoryFileAbstractionInstance; + +/// An abstraction trait over filesystem actions +pub trait FileAbstraction : Debug { + fn remove_file(&self, path: &PathBuf) -> Result<(), SE>; + fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE>; + fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE>; + fn create_dir_all(&self, _: &PathBuf) -> Result<(), SE>; + + fn new_instance(&self, p: PathBuf) -> Box; +} + +/// An abstraction trait over actions on files +pub trait FileAbstractionInstance : Debug { + fn get_file_content(&mut self) -> Result; + fn write_file_content(&mut self, buf: &[u8]) -> Result<(), SE>; +} + +mod fs; +mod inmemory; +mod stdio; + +#[cfg(test)] +mod test { + use super::FileAbstractionInstance; + use super::inmemory::InMemoryFileAbstraction; + use super::inmemory::InMemoryFileAbstractionInstance; + use std::path::PathBuf; + + #[test] + fn lazy_file() { + let fs = InMemoryFileAbstraction::new(); + + let mut path = PathBuf::from("/tests"); + path.set_file_name("test1"); + let mut lf = InMemoryFileAbstractionInstance::new(fs.backend().clone(), path); + lf.write_file_content(b"Hello World").unwrap(); + let bah = lf.get_file_content().unwrap(); + assert_eq!(bah, "Hello World"); + } + +} + diff --git a/libimagstore/src/file_abstraction/stdio/mapper/mod.rs b/libimagstore/src/file_abstraction/stdio/mapper/mod.rs new file mode 100644 index 00000000..cdc40121 --- /dev/null +++ b/libimagstore/src/file_abstraction/stdio/mapper/mod.rs @@ -0,0 +1,30 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015, 2016 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use std::collections::HashMap; +use std::io::Cursor; +use std::io::{Read, Write}; +use std::path::PathBuf; +use store::Result; + +pub trait Mapper { + fn read_to_fs(&self, Box, &mut HashMap>>) -> Result<()>; + fn fs_to_write(&self, &mut HashMap>>, &mut Write) -> Result<()>; +} + diff --git a/libimagstore/src/file_abstraction/stdio/mod.rs b/libimagstore/src/file_abstraction/stdio/mod.rs new file mode 100644 index 00000000..c6b7c671 --- /dev/null +++ b/libimagstore/src/file_abstraction/stdio/mod.rs @@ -0,0 +1,127 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015, 2016 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use std::cell::RefCell; +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Error as FmtError; +use std::fmt::Formatter; +use std::io::Cursor; +use std::io::{Read, Write}; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; + +use libimagerror::into::IntoError; +use libimagerror::trace::*; + +use error::StoreErrorKind as SEK; +use error::StoreError as SE; +use super::FileAbstraction; +use super::FileAbstractionInstance; +use super::InMemoryFileAbstraction; + +pub mod mapper; +use self::mapper::Mapper; + +// Because this is not exported in super::inmemory; +type Backend = Arc>>>>>; + +pub struct StdIoFileAbstraction { + mapper: M, + mem: InMemoryFileAbstraction, + out: Box, +} + +impl Debug for StdIoFileAbstraction + where M: Mapper +{ + fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> { + write!(f, "StdIoFileAbstraction({:?}", self.mem) + } +} + +impl StdIoFileAbstraction + where M: Mapper +{ + + pub fn new(in_stream: Box, out_stream: Box, mapper: M) -> Result, SE> { + let mem = InMemoryFileAbstraction::new(); + + { + let fill_res = match mem.backend().lock() { + Err(_) => Err(SEK::LockError.into_error()), + Ok(mut mtx) => mapper.read_to_fs(in_stream, mtx.get_mut()) + }; + let _ = try!(fill_res); + } + + Ok(StdIoFileAbstraction { + mapper: mapper, + mem: mem, + out: out_stream, + }) + } + + pub fn backend(&self) -> &Backend { + &self.mem.backend() + } + +} + +impl Drop for StdIoFileAbstraction + where M: Mapper +{ + fn drop(&mut self) { + let fill_res = match self.mem.backend().lock() { + Err(_) => Err(SEK::LockError.into_error()), + Ok(mut mtx) => self.mapper.fs_to_write(mtx.get_mut(), &mut *self.out) + }; + + // We can do nothing but end this here with a trace. + // As this drop gets called when imag almost exits, there is no point in exit()ing here + // again. + let _ = fill_res.map_err_trace(); + } +} + +impl FileAbstraction for StdIoFileAbstraction { + + fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { + self.mem.remove_file(path) + } + + fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { + self.mem.copy(from, to) + } + + fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { + self.mem.rename(from, to) + } + + fn create_dir_all(&self, pb: &PathBuf) -> Result<(), SE> { + self.mem.create_dir_all(pb) + } + + fn new_instance(&self, p: PathBuf) -> Box { + self.mem.new_instance(p) + } +} + + From af2b629a8617ac29f7357830245fc685f71ff548 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 19:37:27 +0200 Subject: [PATCH 04/13] Add VersionError kind --- libimagstore/src/error.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libimagstore/src/error.rs b/libimagstore/src/error.rs index 178a5e8f..00e47681 100644 --- a/libimagstore/src/error.rs +++ b/libimagstore/src/error.rs @@ -34,6 +34,8 @@ generate_custom_error_types!(StoreError, StoreErrorKind, CustomErrorData, ConfigTypeError => "Store configuration type error", ConfigKeyMissingError => "Configuration Key missing", + VersionError => "Incompatible store versions detected", + CreateStoreDirDenied => "Creating store directory implicitely denied", FileError => "File Error", IoError => "IO Error", From 08f9eb3d83a3a0921325de1e77757bbe8f3b3944 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 19:38:52 +0200 Subject: [PATCH 05/13] Add new dependencies: Serde* --- libimagstore/Cargo.toml | 3 +++ libimagstore/src/lib.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/libimagstore/Cargo.toml b/libimagstore/Cargo.toml index acee580b..66ce38d2 100644 --- a/libimagstore/Cargo.toml +++ b/libimagstore/Cargo.toml @@ -26,6 +26,9 @@ crossbeam = "0.2.*" walkdir = "1.0.*" itertools = "0.6.*" is-match = "0.1" +serde = "1.0" +serde_json = "1.0" +serde_derive = "1.0" [dependencies.libimagerror] path = "../libimagerror" diff --git a/libimagstore/src/lib.rs b/libimagstore/src/lib.rs index 2083ce72..b5114f78 100644 --- a/libimagstore/src/lib.rs +++ b/libimagstore/src/lib.rs @@ -44,6 +44,9 @@ extern crate crossbeam; extern crate walkdir; extern crate itertools; #[macro_use] extern crate is_match; +#[macro_use] extern crate serde; +#[macro_use] extern crate serde_json; +#[macro_use] extern crate serde_derive; #[macro_use] extern crate libimagerror; extern crate libimagutil; From aede9a112b0743be2646f481b18b734fb10dc78a Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 19:37:54 +0200 Subject: [PATCH 06/13] Add first code draft for JSON mapper implementation --- .../src/file_abstraction/stdio/mapper/json.rs | 136 ++++++++++++++++++ .../src/file_abstraction/stdio/mapper/mod.rs | 2 + 2 files changed, 138 insertions(+) create mode 100644 libimagstore/src/file_abstraction/stdio/mapper/json.rs diff --git a/libimagstore/src/file_abstraction/stdio/mapper/json.rs b/libimagstore/src/file_abstraction/stdio/mapper/json.rs new file mode 100644 index 00000000..9b300122 --- /dev/null +++ b/libimagstore/src/file_abstraction/stdio/mapper/json.rs @@ -0,0 +1,136 @@ +// +// imag - the personal information management suite for the commandline +// Copyright (C) 2015, 2016 Matthias Beyer and contributors +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +use std::collections::HashMap; +use std::io::Cursor; +use std::io::{Read, Write}; +use std::path::PathBuf; + +use serde_json; +use toml; + +use error::StoreErrorKind as SEK; +use error::MapErrInto; +use super::Mapper; +use store::Result; + +use libimagerror::into::IntoError; + +#[derive(Deserialize, Serialize)] +struct Entry { + header: serde_json::Value, + content: String, +} + +impl Entry { + + fn to_string(self) -> Result { + toml::to_string(&self.header) + .map_err_into(SEK::IoError) + .map(|hdr| { + format!("---\n{header}---\n{content}", + header = hdr, + content = self.content) + }) + } + +} + +#[derive(Deserialize, Serialize)] +struct Document { + version: String, + store: HashMap, +} + +pub struct JsonMapper; + +impl JsonMapper { + + pub fn new() -> JsonMapper { + JsonMapper + } + +} + +impl Mapper for JsonMapper { + fn read_to_fs(&self, mut r: Box, hm: &mut HashMap>>) -> Result<()> { + let mut document = { + let mut s = String::new(); + r.read_to_string(&mut s).expect("Reading failed"); + let doc : Document = serde_json::from_str(&s).expect("Mapping error"); + doc + }; + + let _ = try!(::semver::Version::parse(&document.version) + .map_err_into(SEK::VersionError) + .and_then(|doc_vers| { + // safe because cargo does not compile if crate version is not valid + let crate_version = ::semver::Version::parse(version!()).unwrap(); + + if doc_vers > crate_version { + Err(SEK::VersionError.into_error()) + } else { + Ok(()) + } + })); + + for (key, val) in document.store.drain() { + let res = val + .to_string() + .map(|vals| hm.insert(key, Cursor::new(vals.into_bytes()))) + .map(|_| ()); + + let _ = try!(res); + } + + Ok(()) + } + + fn fs_to_write(&self, hm: &mut HashMap>>, out: &mut Write) -> Result<()> { + #[derive(Serialize)] + struct OutDocument { + version: String, + store: HashMap, + } + + let mut doc = OutDocument { + version: String::from(version!()), + store: HashMap::new(), + }; + + for (key, value) in hm.drain() { + let res = String::from_utf8(value.into_inner()) + .map_err_into(SEK::IoError) + .map(|entrystr| { + doc.store.insert(key, entrystr); + }) + .map(|_| ()); + + let _ = try!(res); + } + + serde_json::to_string(&doc) + .map_err_into(SEK::IoError) + .and_then(|json| out.write(&json.into_bytes()).map_err_into(SEK::IoError)) + .and_then(|_| out.flush().map_err_into(SEK::IoError)) + .map(|_| ()) + } +} + + diff --git a/libimagstore/src/file_abstraction/stdio/mapper/mod.rs b/libimagstore/src/file_abstraction/stdio/mapper/mod.rs index cdc40121..bd4b62df 100644 --- a/libimagstore/src/file_abstraction/stdio/mapper/mod.rs +++ b/libimagstore/src/file_abstraction/stdio/mapper/mod.rs @@ -28,3 +28,5 @@ pub trait Mapper { fn fs_to_write(&self, &mut HashMap>>, &mut Write) -> Result<()>; } +pub mod json; + From e5c8e9a1ac702f3623b04edaa31698b1405b36a1 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 20:26:32 +0200 Subject: [PATCH 07/13] Add simple tests --- .../src/file_abstraction/stdio/mapper/json.rs | 79 ++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/libimagstore/src/file_abstraction/stdio/mapper/json.rs b/libimagstore/src/file_abstraction/stdio/mapper/json.rs index 9b300122..7e4a8612 100644 --- a/libimagstore/src/file_abstraction/stdio/mapper/json.rs +++ b/libimagstore/src/file_abstraction/stdio/mapper/json.rs @@ -72,8 +72,8 @@ impl Mapper for JsonMapper { fn read_to_fs(&self, mut r: Box, hm: &mut HashMap>>) -> Result<()> { let mut document = { let mut s = String::new(); - r.read_to_string(&mut s).expect("Reading failed"); - let doc : Document = serde_json::from_str(&s).expect("Mapping error"); + try!(r.read_to_string(&mut s).map_err_into(SEK::IoError)); + let doc : Document = try!(serde_json::from_str(&s).map_err_into(SEK::IoError)); doc }; @@ -133,4 +133,79 @@ impl Mapper for JsonMapper { } } +#[cfg(test)] +mod test { + use std::io::Cursor; + + use super::*; + + #[test] + fn test_json_to_fs() { + let json = r#" + { "version": "0.3.0", + "store": { + "/example": { + "header": { + "imag": { + "version": "0.3.0" + } + }, + "content": "test" + } + } + } + "#; + let json = Cursor::new(String::from(json).into_bytes()); + let mapper = JsonMapper::new(); + let mut hm = HashMap::new(); + + let io_res = mapper.read_to_fs(Box::new(json), &mut hm); + assert!(io_res.is_ok()); + + assert_eq!(1, hm.len()); // we should have exactly one entry + } + + #[test] + fn test_fs_to_json() { + let mapper = JsonMapper::new(); + let mut out : Cursor> = Cursor::new(vec![]); + + let mut hm = { + let mut hm = HashMap::new(); + let content = r#"--- +[imag] +version = "0.3.0" +--- +hi there!"#; + hm.insert(PathBuf::from("/example"), Cursor::new(String::from(content).into_bytes())); + hm + }; + + let io_res = mapper.fs_to_write(&mut hm, &mut out); + assert!(io_res.is_ok()); + + let example = r#" + { + "version": "0.3.0", + "store": { + "/example": { + "header": { + "imag": { + "version": "0.3.0" + } + }, + "content": "hi there!" + } + } + } + "#; + + let example_json : ::serde_json::Value = ::serde_json::from_str(example).unwrap(); + + let output_json = String::from_utf8(out.into_inner()).unwrap(); + let output_json : ::serde_json::Value = ::serde_json::from_str(&output_json).unwrap(); + + assert_eq!(example_json, output_json); + } +} From 76dfe23f65322d431f133ae4ddc301e910efd3fa Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 20:34:08 +0200 Subject: [PATCH 08/13] Add StdIoFileAbstraction module documentation --- .../src/file_abstraction/stdio/mod.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/libimagstore/src/file_abstraction/stdio/mod.rs b/libimagstore/src/file_abstraction/stdio/mod.rs index c6b7c671..c475e106 100644 --- a/libimagstore/src/file_abstraction/stdio/mod.rs +++ b/libimagstore/src/file_abstraction/stdio/mod.rs @@ -17,6 +17,25 @@ // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // +//! The StdIo backend +//! +//! Sidenote: The name is "StdIo" because its main purpose is Stdin/Stdio, but it is abstracted over +//! Read/Write actually, so it is also possible to use this backend in other ways, too. +//! +//! So what is this about? This is a backend for the imag store which is created from stdin, by +//! piping contents into the store (via JSON or TOML) and piping the store contents (as JSON or +//! TOML) to stdout when the the backend is destructed. +//! +//! This is one of some components which make command-chaining in imag possible. With this, the +//! application does not have to know whether the store actually lives on the filesystem or just "in +//! memory". +//! +//! The backend contains a "Mapper" which defines how the contents get mapped into the in-memory +//! store representation: A JSON implementation or a TOML implementation are possible. +//! +//! In fact, a JSON implementation exists in the "json" submodule of this module. +//! + use std::cell::RefCell; use std::collections::HashMap; use std::fmt::Debug; From c4b2287876ac8abc284aa60359a3d1e905572e32 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 21:07:30 +0200 Subject: [PATCH 09/13] Move the documentation to the docs --- doc/src/02000-store.md | 167 ++++++++++++++++++ libimagstore/src/file_abstraction/mod.rs | 56 ------ .../src/file_abstraction/stdio/mod.rs | 19 -- 3 files changed, 167 insertions(+), 75 deletions(-) diff --git a/doc/src/02000-store.md b/doc/src/02000-store.md index eb4fa860..2f484758 100644 --- a/doc/src/02000-store.md +++ b/doc/src/02000-store.md @@ -146,3 +146,170 @@ moment. If the module "B" gets updated, it might update its entries in the store as well. The link from the "a" should never get invalid in this case, though it is not ensured by the core of imag itself. +## Backends {#sec:thestore:backends} + +The store itself also has a backend. This backend is the "filesystem +abstraction" code. + +Note: This is a very core thing. Casual users might want to skip this section. + +### Problem {#sec:thestore:backends:problem} + +First, we had a compiletime backend for the store. +This means that the actual filesystem operations were compiled into the stores +either as real filesystem operations (in a normal debug or release build) but as +a in-memory variant in the 'test' case. +So tests did not hit the filesystem when running. +This gave us us the possibility to run tests concurrently with multiple stores +that did not interfere with eachother. + +This approach worked perfectly well until we started to test not the +store itself but crates that depend on the store implementation. +When running tests in a crate that depends on the store, the store +itself was compiled with the filesystem-hitting-backend. +This was problematic, as tests could not be implemented without hitting +the filesystem. + +Hence we implemented this. + +### Implementation {#sec:thestore:backends:implementation} + +The filesystem is abstracted via a trait `FileAbstraction` which +contains the essential functions for working with the filesystem. + +Two implementations are provided in the code: + +* FSFileAbstraction +* InMemoryFileAbstraction + +whereas the first actually works with the filesystem and the latter +works with an in-memory HashMap that is used as filesystem. + +Further, the trait `FileAbstractionInstance` was introduced for +functions which are executed on actual instances of content from the +filesystem, which was previousely tied into the general abstraction +mechanism. + +So, the `FileAbstraction` trait is for working with the filesystem, the +`FileAbstractionInstance` trait is for working with instances of content +from the filesystem (speak: actual Files). + +In case of the `FSFileAbstractionInstance`, which is the implementation +of the `FileAbstractionInstance` for the actual filesystem-hitting code, +the underlying resource is managed like with the old code before. +The `InMemoryFileAbstractionInstance` implementation is corrosponding to +the `InMemoryFileAbstraction` implementation - for the in-memory +"filesystem". + +The implementation of the `get_file_content()` function had to be +changed to return a `String` rather than a `&mut Read` because of +lifetime issues. +This change is store-internally and the API of the store itself was not +affected. + +## The StdIo backend {#sec:thestore:backends:stdio} + +Sidenote: The name is "StdIo" because its main purpose is Stdin/Stdio, but it +is abstracted over Read/Write actually, so it is also possible to use this +backend in other ways, too. + +### Why? {#sec:thestore:backends:stdio:why} + +This is a backend for the imag store which is created +from stdin, by piping contents into the store (via JSON or TOML) and piping the +store contents (as JSON or TOML) to stdout when the the backend is destructed. + +This is one of some components which make command-chaining in imag possible. +With this, the application does not have to know whether the store actually +lives on the filesystem or just "in memory". + +### Mappers {#sec:thestore:backends:stdio:mappers} + +The backend contains a "Mapper" which defines how the contents get mapped into +the in-memory store representation: A JSON implementation or a TOML +implementation are possible. + +The following section assumes a JSON mapper. + +The backends themselves do not know "header" and "content" - they know only +blobs which live in pathes. +Indeed, this "backend" code does not serve "content" or "header" to the `Store` +implementation, but only a blob of bytes. +Anyways, the JSON-protocol for passing a store around _does_ know about content +and header (see @sec:thestore:backends:stdio:json for the JSON format). + +So the mapper reads the JSON, parses it (thanks serde!) and translates it to +TOML, because TOML is the Store representation of a header. +But because the backend does not serve header and content, but only a blob, +this TOML is then translated (with the content of the respective file) to a +blob. + +This is then made available to the store codebase. +This is complex and probably slow, we know. + +To summarize what we do right now, lets have a look at the awesome ascii-art +below: + +``` + libimag* + | + v + IO Mapper FS Store FS Mapper IO ++--+-------------+---------+--------+---------+--------------+--+ +| | | | | | | | + JSON -> TOML -> String -> Entry -> String -> TOML -> JSON + + TOML + + Content +``` + +This is what gets translated where for one imag call with a stdio store backend. + +The rationale behind this implementation is that this is the best implementation +we can have in a relatively short amount of time. + +### The JSON Mapper {#sec:thestore:backends:stdio:json} + +The JSON mapper maps JSON which is read from a source into a HashMap which +represents the in-memory filesystem. + +The strucure is as follows: + +```json +{ + "version": "0.3.0", + "store": { + "/example": { + "header": { + "imag": { + "version": "0.3.0", + }, + }, + "content": "hi there!", + }, + }, +} +``` + +### TODO {#sec:thestore:backends:todo} + +Of course, the above is not optimal. +The TODO here is almost visible: Implement a proper backend where we do not need +to translate between types all the time. + +The first goal would be to reduce the above figure to: + +``` + libimag* + | + v + IO Mapper Store Mapper IO ++--+------+--------+-------+--+ +| | | | | | + JSON -> Entry -> JSON + + TOML + + Content +``` + +and the second step would be to abstract all the things away so the `libimag*` +crates handle the header without knowing whether it is JSON or TOML. + diff --git a/libimagstore/src/file_abstraction/mod.rs b/libimagstore/src/file_abstraction/mod.rs index 6b98c83d..a961503e 100644 --- a/libimagstore/src/file_abstraction/mod.rs +++ b/libimagstore/src/file_abstraction/mod.rs @@ -17,62 +17,6 @@ // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // -//! The filesystem abstraction code -//! -//! # Problem -//! -//! First, we had a compiletime backend for the store. This means that the actual filesystem -//! operations were compiled into the store either as real filesystem operations (in a normal debug -//! or release build) but as a in-memory variant in the 'test' case. -//! So tests did not hit the filesystem when running. -//! This gave us us the possibility to run tests concurrently with multiple -//! stores that did not interfere with eachother. -//! -//! This approach worked perfectly well until we started to test not the -//! store itself but crates that depend on the store implementation. -//! When running tests in a crate that depends on the store, the store -//! itself was compiled with the filesystem-hitting-backend. -//! This was problematic, as tests could not be implemented without hitting -//! the filesystem. -//! -//! Hence we implemented this. -//! -//! # Implementation -//! -//! The filesystem is abstracted via a trait `FileAbstraction` which -//! contains the essential functions for working with the filesystem. -//! -//! Two implementations are provided in the code: -//! -//! * FSFileAbstraction -//! * InMemoryFileAbstraction -//! -//! whereas the first actually works with the filesystem and the latter -//! works with an in-memory HashMap that is used as filesystem. -//! -//! Further, the trait `FileAbstractionInstance` was introduced for -//! functions which are executed on actual instances of content from the -//! filesystem, which was previousely tied into the general abstraction -//! mechanism. -//! -//! So, the `FileAbstraction` trait is for working with the filesystem, the -//! `FileAbstractionInstance` trait is for working with instances of content -//! from the filesystem (speak: actual Files). -//! -//! In case of the `FSFileAbstractionInstance`, which is the implementation -//! of the `FileAbstractionInstance` for the actual filesystem-hitting code, -//! the underlying resource is managed like with the old code before. -//! The `InMemoryFileAbstractionInstance` implementation is corrosponding to -//! the `InMemoryFileAbstraction` implementation - for the in-memory -//! "filesystem". -//! -//! The implementation of the `get_file_content()` function had to be -//! changed to return a `String` rather than a `&mut Read` because of -//! lifetime issues. -//! This change is store-internally and the API of the store itself was not -//! affected. -//! - use std::path::PathBuf; use std::fmt::Debug; diff --git a/libimagstore/src/file_abstraction/stdio/mod.rs b/libimagstore/src/file_abstraction/stdio/mod.rs index c475e106..c6b7c671 100644 --- a/libimagstore/src/file_abstraction/stdio/mod.rs +++ b/libimagstore/src/file_abstraction/stdio/mod.rs @@ -17,25 +17,6 @@ // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // -//! The StdIo backend -//! -//! Sidenote: The name is "StdIo" because its main purpose is Stdin/Stdio, but it is abstracted over -//! Read/Write actually, so it is also possible to use this backend in other ways, too. -//! -//! So what is this about? This is a backend for the imag store which is created from stdin, by -//! piping contents into the store (via JSON or TOML) and piping the store contents (as JSON or -//! TOML) to stdout when the the backend is destructed. -//! -//! This is one of some components which make command-chaining in imag possible. With this, the -//! application does not have to know whether the store actually lives on the filesystem or just "in -//! memory". -//! -//! The backend contains a "Mapper" which defines how the contents get mapped into the in-memory -//! store representation: A JSON implementation or a TOML implementation are possible. -//! -//! In fact, a JSON implementation exists in the "json" submodule of this module. -//! - use std::cell::RefCell; use std::collections::HashMap; use std::fmt::Debug; From 73c1e790849b2f281e855f9286c2d85cef4cf6e9 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 21:46:01 +0200 Subject: [PATCH 10/13] Implement high-level store test with IO backend --- libimagstore/src/file_abstraction/mod.rs | 9 +-- libimagstore/src/store.rs | 90 ++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/libimagstore/src/file_abstraction/mod.rs b/libimagstore/src/file_abstraction/mod.rs index a961503e..82e3ce95 100644 --- a/libimagstore/src/file_abstraction/mod.rs +++ b/libimagstore/src/file_abstraction/mod.rs @@ -22,6 +22,11 @@ use std::fmt::Debug; use error::StoreError as SE; + +mod fs; +mod inmemory; +pub mod stdio; + pub use self::fs::FSFileAbstraction; pub use self::fs::FSFileAbstractionInstance; pub use self::inmemory::InMemoryFileAbstraction; @@ -43,10 +48,6 @@ pub trait FileAbstractionInstance : Debug { fn write_file_content(&mut self, buf: &[u8]) -> Result<(), SE>; } -mod fs; -mod inmemory; -mod stdio; - #[cfg(test)] mod test { use super::FileAbstractionInstance; diff --git a/libimagstore/src/store.rs b/libimagstore/src/store.rs index b97dd1d8..3bbeae05 100644 --- a/libimagstore/src/store.rs +++ b/libimagstore/src/store.rs @@ -1262,6 +1262,96 @@ mod store_tests { } } + #[test] + fn test_store_create_with_io_backend() { + use std::io::Cursor; + use std::rc::Rc; + use std::cell::RefCell; + use serde_json::Value; + + //let sink = vec![]; + //let output : Cursor<&mut [u8]> = Cursor::new(&mut sink); + //let output = Rc::new(RefCell::new(output)); + let output = Rc::new(RefCell::new(vec![])); + + { + let store = { + use file_abstraction::stdio::StdIoFileAbstraction; + use file_abstraction::stdio::mapper::json::JsonMapper; + + // Lets have an empty store as input + let mut input = Cursor::new(r#" + { "version": "0.3.0", + "store": { } + } + "#); + + let mapper = JsonMapper::new(); + let backend = StdIoFileAbstraction::new(&mut input, output.clone(), mapper).unwrap(); + let backend = Box::new(backend); + + Store::new_with_backend(PathBuf::from("/"), None, backend).unwrap() + }; + + for n in 1..100 { + let s = format!("test-{}", n); + let entry = store.create(PathBuf::from(s.clone())).unwrap(); + assert!(entry.verify().is_ok()); + let loc = entry.get_location().clone().into_pathbuf().unwrap(); + assert!(loc.starts_with("/")); + assert!(loc.ends_with(s)); + } + } + + let vec = Rc::try_unwrap(output).unwrap().into_inner(); + + let errstr = format!("Not UTF8: '{:?}'", vec); + let string = String::from_utf8(vec); + assert!(string.is_ok(), errstr); + let string = string.unwrap(); + + assert!(!string.is_empty(), format!("Expected not to be empty: '{}'", string)); + + let json : ::serde_json::Value = ::serde_json::from_str(&string).unwrap(); + + match json { + Value::Object(ref map) => { + assert!(map.get("version").is_some(), format!("No 'version' in JSON")); + match map.get("version").unwrap() { + &Value::String(ref s) => assert_eq!("0.3.0", s), + _ => panic!("Wrong type in JSON at 'version'"), + } + + assert!(map.get("store").is_some(), format!("No 'store' in JSON")); + match map.get("store").unwrap() { + &Value::Object(ref objs) => { + for n in 1..100 { + let s = format!("/test-{}", n); + assert!(objs.get(&s).is_some(), format!("No entry: '{}'", s)); + match objs.get(&s).unwrap() { + &Value::Object(ref entry) => { + match entry.get("header").unwrap() { + &Value::Object(_) => assert!(true), + _ => panic!("Wrong type in JSON at 'store.'{}'.header'", s), + } + + match entry.get("content").unwrap() { + &Value::String(_) => assert!(true), + _ => panic!("Wrong type in JSON at 'store.'{}'.content'", s), + } + }, + _ => panic!("Wrong type in JSON at 'store.'{}''", s), + } + } + }, + _ => panic!("Wrong type in JSON at 'store'"), + } + }, + _ => panic!("Wrong type in JSON at top level"), + } + + } + #[test] fn test_store_get_create_get_delete_get() { let store = get_store(); From 91c427925b48371482b5b091d715d71ce7dec31f Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sat, 17 Jun 2017 23:06:19 +0200 Subject: [PATCH 11/13] Make backend generic over Read/Write --- .../src/file_abstraction/stdio/mapper/json.rs | 12 ++++---- .../src/file_abstraction/stdio/mapper/mod.rs | 4 +-- .../src/file_abstraction/stdio/mod.rs | 30 ++++++++++++------- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/libimagstore/src/file_abstraction/stdio/mapper/json.rs b/libimagstore/src/file_abstraction/stdio/mapper/json.rs index 7e4a8612..5411d002 100644 --- a/libimagstore/src/file_abstraction/stdio/mapper/json.rs +++ b/libimagstore/src/file_abstraction/stdio/mapper/json.rs @@ -69,7 +69,7 @@ impl JsonMapper { } impl Mapper for JsonMapper { - fn read_to_fs(&self, mut r: Box, hm: &mut HashMap>>) -> Result<()> { + fn read_to_fs(&self, r: &mut R, hm: &mut HashMap>>) -> Result<()> { let mut document = { let mut s = String::new(); try!(r.read_to_string(&mut s).map_err_into(SEK::IoError)); @@ -102,7 +102,7 @@ impl Mapper for JsonMapper { Ok(()) } - fn fs_to_write(&self, hm: &mut HashMap>>, out: &mut Write) -> Result<()> { + fn fs_to_write(&self, hm: &mut HashMap>>, out: &mut W) -> Result<()> { #[derive(Serialize)] struct OutDocument { version: String, @@ -155,11 +155,11 @@ mod test { } } "#; - let json = Cursor::new(String::from(json).into_bytes()); - let mapper = JsonMapper::new(); - let mut hm = HashMap::new(); + let mut json = Cursor::new(String::from(json).into_bytes()); + let mapper = JsonMapper::new(); + let mut hm = HashMap::new(); - let io_res = mapper.read_to_fs(Box::new(json), &mut hm); + let io_res = mapper.read_to_fs(&mut json, &mut hm); assert!(io_res.is_ok()); assert_eq!(1, hm.len()); // we should have exactly one entry diff --git a/libimagstore/src/file_abstraction/stdio/mapper/mod.rs b/libimagstore/src/file_abstraction/stdio/mapper/mod.rs index bd4b62df..591002bc 100644 --- a/libimagstore/src/file_abstraction/stdio/mapper/mod.rs +++ b/libimagstore/src/file_abstraction/stdio/mapper/mod.rs @@ -24,8 +24,8 @@ use std::path::PathBuf; use store::Result; pub trait Mapper { - fn read_to_fs(&self, Box, &mut HashMap>>) -> Result<()>; - fn fs_to_write(&self, &mut HashMap>>, &mut Write) -> Result<()>; + fn read_to_fs(&self, &mut R, &mut HashMap>>) -> Result<()>; + fn fs_to_write(&self, &mut HashMap>>, &mut W) -> Result<()>; } pub mod json; diff --git a/libimagstore/src/file_abstraction/stdio/mod.rs b/libimagstore/src/file_abstraction/stdio/mod.rs index c6b7c671..88072241 100644 --- a/libimagstore/src/file_abstraction/stdio/mod.rs +++ b/libimagstore/src/file_abstraction/stdio/mod.rs @@ -17,6 +17,7 @@ // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // +use std::rc::Rc; use std::cell::RefCell; use std::collections::HashMap; use std::fmt::Debug; @@ -43,25 +44,27 @@ use self::mapper::Mapper; // Because this is not exported in super::inmemory; type Backend = Arc>>>>>; -pub struct StdIoFileAbstraction { +pub struct StdIoFileAbstraction { mapper: M, mem: InMemoryFileAbstraction, - out: Box, + out: Rc>, } -impl Debug for StdIoFileAbstraction - where M: Mapper +impl Debug for StdIoFileAbstraction + where M: Mapper, + W: Write { fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> { write!(f, "StdIoFileAbstraction({:?}", self.mem) } } -impl StdIoFileAbstraction - where M: Mapper +impl StdIoFileAbstraction + where M: Mapper, + W: Write { - pub fn new(in_stream: Box, out_stream: Box, mapper: M) -> Result, SE> { + pub fn new(in_stream: &mut R, out_stream: Rc>, mapper: M) -> Result, SE> { let mem = InMemoryFileAbstraction::new(); { @@ -85,13 +88,18 @@ impl StdIoFileAbstraction } -impl Drop for StdIoFileAbstraction - where M: Mapper +impl Drop for StdIoFileAbstraction + where M: Mapper, + W: Write { fn drop(&mut self) { + use std::ops::DerefMut; + let fill_res = match self.mem.backend().lock() { Err(_) => Err(SEK::LockError.into_error()), - Ok(mut mtx) => self.mapper.fs_to_write(mtx.get_mut(), &mut *self.out) + Ok(mut mtx) => { + self.mapper.fs_to_write(mtx.get_mut(), self.out.borrow_mut().deref_mut()) + }, }; // We can do nothing but end this here with a trace. @@ -101,7 +109,7 @@ impl Drop for StdIoFileAbstraction } } -impl FileAbstraction for StdIoFileAbstraction { +impl FileAbstraction for StdIoFileAbstraction { fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { self.mem.remove_file(path) From c013ca80257a182b26f47c4416e517c4e11fe47c Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sun, 18 Jun 2017 00:50:18 +0200 Subject: [PATCH 12/13] Outsource header/content parsing from store.rs to util.rs for reusability --- libimagstore/src/store.rs | 29 ++++------------------------- libimagstore/src/util.rs | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/libimagstore/src/store.rs b/libimagstore/src/store.rs index 3bbeae05..41c3b85b 100644 --- a/libimagstore/src/store.rs +++ b/libimagstore/src/store.rs @@ -24,7 +24,6 @@ use std::result::Result as RResult; use std::sync::Arc; use std::sync::RwLock; use std::io::Read; -use std::convert::From; use std::convert::Into; use std::ops::Deref; use std::ops::DerefMut; @@ -33,7 +32,6 @@ use std::fmt::Debug; use std::fmt::Error as FMTError; use toml::Value; -use regex::Regex; use glob::glob; use walkdir::WalkDir; use walkdir::Iter as WalkDirIter; @@ -919,33 +917,14 @@ impl Entry { /// - Header cannot be parsed into a TOML object /// pub fn from_str(loc: S, s: &str) -> Result { - debug!("Building entry from string"); - lazy_static! { - static ref RE: Regex = Regex::new(r"(?smx) - ^---$ - (?P
.*) # Header - ^---$\n - (?P.*) # Content - ").unwrap(); - } + use util::entry_buffer_to_header_content; - let matches = match RE.captures(s) { - None => return Err(SE::new(SEK::MalformedEntry, None)), - Some(s) => s, - }; + let (header, content) = try!(entry_buffer_to_header_content(s)); - let header = match matches.name("header") { - None => return Err(SE::new(SEK::MalformedEntry, None)), - Some(s) => s - }; - - let content = matches.name("content").map(|r| r.as_str()).unwrap_or(""); - - debug!("Header and content found. Yay! Building Entry object now"); Ok(Entry { location: try!(loc.into_storeid()), - header: try!(Value::parse(header.as_str())), - content: String::from(content), + header: header, + content: content, }) } diff --git a/libimagstore/src/util.rs b/libimagstore/src/util.rs index 9ff4a145..51462ab1 100644 --- a/libimagstore/src/util.rs +++ b/libimagstore/src/util.rs @@ -17,6 +17,15 @@ // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // +use regex::Regex; +use toml::Value; + +use libimagerror::into::IntoError; + +use store::Result; +use error::StoreErrorKind as SEK; +use toml_ext::Header; + #[cfg(feature = "early-panic")] #[macro_export] macro_rules! if_cfg_panic { @@ -33,3 +42,29 @@ macro_rules! if_cfg_panic { ($fmt:expr, $($arg:tt)+) => { }; } +pub fn entry_buffer_to_header_content(buf: &str) -> Result<(Value, String)> { + debug!("Building entry from string"); + lazy_static! { + static ref RE: Regex = Regex::new(r"(?smx) + ^---$ + (?P
.*) # Header + ^---$\n + (?P.*) # Content + ").unwrap(); + } + + let matches = match RE.captures(buf) { + None => return Err(SEK::MalformedEntry.into_error()), + Some(s) => s, + }; + + let header = match matches.name("header") { + None => return Err(SEK::MalformedEntry.into_error()), + Some(s) => s + }; + + let content = matches.name("content").map(|r| r.as_str()).unwrap_or(""); + + Ok((try!(Value::parse(header.as_str())), String::from(content))) +} + From 3af104259308b507de6ee0a05be3024c617a7b61 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Sun, 18 Jun 2017 00:50:49 +0200 Subject: [PATCH 13/13] Fix fs_to_write() to serialize headers correctly --- .../src/file_abstraction/stdio/mapper/json.rs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/libimagstore/src/file_abstraction/stdio/mapper/json.rs b/libimagstore/src/file_abstraction/stdio/mapper/json.rs index 5411d002..761e290b 100644 --- a/libimagstore/src/file_abstraction/stdio/mapper/json.rs +++ b/libimagstore/src/file_abstraction/stdio/mapper/json.rs @@ -103,10 +103,18 @@ impl Mapper for JsonMapper { } fn fs_to_write(&self, hm: &mut HashMap>>, out: &mut W) -> Result<()> { + use util::entry_buffer_to_header_content; + + #[derive(Serialize, Deserialize)] + struct Entry { + header: ::toml::Value, + content: String, + } + #[derive(Serialize)] struct OutDocument { version: String, - store: HashMap, + store: HashMap, } let mut doc = OutDocument { @@ -117,8 +125,14 @@ impl Mapper for JsonMapper { for (key, value) in hm.drain() { let res = String::from_utf8(value.into_inner()) .map_err_into(SEK::IoError) - .map(|entrystr| { - doc.store.insert(key, entrystr); + .and_then(|buf| entry_buffer_to_header_content(&buf)) + .map(|(header, content)| { + let entry = Entry { + header: header, + content: content + }; + + doc.store.insert(key, entry); }) .map(|_| ());