Merge pull request #975 from matthiasbeyer/libimagstore/backend-replacement

Libimagstore/backend replacement
This commit is contained in:
Matthias Beyer 2017-06-18 19:44:32 +02:00 committed by GitHub
commit f8ed6794c2
7 changed files with 472 additions and 54 deletions

View file

@ -343,6 +343,31 @@ impl<'a> Runtime<'a> {
&self.store
}
/// Change the store backend to stdout
///
/// For the documentation on purpose and cavecats, have a look at the documentation of the
/// `Store::reset_backend()` function.
///
pub fn store_backend_to_stdio(&mut self) -> Result<(), RuntimeError> {
use libimagstore::file_abstraction::stdio::*;
use libimagstore::file_abstraction::stdio::mapper::json::JsonMapper;
use std::rc::Rc;
use std::cell::RefCell;
let mut input = ::std::io::empty();
let output = ::std::io::stdout();
let output = Rc::new(RefCell::new(output));
let mapper = JsonMapper::new();
StdIoFileAbstraction::new(&mut input, output, mapper)
.map_err_into(RuntimeErrorKind::Instantiate)
.and_then(|backend| {
self.store
.reset_backend(Box::new(backend))
.map_err_into(RuntimeErrorKind::Instantiate)
})
}
/// Get a editor command object which can be called to open the $EDITOR
pub fn editor(&self) -> Option<Command> {
self.cli()

View file

@ -25,6 +25,7 @@ use error::{MapErrInto, StoreError as SE, StoreErrorKind as SEK};
use super::FileAbstraction;
use super::FileAbstractionInstance;
use super::Drain;
use store::Entry;
use storeid::StoreId;
@ -131,6 +132,20 @@ impl FileAbstraction for FSFileAbstraction {
fn new_instance(&self, p: PathBuf) -> Box<FileAbstractionInstance> {
Box::new(FSFileAbstractionInstance::Absent(p))
}
/// We return nothing from the FS here.
fn drain(&self) -> Result<Drain, SE> {
Ok(Drain::empty())
}
/// FileAbstraction::fill implementation that consumes the Drain and writes everything to the
/// filesystem
fn fill(&mut self, mut d: Drain) -> Result<(), SE> {
d.iter()
.fold(Ok(()), |acc, (path, element)| {
acc.and_then(|_| self.new_instance(path).write_file_content(&element))
})
}
}
fn open_file<A: AsRef<Path>>(p: A) -> ::std::io::Result<File> {

View file

@ -24,11 +24,13 @@ use std::collections::HashMap;
use std::sync::Mutex;
use std::cell::RefCell;
use std::sync::Arc;
use std::ops::Deref;
use libimagerror::into::IntoError;
use super::FileAbstraction;
use super::FileAbstractionInstance;
use super::Drain;
use store::Entry;
use storeid::StoreId;
@ -104,6 +106,13 @@ impl InMemoryFileAbstraction {
&self.virtual_filesystem
}
fn backend_cloned<'a>(&'a self) -> Result<HashMap<PathBuf, Entry>, SE> {
self.virtual_filesystem
.lock()
.map_err(|_| SEK::LockError.into_error())
.map(|mtx| mtx.deref().borrow().clone())
}
}
impl FileAbstraction for InMemoryFileAbstraction {
@ -148,5 +157,22 @@ impl FileAbstraction for InMemoryFileAbstraction {
fn new_instance(&self, p: PathBuf) -> Box<FileAbstractionInstance> {
Box::new(InMemoryFileAbstractionInstance::new(self.backend().clone(), p))
}
fn drain(&self) -> Result<Drain, SE> {
self.backend_cloned().map(Drain::new)
}
fn fill<'a>(&'a mut self, mut d: Drain) -> Result<(), SE> {
debug!("Draining into : {:?}", self);
let mut mtx = try!(self.backend().lock().map_err(|_| SEK::LockError.into_error()));
let mut backend = mtx.get_mut();
for (path, element) in d.iter() {
debug!("Drain into {:?}: {:?}", self, path);
backend.insert(path, element);
}
Ok(())
}
}

View file

@ -19,6 +19,7 @@
use std::path::PathBuf;
use std::fmt::Debug;
use std::collections::HashMap;
use error::StoreError as SE;
use store::Entry;
@ -41,6 +42,9 @@ pub trait FileAbstraction : Debug {
fn create_dir_all(&self, _: &PathBuf) -> Result<(), SE>;
fn new_instance(&self, p: PathBuf) -> Box<FileAbstractionInstance>;
fn drain(&self) -> Result<Drain, SE>;
fn fill<'a>(&'a mut self, d: Drain) -> Result<(), SE>;
}
/// An abstraction trait over actions on files
@ -54,6 +58,34 @@ pub trait FileAbstractionInstance : Debug {
fn write_file_content(&mut self, buf: &Entry) -> Result<(), SE>;
}
pub struct Drain(HashMap<PathBuf, Entry>);
impl Drain {
pub fn new(hm: HashMap<PathBuf, Entry>) -> Drain {
Drain(hm)
}
pub fn empty() -> Drain {
Drain::new(HashMap::new())
}
pub fn iter<'a>(&'a mut self) -> DrainIter<'a> {
DrainIter(self.0.drain())
}
}
pub struct DrainIter<'a>(::std::collections::hash_map::Drain<'a, PathBuf, Entry>);
impl<'a> Iterator for DrainIter<'a> {
type Item = (PathBuf, Entry);
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
}
#[cfg(test)]
mod test {
use std::path::PathBuf;

View file

@ -20,44 +20,34 @@
use std::rc::Rc;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Error as FmtError;
use std::fmt::Formatter;
use std::io::{Read, Write};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use std::ops::Deref;
use std::fmt::Debug;
use std::fmt::Error as FmtError;
use std::fmt::Formatter;
use libimagerror::into::IntoError;
use libimagerror::trace::*;
use error::StoreErrorKind as SEK;
use error::StoreError as SE;
use super::FileAbstraction;
use super::FileAbstractionInstance;
use super::Drain;
use super::InMemoryFileAbstraction;
use store::Entry;
pub mod mapper;
pub mod out;
use self::mapper::Mapper;
use self::out::StdoutFileAbstraction;
// Because this is not exported in super::inmemory;
type Backend = Arc<Mutex<RefCell<HashMap<PathBuf, Entry>>>>;
pub struct StdIoFileAbstraction<W: Write, M: Mapper> {
mapper: M,
mem: InMemoryFileAbstraction,
out: Rc<RefCell<W>>,
}
impl<W, M> Debug for StdIoFileAbstraction<W, M>
where M: Mapper,
W: Write
{
fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> {
write!(f, "StdIoFileAbstraction({:?}", self.mem)
}
}
pub struct StdIoFileAbstraction<W: Write, M: Mapper>(StdoutFileAbstraction<W, M>);
impl<W, M> StdIoFileAbstraction<W, M>
where M: Mapper,
@ -65,71 +55,73 @@ impl<W, M> StdIoFileAbstraction<W, M>
{
pub fn new<R: Read>(in_stream: &mut R, out_stream: Rc<RefCell<W>>, mapper: M) -> Result<StdIoFileAbstraction<W, M>, SE> {
let mem = InMemoryFileAbstraction::new();
{
let fill_res = match mem.backend().lock() {
StdoutFileAbstraction::new(out_stream, mapper)
.and_then(|out| {
let fill_res = match out.backend().lock() {
Err(_) => Err(SEK::LockError.into_error()),
Ok(mut mtx) => mapper.read_to_fs(in_stream, mtx.get_mut())
Ok(mut mtx) => out.mapper().read_to_fs(in_stream, mtx.get_mut())
};
let _ = try!(fill_res);
}
Ok(StdIoFileAbstraction {
mapper: mapper,
mem: mem,
out: out_stream,
Ok(StdIoFileAbstraction(out))
})
}
pub fn backend(&self) -> &Backend {
self.mem.backend()
self.0.backend()
}
}
impl<W, M> Drop for StdIoFileAbstraction<W, M>
impl<W, M> Debug for StdIoFileAbstraction<W, M>
where M: Mapper,
W: Write
{
fn drop(&mut self) {
use std::ops::DerefMut;
let fill_res = match self.mem.backend().lock() {
Err(_) => Err(SEK::LockError.into_error()),
Ok(mut mtx) => {
self.mapper.fs_to_write(mtx.get_mut(), self.out.borrow_mut().deref_mut())
},
};
// We can do nothing but end this here with a trace.
// As this drop gets called when imag almost exits, there is no point in exit()ing here
// again.
let _ = fill_res.map_err_trace();
fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> {
write!(f, "StdIoFileAbstraction({:?}", self.0)
}
}
impl<W, M> Deref for StdIoFileAbstraction<W, M>
where M: Mapper,
W: Write
{
type Target = StdoutFileAbstraction<W, M>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
// basically #[derive(FileAbstraction)]
impl<W: Write, M: Mapper> FileAbstraction for StdIoFileAbstraction<W, M> {
fn remove_file(&self, path: &PathBuf) -> Result<(), SE> {
self.mem.remove_file(path)
self.0.remove_file(path)
}
fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> {
self.mem.copy(from, to)
self.0.copy(from, to)
}
fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> {
self.mem.rename(from, to)
self.0.rename(from, to)
}
fn create_dir_all(&self, pb: &PathBuf) -> Result<(), SE> {
self.mem.create_dir_all(pb)
self.0.create_dir_all(pb)
}
fn new_instance(&self, p: PathBuf) -> Box<FileAbstractionInstance> {
self.mem.new_instance(p)
self.0.new_instance(p)
}
fn drain(&self) -> Result<Drain, SE> {
self.0.drain()
}
fn fill(&mut self, d: Drain) -> Result<(), SE> {
self.0.fill(d)
}
}

View file

@ -0,0 +1,157 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
//! A StdIoFileAbstraction which does not read from stdin.
use std::rc::Rc;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Error as FmtError;
use std::fmt::Formatter;
use std::io::Write;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use std::ops::Deref;
use libimagerror::into::IntoError;
use libimagerror::trace::*;
use error::StoreErrorKind as SEK;
use error::StoreError as SE;
use super::FileAbstraction;
use super::FileAbstractionInstance;
use super::Drain;
use super::InMemoryFileAbstraction;
use store::Entry;
use super::mapper::Mapper;
// Because this is not exported in super::inmemory;
type Backend = Arc<Mutex<RefCell<HashMap<PathBuf, Entry>>>>;
pub struct StdoutFileAbstraction<W: Write, M: Mapper> {
mapper: M,
mem: InMemoryFileAbstraction,
out: Rc<RefCell<W>>,
}
impl<W, M> StdoutFileAbstraction<W, M>
where M: Mapper,
W: Write
{
pub fn new(out_stream: Rc<RefCell<W>>, mapper: M) -> Result<StdoutFileAbstraction<W, M>, SE> {
Ok(StdoutFileAbstraction {
mapper: mapper,
mem: InMemoryFileAbstraction::new(),
out: out_stream,
})
}
pub fn backend(&self) -> &Backend {
self.mem.backend()
}
fn backend_cloned(&self) -> Result<HashMap<PathBuf, Entry>, SE> {
self.mem
.backend()
.lock()
.map_err(|_| SEK::LockError.into_error())
.map(|mtx| mtx.deref().borrow().clone())
}
pub fn mapper(&self) -> &M {
&self.mapper
}
}
impl<W, M> Debug for StdoutFileAbstraction<W, M>
where M: Mapper,
W: Write
{
fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> {
write!(f, "StdoutFileAbstraction({:?}", self.mem)
}
}
impl<W, M> Drop for StdoutFileAbstraction<W, M>
where M: Mapper,
W: Write
{
fn drop(&mut self) {
use std::ops::DerefMut;
let fill_res = match self.mem.backend().lock() {
Err(_) => Err(SEK::LockError.into_error()),
Ok(mut mtx) => {
self.mapper.fs_to_write(mtx.get_mut(), self.out.borrow_mut().deref_mut())
},
};
// We can do nothing but end this here with a trace.
// As this drop gets called when imag almost exits, there is no point in exit()ing here
// again.
let _ = fill_res.map_err_trace();
}
}
impl<W: Write, M: Mapper> FileAbstraction for StdoutFileAbstraction<W, M> {
fn remove_file(&self, path: &PathBuf) -> Result<(), SE> {
self.mem.remove_file(path)
}
fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> {
self.mem.copy(from, to)
}
fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> {
self.mem.rename(from, to)
}
fn create_dir_all(&self, pb: &PathBuf) -> Result<(), SE> {
self.mem.create_dir_all(pb)
}
fn new_instance(&self, p: PathBuf) -> Box<FileAbstractionInstance> {
self.mem.new_instance(p)
}
fn drain(&self) -> Result<Drain, SE> {
self.backend_cloned().map(Drain::new)
}
fn fill(&mut self, mut d: Drain) -> Result<(), SE> {
debug!("Draining into : {:?}", self);
let mut mtx = try!(self.backend().lock().map_err(|_| SEK::IoError.into_error()));
let mut backend = mtx.get_mut();
for (path, element) in d.iter() {
debug!("Drain into {:?}: {:?}", self, path);
backend.insert(path, element);
}
Ok(())
}
}

View file

@ -295,6 +295,33 @@ impl Store {
Ok(store)
}
/// Reset the backend of the store during runtime
///
/// # Warning
///
/// This is dangerous!
/// You should not be able to do that in application code, only the libimagrt should be used to
/// do this via safe and careful wrapper functions!
///
/// If you are able to do this without using `libimagrt`, please file an issue report.
///
/// # Purpose
///
/// With the I/O backend of the store, the store is able to pipe itself out via (for example)
/// JSON. But because we need a functionality where we load contents from the filesystem and
/// then pipe it to stdout, we need to be able to replace the backend during runtime.
///
/// This also applies the other way round: If we get the store from stdin and have to persist it
/// to stdout, we need to be able to replace the in-memory backend with the real filesystem
/// backend.
///
pub fn reset_backend(&mut self, mut backend: Box<FileAbstraction>) -> Result<()> {
self.backend
.drain()
.and_then(|drain| backend.fill(drain))
.map(|_| self.backend = backend)
}
/// Get the store configuration
pub fn config(&self) -> Option<&Value> {
self.configuration.as_ref()
@ -1544,5 +1571,149 @@ mod store_tests {
}
}
#[test]
fn test_swap_backend_during_runtime() {
use file_abstraction::InMemoryFileAbstraction;
let mut store = {
let backend = InMemoryFileAbstraction::new();
let backend = Box::new(backend);
Store::new_with_backend(PathBuf::from("/"), None, backend).unwrap()
};
for n in 1..100 {
let s = format!("test-{}", n);
let entry = store.create(PathBuf::from(s.clone())).unwrap();
assert!(entry.verify().is_ok());
let loc = entry.get_location().clone().into_pathbuf().unwrap();
assert!(loc.starts_with("/"));
assert!(loc.ends_with(s));
}
{
let other_backend = InMemoryFileAbstraction::new();
let other_backend = Box::new(other_backend);
assert!(store.reset_backend(other_backend).is_ok())
}
for n in 1..100 {
let s = format!("test-{}", n);
let entry = store.get(PathBuf::from(s.clone()));
assert!(entry.is_ok());
let entry = entry.unwrap();
assert!(entry.is_some());
let entry = entry.unwrap();
assert!(entry.verify().is_ok());
let loc = entry.get_location().clone().into_pathbuf().unwrap();
assert!(loc.starts_with("/"));
assert!(loc.ends_with(s));
}
}
#[test]
fn test_swap_backend_during_runtime_with_io() {
use std::io::Cursor;
use std::rc::Rc;
use std::cell::RefCell;
use serde_json::Value;
use file_abstraction::stdio::out::StdoutFileAbstraction;
use file_abstraction::stdio::mapper::json::JsonMapper;
// The output we later read from and check whether there is an entry
let output = Rc::new(RefCell::new(vec![]));
{
let mut store = {
use file_abstraction::stdio::StdIoFileAbstraction;
use file_abstraction::stdio::mapper::json::JsonMapper;
// Lets have an empty store as input
let mut input = Cursor::new(r#"
{ "version": "0.3.0",
"store": {
"example": {
"header": {
"imag": {
"version": "0.3.0"
}
},
"content": "foobar"
}
}
}
"#);
let output = Rc::new(RefCell::new(::std::io::sink()));
let mapper = JsonMapper::new();
let backend = StdIoFileAbstraction::new(&mut input, output, mapper).unwrap();
let backend = Box::new(backend);
Store::new_with_backend(PathBuf::from("/"), None, backend).unwrap()
};
// Replacing the backend
{
let mapper = JsonMapper::new();
let backend = StdoutFileAbstraction::new(output.clone(), mapper);
let _ = assert!(backend.is_ok(), format!("Should be ok: {:?}", backend));
let backend = backend.unwrap();
let backend = Box::new(backend);
assert!(store.reset_backend(backend).is_ok());
}
}
let vec = Rc::try_unwrap(output).unwrap().into_inner();
let errstr = format!("Not UTF8: '{:?}'", vec);
let string = String::from_utf8(vec);
assert!(string.is_ok(), errstr);
let string = string.unwrap();
assert!(!string.is_empty(), format!("Expected not to be empty: '{}'", string));
let json : ::serde_json::Value = ::serde_json::from_str(&string).unwrap();
match json {
Value::Object(ref map) => {
assert!(map.get("version").is_some(), format!("No 'version' in JSON"));
match map.get("version").unwrap() {
&Value::String(ref s) => assert_eq!("0.3.0", s),
_ => panic!("Wrong type in JSON at 'version'"),
}
assert!(map.get("store").is_some(), format!("No 'store' in JSON"));
match map.get("store").unwrap() {
&Value::Object(ref objs) => {
let s = String::from("example");
assert!(objs.get(&s).is_some(), format!("No entry: '{}' in \n{:?}", s, objs));
match objs.get(&s).unwrap() {
&Value::Object(ref entry) => {
match entry.get("header").unwrap() {
&Value::Object(_) => assert!(true),
_ => panic!("Wrong type in JSON at 'store.'{}'.header'", s),
}
match entry.get("content").unwrap() {
&Value::String(_) => assert!(true),
_ => panic!("Wrong type in JSON at 'store.'{}'.content'", s),
}
},
_ => panic!("Wrong type in JSON at 'store.'{}''", s),
}
},
_ => panic!("Wrong type in JSON at 'store'"),
}
},
_ => panic!("Wrong type in JSON at top level"),
}
}
}