From 563c76c375eb704a16e50082d134f8b1bb243a7a Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Tue, 24 Apr 2018 22:08:31 +0200 Subject: [PATCH] Remove "stdio" file abstraction from store implementation --- doc/src/02000-store.md | 77 ------ .../libimagstore/src/file_abstraction/mod.rs | 1 - .../src/file_abstraction/stdio/mapper/json.rs | 250 ------------------ .../src/file_abstraction/stdio/mapper/mod.rs | 32 --- .../src/file_abstraction/stdio/mod.rs | 137 ---------- .../src/file_abstraction/stdio/out.rs | 169 ------------ 6 files changed, 666 deletions(-) delete mode 100644 lib/core/libimagstore/src/file_abstraction/stdio/mapper/json.rs delete mode 100644 lib/core/libimagstore/src/file_abstraction/stdio/mapper/mod.rs delete mode 100644 lib/core/libimagstore/src/file_abstraction/stdio/mod.rs delete mode 100644 lib/core/libimagstore/src/file_abstraction/stdio/out.rs diff --git a/doc/src/02000-store.md b/doc/src/02000-store.md index acb2c6d8..7a0b6ac4 100644 --- a/doc/src/02000-store.md +++ b/doc/src/02000-store.md @@ -178,80 +178,3 @@ The `InMemoryFileAbstractionInstance` implementation is corrosponding to the `InMemoryFileAbstraction` implementation - for the in-memory "filesystem". -## The StdIo backend {#sec:thestore:backends:stdio} - -Sidenote: The name is "StdIo" because its main purpose is Stdin/Stdio, but it -is abstracted over Read/Write actually, so it is also possible to use this -backend in other ways, too. - -### Why? {#sec:thestore:backends:stdio:why} - -This is a backend for the imag store which is created -from stdin, by piping contents into the store (via JSON or TOML) and piping the -store contents (as JSON or TOML) to stdout when the backend is destructed. - -This is one of some components which make command-chaining in imag possible. -With this, the application does not have to know whether the store actually -lives on the filesystem or just "in memory". - -### Mappers {#sec:thestore:backends:stdio:mappers} - -The backend contains a "Mapper" which defines how the contents get mapped into -the in-memory store representation: A JSON implementation or a TOML -implementation are possible. - -The following section assumes a JSON mapper. - -The mapper reads the JSON, parses it and translates it to a `Entry`. -Then, the entry is made available to the store codebase. -To summarize what we do right now, lets have a look at the awesome ascii-art -below: - -``` - libimag* - | - v - IO Mapper Store Mapper IO -+--+---------+----------------+--------+--+ -| | | | | | - JSON -> Entry -> JSON -``` - -This is what gets translated where for one imag call with a stdio store backend. - -### The JSON Mapper {#sec:thestore:backends:stdio:json} - -The JSON mapper maps JSON which is read from a source into a HashMap which -represents the in-memory filesystem. - -The strucure is as follows: - -```json -{ - "version": "0.8.0", - "store": { - "example": { - "header": { - "imag": { - "version": "0.8.0", - }, - }, - "content": "hi there!", - }, - }, -} -``` - -### TODO {#sec:thestore:backends:todo} - -If you look at the version history of this file you will see that this -implementation has grown from something complex and probably slow to what we -have today. - -Still, there's one improvement we could make: abstract all the things away so -the `libimag*` crates handle the header without knowing whether it is JSON or -TOML. -With this, we would not even have to translate JSON to TOML anymore. -We should measure whether this would have actually any performance impact before -implementing it. - diff --git a/lib/core/libimagstore/src/file_abstraction/mod.rs b/lib/core/libimagstore/src/file_abstraction/mod.rs index 6c3807ea..b6ca5e00 100644 --- a/lib/core/libimagstore/src/file_abstraction/mod.rs +++ b/lib/core/libimagstore/src/file_abstraction/mod.rs @@ -28,7 +28,6 @@ use storeid::StoreId; mod fs; mod inmemory; mod iter; -pub mod stdio; pub use self::fs::FSFileAbstraction; pub use self::fs::FSFileAbstractionInstance; diff --git a/lib/core/libimagstore/src/file_abstraction/stdio/mapper/json.rs b/lib/core/libimagstore/src/file_abstraction/stdio/mapper/json.rs deleted file mode 100644 index 8b6052f0..00000000 --- a/lib/core/libimagstore/src/file_abstraction/stdio/mapper/json.rs +++ /dev/null @@ -1,250 +0,0 @@ -// -// imag - the personal information management suite for the commandline -// Copyright (C) 2015-2018 Matthias Beyer and contributors -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; version -// 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// - -use std::collections::HashMap; -use std::io::{Read, Write}; -use std::path::PathBuf; - -use serde_json; -use toml; - -use error::StoreErrorKind as SEK; -use error::StoreError as SE; -use error::ResultExt; -use super::Mapper; -use store::Result; -use store::Entry; -use storeid::StoreId; - -#[derive(Debug, Deserialize, Serialize)] -struct BackendEntry { - header: serde_json::Value, - content: String, -} - -impl BackendEntry { - - fn to_string(self) -> Result { - toml::to_string(&self.header) - .chain_err(|| SEK::IoError) - .map(|hdr| { - format!("---\n{header}---\n{content}", - header = hdr, - content = self.content) - }) - } - -} - -#[derive(Debug, Deserialize, Serialize)] -struct Document { - version: String, - store: HashMap, -} - -pub struct JsonMapper; - -impl JsonMapper { - - pub fn new() -> JsonMapper { - JsonMapper - } - -} - -impl Mapper for JsonMapper { - fn read_to_fs(&self, r: &mut R, hm: &mut HashMap) -> Result<()> { - let mut document = { - debug!("Reading Document"); - let mut s = String::new(); - r.read_to_string(&mut s).chain_err(|| SEK::IoError)?; - debug!("Document = {:?}", s); - debug!("Parsing Document"); - let doc : Document = serde_json::from_str(&s).chain_err(|| SEK::IoError)?; - debug!("Document = {:?}", doc); - doc - }; - - let _ = ::semver::Version::parse(&document.version) - .chain_err(|| SEK::VersionError) - .and_then(|doc_vers| { - // safe because cargo does not compile if crate version is not valid - let crate_version = ::semver::Version::parse(env!("CARGO_PKG_VERSION")).unwrap(); - - debug!("Document version vs. own version: {doc_vers} > {crate_vers}", - doc_vers = doc_vers, - crate_vers = crate_version); - - if doc_vers > crate_version { - Err(SE::from_kind(SEK::VersionError)) - } else { - Ok(()) - } - })?; - - for (key, val) in document.store.drain() { - debug!("(key, value) ({:?}, {:?})", key, val); - let res = val - .to_string() - .and_then(|vals| { - debug!("value string = {:?}", vals); - StoreId::new_baseless(key.clone()) - .and_then(|id| Entry::from_str(id, &vals)) - .map(|entry| hm.insert(key, entry)) - }) - .map(|_| ()); - - let _ = res?; - } - - Ok(()) - } - - fn fs_to_write(&self, hm: &mut HashMap, out: &mut W) -> Result<()> { - #[derive(Serialize)] - struct BackendEntry { - header: ::toml::Value, - content: String, - } - - impl BackendEntry { - fn construct_from(e: Entry) -> BackendEntry { - BackendEntry { - header: e.get_header().clone(), - content: e.get_content().clone(), - } - } - } - - #[derive(Serialize)] - struct OutDocument { - version: String, - store: HashMap, - } - - let mut store = HashMap::new(); - for (key, value) in hm.drain() { - store.insert(key, BackendEntry::construct_from(value)); - } - - let doc = OutDocument { - version: String::from(env!("CARGO_PKG_VERSION")), - store: store, - }; - - serde_json::to_string(&doc) - .chain_err(|| SEK::IoError) - .and_then(|json| out.write(&json.into_bytes()).chain_err(|| SEK::IoError)) - .and_then(|_| out.flush().chain_err(|| SEK::IoError)) - .map(|_| ()) - } -} - -#[cfg(test)] -mod test { - use std::io::Cursor; - - use super::*; - - #[test] - fn test_empty_json_to_fs() { - let json = format!(r#"{{"version":"{version}","store":{{}}}}"#, - version = env!("CARGO_PKG_VERSION")); - let mut json = Cursor::new(String::from(json).into_bytes()); - let mapper = JsonMapper::new(); - let mut hm = HashMap::new(); - - let io_res = mapper.read_to_fs(&mut json, &mut hm); - assert!(io_res.is_ok()); - assert!(hm.is_empty()); - } - - #[test] - fn test_json_to_fs() { - let json = format!(r#" - {{ "version": "{version}", - "store": {{ - "example": {{ - "header": {{ - "imag": {{ - "version": "{version}" - }} - }}, - "content": "test" - }} - }} - }} - "#, version = env!("CARGO_PKG_VERSION")); - let mut json = Cursor::new(String::from(json).into_bytes()); - let mapper = JsonMapper::new(); - let mut hm = HashMap::new(); - - let io_res = mapper.read_to_fs(&mut json, &mut hm); - assert!(io_res.is_ok()); - - assert_eq!(1, hm.len()); // we should have exactly one entry - } - - #[test] - fn test_fs_to_json() { - let mapper = JsonMapper::new(); - let mut out : Cursor> = Cursor::new(vec![]); - - let mut hm = { - let mut hm = HashMap::new(); - let content = format!(r#"--- -[imag] -version = "{}" ---- -hi there!"#, env!("CARGO_PKG_VERSION")); - - let id = PathBuf::from("example"); - let entry = Entry::from_str(id.clone(), &content).unwrap(); - hm.insert(id, entry); - hm - }; - - let io_res = mapper.fs_to_write(&mut hm, &mut out); - assert!(io_res.is_ok()); - - let example = format!(r#" - {{ - "version": "{version}", - "store": {{ - "example": {{ - "header": {{ - "imag": {{ - "version": "{version}" - }} - }}, - "content": "hi there!" - }} - }} - }} - "#, version = env!("CARGO_PKG_VERSION")); - - let example_json : ::serde_json::Value = ::serde_json::from_str(&example).unwrap(); - - let output_json = String::from_utf8(out.into_inner()).unwrap(); - let output_json : ::serde_json::Value = ::serde_json::from_str(&output_json).unwrap(); - - assert_eq!(example_json, output_json); - } -} - diff --git a/lib/core/libimagstore/src/file_abstraction/stdio/mapper/mod.rs b/lib/core/libimagstore/src/file_abstraction/stdio/mapper/mod.rs deleted file mode 100644 index 0543feb1..00000000 --- a/lib/core/libimagstore/src/file_abstraction/stdio/mapper/mod.rs +++ /dev/null @@ -1,32 +0,0 @@ -// -// imag - the personal information management suite for the commandline -// Copyright (C) 2015-2018 Matthias Beyer and contributors -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; version -// 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// - -use std::collections::HashMap; -use std::io::{Read, Write}; -use std::path::PathBuf; -use store::Result; -use store::Entry; - -pub trait Mapper { - fn read_to_fs(&self, &mut R, &mut HashMap) -> Result<()>; - fn fs_to_write(&self, &mut HashMap, &mut W) -> Result<()>; -} - -pub mod json; - diff --git a/lib/core/libimagstore/src/file_abstraction/stdio/mod.rs b/lib/core/libimagstore/src/file_abstraction/stdio/mod.rs deleted file mode 100644 index a9ea379c..00000000 --- a/lib/core/libimagstore/src/file_abstraction/stdio/mod.rs +++ /dev/null @@ -1,137 +0,0 @@ -// -// imag - the personal information management suite for the commandline -// Copyright (C) 2015-2018 Matthias Beyer and contributors -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; version -// 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// - -use std::rc::Rc; -use std::cell::RefCell; -use std::collections::HashMap; -use std::io::{Read, Write}; -use std::path::PathBuf; -use std::sync::Arc; -use std::sync::Mutex; -use std::ops::Deref; -use std::fmt::Debug; -use std::fmt::Error as FmtError; -use std::fmt::Formatter; - -use error::StoreErrorKind as SEK; -use error::StoreError as SE; -use super::FileAbstraction; -use super::FileAbstractionInstance; -use super::Drain; -use super::InMemoryFileAbstraction; -use store::Entry; -use file_abstraction::iter::PathIterator; - -pub mod mapper; -pub mod out; -use self::mapper::Mapper; -use self::out::StdoutFileAbstraction; - -// Because this is not exported in super::inmemory; -type Backend = Arc>>>; - -pub struct StdIoFileAbstraction(StdoutFileAbstraction); - -impl StdIoFileAbstraction - where M: Mapper, - W: Write -{ - - pub fn new(in_stream: &mut R, out_stream: Rc>, mapper: M) -> Result, SE> { - StdoutFileAbstraction::new(out_stream, mapper) - .and_then(|out| { - let _ = out.backend() - .lock() - .map_err(|_| SE::from_kind(SEK::LockError)) - .map(|mut mtx| out.mapper().read_to_fs(in_stream, mtx.get_mut()))?; - - Ok(StdIoFileAbstraction(out)) - }) - } - - pub fn backend(&self) -> &Backend { - self.0.backend() - } - -} - -impl Debug for StdIoFileAbstraction - where M: Mapper, - W: Write -{ - fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> { - write!(f, "StdIoFileAbstraction({:?}", self.0) - } -} - -impl Deref for StdIoFileAbstraction - where M: Mapper, - W: Write -{ - type Target = StdoutFileAbstraction; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -// basically #[derive(FileAbstraction)] -impl FileAbstraction for StdIoFileAbstraction { - - fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { - self.0.remove_file(path) - } - - fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - self.0.copy(from, to) - } - - fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - self.0.rename(from, to) - } - - fn create_dir_all(&self, pb: &PathBuf) -> Result<(), SE> { - self.0.create_dir_all(pb) - } - - fn new_instance(&self, p: PathBuf) -> Box { - self.0.new_instance(p) - } - - fn exists(&self, p: &PathBuf) -> Result { - self.0.exists(p) - } - - fn is_file(&self, p: &PathBuf) -> Result { - self.0.is_file(p) - } - - fn drain(&self) -> Result { - self.0.drain() - } - - fn fill(&mut self, d: Drain) -> Result<(), SE> { - self.0.fill(d) - } - - fn pathes_recursively(&self, basepath: PathBuf) -> Result { - self.0.pathes_recursively(basepath) - } -} - diff --git a/lib/core/libimagstore/src/file_abstraction/stdio/out.rs b/lib/core/libimagstore/src/file_abstraction/stdio/out.rs deleted file mode 100644 index e534fa9c..00000000 --- a/lib/core/libimagstore/src/file_abstraction/stdio/out.rs +++ /dev/null @@ -1,169 +0,0 @@ -// -// imag - the personal information management suite for the commandline -// Copyright (C) 2015-2018 Matthias Beyer and contributors -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; version -// 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// - -//! A StdIoFileAbstraction which does not read from stdin. - -use std::rc::Rc; -use std::cell::RefCell; -use std::collections::HashMap; -use std::fmt::Debug; -use std::fmt::Error as FmtError; -use std::fmt::Formatter; -use std::io::Write; -use std::path::PathBuf; -use std::sync::Arc; -use std::sync::Mutex; -use std::ops::Deref; - -use libimagerror::trace::*; - -use error::StoreErrorKind as SEK; -use error::StoreError as SE; -use super::FileAbstraction; -use super::FileAbstractionInstance; -use super::Drain; -use super::InMemoryFileAbstraction; -use store::Entry; -use file_abstraction::iter::PathIterator; - -use super::mapper::Mapper; - -// Because this is not exported in super::inmemory; -type Backend = Arc>>>; - -pub struct StdoutFileAbstraction { - mapper: M, - mem: InMemoryFileAbstraction, - out: Rc>, -} - -impl StdoutFileAbstraction - where M: Mapper, - W: Write -{ - - pub fn new(out_stream: Rc>, mapper: M) -> Result, SE> { - Ok(StdoutFileAbstraction { - mapper: mapper, - mem: InMemoryFileAbstraction::new(), - out: out_stream, - }) - } - - pub fn backend(&self) -> &Backend { - self.mem.backend() - } - - fn backend_cloned(&self) -> Result, SE> { - self.mem - .backend() - .lock() - .map_err(|_| SE::from_kind(SEK::LockError)) - .map(|mtx| mtx.deref().borrow().clone()) - } - - pub fn mapper(&self) -> &M { - &self.mapper - } - -} - -impl Debug for StdoutFileAbstraction - where M: Mapper, - W: Write -{ - fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> { - write!(f, "StdoutFileAbstraction({:?}", self.mem) - } -} - -impl Drop for StdoutFileAbstraction - where M: Mapper, - W: Write -{ - fn drop(&mut self) { - use std::ops::DerefMut; - - let fill_res = match self.mem.backend().lock() { - Err(_) => Err(SE::from_kind(SEK::LockError)), - Ok(mut mtx) => { - self.mapper.fs_to_write(mtx.get_mut(), self.out.borrow_mut().deref_mut()) - }, - }; - - // We can do nothing but end this here with a trace. - // As this drop gets called when imag almost exits, there is no point in exit()ing here - // again. - let _ = fill_res.map_err_trace(); - } -} - -impl FileAbstraction for StdoutFileAbstraction { - - fn remove_file(&self, path: &PathBuf) -> Result<(), SE> { - self.mem.remove_file(path) - } - - fn copy(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - self.mem.copy(from, to) - } - - fn rename(&self, from: &PathBuf, to: &PathBuf) -> Result<(), SE> { - self.mem.rename(from, to) - } - - fn create_dir_all(&self, pb: &PathBuf) -> Result<(), SE> { - self.mem.create_dir_all(pb) - } - - fn new_instance(&self, p: PathBuf) -> Box { - self.mem.new_instance(p) - } - - fn exists(&self, p: &PathBuf) -> Result { - self.mem.exists(p) - } - - fn is_file(&self, p: &PathBuf) -> Result { - self.mem.is_file(p) - } - - fn drain(&self) -> Result { - self.backend_cloned().map(Drain::new) - } - - fn fill(&mut self, mut d: Drain) -> Result<(), SE> { - debug!("Draining into : {:?}", self); - let mut mtx = self.backend().lock().map_err(|_| SE::from_kind(SEK::IoError))?; - let backend = mtx.get_mut(); - - for (path, element) in d.iter() { - debug!("Drain into {:?}: {:?}", self, path); - backend.insert(path, element); - } - Ok(()) - } - - fn pathes_recursively(&self, basepath: PathBuf) -> Result { - self.mem.pathes_recursively(basepath) - } - -} - -