Merge pull request #1416 from matthiasbeyer/libimagstore/remove-retrieve-for-module

Remove: Store::retrieve_for_module

bin/domain/imag-mail/src/main.rs

@@ -141,8 +141,9 @@ fn list(rt: &Runtime) {
     }
 
     let _ = rt.store()
-        .retrieve_for_module("ref")
+        .entries()
         .map_err_trace_exit_unwrap(1)
+        .filter(|id| id.is_in_collection(&["mail"]))
         .filter_map(|id| {
             rt.store()
                 .get(id)

bin/domain/imag-timetrack/src/cont.rs

@@ -37,8 +37,6 @@ pub fn cont(rt: &Runtime) -> i32 {
         .get_timetrackings()
         .map_err_trace_exit_unwrap(1)
         .trace_unwrap()
-        .filter(Option::is_some)
-        .map(Option::unwrap)
         .filter(|e| has_end_time.filter(&e))
         .group_by(|elem| match elem.get_end_datetime() { // Now group them by the end time
             Ok(Some(dt)) => dt,

bin/domain/imag-timetrack/src/day.rs

@@ -90,8 +90,6 @@ pub fn day(rt: &Runtime) -> i32 {
         .get_timetrackings()
         .map_err_trace_exit_unwrap(1)
         .trace_unwrap()
-        .filter(Option::is_some)
-        .map(Option::unwrap)
         .filter(|e| filter.filter(e))
         .map(|e| -> Result<_, TTE> {
             debug!("Processing {:?}", e.get_location());

bin/domain/imag-timetrack/src/list.rs

@@ -121,8 +121,6 @@ pub fn list_impl(rt: &Runtime,
         .get_timetrackings()
         .map_err_trace_exit_unwrap(1)
         .trace_unwrap()
-        .filter(Option::is_some)
-        .map(Option::unwrap)
         .filter(|e| filter.filter(e))
         .fold(Ok(table), |acc: Result<_>, e| {
             acc.and_then(|mut tab: Table| {

bin/domain/imag-timetrack/src/month.rs

@@ -105,8 +105,6 @@ pub fn month(rt: &Runtime) -> i32 {
         .get_timetrackings()
         .map_err_trace_exit_unwrap(1)
         .trace_unwrap()
-        .filter(Option::is_some)
-        .map(Option::unwrap)
         .filter(|e| filter.filter(e))
         .map(|e| -> Result<_, TTE> {
             debug!("Processing {:?}", e.get_location());

bin/domain/imag-timetrack/src/stop.rs

@@ -60,8 +60,6 @@ pub fn stop(rt: &Runtime) -> i32 {
                 .get_timetrackings()
                 .map_err_trace_exit_unwrap(1)
                 .trace_unwrap()
-                .filter(Option::is_some)
-                .map(Option::unwrap)
                 .filter_map(|tracking| {
                     let is_none = tracking
                         .get_end_datetime()
@@ -87,8 +85,6 @@ pub fn stop(rt: &Runtime) -> i32 {
         .map_warn_err_str("Getting timetrackings failed")
         .map_err_trace_exit_unwrap(1)
         .trace_unwrap()
-        .filter(Option::is_some)
-        .map(Option::unwrap)
 
         // Filter all timetrackings for the ones that are not yet ended.
         .filter(|e| filter.filter(e))

bin/domain/imag-timetrack/src/week.rs

@@ -103,8 +103,6 @@ pub fn week(rt: &Runtime) -> i32 {
         .get_timetrackings()
         .map_err_trace_exit_unwrap(1)
         .trace_unwrap()
-        .filter(Option::is_some)
-        .map(Option::unwrap)
         .filter(|e| filter.filter(e))
         .map(|e| -> Result<_, TTE> {
             debug!("Processing {:?}", e.get_location());

bin/domain/imag-timetrack/src/year.rs

@@ -103,8 +103,6 @@ pub fn year(rt: &Runtime) -> i32 {
         .get_timetrackings()
         .map_err_trace_exit_unwrap(1)
         .trace_unwrap()
-        .filter(Option::is_some)
-        .map(Option::unwrap)
         .filter(|e| filter.filter(e))
         .map(|e| -> Result<_, TTE> {
             debug!("Processing {:?}", e.get_location());

lib/core/libimagstore/src/store.rs

@@ -32,7 +32,6 @@ use std::fmt::Debug;
 use std::fmt::Error as FMTError;
 
 use toml::Value;
-use glob::glob;
 use walkdir::WalkDir;
 use walkdir::Iter as WalkDirIter;
 use toml_query::read::TomlValueReadExt;
@@ -40,7 +39,7 @@ use toml_query::read::TomlValueReadTypeExt;
 
 use error::{StoreError as SE, StoreErrorKind as SEK};
 use error::ResultExt;
-use storeid::{IntoStoreId, StoreId, StoreIdIterator, StoreIdIteratorWithStore};
+use storeid::{IntoStoreId, StoreId, StoreIdIteratorWithStore};
 use file_abstraction::FileAbstractionInstance;
 
 // We re-export the following things so tests can use them
@@ -51,8 +50,6 @@ pub use file_abstraction::InMemoryFileAbstraction;
 use libimagerror::trace::trace_error;
 use libimagutil::debug_result::*;
 
-use self::glob_store_iter::*;
-
 /// The Result Type returned by any interaction with the store that could fail
 pub type Result<T> = RResult<T, SE>;
 
@@ -425,39 +422,6 @@ impl Store {
         self.retrieve(id).map(Some).chain_err(|| SEK::GetCallError)
     }
 
-    /// Iterate over all StoreIds for one module name
-    ///
-    /// # Returns
-    ///
-    /// On success: An iterator over all entries in the module
-    ///
-    /// On failure:
-    ///  - (if the glob or one of the intermediate fail)
-    ///  - RetrieveForModuleCallError(GlobError(EncodingError())) if the path string cannot be
-    ///    encoded
-    ///  - GRetrieveForModuleCallError(GlobError(lobError())) if the glob() failed.
-    ///
-    pub fn retrieve_for_module(&self, mod_name: &str) -> Result<StoreIdIterator> {
-        let mut path = self.path().clone();
-        path.push(mod_name);
-
-        debug!("Retrieving for module: '{}'", mod_name);
-
-        path.to_str()
-            .ok_or(SE::from_kind(SEK::EncodingError))
-            .and_then(|path| {
-                let path = [ path, "/**/*" ].join("");
-                debug!("glob()ing with '{}'", path);
-                glob(&path[..]).map_err(From::from)
-            })
-            .and_then(|paths| {
-                GlobStoreIdIterator::new(paths, self.path().clone())
-                    .collect::<Result<Vec<_>>>()
-            })
-            .map(|v| StoreIdIterator::new(Box::new(v.into_iter())))
-            .chain_err(|| SEK::RetrieveForModuleCallError)
-    }
-
     /// Walk the store tree for the module
     ///
     /// The difference between a `Walk` and a `StoreIdIterator` is that with a `Walk`, one can find
@@ -988,69 +952,6 @@ impl PartialEq for Entry {
 
 }
 
-mod glob_store_iter {
-    use std::fmt::{Debug, Formatter};
-    use std::fmt::Error as FmtError;
-    use std::path::PathBuf;
-    use std::result::Result as RResult;
-    use glob::Paths;
-    use storeid::StoreId;
-    use error::Result;
-
-    use error::StoreErrorKind as SEK;
-    use error::ResultExt;
-
-    /// An iterator which is constructed from a `glob()` and returns valid `StoreId` objects or
-    /// errors
-    pub struct GlobStoreIdIterator {
-        store_path: PathBuf,
-        paths: Paths,
-    }
-
-    impl Debug for GlobStoreIdIterator {
-
-        fn fmt(&self, fmt: &mut Formatter) -> RResult<(), FmtError> {
-            write!(fmt, "GlobStoreIdIterator")
-        }
-
-    }
-
-    impl GlobStoreIdIterator {
-
-        pub fn new(paths: Paths, store_path: PathBuf) -> GlobStoreIdIterator {
-            debug!("Create a GlobStoreIdIterator(store_path = {:?}, /* ... */)", store_path);
-
-            GlobStoreIdIterator {
-                store_path: store_path,
-                paths: paths,
-            }
-        }
-
-    }
-
-    impl Iterator for GlobStoreIdIterator {
-        type Item = Result<StoreId>;
-
-        fn next(&mut self) -> Option<Self::Item> {
-            while let Some(o) = self.paths.next() {
-                debug!("GlobStoreIdIterator::next() => {:?}", o);
-                match o.chain_err(|| SEK::StoreIdHandlingError) {
-                    Err(e)   => return Some(Err(e)),
-                    Ok(path) => if path.exists() && path.is_file() {
-                        return Some(StoreId::from_full_path(&self.store_path, path));
-                    /* } else { */
-                        /* continue */
-                    }
-                }
-            }
-
-            None
-        }
-
-    }
-
-}
-
 /// Extension trait for top-level toml::Value::Table, will only yield correct results on the
 /// top-level Value::Table, but not on intermediate tables.
 pub trait Header {
@@ -1511,55 +1412,6 @@ mod store_tests {
         }
     }
 
-    // Disabled because we cannot test this by now, as we rely on glob() in
-    // Store::retieve_for_module(), which accesses the filesystem and tests run in-memory, so there
-    // are no files on the filesystem in this test after Store::create().
-    //
-    // #[test]
-    // fn test_retrieve_for_module() {
-    //     let pathes = vec![
-    //         "foo/1", "foo/2", "foo/3", "foo/4", "foo/5",
-    //         "bar/1", "bar/2", "bar/3", "bar/4", "bar/5",
-    //         "bla/1", "bla/2", "bla/3", "bla/4", "bla/5",
-    //         "boo/1", "boo/2", "boo/3", "boo/4", "boo/5",
-    //         "glu/1", "glu/2", "glu/3", "glu/4", "glu/5",
-    //     ];
-
-    //     fn test(store: &Store, modulename: &str) {
-    //         use std::path::Component;
-    //         use storeid::StoreId;
-
-    //         let retrieved = store.retrieve_for_module(modulename);
-    //         assert!(retrieved.is_ok());
-    //         let v : Vec<StoreId> = retrieved.unwrap().collect();
-    //         println!("v = {:?}", v);
-    //         assert!(v.len() == 5);
-
-    //         let retrieved = store.retrieve_for_module(modulename);
-    //         assert!(retrieved.is_ok());
-
-    //         assert!(retrieved.unwrap().all(|e| {
-    //             let first = e.components().next();
-    //             assert!(first.is_some());
-    //             match first.unwrap() {
-    //                 Component::Normal(s) => s == modulename,
-    //                 _                    => false,
-    //             }
-    //         }))
-    //     }
-
-    //     let store = get_store();
-    //     for path in pathes {
-    //         assert!(store.create(PathBuf::from(path)).is_ok());
-    //     }
-
-    //     test(&store, "foo");
-    //     test(&store, "bar");
-    //     test(&store, "bla");
-    //     test(&store, "boo");
-    //     test(&store, "glu");
-    // }
-
     #[test]
     fn test_store_move_moves_in_hm() {
         use storeid::StoreId;

lib/domain/libimagdiary/src/diary.rs

@@ -134,9 +134,9 @@ impl Diary for Store {
 
     /// Get all diary names
     fn diary_names(&self) -> Result<DiaryNameIterator> {
-        self.retrieve_for_module("diary")
+        self.entries()
-            .chain_err(|| DEK::StoreReadError)
+            .map(|it| DiaryNameIterator::new(it.without_store()))
-            .map(DiaryNameIterator::new)
+            .map_err(::error::DiaryError::from)
     }
 
 }

lib/domain/libimagdiary/src/error.rs

@@ -23,6 +23,7 @@ error_chain! {
     }
 
     links {
+        StoreError(::libimagstore::error::StoreError, ::libimagstore::error::StoreErrorKind);
         EntryUtilError(::libimagentryutil::error::EntryUtilError, ::libimagentryutil::error::EntryUtilErrorKind);
     }
 

lib/domain/libimagdiary/src/iter.rs

@@ -140,18 +140,21 @@ impl Iterator for DiaryNameIterator {
     type Item = Result<String>;
 
     fn next(&mut self) -> Option<Self::Item> {
-        self.0
+        while let Some(next) = self.0.next() {
-            .next()
+            if next.is_in_collection(&["diary"]) {
-            .map(|s| {
+                return Some(next
-                s.to_str()
+                    .to_str()
                     .chain_err(|| DEK::DiaryNameFindingError)
                     .and_then(|s| {
                         s.split("diary/")
                             .nth(1)
                             .and_then(|n| n.split("/").nth(0).map(String::from))
                             .ok_or(DE::from_kind(DEK::DiaryNameFindingError))
-                    })
+                    }))
-            })
+            }
+        }
+
+        None
     }
 
 }

lib/domain/libimagnotes/src/notestore.rs

@@ -84,7 +84,8 @@ impl<'a> NoteStore<'a> for Store {
     }
 
     fn all_notes(&'a self) -> Result<NoteIterator> {
-        self.retrieve_for_module("notes")
+        self.entries()
+            .map(|it| it.without_store())
             .map(NoteIterator::new)
             .map_err(NE::from)
     }

lib/domain/libimagtimetrack/src/iter/get.rs

@@ -0,0 +1,53 @@
+//
+// imag - the personal information management suite for the commandline
+// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; version
+// 2.1 of the License.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+//
+
+use libimagstore::storeid::StoreIdIteratorWithStore;
+use libimagstore::store::Store;
+use libimagstore::store::Result as StoreResult;
+use libimagstore::store::FileLockEntry;
+
+use constants::*;
+
+pub struct TimeTrackingsGetIterator<'a>(StoreIdIteratorWithStore<'a>, &'a Store);
+
+impl<'a> TimeTrackingsGetIterator<'a> {
+    pub fn new(sit: StoreIdIteratorWithStore<'a>, store: &'a Store) -> Self {
+        TimeTrackingsGetIterator(sit, store)
+    }
+}
+
+impl<'a> Iterator for TimeTrackingsGetIterator<'a> {
+    type Item = StoreResult<FileLockEntry<'a>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        while let Some(next) = self.0.next() {
+            if next.is_in_collection(&[CRATE_NAME]) {
+                return match self.1.get(next) {
+                    Ok(Some(fle)) => Some(Ok(fle)),
+                    Ok(None)      => continue,
+                    Err(e)        => Some(Err(e))
+                };
+            }
+        }
+
+        None
+    }
+
+}
+

lib/domain/libimagtimetrack/src/iter/mod.rs

@@ -19,6 +19,7 @@
 
 pub mod create;
 pub mod filter;
+pub mod get;
 pub mod setendtime;
 pub mod storeid;
 pub mod tag;

lib/domain/libimagtimetrack/src/timetrackingstore.rs

@@ -28,12 +28,11 @@ use toml_query::insert::TomlValueInsertExt;
 
 use libimagstore::store::Store;
 use libimagstore::store::FileLockEntry;
-use libimagstore::iter::get::StoreGetIterator;
-use libimagstore::iter::get::StoreIdGetIteratorExtension;
 use libimagentrydatetime::datepath::compiler::DatePathCompiler;
 
 use error::Result;
 use constants::*;
+use iter::get::TimeTrackingsGetIterator;
 
 use tag::TimeTrackingTag as TTT;
 
@@ -43,7 +42,7 @@ pub trait TimeTrackStore<'a> {
     fn create_timetracking_at(&'a self, start: &NDT, ts: &TTT)         -> Result<FileLockEntry<'a>>;
     fn create_timetracking(&'a self, start: &NDT, end: &NDT, ts: &TTT) -> Result<FileLockEntry<'a>>;
 
-    fn get_timetrackings(&'a self) -> Result<StoreGetIterator<'a>>;
+    fn get_timetrackings(&'a self) -> Result<TimeTrackingsGetIterator<'a>>;
 }
 
 fn now() -> NDT {
@@ -103,10 +102,8 @@ impl<'a> TimeTrackStore<'a> for Store {
             })
     }
 
-    fn get_timetrackings(&'a self) -> Result<StoreGetIterator<'a>> {
+    fn get_timetrackings(&'a self) -> Result<TimeTrackingsGetIterator<'a>> {
-        self.retrieve_for_module(CRATE_NAME)
+        Ok(TimeTrackingsGetIterator::new(self.entries()?, self))
-            .map_err(From::from)
-            .map(|iter| iter.into_get_iter(self))
     }
 
 }

lib/domain/libimagtodo/src/iter.rs

@@ -0,0 +1,48 @@
+//
+// imag - the personal information management suite for the commandline
+// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; version
+// 2.1 of the License.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+//
+
+use libimagstore::storeid::StoreIdIterator;
+use libimagstore::storeid::StoreId;
+
+pub struct TaskIdIterator(StoreIdIterator);
+
+impl TaskIdIterator {
+
+    pub fn new(inner: StoreIdIterator) -> Self {
+        TaskIdIterator(inner)
+    }
+
+}
+
+impl Iterator for TaskIdIterator {
+    type Item = StoreId;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            match self.0.next() {
+                None    => return None,
+                Some(n) => if n.is_in_collection(&["todo", "taskwarrior"]) {
+                    return Some(n)
+                }, // else continue
+            }
+        }
+    }
+
+}
+

lib/domain/libimagtodo/src/lib.rs

@@ -51,4 +51,5 @@ module_entry_path_mod!("todo");
 pub mod error;
 pub mod task;
 pub mod taskstore;
+pub mod iter;
 

lib/domain/libimagtodo/src/taskstore.rs

@@ -28,13 +28,14 @@ use task_hookrs::task::Task as TTask;
 use task_hookrs::import::{import_task, import_tasks};
 
 use libimagstore::store::{FileLockEntry, Store};
-use libimagstore::storeid::{IntoStoreId, StoreIdIterator};
+use libimagstore::storeid::IntoStoreId;
 use module_path::ModuleEntryPath;
 
 use error::TodoErrorKind as TEK;
 use error::TodoError as TE;
 use error::Result;
 use error::ResultExt;
+use iter::TaskIdIterator;
 
 /// Task struct containing a `FileLockEntry`
 pub trait TaskStore<'a> {
@@ -46,7 +47,7 @@ pub trait TaskStore<'a> {
     fn retrieve_task_from_string(&'a self, s: String) -> Result<FileLockEntry<'a>>;
     fn delete_tasks_by_imports<R: BufRead>(&self, r: R) -> Result<()>;
     fn delete_task_by_uuid(&self, uuid: Uuid) -> Result<()>;
-    fn all_tasks(&self) -> Result<StoreIdIterator>;
+    fn all_tasks(&self) -> Result<TaskIdIterator>;
     fn new_from_twtask(&'a self, task: TTask) -> Result<FileLockEntry<'a>>;
 }
 
@@ -167,8 +168,9 @@ impl<'a> TaskStore<'a> for Store {
             .chain_err(|| TEK::StoreError)
     }
 
-    fn all_tasks(&self) -> Result<StoreIdIterator> {
+    fn all_tasks(&self) -> Result<TaskIdIterator> {
-        self.retrieve_for_module("todo/taskwarrior")
+        self.entries()
+            .map(|i| TaskIdIterator::new(i.without_store()))
             .chain_err(|| TEK::StoreError)
     }
 

lib/entry/libimagentryannotation/src/annotation_fetcher.rs

@@ -31,9 +31,7 @@ pub trait AnnotationFetcher<'a> {
 impl<'a> AnnotationFetcher<'a> for Store {
 
     fn all_annotations(&'a self) -> Result<AnnotationIter<'a>> {
-        self.retrieve_for_module("annotations")
+        Ok(AnnotationIter::new(self.entries()?.without_store(), self))
-            .map(|iter| AnnotationIter::new(iter, self))
-            .map_err(Into::into)
     }
 
 }

lib/entry/libimagentrycategory/src/register.rs

@@ -101,9 +101,7 @@ impl CategoryRegister for Store {
 
     /// Get all category names
     fn all_category_names(&self) -> Result<CategoryNameIter> {
-        self.retrieve_for_module("category")
+        Ok(CategoryNameIter::new(self, self.entries()?.without_store()))
-            .chain_err(|| CEK::StoreReadError)
-            .map(|iter| CategoryNameIter::new(self, iter))
     }
 
     /// Get a category by its name
@@ -268,10 +266,10 @@ impl<'a> Iterator for CategoryNameIter<'a> {
         // TODO: Optimize me with lazy_static
         let query = CATEGORY_REGISTER_NAME_FIELD_PATH;
 
-        self.1
+        while let Some(sid) = self.1.next() {
-            .next()
+            if sid.is_in_collection(&["category"]) {
-            .map(|sid| {
+                let func = |store: &Store| { // hack for returning Some(Result<_, _>)
-                self.0
+                    store
                         .get(sid)?
                         .ok_or_else(|| CE::from_kind(CEK::StoreReadError))?
                         .get_header()
@@ -279,7 +277,13 @@ impl<'a> Iterator for CategoryNameIter<'a> {
                         .chain_err(|| CEK::HeaderReadError)?
                         .map(Category::from)
                         .ok_or_else(|| CE::from_kind(CEK::StoreReadError))
-            })
+                };
+
+                return Some(func(&self.0))
+            } // else continue
+        }
+
+        None
     }
 }