From f4556f39839aa19e687767b8ab7e17cb92c4ba63 Mon Sep 17 00:00:00 2001
From: Matthias Beyer <mail@beyermatthias.de>
Date: Mon, 30 Apr 2018 15:42:20 +0200
Subject: [PATCH] Rewrite backend to not collect on pathes_recursively()

---
 lib/core/libimagstore/src/file_abstraction/fs.rs  | 15 +++++----------
 .../libimagstore/src/file_abstraction/inmemory.rs |  6 +++---
 .../libimagstore/src/file_abstraction/iter.rs     |  8 +++++---
 3 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/lib/core/libimagstore/src/file_abstraction/fs.rs b/lib/core/libimagstore/src/file_abstraction/fs.rs
index a41a2239..8ee7ee9d 100644
--- a/lib/core/libimagstore/src/file_abstraction/fs.rs
+++ b/lib/core/libimagstore/src/file_abstraction/fs.rs
@@ -168,20 +168,15 @@ impl FileAbstraction for FSFileAbstraction {
     fn pathes_recursively(&self, basepath: PathBuf) -> Result<PathIterator, SE> {
         use walkdir::WalkDir;
 
-        let i : Result<Vec<PathBuf>, SE> = WalkDir::new(basepath)
+        let i = WalkDir::new(basepath)
             .min_depth(1)
+            .max_open(100)
             .into_iter()
             .map(|r| {
-                r.map(|e| PathBuf::from(e.path()))
-                    .chain_err(|| SE::from_kind(SEK::FileError))
-            })
-            .fold(Ok(vec![]), |acc, e| {
-                acc.and_then(move |mut a| {
-                    a.push(e?);
-                    Ok(a)
-                })
+                r.map(|e| PathBuf::from(e.path())).chain_err(|| SE::from_kind(SEK::FileError))
             });
-        Ok(PathIterator::new(Box::new(i?.into_iter())))
+
+        Ok(PathIterator::new(Box::new(i)))
     }
 }
 
diff --git a/lib/core/libimagstore/src/file_abstraction/inmemory.rs b/lib/core/libimagstore/src/file_abstraction/inmemory.rs
index efe0c4f4..495286fd 100644
--- a/lib/core/libimagstore/src/file_abstraction/inmemory.rs
+++ b/lib/core/libimagstore/src/file_abstraction/inmemory.rs
@@ -183,15 +183,15 @@ impl FileAbstraction for InMemoryFileAbstraction {
 
     fn pathes_recursively(&self, _basepath: PathBuf) -> Result<PathIterator, SE> {
         debug!("Getting all pathes");
-        let keys : Vec<PathBuf> = self
+        let keys : Vec<Result<PathBuf, SE>> = self
             .backend()
             .lock()
             .map_err(|_| SE::from_kind(SEK::FileError))?
             .get_mut()
             .keys()
             .map(PathBuf::from)
-            .collect();
-        // we collect here as this happens only in tests and in memory anyways, so no problem
+            .map(Ok)
+            .collect(); // we have to collect() because of the lock() above.
 
         Ok(PathIterator::new(Box::new(keys.into_iter())))
     }
diff --git a/lib/core/libimagstore/src/file_abstraction/iter.rs b/lib/core/libimagstore/src/file_abstraction/iter.rs
index ca2862b3..c0bd8b5f 100644
--- a/lib/core/libimagstore/src/file_abstraction/iter.rs
+++ b/lib/core/libimagstore/src/file_abstraction/iter.rs
@@ -19,19 +19,21 @@
 
 use std::path::PathBuf;
 
+use error::Result;
+
 /// A wrapper for an iterator over `PathBuf`s
-pub struct PathIterator(Box<Iterator<Item = PathBuf>>);
+pub struct PathIterator(Box<Iterator<Item = Result<PathBuf>>>);
 
 impl PathIterator {
 
-    pub fn new(iter: Box<Iterator<Item = PathBuf>>) -> PathIterator {
+    pub fn new(iter: Box<Iterator<Item = Result<PathBuf>>>) -> PathIterator {
         PathIterator(iter)
     }
 
 }
 
 impl Iterator for PathIterator {
-    type Item = PathBuf;
+    type Item = Result<PathBuf>;
 
     fn next(&mut self) -> Option<Self::Item> {
         self.0.next()