From eed57af0bb498c618069767e4e69fa5e36d0da75 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 30 Dec 2015 11:32:42 +0100 Subject: [PATCH 1/4] module header helpers: Add function to get NAME from header --- src/module/helpers/header/mod.rs | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/module/helpers/header/mod.rs b/src/module/helpers/header/mod.rs index 04fdc834..548dc167 100644 --- a/src/module/helpers/header/mod.rs +++ b/src/module/helpers/header/mod.rs @@ -52,5 +52,46 @@ pub mod data { } } + /** + * Get an NAME from a header, whereas the header has to have the following format: + * + * { ..., "NAME": "", ... } + * + * Does no spec verification. + */ + pub fn get_name_from_header(header: &FHD) -> Option { + match header { + &FHD::Map{keys: ref ks} => { + let mut keys : Vec = ks.clone(); + keys.iter().find(|k| { + match k.deref() { + &FHD::Key{name: ref n, value: ref v} => n == "NAME", + _ => false + } + }).and_then(|urlkey| { + match urlkey.deref().clone() { + FHD::Key{name: _, value: ref v} => { + match v.deref().clone() { + FHD::Text(s) => Some(s), + _ => { + warn!("Malformed Header Data: Expected Text, found non-Text"); + None + }, + } + } + _ => { + warn!("Malformed Header Data: Expected Text, found non-Text"); + None + }, + } + }) + }, + _ => { + warn!("Malformed Header Data: Expected Map, found non-Map"); + None + } + } + } + } From d15d50d8a27e66958aca933ff97e1311f95bf52e Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 30 Dec 2015 11:37:17 +0100 Subject: [PATCH 2/4] Refactor: Header helper code can use generic function --- src/module/helpers/header/mod.rs | 46 ++++++++++---------------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/src/module/helpers/header/mod.rs b/src/module/helpers/header/mod.rs index 548dc167..ffbbd6c0 100644 --- a/src/module/helpers/header/mod.rs +++ b/src/module/helpers/header/mod.rs @@ -19,37 +19,7 @@ pub mod data { * Does no spec verification. */ pub fn get_url_from_header(header: &FHD) -> Option { - match header { - &FHD::Map{keys: ref ks} => { - let mut keys : Vec = ks.clone(); - keys.iter().find(|k| { - match k.deref() { - &FHD::Key{name: ref n, value: ref v} => n == "URL", - _ => false - } - }).and_then(|urlkey| { - match urlkey.deref().clone() { - FHD::Key{name: _, value: ref v} => { - match v.deref().clone() { - FHD::Text(s) => Some(s), - _ => { - warn!("Malformed Header Data: Expected Text, found non-Text"); - None - }, - } - } - _ => { - warn!("Malformed Header Data: Expected Text, found non-Text"); - None - }, - } - }) - }, - _ => { - warn!("Malformed Header Data: Expected Map, found non-Map"); - None - } - } + get_named_text_from_header("URL", header) } /** @@ -60,12 +30,24 @@ pub mod data { * Does no spec verification. */ pub fn get_name_from_header(header: &FHD) -> Option { + get_named_text_from_header("NAME", header) + } + + + /** + * Get a named field from the header, which has to be of this format + * + * { ..., "": "", ... } + * + * Does no spec verification. + */ + pub fn get_named_text_from_header(name: &'static str, header: &FHD) -> Option { match header { &FHD::Map{keys: ref ks} => { let mut keys : Vec = ks.clone(); keys.iter().find(|k| { match k.deref() { - &FHD::Key{name: ref n, value: ref v} => n == "NAME", + &FHD::Key{name: ref n, value: ref v} => n == name, _ => false } }).and_then(|urlkey| { From 09aa7d9ec336a4aa37126353b55b9c249b442238 Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 30 Dec 2015 14:57:46 +0100 Subject: [PATCH 3/4] Add debug output to text-from-header fetcher function --- src/module/helpers/header/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/module/helpers/header/mod.rs b/src/module/helpers/header/mod.rs index ffbbd6c0..367f845c 100644 --- a/src/module/helpers/header/mod.rs +++ b/src/module/helpers/header/mod.rs @@ -52,11 +52,12 @@ pub mod data { } }).and_then(|urlkey| { match urlkey.deref().clone() { - FHD::Key{name: _, value: ref v} => { + FHD::Key{name: ref n, value: ref v} => { match v.deref().clone() { FHD::Text(s) => Some(s), _ => { warn!("Malformed Header Data: Expected Text, found non-Text"); + debug!(" in {}", n); None }, } From 7bbe1bb6d0c865b4aa1248a0f45b3f383aa6655a Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Wed, 30 Dec 2015 12:30:45 +0100 Subject: [PATCH 4/4] Refactor: Provide generic cli-to-filter-function generator The refactors functions from the BM module into more generic functions for all modules. The previous state was, that there were functions in the BM module which were used to load from the store and applying some filters. The new state is that there are functions to generate the filter functions which are then used to filter the loaded files from the store, so we can apply some more functionality on the filtered list. --- src/module/bm/mod.rs | 123 ++++--------------------- src/module/helpers/cli.rs | 189 ++++++++++++++++++++++++++++++++++++++ src/module/helpers/mod.rs | 1 + 3 files changed, 208 insertions(+), 105 deletions(-) create mode 100644 src/module/helpers/cli.rs diff --git a/src/module/bm/mod.rs b/src/module/bm/mod.rs index 9b500ec3..a200a25d 100644 --- a/src/module/bm/mod.rs +++ b/src/module/bm/mod.rs @@ -18,6 +18,8 @@ use storage::file::File; use storage::parser::FileHeaderParser; use storage::parser::Parser; use storage::json::parser::JsonHeaderParser; +use module::helpers::cli::get_file_filter_by_cli; +use module::helpers::cli::CliFileFilter; mod header; @@ -136,15 +138,13 @@ impl<'a> BM<'a> { fn command_remove(&self, matches: &ArgMatches) -> bool { use std::process::exit; - let (filtered, files) = self.get_files(matches, "id", "match", "tags"); - - if !filtered { - error!("Unexpected error. Exiting"); - exit(1); - } - - let result = files + let parser = Parser::new(JsonHeaderParser::new(None)); + let filter : Box = get_file_filter_by_cli(&parser, matches, "id", "match", "tags", None); + let result = self.rt + .store() + .load_for_module(self, &parser) .iter() + .filter(|file| filter.filter_file(file)) .map(|file| { debug!("File loaded, can remove now: {:?}", file); let f = file.deref().borrow(); @@ -209,15 +209,18 @@ impl<'a> BM<'a> { }) .unwrap_or(vec![]); - let (filter, files) = self.get_files(matches, "with_id", "with_match", "with_tags"); - - if !filter { - warn!("There were no filter applied when loading the files"); - } - let parser = Parser::new(JsonHeaderParser::new(None)); - files + let filter : Box = get_file_filter_by_cli(&parser, + matches, + "with_id", + "with_match", + "with_tags", + None); + self.rt + .store() + .load_for_module(self, &parser) .into_iter() + .filter(|file| filter.filter_file(file)) .map(|file| { debug!("Remove tags from file: {:?}", file); @@ -250,96 +253,6 @@ impl<'a> BM<'a> { .all(|x| x) } - /** - * Helper function to get files from the store filtered by the constraints passed via the - * CLI - */ - fn get_files(&self, - matches: &ArgMatches, - id_key: &'static str, - match_key: &'static str, - tag_key: &'static str) - -> (bool, Vec>>) - { - if matches.is_present(id_key) { - let hash = FileHash::from(matches.value_of(id_key).unwrap()); - (true, self.get_files_by_id(hash)) - } else if matches.is_present(match_key) { - let matcher = String::from(matches.value_of(match_key).unwrap()); - (true, self.get_files_by_match(matcher)) - } else if matches.is_present(tag_key) { - let tags = matches.value_of(tag_key) - .unwrap() - .split(",") - .map(String::from) - .collect::>(); - (true, self.get_files_by_tags(tags)) - } else { - // get all files - let parser = Parser::new(JsonHeaderParser::new(None)); - (false, self.rt.store().load_for_module(self, &parser)) - } - } - - /** - * Get files from the store, filtere by ID - */ - fn get_files_by_id(&self, hash: FileHash) -> Vec>> { - let parser = Parser::new(JsonHeaderParser::new(None)); - self.rt - .store() - .load_by_hash(self, &parser, hash) - .map(|f| vec![f]) - .unwrap_or(vec![]) - } - - /** - * Get files from the store, filtere by Regex - */ - fn get_files_by_match(&self, matcher: String) -> Vec>> { - let parser = Parser::new(JsonHeaderParser::new(None)); - let re = Regex::new(&matcher[..]).unwrap_or_else(|e| { - error!("Cannot build regex out of '{}'", matcher); - error!("{}", e); - exit(1); - }); - - debug!("Compiled '{}' to regex: '{:?}'", matcher, re); - - self.rt - .store() - .load_for_module(self, &parser) - .into_iter() - .filter(|file| { - let f = file.deref().borrow(); - let url = get_url_from_header(f.header()); - debug!("url = {:?}", url); - url.map(|u| { - debug!("Matching '{}' ~= '{}'", re.as_str(), u); - re.is_match(&u[..]) - }).unwrap_or(false) - }) - .collect() - } - - /** - * Get files from the store, filtere by tags - */ - fn get_files_by_tags(&self, tags: Vec) -> Vec>> { - let parser = Parser::new(JsonHeaderParser::new(None)); - self.rt - .store() - .load_for_module(self, &parser) - .into_iter() - .filter(|file| { - let f = file.deref().borrow(); - get_tags_from_header(f.header()).iter().any(|tag| { - tags.iter().any(|remtag| remtag == tag) - }) - }) - .collect() - } - } /** diff --git a/src/module/helpers/cli.rs b/src/module/helpers/cli.rs new file mode 100644 index 00000000..7d96cef1 --- /dev/null +++ b/src/module/helpers/cli.rs @@ -0,0 +1,189 @@ +use std::rc::Rc; +use std::cell::RefCell; +use std::ops::Deref; +use std::process::exit; + +use clap::ArgMatches; +use regex::Regex; + +use storage::file::File; +use storage::file::hash::FileHash; +use storage::file::header::data::FileHeaderData; +use storage::file::id::FileID; +use storage::json::parser::JsonHeaderParser; +use storage::parser::FileHeaderParser; +use storage::parser::Parser; + +pub trait CliFileFilter { + + fn filter_file(&self, &Rc>) -> bool; + +} + +struct CliFileFilterDefault { + default: bool, +} + +impl CliFileFilter for CliFileFilterDefault { + + fn filter_file(&self, _: &Rc>) -> bool { + debug!("Filtering file with default value = {}", self.default); + return self.default + } + +} + +struct CliFileFilterByHash { + hash: FileHash, +} + +impl CliFileFilter for CliFileFilterByHash { + + fn filter_file(&self, file: &Rc>) -> bool { + debug!("Filtering file with hash = {}", self.hash); + let f = file.deref().borrow(); + f.id().get_id() == self.hash + } + +} + +struct CliFileFilterByDataRegex { + regex: Regex, +} + +impl CliFileFilter for CliFileFilterByDataRegex { + + fn filter_file(&self, file: &Rc>) -> bool { + debug!("Filtering file with regex = {:?}", self.regex); + let f = file.deref().borrow(); + self.regex.is_match(&f.data()[..]) + } + +} + +struct CliFileFilterByHeaderRegex { + header_field_name: &'static str, + regex: Regex, +} + +impl CliFileFilter for CliFileFilterByHeaderRegex { + + fn filter_file(&self, file: &Rc>) -> bool { + use module::helpers::header::data::get_named_text_from_header; + + debug!("Filtering file (header field = {}) with regex = {:?}", + self.header_field_name, + self.regex); + + let f = file.deref().borrow(); + get_named_text_from_header(self.header_field_name, f.header()) + .map(|headerfield| self.regex.is_match(&headerfield[..])) + .unwrap_or(false) + } + +} + +struct CliFileFilterByTags { + tags: Vec, +} + +impl CliFileFilter for CliFileFilterByTags { + + fn filter_file(&self, file: &Rc>) -> bool { + use module::helpers::header::tags::data::get_tags_from_header; + + debug!("Filtering file with tags = {:?}", self.tags); + + let f = file.deref().borrow(); + get_tags_from_header(f.header()) + .iter() + .any(|tag| self.tags.iter().any(|remtag| remtag == tag)) + } + +} + +/** + * Helper function to get files from the store filtered by the constraints passed via the + * CLI + */ +pub fn get_file_filter_by_cli(parser: &Parser, + matches: &ArgMatches, + id_key: &'static str, + match_key: &'static str, + tag_key: &'static str, + header_field_name: Option<&'static str>) + -> Box + where HP: FileHeaderParser, +{ + if matches.is_present(id_key) { + Box::new(CliFileFilterByHash { hash: FileHash::from(matches.value_of(id_key).unwrap()) }) + } else if matches.is_present(match_key) { + let matcher = String::from(matches.value_of(match_key).unwrap()); + header_field_name + .and_then(|header_field_name| { + Some(get_files_by_header_field_match_filter(parser, + &matcher, + header_field_name)) + }) + .unwrap_or(get_file_by_match_filter(parser, &matcher)) + } else if matches.is_present(tag_key) { + let tags = matches.value_of(tag_key) + .unwrap() + .split(",") + .map(String::from) + .collect::>(); + get_file_by_tags_filter(tags) + } else { + Box::new(CliFileFilterDefault { default: true }) + } +} + +/** + * Get files from the store, filtere by Regex + */ +fn get_file_by_match_filter(parser: &Parser, matcher: &String) + -> Box + where HP: FileHeaderParser +{ + let parser = Parser::new(JsonHeaderParser::new(None)); + let re = Regex::new(&matcher[..]).unwrap_or_else(|e| { + error!("Cannot build regex out of '{}'", matcher); + error!("{}", e); + exit(1); + }); + + debug!("Compiled '{}' to regex: '{:?}'", matcher, re); + + Box::new(CliFileFilterByDataRegex { regex: re }) +} + +fn get_files_by_header_field_match_filter(parser: &Parser, + matcher: &String, + header_field_name: &'static str) + -> Box + where HP: FileHeaderParser, +{ + let parser = Parser::new(JsonHeaderParser::new(None)); + let re = Regex::new(&matcher[..]).unwrap_or_else(|e| { + error!("Cannot build regex out of '{}'", matcher); + error!("{}", e); + exit(1); + }); + + debug!("Compiled '{}' to regex: '{:?}'", matcher, re); + + Box::new(CliFileFilterByHeaderRegex { + header_field_name: header_field_name, + regex: re + }) +} + +/** + * Get files from the store, filtere by tags + */ +fn get_file_by_tags_filter(tags: Vec) + -> Box +{ + Box::new(CliFileFilterByTags { tags: tags }) +} + diff --git a/src/module/helpers/mod.rs b/src/module/helpers/mod.rs index f260114b..b836be85 100644 --- a/src/module/helpers/mod.rs +++ b/src/module/helpers/mod.rs @@ -2,6 +2,7 @@ * Utility helpers for modules */ +pub mod cli; pub mod header; pub mod utils;