diff --git a/src/crawl.rs b/src/crawl.rs index c3d6a88..063d1e0 100644 --- a/src/crawl.rs +++ b/src/crawl.rs @@ -10,13 +10,14 @@ use std::collections::VecDeque; pub async fn crawl( start_instances: Vec, max_depth: i32, -) -> Result, Error> { +) -> Result<(Vec, i32), Error> { let mut pending_instances: VecDeque = start_instances .iter() .map(|s| CrawlInstance::new(s.to_string(), 0)) .collect(); let mut crawled_instances = vec![]; let mut instance_details = vec![]; + let mut failed_instances = 0; while let Some(current_instance) = pending_instances.pop_back() { crawled_instances.push(current_instance.domain.clone()); if current_instance.depth > max_depth { @@ -34,11 +35,14 @@ pub async fn crawl( } } } - Err(e) => eprintln!("Failed to crawl {}: {}", current_instance.domain, e), + Err(e) => { + failed_instances += 1; + eprintln!("Failed to crawl {}: {}", current_instance.domain, e) + }, } } - Ok(instance_details) + Ok((instance_details, failed_instances)) } #[derive(Serialize, Clone)] diff --git a/src/main.rs b/src/main.rs index 5742be8..4c16c2b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,8 +32,8 @@ pub async fn main() -> Result<(), Error> { let start_instances = trusted_instances.iter().map(|s| s.to_string()).collect(); eprintln!("Crawling..."); - let instance_details = crawl(start_instances, max_crawl_depth).await?; - let total_stats = aggregate(instance_details); + let (instance_details, failed_instances) = crawl(start_instances, max_crawl_depth).await?; + let total_stats = aggregate(instance_details, failed_instances); println!("{}", serde_json::to_string_pretty(&total_stats)?); Ok(()) @@ -41,23 +41,25 @@ pub async fn main() -> Result<(), Error> { #[derive(Serialize)] struct TotalStats { - total_instances: i32, + crawled_instances: i32, + failed_instances: i32, total_users: i64, total_online_users: i32, instance_details: Vec, } -fn aggregate(instance_details: Vec) -> TotalStats { - let mut total_instances = 0; +fn aggregate(instance_details: Vec, failed_instances: i32) -> TotalStats { + let mut crawled_instances = 0; let mut total_users = 0; let mut total_online_users = 0; for i in &instance_details { - total_instances += 1; + crawled_instances += 1; total_users += i.total_users; total_online_users += i.online_users; } TotalStats { - total_instances, + crawled_instances, + failed_instances, total_users, total_online_users, instance_details,