LemmyNet/lemmy-stats-crawler
2
1
Fork 0
Code Issues 3 Pull requests Projects Releases Wiki Activity

Include number of failed instances

Browse source
This commit is contained in:
Felix Ableitner 2021-03-17 03:04:57 +01:00
parent 5d5a9b863d
commit 95107b6a39
2 changed files with 16 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/crawl.rs
View file

@ -10,13 +10,14 @@ use std::collections::VecDeque;
pub async fn crawl( pub async fn crawl(
start_instances: Vec<String>, start_instances: Vec<String>,
max_depth: i32, max_depth: i32,
) -> Result<Vec<InstanceDetails>, Error> { ) -> Result<(Vec<InstanceDetails>, i32), Error> {
let mut pending_instances: VecDeque<CrawlInstance> = start_instances let mut pending_instances: VecDeque<CrawlInstance> = start_instances
.iter() .iter()
.map(|s| CrawlInstance::new(s.to_string(), 0)) .map(|s| CrawlInstance::new(s.to_string(), 0))
.collect(); .collect();
let mut crawled_instances = vec![]; let mut crawled_instances = vec![];
let mut instance_details = vec![]; let mut instance_details = vec![];
let mut failed_instances = 0;
while let Some(current_instance) = pending_instances.pop_back() { while let Some(current_instance) = pending_instances.pop_back() {
crawled_instances.push(current_instance.domain.clone()); crawled_instances.push(current_instance.domain.clone());
if current_instance.depth > max_depth { if current_instance.depth > max_depth {
@ -34,11 +35,14 @@ pub async fn crawl(
} }
} }
} }
Err(e) => eprintln!("Failed to crawl {}: {}", current_instance.domain, e), Err(e) => {
failed_instances += 1;
eprintln!("Failed to crawl {}: {}", current_instance.domain, e)
},
} }
} }
Ok(instance_details) Ok((instance_details, failed_instances))
} }
#[derive(Serialize, Clone)] #[derive(Serialize, Clone)]

16
src/main.rs
View file

@ -32,8 +32,8 @@ pub async fn main() -> Result<(), Error> {
let start_instances = trusted_instances.iter().map(|s| s.to_string()).collect(); let start_instances = trusted_instances.iter().map(|s| s.to_string()).collect();
eprintln!("Crawling..."); eprintln!("Crawling...");
let instance_details = crawl(start_instances, max_crawl_depth).await?; let (instance_details, failed_instances) = crawl(start_instances, max_crawl_depth).await?;
let total_stats = aggregate(instance_details); let total_stats = aggregate(instance_details, failed_instances);
println!("{}", serde_json::to_string_pretty(&total_stats)?); println!("{}", serde_json::to_string_pretty(&total_stats)?);
Ok(()) Ok(())
@ -41,23 +41,25 @@ pub async fn main() -> Result<(), Error> {
#[derive(Serialize)] #[derive(Serialize)]
struct TotalStats { struct TotalStats {
total_instances: i32, crawled_instances: i32,
failed_instances: i32,
total_users: i64, total_users: i64,
total_online_users: i32, total_online_users: i32,
instance_details: Vec<InstanceDetails>, instance_details: Vec<InstanceDetails>,
} }
fn aggregate(instance_details: Vec<InstanceDetails>) -> TotalStats { fn aggregate(instance_details: Vec<InstanceDetails>, failed_instances: i32) -> TotalStats {
let mut total_instances = 0; let mut crawled_instances = 0;
let mut total_users = 0; let mut total_users = 0;
let mut total_online_users = 0; let mut total_online_users = 0;
for i in &instance_details { for i in &instance_details {
total_instances += 1; crawled_instances += 1;
total_users += i.total_users; total_users += i.total_users;
total_online_users += i.online_users; total_online_users += i.online_users;
} }
TotalStats { TotalStats {
total_instances, crawled_instances,
failed_instances,
total_users, total_users,
total_online_users, total_online_users,
instance_details, instance_details,