LemmyNet/lemmy-stats-crawler
2
1
Fork 0
Code Issues 3 Pull requests Projects Releases Wiki Activity

Improved crawling code, allow null icon

Browse source
This commit is contained in:
Felix Ableitner 2021-03-11 02:41:24 +01:00
parent 342fe6ded8
commit 4426b0edf9
2 changed files with 17 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/federated_instances.rs
View file

@ -22,5 +22,5 @@ pub struct SiteView {
#[derive(Deserialize, Debug, Clone)] #[derive(Deserialize, Debug, Clone)]
pub struct Site { pub struct Site {
pub name: String, pub name: String,
pub icon: String, pub icon: Option<String>,
} }

29
src/main.rs
View file

@ -52,23 +52,26 @@ async fn crawl(start_instances: Vec<String>) -> Result<Vec<InstanceDetails>, Err
let mut pending_instances = start_instances; let mut pending_instances = start_instances;
let mut crawled_instances = vec![]; let mut crawled_instances = vec![];
let mut instance_details = vec![]; let mut instance_details = vec![];
while let Some(pi) = pending_instances.iter().next() { while let Some(current_instance) = pending_instances.to_owned().first() {
crawled_instances.push(pi.to_owned()); crawled_instances.push(current_instance.to_owned());
let current_instance_details = fetch_instance_details(&pi).await.ok(); // remove curent instance from pending
pending_instances = pending_instances pending_instances = pending_instances
.iter() .iter()
.filter(|i| i != &pi) .filter(|i| i != &current_instance)
.map(|i| i.to_owned()) .map(|i| i.to_owned())
.collect(); .collect();
if let Some(details) = current_instance_details { match fetch_instance_details(&current_instance).await {
instance_details.push(details.to_owned()); Ok(details) => {
// add all unknown, linked instances to pending instance_details.push(details.to_owned());
for ci in details.linked_instances { // add all unknown, linked instances to pending
if !crawled_instances.contains(&ci) { for i in details.linked_instances {
pending_instances.push(ci); if !crawled_instances.contains(&i) {
} pending_instances.push(i);
} }
}
},
Err(e) => eprintln!("Failed to crawl {}: {}", current_instance, e)
} }
} }
@ -79,7 +82,7 @@ async fn crawl(start_instances: Vec<String>) -> Result<Vec<InstanceDetails>, Err
struct InstanceDetails { struct InstanceDetails {
domain: String, domain: String,
name: String, name: String,
icon: String, icon: Option<String>,
online_users: i32, online_users: i32,
total_users: i64, total_users: i64,
users_active_halfyear: i64, users_active_halfyear: i64,