LemmyNet/lemmy-stats-crawler
2
1
Fork 0
Code Issues 3 Pull requests Projects Releases Wiki Activity

Improved crawling code, allow null icon

Browse source
This commit is contained in:
Felix Ableitner 2021-03-11 02:41:24 +01:00
parent 342fe6ded8
commit 4426b0edf9
2 changed files with 17 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/federated_instances.rs
View file

@ -22,5 +22,5 @@ pub struct SiteView {
#[derive(Deserialize, Debug, Clone)]
pub struct Site {
pub name: String,
pub icon: String,
pub icon: Option<String>,
}

21
src/main.rs
View file

@ -52,23 +52,26 @@ async fn crawl(start_instances: Vec<String>) -> Result<Vec<InstanceDetails>, Err
let mut pending_instances = start_instances;
let mut crawled_instances = vec![];
let mut instance_details = vec![];
while let Some(pi) = pending_instances.iter().next() {
crawled_instances.push(pi.to_owned());
let current_instance_details = fetch_instance_details(&pi).await.ok();
while let Some(current_instance) = pending_instances.to_owned().first() {
crawled_instances.push(current_instance.to_owned());
// remove curent instance from pending
pending_instances = pending_instances
.iter()
.filter(|i| i != &pi)
.filter(|i| i != &current_instance)
.map(|i| i.to_owned())
.collect();
if let Some(details) = current_instance_details {
match fetch_instance_details(&current_instance).await {
Ok(details) => {
instance_details.push(details.to_owned());
// add all unknown, linked instances to pending
for ci in details.linked_instances {
if !crawled_instances.contains(&ci) {
pending_instances.push(ci);
for i in details.linked_instances {
if !crawled_instances.contains(&i) {
pending_instances.push(i);
}
}
},
Err(e) => eprintln!("Failed to crawl {}: {}", current_instance, e)
}
}
@ -79,7 +82,7 @@ async fn crawl(start_instances: Vec<String>) -> Result<Vec<InstanceDetails>, Err
struct InstanceDetails {
domain: String,
name: String,
icon: String,
icon: Option<String>,
online_users: i32,
total_users: i64,
users_active_halfyear: i64,