From 4426b0edf92c3c85df2bd913e62d2da7ba1d74bc Mon Sep 17 00:00:00 2001
From: Felix Ableitner <me@nutomic.com>
Date: Thu, 11 Mar 2021 02:41:24 +0100
Subject: [PATCH] Improved crawling code, allow null icon

---
 src/federated_instances.rs |  2 +-
 src/main.rs                | 29 ++++++++++++++++-------------
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/src/federated_instances.rs b/src/federated_instances.rs
index 018f20e..fb0a83f 100644
--- a/src/federated_instances.rs
+++ b/src/federated_instances.rs
@@ -22,5 +22,5 @@ pub struct SiteView {
 #[derive(Deserialize, Debug, Clone)]
 pub struct Site {
     pub name: String,
-    pub icon: String,
+    pub icon: Option<String>,
 }
diff --git a/src/main.rs b/src/main.rs
index 0c008e1..78bf0fa 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -52,23 +52,26 @@ async fn crawl(start_instances: Vec<String>) -> Result<Vec<InstanceDetails>, Err
     let mut pending_instances = start_instances;
     let mut crawled_instances = vec![];
     let mut instance_details = vec![];
-    while let Some(pi) = pending_instances.iter().next() {
-        crawled_instances.push(pi.to_owned());
-        let current_instance_details = fetch_instance_details(&pi).await.ok();
+    while let Some(current_instance) = pending_instances.to_owned().first() {
+        crawled_instances.push(current_instance.to_owned());
+        // remove curent instance from pending
         pending_instances = pending_instances
             .iter()
-            .filter(|i| i != &pi)
+            .filter(|i| i != &current_instance)
             .map(|i| i.to_owned())
             .collect();
 
-        if let Some(details) = current_instance_details {
-            instance_details.push(details.to_owned());
-            // add all unknown, linked instances to pending
-            for ci in details.linked_instances {
-                if !crawled_instances.contains(&ci) {
-                    pending_instances.push(ci);
-                }
-            }
+        match fetch_instance_details(&current_instance).await {
+           Ok(details) => {
+               instance_details.push(details.to_owned());
+               // add all unknown, linked instances to pending
+               for i in details.linked_instances {
+                   if !crawled_instances.contains(&i) {
+                       pending_instances.push(i);
+                   }
+               }
+           },
+           Err(e) => eprintln!("Failed to crawl {}: {}", current_instance, e)
         }
     }
 
@@ -79,7 +82,7 @@ async fn crawl(start_instances: Vec<String>) -> Result<Vec<InstanceDetails>, Err
 struct InstanceDetails {
     domain: String,
     name: String,
-    icon: String,
+    icon: Option<String>,
     online_users: i32,
     total_users: i64,
     users_active_halfyear: i64,