Use jq to filter instance list and avoid js crashes (fixes #216) (#217)

* Use jq to filter instance list and avoid js crashes (fixes #216)

* prettier

* prettier 2

crawl.mjs

@@ -19,16 +19,20 @@ try {
       all_recommended.push(...recommended_instances[k]);
     }
   }
+  // Run Rust crawler with given params. Then pipe output directly into jq, to filter
+  // out fields with lots of data which we dont need. This is necessary because otherwise
+  // Javascript may crash when loading the crawl output.
   const run = spawn(
-    "cargo",
+    "sh",
     [
-      "run",
+      "-c",
-      "--",
+      `cargo run -- --json --start-instances ${all_recommended} \
-      "--json",
+      --exclude-instances ${recommended_instances.exclude} | \
-      "--start-instances",
+      jq 'del(.instance_details[].federated_instances, \
-      all_recommended,
+        .instance_details[].site_info.all_languages, \
-      "--exclude-instances",
+        .instance_details[].site_info.discussion_languages, \
-      recommended_instances.exclude,
+        .instance_details[].site_info.admins, .instance_details[].site_info.taglines, \
+        .instance_details[].site_info.custom_emojis)'`,
     ],
     {
       cwd: "lemmy-stats-crawler",
@@ -62,15 +66,6 @@ try {
         i =>
           i.site_info.site_view.counts.users_active_month > min_monthly_users,
       );
-    // Exclude unnecessary data
-    stats.instance_details.forEach(i => {
-      delete i.site_info.admins;
-      delete i.site_info.all_languages;
-      delete i.site_info.discussion_languages;
-      delete i.site_info.taglines;
-      delete i.site_info.custom_emojis;
-      delete i.federated_instances;
-    });
 
     let stats2 = {
       stats: stats,