Filter instances at crawl time (#206)

* Filter instance list at crawl time

* fixes

* exclude more data

* no max crawl

* prettier

* update instance stats

crawl.mjs

@@ -1,11 +1,10 @@
 import fs from "fs";
-import path from "path";
-import { exit } from "process";
 import { spawn } from "child_process";
 
 const outDir = "src/shared/translations/";
 const recommendationsFile = "recommended-instances.json";
 const instanceStatsFile = "src/shared/instance_stats.ts";
+const min_monthly_users = 5;
 
 fs.mkdirSync(outDir, { recursive: true });
 
@@ -50,7 +49,27 @@ try {
   });
 
   run.on("close", exitCode => {
-    const stats = JSON.parse(savedOutput);
+    var stats = JSON.parse(savedOutput);
+    // Crawl results from all instances include tons of data which needs to be compiled.
+    // If it is too much data it breaks the build, so we need to exclude as much as possible.
+    stats.instance_details = stats.instance_details
+      // Exclude instances with closed registration
+      .filter(
+        i => i.site_info.site_view.local_site.registration_mode != "closed"
+      )
+      // Exclude instances with few active users
+      .filter(
+        i => i.site_info.site_view.counts.users_active_month > min_monthly_users
+      );
+    // Exclude unnecessary data
+    stats.instance_details.forEach(i => {
+      delete i.site_info.admins;
+      delete i.site_info.all_languages;
+      delete i.site_info.discussion_languages;
+      delete i.site_info.taglines;
+      delete i.site_info.custom_emojis;
+      delete i.federated_instances;
+    });
 
     let stats2 = {
       stats: stats,

lemmy-js-client

@@ -1 +1 @@
-Subproject commit 2eac19b2a21d77d711b8a0b0a534affe0d4d6851
+Subproject commit 4553c749cb0fb74d62fd156ebd119dc479693dfd

lemmy-stats-crawler

@@ -1 +1 @@
-Subproject commit ed6a393329b79fd23964231e26cfa00347b8fc83
+Subproject commit 788a3dd6e02fbe153e6d7c6315601ade15637f8c

lemmy-translations

@@ -1 +1 @@
-Subproject commit a241fe1255a6363c7ae1ec5a09520c066745e6ce
+Subproject commit 5a9d44656e2658ab7cb2dbec3fd1bfaf57654533

src/shared/components/instances.tsx

@@ -4,8 +4,6 @@ import { i18n } from "../i18next";
 import { instance_stats } from "../instance_stats";
 import { numToSI } from "../utils";
 
-const min_monthly_users = 5;
-
 export class Instances extends Component<any, any> {
   constructor(props: any, context: any) {
     super(props, context);
@@ -101,17 +99,7 @@ export class Instances extends Component<any, any> {
       <div>
         <h2>{header}</h2>
         <div class="row">
-          {instances
-            .filter(
-              i =>
-                i.site_info.site_view.local_site.registration_mode != "closed"
-            )
-            .filter(
-              i =>
-                i.site_info.site_view.counts.users_active_month >
-                min_monthly_users
-            )
-            .map(instance => {
+          {instances.map(instance => {
             let domain = instance.domain;
             let description = instance.site_info.site_view.site.description;
             let icon = instance.site_info.site_view.site.icon;