Filter instances at crawl time (#206)

* Filter instance list at crawl time

* fixes

* exclude more data

* no max crawl

* prettier

* update instance stats

crawl.mjs

@@ -1,11 +1,10 @@
 import fs from "fs";
-import path from "path";
-import { exit } from "process";
 import { spawn } from "child_process";
 
 const outDir = "src/shared/translations/";
 const recommendationsFile = "recommended-instances.json";
 const instanceStatsFile = "src/shared/instance_stats.ts";
+const min_monthly_users = 5;
 
 fs.mkdirSync(outDir, { recursive: true });
 
@@ -50,7 +49,27 @@ try {
   });
 
   run.on("close", exitCode => {
-    const stats = JSON.parse(savedOutput);
+    var stats = JSON.parse(savedOutput);
+    // Crawl results from all instances include tons of data which needs to be compiled.
+    // If it is too much data it breaks the build, so we need to exclude as much as possible.
+    stats.instance_details = stats.instance_details
+      // Exclude instances with closed registration
+      .filter(
+        i => i.site_info.site_view.local_site.registration_mode != "closed"
+      )
+      // Exclude instances with few active users
+      .filter(
+        i => i.site_info.site_view.counts.users_active_month > min_monthly_users
+      );
+    // Exclude unnecessary data
+    stats.instance_details.forEach(i => {
+      delete i.site_info.admins;
+      delete i.site_info.all_languages;
+      delete i.site_info.discussion_languages;
+      delete i.site_info.taglines;
+      delete i.site_info.custom_emojis;
+      delete i.federated_instances;
+    });
 
     let stats2 = {
       stats: stats,

lemmy-js-client

@@ -1 +1 @@
-Subproject commit 2eac19b2a21d77d711b8a0b0a534affe0d4d6851
+Subproject commit 4553c749cb0fb74d62fd156ebd119dc479693dfd

lemmy-stats-crawler

@@ -1 +1 @@
-Subproject commit ed6a393329b79fd23964231e26cfa00347b8fc83
+Subproject commit 788a3dd6e02fbe153e6d7c6315601ade15637f8c

lemmy-translations

@@ -1 +1 @@
-Subproject commit a241fe1255a6363c7ae1ec5a09520c066745e6ce
+Subproject commit 5a9d44656e2658ab7cb2dbec3fd1bfaf57654533

src/shared/components/instances.tsx

@@ -4,8 +4,6 @@ import { i18n } from "../i18next";
 import { instance_stats } from "../instance_stats";
 import { numToSI } from "../utils";
 
-const min_monthly_users = 5;
-
 export class Instances extends Component<any, any> {
   constructor(props: any, context: any) {
     super(props, context);
@@ -101,43 +99,33 @@ export class Instances extends Component<any, any> {
       <div>
         <h2>{header}</h2>
         <div class="row">
-          {instances
-            .filter(
-              i =>
-                i.site_info.site_view.local_site.registration_mode != "closed"
-            )
-            .filter(
-              i =>
-                i.site_info.site_view.counts.users_active_month >
-                min_monthly_users
-            )
-            .map(instance => {
-              let domain = instance.domain;
-              let description = instance.site_info.site_view.site.description;
-              let icon = instance.site_info.site_view.site.icon;
-              return (
-                <div class="card col-6">
-                  <header>
-                    <div class="row">
-                      <h4 class="col">{domain}</h4>
-                    </div>
-                  </header>
-                  <div class="is-center">
-                    <img
-                      class="join-banner"
-                      src={icon || "/static/assets/images/lemmy.svg"}
-                    />
+          {instances.map(instance => {
+            let domain = instance.domain;
+            let description = instance.site_info.site_view.site.description;
+            let icon = instance.site_info.site_view.site.icon;
+            return (
+              <div class="card col-6">
+                <header>
+                  <div class="row">
+                    <h4 class="col">{domain}</h4>
                   </div>
-                  <br />
-                  <p class="join-desc">{description}</p>
-                  <footer>
-                    <a class="button primary" href={`https://${domain}`}>
-                      {i18n.t("browse_instance")}
-                    </a>
-                  </footer>
+                </header>
+                <div class="is-center">
+                  <img
+                    class="join-banner"
+                    src={icon || "/static/assets/images/lemmy.svg"}
+                  />
                 </div>
-              );
-            })}
+                <br />
+                <p class="join-desc">{description}</p>
+                <footer>
+                  <a class="button primary" href={`https://${domain}`}>
+                    {i18n.t("browse_instance")}
+                  </a>
+                </footer>
+              </div>
+            );
+          })}
         </div>
       </div>
     );