Update crawler to latest version (#103)

crawl.mjs

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import { exit } from 'process';
-import { spawnSync } from 'child_process';
+import { spawn } from 'child_process';
 
 const outDir = "src/shared/translations/";
 const recommendationsFile = "recommended-instances.json";
@@ -15,25 +15,41 @@ try {
   var all_recommended = [];
   for (var k in recommended_instances) {
     if (k != "exclude") {
-    all_recommended.push(...recommended_instances[k]);
+      all_recommended.push(...recommended_instances[k]);
     }
   }
-  const run = spawnSync("cargo", 
-  ["run", "--", "--start-instances", all_recommended, "--exclude", recommended_instances.exclude], {
-    cwd: "../lemmy-stats-crawler/",
-    encoding : 'utf8' 
+  const run = spawn("cargo",
+      ["run", "--", "--start-instances", all_recommended,
+          "--exclude-instances", recommended_instances.exclude], {
+        cwd: "../lemmy-stats-crawler",
+        encoding : 'utf8'
+      });
+  let savedOutput = '';
+
+  run.stdout.on('data', data => {
+     const strData = data.toString();
+     process.stdout.write(strData);
+     savedOutput += strData;
   });
-  //console.log("crawler run output: ", run.output);
-  const stats = JSON.parse(run.stdout);
 
-  let stats2 = {
-    stats: stats,
-    recommended: recommended_instances
-  }
+  run.stderr.on('data', data => {
+     const strData = data.toString();
+     process.stdout.write(strData);
+  });
 
-  let data = `export const instance_stats = \n `;
-  data += JSON.stringify(stats2, null, 2) + ";";
-  fs.writeFileSync(instanceStatsFile, data);
+  run.on('close', exitCode => {
+    const stats = JSON.parse(savedOutput);
+
+    let stats2 = {
+      stats: stats,
+      recommended: recommended_instances
+    }
+
+    let data = `export const instance_stats = \n `;
+    data += JSON.stringify(stats2, null, 2) + ";";
+    fs.writeFileSync(instanceStatsFile, data);
+  });
+    run.await;
 } catch (err) {
   console.error(err);
 }

src/shared/components/instances.tsx

@@ -16,12 +16,11 @@ export class Instances extends Component<any, any> {
     if (!recommended_instances) {
       recommended_instances = instance_stats.recommended["en"];
     }
-    console.log(recommended_instances);
+
     var recommended = [];
     var remaining = [];
     for (var i of instance_stats.stats.instance_details) {
       if (recommended_instances.indexOf(i.domain) > -1) {
-        console.log(i.domain);
         recommended.push(i);
       } else {
         remaining.push(i);
@@ -69,36 +68,40 @@ export class Instances extends Component<any, any> {
         <div class="container">
           <h1 class="is-marginless">{header}</h1>
 
-          {instances.map(i => (
-            <div class="card">
-              <img
-                class="join-icon"
-                src={i.icon || "/static/assets/images/lemmy.svg"}
-              />
-              <div class="join-text">
-                <div class="join-header">
-                  <h4 class="join-title">{i.name}</h4>
-                  <i>
-                    {numToSI(i.users_active_month)} {i18n.t("users")} /{" "}
-                    {i18n.t("month")}
-                  </i>
+          {instances.map(instance => {
+            const site = instance.site_info.site_view.site;
+            const counts = instance.site_info.site_view.counts;
+            return (
+              <div class="card">
+                <img
+                  class="join-icon"
+                  src={site.icon || "/static/assets/images/lemmy.svg"}
+                />
+                <div class="join-text">
+                  <div class="join-header">
+                    <h4 class="join-title">{site.name}</h4>
+                    <i>
+                      {numToSI(counts.users_active_month)} {i18n.t("users")} /{" "}
+                      {i18n.t("month")}
+                    </i>
+                  </div>
+                  <p class="join-desc">{site.description}</p>
                 </div>
-                <p class="join-desc">{i.description}</p>
+                {site.require_application ? (
+                  <a
+                    class="button primary button-yellow"
+                    href={`https://${instance.domain}`}
+                  >
+                    {i18n.t("apply_to_join")}
+                  </a>
+                ) : (
+                  <a class="button primary" href={`https://${instance.domain}`}>
+                    {i18n.t("join")}
+                  </a>
+                )}
               </div>
-              {i.require_application ? (
-                <a
-                  class="button primary button-yellow"
-                  href={`https://${i.domain}`}
-                >
-                  {i18n.t("apply_to_join")}
-                </a>
-              ) : (
-                <a class="button primary" href={`https://${i.domain}`}>
-                  {i18n.t("join")}
-                </a>
-              )}
-            </div>
-          ))}
+            );
+          })}
         </div>
       </div>
     );

update_submodules.sh

@@ -11,11 +11,6 @@ git fetch weblate
 git merge weblate/main
 git push
 
-popd
-pushd ../lemmy-instance-stats
-./update.sh
-popd
-
 git submodule update --remote
 git add joinlemmy-translations
 git add lemmy-translations