Merge branch 'main' of https://github.com/LemmyNet/joinlemmy-site

.drone.yml

@@ -29,6 +29,16 @@ steps:
     commands:
       - yarn build:dev
 
+  - name: run instance crawl
+    image: node:14-alpine
+    commands:
+      # libpq and openssl can probably be removed after lemmy dep is upgraded to 0.16.4+
+      - apk add cargo pkgconfig openssl openssl-dev libpq libpq-dev
+      - yarn crawl
+    when:
+      event:
+        - cron
+
   - name: make release build and push to docker hub
     image: plugins/docker
     settings:
@@ -41,5 +51,5 @@ steps:
       password:
         from_secret: docker_password
     when:
-      ref:
-        - refs/tags/*
+      event:
+        - cron

.gitmodules

@@ -10,11 +10,10 @@
 	path = lemmy-translations
 	url = https://github.com/lemmynet/lemmy-translations
 	branch = main
-[submodule "lemmy-instance-stats"]
-	path = lemmy-instance-stats
-	url = https://github.com/LemmyNet/lemmy-instance-stats
-	branch = main
 [submodule "lemmy-js-client"]
 	path = lemmy-js-client
 	url = https://github.com/LemmyNet/lemmy-js-client
 	branch = main
+[submodule "lemmy-stats-crawler"]
+	path = lemmy-stats-crawler
+	url = https://yerbamate.ml/LemmyNet/lemmy-stats-crawler.git

Dockerfile

@@ -44,7 +44,6 @@ COPY tsconfig.json \
 
 COPY joinlemmy-translations joinlemmy-translations
 COPY lemmy-translations lemmy-translations
-COPY lemmy-instance-stats lemmy-instance-stats
 COPY src src
 
 # Copy the docs and API

crawl.mjs

@@ -0,0 +1,55 @@
+import fs from 'fs';
+import path from 'path';
+import { exit } from 'process';
+import { spawn } from 'child_process';
+
+const outDir = "src/shared/translations/";
+const recommendationsFile = "recommended-instances.json";
+const instanceStatsFile = "src/shared/instance_stats.ts";
+
+fs.mkdirSync(outDir, { recursive: true });
+
+// crawl instance stats
+try {
+  const recommended_instances = JSON.parse(fs.readFileSync(recommendationsFile, "utf8"));
+  var all_recommended = [];
+  for (var k in recommended_instances) {
+    if (k != "exclude") {
+      all_recommended.push(...recommended_instances[k]);
+    }
+  }
+  const run = spawn("cargo",
+      ["run", "--", "--start-instances", all_recommended,
+          "--exclude-instances", recommended_instances.exclude], {
+        cwd: "lemmy-stats-crawler",
+        encoding : 'utf8'
+      });
+  let savedOutput = '';
+
+  run.stdout.on('data', data => {
+     const strData = data.toString();
+     process.stdout.write(strData);
+     savedOutput += strData;
+  });
+
+  run.stderr.on('data', data => {
+     const strData = data.toString();
+     process.stdout.write(strData);
+  });
+
+  run.on('close', exitCode => {
+    const stats = JSON.parse(savedOutput);
+
+    let stats2 = {
+      stats: stats,
+      recommended: recommended_instances
+    }
+
+    let data = `export const instance_stats = \n `;
+    data += JSON.stringify(stats2, null, 2) + ";";
+    fs.writeFileSync(instanceStatsFile, data);
+  });
+    run.await;
+} catch (err) {
+  console.error(err);
+}

deploy.sh

@@ -3,6 +3,10 @@
 # Update all the submodules and translations
 ./update_submodules.sh
 
+yarn crawl
+git add "src/shared/instance_stats.ts"
+git commit -m "Crawl instance statistics"
+
 # look for unused translations
 for langfile in joinlemmy-translations/translations/*.json; do
     lang=$(basename $langfile .json)

generate_translations.mjs

@@ -4,39 +4,10 @@ import path from 'path';
 const translationDir = "joinlemmy-translations/translations/";
 const outDir = "src/shared/translations/";
 const translatorsJsonFile = "lemmy-translations/translators.json";
-const statsFile = "lemmy-instance-stats/stats.json";
-const recommendationsFile = "lemmy-instance-stats/recommended-instances.csv";
 const newsDir = "src/assets/news";
 
 fs.mkdirSync(outDir, { recursive: true });
 
-// Write the stats file
-try {
-  const stats = JSON.parse(fs.readFileSync(statsFile, "utf8"));
-  const recommended_domains = fs.readFileSync(recommendationsFile, "utf8").trim().split(',');
-  console.log(recommended_domains);
-  const recommended = stats.instance_details.filter(i => 
-    recommended_domains.includes(i.domain)
-  );
-  const remaining = stats.instance_details.filter(i => 
-    !recommended_domains.includes(i.domain)
-  );
-
-  let stats2 = {
-    crawled_instances: stats.crawled_instances,
-    total_users: stats.total_users,
-    recommended: recommended,
-    remaining: remaining,
-  }
-
-  let data = `export const instance_stats = \n `;
-  data += JSON.stringify(stats2, null, 2) + ";";
-  const target = outDir + "instance_stats.ts";
-  fs.writeFileSync(target, data);
-} catch (err) {
-  console.error(err);
-}
-
 // Write the news file
 try {
   let files = fs.readdirSync(newsDir);

lemmy-instance-stats

@@ -1 +0,0 @@
-Subproject commit 75824524a2a86094e863f0333ea22fb3aa39ce46

lemmy-stats-crawler

@@ -0,0 +1 @@
+Subproject commit d11febc7e80ae7ea37bd57f00163d71e4b48a918

package.json

@@ -7,6 +7,7 @@
     "build:dev": "webpack --mode=development",
     "build:prod": "webpack --mode=production",
     "clean": "yarn run rimraf dist",
+    "crawl": "node crawl.mjs",
     "lint": "node generate_translations.mjs && tsc --noEmit && eslint --report-unused-disable-directives --ext .js,.ts,.tsx src",
     "prebuild:dev": "yarn clean && node generate_translations.mjs",
     "prebuild:prod": "yarn clean && node generate_translations.mjs",

recommended-instances.json

@@ -0,0 +1,10 @@
+{
+    "en": ["sopuli.xyz", "beehaw.org"],
+    "de": ["feddit.de"],
+    "pt": ["lemmy.pt"],
+    "pt_BR": ["lemmy.pt"],
+    "eu": ["lemmy.eus"],
+    "ja": ["tabinezumi.net", "lm.korako.me"],
+    "es": ["forum.nobigtech.es"],
+    "exclude": ["lemmy.glasgow.social","ds9.lemmy.ml","voyager.lemmy.ml","enterprise.lemmy.ml"]
+}

src/assets/css/main.css

@@ -14,12 +14,6 @@
   webkit-box-shadow: unset;
   box-shadow: unset;
   border: 1px solid var(--color-darkGrey) !important;
-  padding: 10px;
-  margin: 10px 0 10px 0;
-  height: 100px;
-  display: flex;
-  flex-direction: row;
-  align-items: center;
 }
 .stylized {
   font-family: "CaviarDreams", Fallback, sans-serif;
@@ -38,29 +32,11 @@
 p {
   font: 1.2em/1.62 sans-serif;
 }
-.join-icon {
-  width: 80px; 
-  height: 80px; 
+.join-banner {
+  width: 100%;
+  height: 100px;
   object-fit: scale-down;
 }
-.join-text {
-  display: inline-block; 
-  margin: 0 10px 0 10px;
-  display: flex;
-  flex-direction: column;
-  flex: 1;
-}
-.join-header {
-  display: flex;
-  flex-direction: row;
-}
-.join-title {
-  margin: 0px;
-  flex: 1;
-}
-.join-desc {
-  font-size: 0.9em;
-}
 .app-banner {
   width: 100%;
   height: 300px;
@@ -89,7 +65,3 @@ img {
   margin-top: 7px;
   background-color: #333;
 }
-
-.button-yellow {
-  background-color: #b5932e !important;
-}

src/shared/components/instances.tsx

@@ -1,7 +1,7 @@
 import { Component } from "inferno";
 import { Helmet } from "inferno-helmet";
 import { i18n } from "../i18next";
-import { instance_stats } from "../translations/instance_stats";
+import { instance_stats } from "../instance_stats";
 import { numToSI } from "../utils";
 
 const title = i18n.t("join_title");
@@ -12,9 +12,23 @@ export class Instances extends Component<any, any> {
   }
 
   render() {
+    var recommended_instances = instance_stats.recommended[i18n.language];
+    if (!recommended_instances) {
+      recommended_instances = instance_stats.recommended["en"];
+    }
+
+    var recommended = [];
+    var remaining = [];
+    for (var i of instance_stats.stats.instance_details) {
+      if (recommended_instances.indexOf(i.domain) > -1) {
+        recommended.push(i);
+      } else {
+        remaining.push(i);
+      }
+    }
     // shuffle recommended instances list into random order
     // https://stackoverflow.com/a/46545530
-    let recommended = instance_stats.recommended
+    let recommended2 = recommended
       .map(value => ({ value, sort: Math.random() }))
       .sort((a, b) => a.sort - b.sort)
       .map(({ value }) => value);
@@ -28,8 +42,8 @@ export class Instances extends Component<any, any> {
         {this.header()}
         <br />
         <br />
-        {this.renderList(i18n.t("recommended_instances"), recommended)}
-        {this.renderList(i18n.t("popular_instances"), instance_stats.remaining)}
+        {this.renderList(i18n.t("recommended_instances"), recommended2)}
+        {this.renderList(i18n.t("popular_instances"), remaining)}
       </div>
     );
   }
@@ -38,8 +52,8 @@ export class Instances extends Component<any, any> {
     return (
       <i>
         {i18n.t("instance_totals", {
-          instances: numToSI(instance_stats.crawled_instances),
-          users: numToSI(instance_stats.total_users),
+          instances: numToSI(instance_stats.stats.crawled_instances),
+          users: numToSI(instance_stats.stats.total_users),
         })}
       </i>
     );
@@ -48,42 +62,51 @@ export class Instances extends Component<any, any> {
   renderList(header: string, instances: any[]) {
     return (
       <div>
-        <Helmet title={title}>
-          <meta property={"title"} content={title} />
-        </Helmet>
-        <div class="container">
-          <h1 class="is-marginless">{header}</h1>
-
-          {instances.map(i => (
-            <div class="card">
-              <img
-                class="join-icon"
-                src={i.icon || "/static/assets/images/lemmy.svg"}
-              />
-              <div class="join-text">
-                <div class="join-header">
-                  <h4 class="join-title">{i.name}</h4>
+        <h2>{header}</h2>
+        <div class="row">
+          {instances.map(instance => {
+            let domain = instance.domain;
+            let users_active_month =
+              instance.site_info.site_view.counts.users_active_month;
+            let description = instance.site_info.site_view.site.description;
+            let icon = instance.site_info.site_view.site.icon;
+            let require_application =
+              instance.site_info.site_view.site.require_application;
+            return (
+              <div class="card col-6">
+                <header>
+                  <div class="row">
+                    <h4 class="col">{domain}</h4>
+                    <h4 class="col text-right">
                       <i>
-                    {numToSI(i.users_active_month)} {i18n.t("users")} /{" "}
+                        {numToSI(users_active_month)} {i18n.t("users")} /{" "}
                         {i18n.t("month")}
                       </i>
+                    </h4>
                   </div>
-                <p class="join-desc">{i.description}</p>
+                </header>
+                <div class="is-center">
+                  <img
+                    class="join-banner"
+                    src={icon || "/static/assets/images/lemmy.svg"}
+                  />
                 </div>
-              {i.require_application ? (
-                <a
-                  class="button primary button-yellow"
-                  href={`https://${i.domain}`}
-                >
+                <br />
+                <p class="join-desc">{description}</p>
+                <footer>
+                  {require_application ? (
+                    <a class="button primary" href={`https://${domain}`}>
                       {i18n.t("apply_to_join")}
                     </a>
                   ) : (
-                <a class="button primary" href={`https://${i.domain}`}>
+                    <a class="button primary" href={`https://${domain}`}>
                       {i18n.t("join")}
                     </a>
                   )}
+                </footer>
               </div>
-          ))}
+            );
+          })}
         </div>
       </div>
     );

update_submodules.sh

@@ -11,11 +11,6 @@ git fetch weblate
 git merge weblate/main
 git push
 
-popd
-pushd ../lemmy-instance-stats
-./update.sh
-popd
-
 git submodule update --remote
 git add joinlemmy-translations
 git add lemmy-translations