Merge branch 'main' of https://github.com/LemmyNet/joinlemmy-site

2024-12-22 10:51:26 +00:00 · 2022-05-26 13:29:02 -04:00 · 2022-05-26 13:29:02 -04:00 · 8ae5b474bf
commit 8ae5b474bf
parent 0ea6a17fdd a2d4b0713b
14 changed files with 6402 additions and 113 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -29,6 +29,16 @@ steps:
    commands:
      - yarn build:dev

+  - name: run instance crawl
+    image: node:14-alpine
+    commands:
+      # libpq and openssl can probably be removed after lemmy dep is upgraded to 0.16.4+
+      - apk add cargo pkgconfig openssl openssl-dev libpq libpq-dev
+      - yarn crawl
+    when:
+      event:
+        - cron
+
  - name: make release build and push to docker hub
    image: plugins/docker
    settings:
@ -41,5 +51,5 @@ steps:
      password:
        from_secret: docker_password
    when:
-      ref:
-        - refs/tags/*
+      event:
+        - cron
--- a/.gitmodules
+++ b/.gitmodules
@ -10,11 +10,10 @@
 	path = lemmy-translations
 	url = https://github.com/lemmynet/lemmy-translations
 	branch = main
-[submodule "lemmy-instance-stats"]
-	path = lemmy-instance-stats
-	url = https://github.com/LemmyNet/lemmy-instance-stats
-	branch = main
 [submodule "lemmy-js-client"]
 	path = lemmy-js-client
 	url = https://github.com/LemmyNet/lemmy-js-client
 	branch = main
+[submodule "lemmy-stats-crawler"]
+	path = lemmy-stats-crawler
+	url = https://yerbamate.ml/LemmyNet/lemmy-stats-crawler.git
--- a/1
+++ b/1
@ -44,7 +44,6 @@ COPY tsconfig.json \

 COPY joinlemmy-translations joinlemmy-translations
 COPY lemmy-translations lemmy-translations
-COPY lemmy-instance-stats lemmy-instance-stats
 COPY src src

 # Copy the docs and API
--- a/crawl.mjs
+++ b/crawl.mjs
@ -0,0 +1,55 @@
+import fs from 'fs';
+import path from 'path';
+import { exit } from 'process';
+import { spawn } from 'child_process';
+
+const outDir = "src/shared/translations/";
+const recommendationsFile = "recommended-instances.json";
+const instanceStatsFile = "src/shared/instance_stats.ts";
+
+fs.mkdirSync(outDir, { recursive: true });
+
+// crawl instance stats
+try {
+  const recommended_instances = JSON.parse(fs.readFileSync(recommendationsFile, "utf8"));
+  var all_recommended = [];
+  for (var k in recommended_instances) {
+    if (k != "exclude") {
+      all_recommended.push(...recommended_instances[k]);
+    }
+  }
+  const run = spawn("cargo",
+      ["run", "--", "--start-instances", all_recommended,
+          "--exclude-instances", recommended_instances.exclude], {
+        cwd: "lemmy-stats-crawler",
+        encoding : 'utf8'
+      });
+  let savedOutput = '';
+
+  run.stdout.on('data', data => {
+     const strData = data.toString();
+     process.stdout.write(strData);
+     savedOutput += strData;
+  });
+
+  run.stderr.on('data', data => {
+     const strData = data.toString();
+     process.stdout.write(strData);
+  });
+
+  run.on('close', exitCode => {
+    const stats = JSON.parse(savedOutput);
+
+    let stats2 = {
+      stats: stats,
+      recommended: recommended_instances
+    }
+
+    let data = `export const instance_stats = \n `;
+    data += JSON.stringify(stats2, null, 2) + ";";
+    fs.writeFileSync(instanceStatsFile, data);
+  });
+    run.await;
+} catch (err) {
+  console.error(err);
+}
--- a/deploy.sh
+++ b/deploy.sh
@ -3,6 +3,10 @@
 # Update all the submodules and translations
 ./update_submodules.sh

+yarn crawl
+git add "src/shared/instance_stats.ts"
+git commit -m "Crawl instance statistics"
+
 # look for unused translations
 for langfile in joinlemmy-translations/translations/*.json; do
    lang=$(basename $langfile .json)
--- a/generate_translations.mjs
+++ b/generate_translations.mjs
@ -4,39 +4,10 @@ import path from 'path';
 const translationDir = "joinlemmy-translations/translations/";
 const outDir = "src/shared/translations/";
 const translatorsJsonFile = "lemmy-translations/translators.json";
-const statsFile = "lemmy-instance-stats/stats.json";
-const recommendationsFile = "lemmy-instance-stats/recommended-instances.csv";
 const newsDir = "src/assets/news";

 fs.mkdirSync(outDir, { recursive: true });

-// Write the stats file
-try {
-  const stats = JSON.parse(fs.readFileSync(statsFile, "utf8"));
-  const recommended_domains = fs.readFileSync(recommendationsFile, "utf8").trim().split(',');
-  console.log(recommended_domains);
-  const recommended = stats.instance_details.filter(i => 
-    recommended_domains.includes(i.domain)
-  );
-  const remaining = stats.instance_details.filter(i => 
-    !recommended_domains.includes(i.domain)
-  );
-
-  let stats2 = {
-    crawled_instances: stats.crawled_instances,
-    total_users: stats.total_users,
-    recommended: recommended,
-    remaining: remaining,
-  }
-
-  let data = `export const instance_stats = \n `;
-  data += JSON.stringify(stats2, null, 2) + ";";
-  const target = outDir + "instance_stats.ts";
-  fs.writeFileSync(target, data);
-} catch (err) {
-  console.error(err);
-}
-
 // Write the news file
 try {
  let files = fs.readdirSync(newsDir);
--- a/1
+++ b/1
@ -1 +0,0 @@
-Subproject commit 75824524a2a86094e863f0333ea22fb3aa39ce46
--- a/1
+++ b/1
@ -0,0 +1 @@
+Subproject commit d11febc7e80ae7ea37bd57f00163d71e4b48a918
--- a/package.json
+++ b/package.json
@ -7,6 +7,7 @@
    "build:dev": "webpack --mode=development",
    "build:prod": "webpack --mode=production",
    "clean": "yarn run rimraf dist",
+    "crawl": "node crawl.mjs",
    "lint": "node generate_translations.mjs && tsc --noEmit && eslint --report-unused-disable-directives --ext .js,.ts,.tsx src",
    "prebuild:dev": "yarn clean && node generate_translations.mjs",
    "prebuild:prod": "yarn clean && node generate_translations.mjs",
--- a/recommended-instances.json
+++ b/recommended-instances.json
@ -0,0 +1,10 @@
+{
+    "en": ["sopuli.xyz", "beehaw.org"],
+    "de": ["feddit.de"],
+    "pt": ["lemmy.pt"],
+    "pt_BR": ["lemmy.pt"],
+    "eu": ["lemmy.eus"],
+    "ja": ["tabinezumi.net", "lm.korako.me"],
+    "es": ["forum.nobigtech.es"],
+    "exclude": ["lemmy.glasgow.social","ds9.lemmy.ml","voyager.lemmy.ml","enterprise.lemmy.ml"]
+}
--- a/src/assets/css/main.css
+++ b/src/assets/css/main.css
@ -14,12 +14,6 @@
  webkit-box-shadow: unset;
  box-shadow: unset;
  border: 1px solid var(--color-darkGrey) !important;
-  padding: 10px;
-  margin: 10px 0 10px 0;
-  height: 100px;
-  display: flex;
-  flex-direction: row;
-  align-items: center;
 }
 .stylized {
  font-family: "CaviarDreams", Fallback, sans-serif;
@ -38,29 +32,11 @@
 p {
  font: 1.2em/1.62 sans-serif;
 }
-.join-icon {
-  width: 80px; 
-  height: 80px; 
+.join-banner {
+  width: 100%;
+  height: 100px;
  object-fit: scale-down;
 }
-.join-text {
-  display: inline-block; 
-  margin: 0 10px 0 10px;
-  display: flex;
-  flex-direction: column;
-  flex: 1;
-}
-.join-header {
-  display: flex;
-  flex-direction: row;
-}
-.join-title {
-  margin: 0px;
-  flex: 1;
-}
-.join-desc {
-  font-size: 0.9em;
-}
 .app-banner {
  width: 100%;
  height: 300px;
@ -89,7 +65,3 @@ img {
  margin-top: 7px;
  background-color: #333;
 }
-
-.button-yellow {
-  background-color: #b5932e !important;
-}
--- a/src/shared/components/instances.tsx
+++ b/src/shared/components/instances.tsx
@ -1,7 +1,7 @@
 import { Component } from "inferno";
 import { Helmet } from "inferno-helmet";
 import { i18n } from "../i18next";
-import { instance_stats } from "../translations/instance_stats";
+import { instance_stats } from "../instance_stats";
 import { numToSI } from "../utils";

 const title = i18n.t("join_title");
@ -12,9 +12,23 @@ export class Instances extends Component<any, any> {
  }

  render() {
+    var recommended_instances = instance_stats.recommended[i18n.language];
+    if (!recommended_instances) {
+      recommended_instances = instance_stats.recommended["en"];
+    }
+
+    var recommended = [];
+    var remaining = [];
+    for (var i of instance_stats.stats.instance_details) {
+      if (recommended_instances.indexOf(i.domain) > -1) {
+        recommended.push(i);
+      } else {
+        remaining.push(i);
+      }
+    }
    // shuffle recommended instances list into random order
    // https://stackoverflow.com/a/46545530
-    let recommended = instance_stats.recommended
+    let recommended2 = recommended
      .map(value => ({ value, sort: Math.random() }))
      .sort((a, b) => a.sort - b.sort)
      .map(({ value }) => value);
@ -28,8 +42,8 @@ export class Instances extends Component<any, any> {
        {this.header()}
        <br />
        <br />
-        {this.renderList(i18n.t("recommended_instances"), recommended)}
-        {this.renderList(i18n.t("popular_instances"), instance_stats.remaining)}
+        {this.renderList(i18n.t("recommended_instances"), recommended2)}
+        {this.renderList(i18n.t("popular_instances"), remaining)}
      </div>
    );
  }
@ -38,8 +52,8 @@ export class Instances extends Component<any, any> {
    return (
      <i>
        {i18n.t("instance_totals", {
-          instances: numToSI(instance_stats.crawled_instances),
-          users: numToSI(instance_stats.total_users),
+          instances: numToSI(instance_stats.stats.crawled_instances),
+          users: numToSI(instance_stats.stats.total_users),
        })}
      </i>
    );
@ -48,42 +62,51 @@ export class Instances extends Component<any, any> {
  renderList(header: string, instances: any[]) {
    return (
      <div>
-        <Helmet title={title}>
-          <meta property={"title"} content={title} />
-        </Helmet>
-        <div class="container">
-          <h1 class="is-marginless">{header}</h1>
-
-          {instances.map(i => (
-            <div class="card">
-              <img
-                class="join-icon"
-                src={i.icon || "/static/assets/images/lemmy.svg"}
-              />
-              <div class="join-text">
-                <div class="join-header">
-                  <h4 class="join-title">{i.name}</h4>
-                  <i>
-                    {numToSI(i.users_active_month)} {i18n.t("users")} /{" "}
-                    {i18n.t("month")}
-                  </i>
+        <h2>{header}</h2>
+        <div class="row">
+          {instances.map(instance => {
+            let domain = instance.domain;
+            let users_active_month =
+              instance.site_info.site_view.counts.users_active_month;
+            let description = instance.site_info.site_view.site.description;
+            let icon = instance.site_info.site_view.site.icon;
+            let require_application =
+              instance.site_info.site_view.site.require_application;
+            return (
+              <div class="card col-6">
+                <header>
+                  <div class="row">
+                    <h4 class="col">{domain}</h4>
+                    <h4 class="col text-right">
+                      <i>
+                        {numToSI(users_active_month)} {i18n.t("users")} /{" "}
+                        {i18n.t("month")}
+                      </i>
+                    </h4>
+                  </div>
+                </header>
+                <div class="is-center">
+                  <img
+                    class="join-banner"
+                    src={icon || "/static/assets/images/lemmy.svg"}
+                  />
                </div>
-                <p class="join-desc">{i.description}</p>
+                <br />
+                <p class="join-desc">{description}</p>
+                <footer>
+                  {require_application ? (
+                    <a class="button primary" href={`https://${domain}`}>
+                      {i18n.t("apply_to_join")}
+                    </a>
+                  ) : (
+                    <a class="button primary" href={`https://${domain}`}>
+                      {i18n.t("join")}
+                    </a>
+                  )}
+                </footer>
              </div>
-              {i.require_application ? (
-                <a
-                  class="button primary button-yellow"
-                  href={`https://${i.domain}`}
-                >
-                  {i18n.t("apply_to_join")}
-                </a>
-              ) : (
-                <a class="button primary" href={`https://${i.domain}`}>
-                  {i18n.t("join")}
-                </a>
-              )}
-            </div>
-          ))}
+            );
+          })}
        </div>
      </div>
    );
--- a/src/shared/instance_stats.ts
+++ b/src/shared/instance_stats.ts
--- a/update_submodules.sh
+++ b/update_submodules.sh
@ -11,11 +11,6 @@ git fetch weblate
 git merge weblate/main
 git push

-popd
-pushd ../lemmy-instance-stats
-./update.sh
-popd
-
 git submodule update --remote
 git add joinlemmy-translations
 git add lemmy-translations
				`@ -1 +0,0 @@`
				`Subproject commit 75824524a2a86094e863f0333ea22fb3aa39ce46`
				`@ -0,0 +1 @@`
				`Subproject commit d11febc7e80ae7ea37bd57f00163d71e4b48a918`