From 2f6caa3f863de6e2821ec967c1ec272da159fff4 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Thu, 28 Apr 2022 14:04:50 +0200 Subject: [PATCH 1/2] Add separate instance recommendations for each language --- .gitmodules | 4 - generate_translations.mjs | 40 +- lemmy-instance-stats | 1 - recommended-instances.json | 11 + src/shared/components/instances.tsx | 27 +- src/shared/instance_stats.ts | 630 ++++++++++++++++++++++++++++ 6 files changed, 685 insertions(+), 28 deletions(-) delete mode 160000 lemmy-instance-stats create mode 100644 recommended-instances.json create mode 100644 src/shared/instance_stats.ts diff --git a/.gitmodules b/.gitmodules index 6025dae..4d5221e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,10 +10,6 @@ path = lemmy-translations url = https://github.com/lemmynet/lemmy-translations branch = main -[submodule "lemmy-instance-stats"] - path = lemmy-instance-stats - url = https://github.com/LemmyNet/lemmy-instance-stats - branch = main [submodule "lemmy-js-client"] path = lemmy-js-client url = https://github.com/LemmyNet/lemmy-js-client diff --git a/generate_translations.mjs b/generate_translations.mjs index 36fdc84..46fc9bd 100644 --- a/generate_translations.mjs +++ b/generate_translations.mjs @@ -1,42 +1,48 @@ import fs from 'fs'; import path from 'path'; +import { exit } from 'process'; +import { spawnSync } from 'child_process'; const translationDir = "joinlemmy-translations/translations/"; const outDir = "src/shared/translations/"; const translatorsJsonFile = "lemmy-translations/translators.json"; -const statsFile = "lemmy-instance-stats/stats.json"; -const recommendationsFile = "lemmy-instance-stats/recommended-instances.csv"; +const recommendationsFile = "recommended-instances.json"; +const instanceStatsFile = "src/shared/instance_stats.ts"; const newsDir = "src/assets/news"; fs.mkdirSync(outDir, { recursive: true }); // Write the stats file try { - const stats = JSON.parse(fs.readFileSync(statsFile, "utf8")); - const recommended_domains = fs.readFileSync(recommendationsFile, "utf8").trim().split(','); - console.log(recommended_domains); - const recommended = stats.instance_details.filter(i => - recommended_domains.includes(i.domain) - ); - const remaining = stats.instance_details.filter(i => - !recommended_domains.includes(i.domain) - ); + const recommended_instances = JSON.parse(fs.readFileSync(recommendationsFile, "utf8")); + var all_recommended = []; + for (var k in recommended_instances) { + if (k != "exclude") { + all_recommended.push(...recommended_instances[k]); + } + } + const run = spawnSync("cargo", + ["run", "--", "--start-instances", all_recommended, "--exclude", recommended_instances.exclude], { + cwd: "../lemmy-stats-crawler/", + encoding : 'utf8' + }); + //console.log("crawler run output: ", run.output); + const stats = JSON.parse(run.stdout); let stats2 = { - crawled_instances: stats.crawled_instances, - total_users: stats.total_users, - recommended: recommended, - remaining: remaining, + stats: stats, + recommended: recommended_instances } let data = `export const instance_stats = \n `; data += JSON.stringify(stats2, null, 2) + ";"; - const target = outDir + "instance_stats.ts"; - fs.writeFileSync(target, data); + fs.writeFileSync(instanceStatsFile, data); } catch (err) { console.error(err); } +exit; + // Write the news file try { let files = fs.readdirSync(newsDir); diff --git a/lemmy-instance-stats b/lemmy-instance-stats deleted file mode 160000 index 7582452..0000000 --- a/lemmy-instance-stats +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 75824524a2a86094e863f0333ea22fb3aa39ce46 diff --git a/recommended-instances.json b/recommended-instances.json new file mode 100644 index 0000000..d20ac23 --- /dev/null +++ b/recommended-instances.json @@ -0,0 +1,11 @@ +{ + "en": ["sopuli.xyz", "beehaw.org"], + "de": ["feddit.de"], + "pt": ["lemmy.pt"], + "pt_BR": ["lemmy.pt"], + "pl": ["szmer.info"], + "eu": ["lemmy.eus"], + "ja": ["tabinezumi.net", "lm.korako.me"], + "es": ["forum.nobigtech.es"], + "exclude": ["lemmy.glasgow.social","ds9.lemmy.ml","voyager.lemmy.ml","enterprise.lemmy.ml"] +} \ No newline at end of file diff --git a/src/shared/components/instances.tsx b/src/shared/components/instances.tsx index fe2db28..f2e3eea 100644 --- a/src/shared/components/instances.tsx +++ b/src/shared/components/instances.tsx @@ -1,7 +1,7 @@ import { Component } from "inferno"; import { Helmet } from "inferno-helmet"; import { i18n } from "../i18next"; -import { instance_stats } from "../translations/instance_stats"; +import { instance_stats } from "../instance_stats"; import { numToSI } from "../utils"; const title = i18n.t("join_title"); @@ -12,9 +12,24 @@ export class Instances extends Component { } render() { + var recommended_instances = instance_stats.recommended[i18n.language]; + if (!recommended_instances) { + recommended_instances = instance_stats.recommended["en"]; + } + console.log(recommended_instances); + var recommended = []; + var remaining = []; + for (var i of instance_stats.stats.instance_details) { + if (recommended_instances.indexOf(i.domain) > -1) { + console.log(i.domain); + recommended.push(i); + } else { + remaining.push(i); + } + } // shuffle recommended instances list into random order // https://stackoverflow.com/a/46545530 - let recommended = instance_stats.recommended + let recommended2 = recommended .map(value => ({ value, sort: Math.random() })) .sort((a, b) => a.sort - b.sort) .map(({ value }) => value); @@ -28,8 +43,8 @@ export class Instances extends Component { {this.header()}

- {this.renderList(i18n.t("recommended_instances"), recommended)} - {this.renderList(i18n.t("popular_instances"), instance_stats.remaining)} + {this.renderList(i18n.t("recommended_instances"), recommended2)} + {this.renderList(i18n.t("popular_instances"), remaining)} ); } @@ -38,8 +53,8 @@ export class Instances extends Component { return ( {i18n.t("instance_totals", { - instances: numToSI(instance_stats.crawled_instances), - users: numToSI(instance_stats.total_users), + instances: numToSI(instance_stats.stats.crawled_instances), + users: numToSI(instance_stats.stats.total_users), })} ); diff --git a/src/shared/instance_stats.ts b/src/shared/instance_stats.ts new file mode 100644 index 0000000..33c3ce5 --- /dev/null +++ b/src/shared/instance_stats.ts @@ -0,0 +1,630 @@ +export const instance_stats = { + stats: { + crawled_instances: 42, + failed_instances: 118, + total_users: 27447, + total_online_users: 355, + instance_details: [ + { + domain: "lemmygrad.ml", + name: "Lemmygrad", + description: + "A collection of leftist communities, for memes, learning, news, discussion, media, or anything you like.", + version: "0.16.3", + icon: "https://lemmygrad.ml/pictrs/image/gB8yP0oFF5.png", + online_users: 151, + total_users: 6373, + users_active_halfyear: 1177, + users_active_month: 748, + open_registrations: true, + linked_instances_count: 20, + require_application: true, + }, + { + domain: "lemmy.ml", + name: "Lemmy", + description: + "A community of leftist privacy and FOSS enthusiasts, run by Lemmy’s developers", + version: "0.16.3", + icon: "https://lemmy.ml/pictrs/image/bhQ7ELa4oq.webp", + online_users: 131, + total_users: 16719, + users_active_halfyear: 1955, + users_active_month: 555, + open_registrations: true, + linked_instances_count: 56, + require_application: true, + }, + { + domain: "szmer.info", + name: "szmer", + description: "polskojęzyczna instancja lemmy-iego. ", + version: "0.16.1", + icon: "https://szmer.info/pictrs/image/XcQWr0D2CU.png", + online_users: 16, + total_users: 1209, + users_active_halfyear: 346, + users_active_month: 94, + open_registrations: true, + linked_instances_count: 8, + require_application: true, + }, + { + domain: "wolfballs.com", + name: "Wolfballs", + description: + "A decentralized federated community of freedom fighting meme farmers ", + version: "0.16.3", + icon: "https://wolfballs.com/pictrs/image/UYZ2wquwlB.png", + online_users: 7, + total_users: 245, + users_active_halfyear: 128, + users_active_month: 42, + open_registrations: true, + linked_instances_count: 13, + require_application: false, + }, + { + domain: "community.xmpp.net", + name: "XMPP Community", + description: + "A community space for projects and users of the Extensible Messaging and Presence Protocol (XMPP).", + version: "0.16.3", + icon: "https://community.xmpp.net/pictrs/image/35c85076-7971-4e2a-b5d1-18ef437366ff.png", + online_users: 6, + total_users: 47, + users_active_halfyear: 34, + users_active_month: 34, + open_registrations: true, + linked_instances_count: 7, + require_application: true, + }, + { + domain: "feddit.de", + name: "Feddit", + description: "Deutschsprachige Lemmy Community", + version: "0.16.3", + icon: "https://feddit.de/pictrs/image/uI7Q7MuePp.png", + online_users: 3, + total_users: 201, + users_active_halfyear: 68, + users_active_month: 33, + open_registrations: true, + linked_instances_count: 78, + require_application: true, + }, + { + domain: "lemmy.perthchat.org", + name: "PerthChat", + description: "The Perth Lemmy Instance", + version: "0.16.3", + icon: "https://lemmy.perthchat.org/pictrs/image/d3728083-9d5f-4311-8c9b-d9dda43f7413.png", + online_users: 0, + total_users: 36, + users_active_halfyear: 26, + users_active_month: 26, + open_registrations: true, + linked_instances_count: 9, + require_application: true, + }, + { + domain: "sopuli.xyz", + name: "Sopuli", + description: + "A general-purpose instance run by a Finn - everyone is welcome here!", + version: "0.16.3", + icon: "https://sopuli.xyz/pictrs/image/AjMQEWabkH.png", + online_users: 2, + total_users: 377, + users_active_halfyear: 83, + users_active_month: 21, + open_registrations: true, + linked_instances_count: 23, + require_application: false, + }, + { + domain: "beehaw.org", + name: "Beehaw", + description: "Aspiring to be(e) a safe, friendly and diverse place.", + version: "0.16.3", + icon: "https://beehaw.org/pictrs/image/33fad824-7820-41c0-9c4a-0d7cf831ddd3.png", + online_users: 2, + total_users: 85, + users_active_halfyear: 47, + users_active_month: 18, + open_registrations: true, + linked_instances_count: 4, + require_application: true, + }, + { + domain: "slrpnk.net", + name: "SLRPNK", + description: "where solarpunks organize for a better world!", + version: "0.16.3", + icon: "https://slrpnk.net/pictrs/image/9d582815-3984-4996-9b10-b18706bebac1.png", + online_users: 6, + total_users: 31, + users_active_halfyear: 18, + users_active_month: 18, + open_registrations: true, + linked_instances_count: 5, + require_application: true, + }, + { + domain: "exploding-heads.com", + name: "Exploding Heads", + description: "Things that make your head explode", + version: "0.16.3", + icon: "https://exploding-heads.com/pictrs/image/f48abd93-f2d9-4a44-a327-380e063744b1.png", + online_users: 1, + total_users: 16, + users_active_halfyear: 15, + users_active_month: 15, + open_registrations: true, + linked_instances_count: 19, + require_application: false, + }, + { + domain: "collapse.cat", + name: "Col·lapse / Colapso", + description: "Col·lapse, emergència climàtica i temes relacionats", + version: "0.15.1", + icon: "https://collapse.cat/pictrs/image/NUpYQehEdv.png", + online_users: 2, + total_users: 75, + users_active_halfyear: 58, + users_active_month: 14, + open_registrations: true, + linked_instances_count: 11, + require_application: false, + }, + { + domain: "mandacaru.caatinga.digital", + name: "Mandacaru", + description: null, + version: "0.14.1", + icon: null, + online_users: 0, + total_users: 35, + users_active_halfyear: 21, + users_active_month: 12, + open_registrations: true, + linked_instances_count: 3, + require_application: false, + }, + { + domain: "lemmy.ca", + name: "Lemmy dot C.A.", + description: + "A canadian-run community, geared towards canadians, but all are welcome!", + version: "0.16.3", + icon: "https://lemmy.ca/pictrs/image/AX0BZv78yT.png", + online_users: 0, + total_users: 311, + users_active_halfyear: 104, + users_active_month: 11, + open_registrations: true, + linked_instances_count: 22, + require_application: true, + }, + { + domain: "lemmy.eus", + name: "Lemmy.eus", + description: + "Euskarazko lehen web-foro irekiak. Software librea, fedibertsoa, euskalmemeak, literatura... (Basque language)", + version: "0.16.2", + icon: "https://lemmy.eus/pictrs/image/R55fPm9RfM.png", + online_users: 9, + total_users: 475, + users_active_halfyear: 150, + users_active_month: 9, + open_registrations: true, + linked_instances_count: 10, + require_application: true, + }, + { + domain: "mander.xyz", + name: "Mander", + description: "An instance dedicated to nature and science.", + version: "0.16.3", + icon: "https://mander.xyz/pictrs/image/2fc86ef3-797e-4bcd-bcca-e75cdf0959f9.png", + online_users: 0, + total_users: 70, + users_active_halfyear: 27, + users_active_month: 8, + open_registrations: true, + linked_instances_count: 23, + require_application: true, + }, + { + domain: "lemmy.pt", + name: "Lemmy Portugal 🇵🇹", + description: + "Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.", + version: "0.16.3", + icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png", + online_users: 0, + total_users: 86, + users_active_halfyear: 21, + users_active_month: 7, + open_registrations: true, + linked_instances_count: 4, + require_application: true, + }, + { + domain: "lemmy.pt", + name: "Lemmy Portugal 🇵🇹", + description: + "Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.", + version: "0.16.3", + icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png", + online_users: 0, + total_users: 86, + users_active_halfyear: 21, + users_active_month: 7, + open_registrations: true, + linked_instances_count: 4, + require_application: true, + }, + { + domain: "gtio.io", + name: "Go Talk It Out", + description: "Politically-neutral forum for serious debate", + version: "0.16.3", + icon: "https://gtio.io/pictrs/image/32f393ce-2895-462c-8f9b-25c31d713a05.png", + online_users: 1, + total_users: 7, + users_active_halfyear: 6, + users_active_month: 6, + open_registrations: true, + linked_instances_count: 3, + require_application: true, + }, + { + domain: "midwest.social", + name: "midwest.social", + description: + "A lemmy server for, but not limited to, leftists in the Midwest USA", + version: "0.16.3", + icon: "https://midwest.social/pictrs/image/Rbal22EuF8.png", + online_users: 1, + total_users: 111, + users_active_halfyear: 26, + users_active_month: 5, + open_registrations: true, + linked_instances_count: 12, + require_application: true, + }, + { + domain: "lemmy.juggler.jp", + name: "Juggler.jp Lemmyサービス", + description: null, + version: "0.16.1", + icon: "https://lemmy.juggler.jp/pictrs/image/v0TGrlnKGS.png", + online_users: 1, + total_users: 30, + users_active_halfyear: 13, + users_active_month: 4, + open_registrations: true, + linked_instances_count: 13, + require_application: false, + }, + { + domain: "baraza.africa", + name: "Baraza", + description: + "A space to share and discusss Africa-related content. Karibu.", + version: "0.16.2", + icon: "https://baraza.africa/pictrs/image/qFpb6BEV2c.png", + online_users: 3, + total_users: 352, + users_active_halfyear: 77, + users_active_month: 4, + open_registrations: true, + linked_instances_count: 16, + require_application: true, + }, + { + domain: "forum.nobigtech.es", + name: "Foro de NoBIGTech", + description: "Foro de NoBIGTech", + version: "0.16.3", + icon: "https://forum.nobigtech.es/pictrs/image/atwGqrgiHu.png", + online_users: 0, + total_users: 8, + users_active_halfyear: 4, + users_active_month: 4, + open_registrations: true, + linked_instances_count: 3, + require_application: true, + }, + { + domain: "fapsi.be", + name: "Fapsi", + description: + "This instance is for all the creative. No matter if you're a writer, author or illustrator.", + version: "0.16.3", + icon: "https://fapsi.be/pictrs/image/jUD7V242rr.png", + online_users: 1, + total_users: 150, + users_active_halfyear: 30, + users_active_month: 3, + open_registrations: true, + linked_instances_count: 14, + require_application: false, + }, + { + domain: "lemmy.cat", + name: "Lemmy CAT", + description: "Cultura catalana", + version: "0.16.3", + icon: "https://lemmy.cat/pictrs/image/4iwqIfDlgI.png", + online_users: 1, + total_users: 73, + users_active_halfyear: 8, + users_active_month: 3, + open_registrations: true, + linked_instances_count: 9, + require_application: true, + }, + { + domain: "tabinezumi.net", + name: "たびねずみのみみ", + description: "日本語圏Lemmyインスタンスのひとつ", + version: "0.16.3", + icon: null, + online_users: 0, + total_users: 35, + users_active_halfyear: 6, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 9, + require_application: true, + }, + { + domain: "lm.korako.me", + name: "鴉は拠り所について語り合う", + description: "日本語話者向けLemmyインスタンスです", + version: "0.16.3", + icon: null, + online_users: 0, + total_users: 17, + users_active_halfyear: 7, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 6, + require_application: true, + }, + { + domain: "buckeyestate.social", + name: "buckeyestate.social", + description: + "A lemmy server for, but not limited to, leftists of the state of Ohio", + version: "0.16.3", + icon: "https://buckeyestate.social/pictrs/image/a9b79b22-7346-40fc-b10c-87d486215cde.png", + online_users: 1, + total_users: 3, + users_active_halfyear: 2, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 3, + require_application: true, + }, + { + domain: "eope.xyz", + name: "EOPE", + description: "Engineers On Planet Earth", + version: "0.15.1", + icon: "https://eope.xyz/pictrs/image/870ec1de-0f6f-4f8e-88a2-c02d2a75adb5.jpeg", + online_users: 0, + total_users: 22, + users_active_halfyear: 4, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 5, + require_application: false, + }, + { + domain: "lemmy.rollenspiel.monster", + name: "RollenspielMonster", + description: + "Eine deutschsprachige Instanz für Rollenspieler. Wir bieten einen platz für Rollenspiel, Pen & Paper, Tabletop, TCG und vieles mehr.", + version: "0.16.3", + icon: "https://lemmy.rollenspiel.monster/pictrs/image/040c05a3-ba7c-4e66-a225-5209dd4893f9.png", + online_users: 2, + total_users: 3, + users_active_halfyear: 2, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 4, + require_application: false, + }, + { + domain: "fuckreddit.tryp.digital", + name: "TRYP", + description: "Now 50% Prod!", + version: "0.16.3", + icon: "https://fuckreddit.tryp.digital/pictrs/image/52dc0c6c-71a4-40f9-a326-b51d54c95173.png", + online_users: 3, + total_users: 5, + users_active_halfyear: 2, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 13, + require_application: true, + }, + { + domain: "info.prou.be", + name: "info.prou.be", + description: "infopoint virtual", + version: "0.16.1", + icon: "https://info.prou.be/pictrs/image/82e736ba-6dee-423b-b06b-b47f994ce6b6.png", + online_users: 0, + total_users: 10, + users_active_halfyear: 4, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 5, + require_application: true, + }, + { + domain: "stammtisch.hallertau.social", + name: "Stammtisch", + description: null, + version: "0.16.3", + icon: null, + online_users: 0, + total_users: 55, + users_active_halfyear: 20, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 9, + require_application: true, + }, + { + domain: "lemmy.otakufarms.com", + name: "Otaku Farms", + description: + "Otaku Farms is a network of Fediverse instances created for the sole purpose of letting individuals express themselves.", + version: "0.14.3", + icon: "https://lemmy.otakufarms.com/pictrs/image/ekmeJ0BD8S.png", + online_users: 0, + total_users: 3, + users_active_halfyear: 3, + users_active_month: 1, + open_registrations: true, + linked_instances_count: 1, + require_application: false, + }, + { + domain: "elgiebety.pl", + name: "elgiebety", + description: null, + version: "0.15.2", + icon: null, + online_users: 1, + total_users: 4, + users_active_halfyear: 1, + users_active_month: 1, + open_registrations: true, + linked_instances_count: 3, + require_application: false, + }, + { + domain: "heapoverflow.ml", + name: "Heap Overflow", + description: + "A place to ask programming questions and share free resources", + version: "0.16.3", + icon: "https://heapoverflow.ml/pictrs/image/f910bf1c-215b-47c0-a9ff-d65f483c02a3.png", + online_users: 1, + total_users: 11, + users_active_halfyear: 1, + users_active_month: 1, + open_registrations: true, + linked_instances_count: 3, + require_application: true, + }, + { + domain: "lemmy.schuerz.at", + name: "lemmy.schuerz.at", + description: null, + version: "0.16.3", + icon: null, + online_users: 2, + total_users: 40, + users_active_halfyear: 2, + users_active_month: 1, + open_registrations: false, + linked_instances_count: 15, + require_application: true, + }, + { + domain: "lemmy.wiredentrypoint.xyz", + name: "Wiredentrypoint", + description: null, + version: "0.16.1", + icon: "https://lemmy.wiredentrypoint.xyz/pictrs/image/826b6251-bf21-41b4-b8f2-7a39a0b09313.png", + online_users: 0, + total_users: 1, + users_active_halfyear: 1, + users_active_month: 1, + open_registrations: false, + linked_instances_count: 4, + require_application: false, + }, + { + domain: "lemmy.mesh.party", + name: "Mesh.Party", + description: "A Lemmy instance for mesh network denizens.", + version: "0.16.3", + icon: "https://lemmy.mesh.party/pictrs/image/3653fdce-1fdc-4514-aceb-38c6ba29006c.png", + online_users: 1, + total_users: 8, + users_active_halfyear: 2, + users_active_month: 1, + open_registrations: true, + linked_instances_count: 4, + require_application: false, + }, + { + domain: "verity.fail", + name: "Verity Fail", + description: null, + version: "0.16.0", + icon: null, + online_users: 0, + total_users: 19, + users_active_halfyear: 9, + users_active_month: 1, + open_registrations: true, + linked_instances_count: 45, + require_application: true, + }, + { + domain: "lemmy.shrieker.net", + name: "美食レミー", + description: null, + version: "0.16.3", + icon: null, + online_users: 0, + total_users: 2, + users_active_halfyear: 1, + users_active_month: 1, + open_registrations: false, + linked_instances_count: 2, + require_application: false, + }, + { + domain: "lem.ph3j.com", + name: "lem.ph3j.com", + description: null, + version: "0.16.3", + icon: null, + online_users: 0, + total_users: 1, + users_active_halfyear: 0, + users_active_month: 0, + open_registrations: false, + linked_instances_count: 7, + require_application: false, + }, + ], + }, + recommended: { + en: ["sopuli.xyz", "beehaw.org"], + de: ["feddit.de"], + pt: ["lemmy.pt"], + pt_BR: ["lemmy.pt"], + pl: ["szmer.info"], + eu: ["lemmy.eus"], + ja: ["tabinezumi.net", "lm.korako.me"], + es: ["forum.nobigtech.es"], + exclude: [ + "lemmy.glasgow.social", + "ds9.lemmy.ml", + "voyager.lemmy.ml", + "enterprise.lemmy.ml", + ], + }, +}; From cd0117188143ba79e1b5f1da1978c425a0e1536f Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 3 May 2022 14:23:37 +0200 Subject: [PATCH 2/2] separate yarn command for crawl, fix docker build --- Dockerfile | 1 - crawl.mjs | 39 ++++ deploy.sh | 4 + generate_translations.mjs | 35 ---- package.json | 1 + recommended-instances.json | 1 - src/shared/instance_stats.ts | 388 +++++++++++++++++++---------------- 7 files changed, 252 insertions(+), 217 deletions(-) create mode 100644 crawl.mjs diff --git a/Dockerfile b/Dockerfile index 0230515..751690c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,6 @@ COPY tsconfig.json \ COPY joinlemmy-translations joinlemmy-translations COPY lemmy-translations lemmy-translations -COPY lemmy-instance-stats lemmy-instance-stats COPY src src # Copy the docs and API diff --git a/crawl.mjs b/crawl.mjs new file mode 100644 index 0000000..1bc6d00 --- /dev/null +++ b/crawl.mjs @@ -0,0 +1,39 @@ +import fs from 'fs'; +import path from 'path'; +import { exit } from 'process'; +import { spawnSync } from 'child_process'; + +const outDir = "src/shared/translations/"; +const recommendationsFile = "recommended-instances.json"; +const instanceStatsFile = "src/shared/instance_stats.ts"; + +fs.mkdirSync(outDir, { recursive: true }); + +// crawl instance stats +try { + const recommended_instances = JSON.parse(fs.readFileSync(recommendationsFile, "utf8")); + var all_recommended = []; + for (var k in recommended_instances) { + if (k != "exclude") { + all_recommended.push(...recommended_instances[k]); + } + } + const run = spawnSync("cargo", + ["run", "--", "--start-instances", all_recommended, "--exclude", recommended_instances.exclude], { + cwd: "../lemmy-stats-crawler/", + encoding : 'utf8' + }); + //console.log("crawler run output: ", run.output); + const stats = JSON.parse(run.stdout); + + let stats2 = { + stats: stats, + recommended: recommended_instances + } + + let data = `export const instance_stats = \n `; + data += JSON.stringify(stats2, null, 2) + ";"; + fs.writeFileSync(instanceStatsFile, data); +} catch (err) { + console.error(err); +} diff --git a/deploy.sh b/deploy.sh index 914f5ae..da5f32a 100755 --- a/deploy.sh +++ b/deploy.sh @@ -3,6 +3,10 @@ # Update all the submodules and translations ./update_submodules.sh +yarn crawl +git add "src/shared/instance_stats.ts" +git commit -m "Crawl instance statistics" + # look for unused translations for langfile in joinlemmy-translations/translations/*.json; do lang=$(basename $langfile .json) diff --git a/generate_translations.mjs b/generate_translations.mjs index 46fc9bd..6889961 100644 --- a/generate_translations.mjs +++ b/generate_translations.mjs @@ -1,48 +1,13 @@ import fs from 'fs'; import path from 'path'; -import { exit } from 'process'; -import { spawnSync } from 'child_process'; const translationDir = "joinlemmy-translations/translations/"; const outDir = "src/shared/translations/"; const translatorsJsonFile = "lemmy-translations/translators.json"; -const recommendationsFile = "recommended-instances.json"; -const instanceStatsFile = "src/shared/instance_stats.ts"; const newsDir = "src/assets/news"; fs.mkdirSync(outDir, { recursive: true }); -// Write the stats file -try { - const recommended_instances = JSON.parse(fs.readFileSync(recommendationsFile, "utf8")); - var all_recommended = []; - for (var k in recommended_instances) { - if (k != "exclude") { - all_recommended.push(...recommended_instances[k]); - } - } - const run = spawnSync("cargo", - ["run", "--", "--start-instances", all_recommended, "--exclude", recommended_instances.exclude], { - cwd: "../lemmy-stats-crawler/", - encoding : 'utf8' - }); - //console.log("crawler run output: ", run.output); - const stats = JSON.parse(run.stdout); - - let stats2 = { - stats: stats, - recommended: recommended_instances - } - - let data = `export const instance_stats = \n `; - data += JSON.stringify(stats2, null, 2) + ";"; - fs.writeFileSync(instanceStatsFile, data); -} catch (err) { - console.error(err); -} - -exit; - // Write the news file try { let files = fs.readdirSync(newsDir); diff --git a/package.json b/package.json index 1021cdf..67f351c 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "build:dev": "webpack --mode=development", "build:prod": "webpack --mode=production", "clean": "yarn run rimraf dist", + "crawl": "node crawl.mjs", "lint": "node generate_translations.mjs && tsc --noEmit && eslint --report-unused-disable-directives --ext .js,.ts,.tsx src", "prebuild:dev": "yarn clean && node generate_translations.mjs", "prebuild:prod": "yarn clean && node generate_translations.mjs", diff --git a/recommended-instances.json b/recommended-instances.json index d20ac23..050fd8e 100644 --- a/recommended-instances.json +++ b/recommended-instances.json @@ -3,7 +3,6 @@ "de": ["feddit.de"], "pt": ["lemmy.pt"], "pt_BR": ["lemmy.pt"], - "pl": ["szmer.info"], "eu": ["lemmy.eus"], "ja": ["tabinezumi.net", "lm.korako.me"], "es": ["forum.nobigtech.es"], diff --git a/src/shared/instance_stats.ts b/src/shared/instance_stats.ts index 33c3ce5..6bcc412 100644 --- a/src/shared/instance_stats.ts +++ b/src/shared/instance_stats.ts @@ -1,9 +1,9 @@ export const instance_stats = { stats: { - crawled_instances: 42, - failed_instances: 118, - total_users: 27447, - total_online_users: 355, + crawled_instances: 44, + failed_instances: 61, + total_users: 27594, + total_online_users: 339, instance_details: [ { domain: "lemmygrad.ml", @@ -12,12 +12,12 @@ export const instance_stats = { "A collection of leftist communities, for memes, learning, news, discussion, media, or anything you like.", version: "0.16.3", icon: "https://lemmygrad.ml/pictrs/image/gB8yP0oFF5.png", - online_users: 151, - total_users: 6373, - users_active_halfyear: 1177, - users_active_month: 748, + online_users: 123, + total_users: 6403, + users_active_halfyear: 1196, + users_active_month: 674, open_registrations: true, - linked_instances_count: 20, + linked_instances_count: 21, require_application: true, }, { @@ -27,12 +27,12 @@ export const instance_stats = { "A community of leftist privacy and FOSS enthusiasts, run by Lemmy’s developers", version: "0.16.3", icon: "https://lemmy.ml/pictrs/image/bhQ7ELa4oq.webp", - online_users: 131, - total_users: 16719, + online_users: 117, + total_users: 16764, users_active_halfyear: 1955, users_active_month: 555, open_registrations: true, - linked_instances_count: 56, + linked_instances_count: 58, require_application: true, }, { @@ -41,12 +41,12 @@ export const instance_stats = { description: "polskojęzyczna instancja lemmy-iego. ", version: "0.16.1", icon: "https://szmer.info/pictrs/image/XcQWr0D2CU.png", - online_users: 16, - total_users: 1209, - users_active_halfyear: 346, - users_active_month: 94, + online_users: 10, + total_users: 1218, + users_active_halfyear: 341, + users_active_month: 85, open_registrations: true, - linked_instances_count: 8, + linked_instances_count: 9, require_application: true, }, { @@ -56,14 +56,28 @@ export const instance_stats = { "A decentralized federated community of freedom fighting meme farmers ", version: "0.16.3", icon: "https://wolfballs.com/pictrs/image/UYZ2wquwlB.png", - online_users: 7, - total_users: 245, + online_users: 14, + total_users: 249, users_active_halfyear: 128, - users_active_month: 42, + users_active_month: 36, open_registrations: true, linked_instances_count: 13, require_application: false, }, + { + domain: "feddit.de", + name: "Feddit", + description: "Deutschsprachige Lemmy Community", + version: "0.16.3", + icon: "https://feddit.de/pictrs/image/uI7Q7MuePp.png", + online_users: 12, + total_users: 209, + users_active_halfyear: 69, + users_active_month: 34, + open_registrations: true, + linked_instances_count: 21, + require_application: true, + }, { domain: "community.xmpp.net", name: "XMPP Community", @@ -71,40 +85,12 @@ export const instance_stats = { "A community space for projects and users of the Extensible Messaging and Presence Protocol (XMPP).", version: "0.16.3", icon: "https://community.xmpp.net/pictrs/image/35c85076-7971-4e2a-b5d1-18ef437366ff.png", - online_users: 6, + online_users: 10, total_users: 47, users_active_halfyear: 34, users_active_month: 34, open_registrations: true, - linked_instances_count: 7, - require_application: true, - }, - { - domain: "feddit.de", - name: "Feddit", - description: "Deutschsprachige Lemmy Community", - version: "0.16.3", - icon: "https://feddit.de/pictrs/image/uI7Q7MuePp.png", - online_users: 3, - total_users: 201, - users_active_halfyear: 68, - users_active_month: 33, - open_registrations: true, - linked_instances_count: 78, - require_application: true, - }, - { - domain: "lemmy.perthchat.org", - name: "PerthChat", - description: "The Perth Lemmy Instance", - version: "0.16.3", - icon: "https://lemmy.perthchat.org/pictrs/image/d3728083-9d5f-4311-8c9b-d9dda43f7413.png", - online_users: 0, - total_users: 36, - users_active_halfyear: 26, - users_active_month: 26, - open_registrations: true, - linked_instances_count: 9, + linked_instances_count: 8, require_application: true, }, { @@ -114,42 +100,56 @@ export const instance_stats = { "A general-purpose instance run by a Finn - everyone is welcome here!", version: "0.16.3", icon: "https://sopuli.xyz/pictrs/image/AjMQEWabkH.png", - online_users: 2, - total_users: 377, - users_active_halfyear: 83, - users_active_month: 21, + online_users: 4, + total_users: 386, + users_active_halfyear: 85, + users_active_month: 20, open_registrations: true, - linked_instances_count: 23, + linked_instances_count: 25, require_application: false, }, - { - domain: "beehaw.org", - name: "Beehaw", - description: "Aspiring to be(e) a safe, friendly and diverse place.", - version: "0.16.3", - icon: "https://beehaw.org/pictrs/image/33fad824-7820-41c0-9c4a-0d7cf831ddd3.png", - online_users: 2, - total_users: 85, - users_active_halfyear: 47, - users_active_month: 18, - open_registrations: true, - linked_instances_count: 4, - require_application: true, - }, { domain: "slrpnk.net", name: "SLRPNK", description: "where solarpunks organize for a better world!", version: "0.16.3", icon: "https://slrpnk.net/pictrs/image/9d582815-3984-4996-9b10-b18706bebac1.png", - online_users: 6, - total_users: 31, - users_active_halfyear: 18, + online_users: 2, + total_users: 33, + users_active_halfyear: 20, users_active_month: 18, open_registrations: true, + linked_instances_count: 6, + require_application: true, + }, + { + domain: "beehaw.org", + name: "Beehaw", + description: "Aspiring to be(e) a safe, friendly and diverse place.", + version: "0.16.3", + icon: "https://beehaw.org/pictrs/image/33fad824-7820-41c0-9c4a-0d7cf831ddd3.png", + online_users: 7, + total_users: 89, + users_active_halfyear: 48, + users_active_month: 17, + open_registrations: true, linked_instances_count: 5, require_application: true, }, + { + domain: "collapse.cat", + name: "Col·lapse / Colapso", + description: "Col·lapse, emergència climàtica i temes relacionats", + version: "0.15.1", + icon: "https://collapse.cat/pictrs/image/NUpYQehEdv.png", + online_users: 3, + total_users: 77, + users_active_halfyear: 61, + users_active_month: 17, + open_registrations: true, + linked_instances_count: 11, + require_application: false, + }, { domain: "exploding-heads.com", name: "Exploding Heads", @@ -157,27 +157,13 @@ export const instance_stats = { version: "0.16.3", icon: "https://exploding-heads.com/pictrs/image/f48abd93-f2d9-4a44-a327-380e063744b1.png", online_users: 1, - total_users: 16, - users_active_halfyear: 15, - users_active_month: 15, + total_users: 17, + users_active_halfyear: 16, + users_active_month: 16, open_registrations: true, linked_instances_count: 19, require_application: false, }, - { - domain: "collapse.cat", - name: "Col·lapse / Colapso", - description: "Col·lapse, emergència climàtica i temes relacionats", - version: "0.15.1", - icon: "https://collapse.cat/pictrs/image/NUpYQehEdv.png", - online_users: 2, - total_users: 75, - users_active_halfyear: 58, - users_active_month: 14, - open_registrations: true, - linked_instances_count: 11, - require_application: false, - }, { domain: "mandacaru.caatinga.digital", name: "Mandacaru", @@ -185,7 +171,7 @@ export const instance_stats = { version: "0.14.1", icon: null, online_users: 0, - total_users: 35, + total_users: 36, users_active_halfyear: 21, users_active_month: 12, open_registrations: true, @@ -199,14 +185,28 @@ export const instance_stats = { "A canadian-run community, geared towards canadians, but all are welcome!", version: "0.16.3", icon: "https://lemmy.ca/pictrs/image/AX0BZv78yT.png", - online_users: 0, - total_users: 311, - users_active_halfyear: 104, + online_users: 3, + total_users: 315, + users_active_halfyear: 102, users_active_month: 11, open_registrations: true, linked_instances_count: 22, require_application: true, }, + { + domain: "lemmy.perthchat.org", + name: "PerthChat", + description: "The Perth Lemmy Instance", + version: "0.16.3", + icon: "https://lemmy.perthchat.org/pictrs/image/d3728083-9d5f-4311-8c9b-d9dda43f7413.png", + online_users: 0, + total_users: 36, + users_active_halfyear: 26, + users_active_month: 9, + open_registrations: true, + linked_instances_count: 9, + require_application: true, + }, { domain: "lemmy.eus", name: "Lemmy.eus", @@ -215,9 +215,9 @@ export const instance_stats = { version: "0.16.2", icon: "https://lemmy.eus/pictrs/image/R55fPm9RfM.png", online_users: 9, - total_users: 475, - users_active_halfyear: 150, - users_active_month: 9, + total_users: 478, + users_active_halfyear: 143, + users_active_month: 8, open_registrations: true, linked_instances_count: 10, require_application: true, @@ -228,44 +228,14 @@ export const instance_stats = { description: "An instance dedicated to nature and science.", version: "0.16.3", icon: "https://mander.xyz/pictrs/image/2fc86ef3-797e-4bcd-bcca-e75cdf0959f9.png", - online_users: 0, - total_users: 70, + online_users: 2, + total_users: 72, users_active_halfyear: 27, users_active_month: 8, open_registrations: true, linked_instances_count: 23, require_application: true, }, - { - domain: "lemmy.pt", - name: "Lemmy Portugal 🇵🇹", - description: - "Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.", - version: "0.16.3", - icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png", - online_users: 0, - total_users: 86, - users_active_halfyear: 21, - users_active_month: 7, - open_registrations: true, - linked_instances_count: 4, - require_application: true, - }, - { - domain: "lemmy.pt", - name: "Lemmy Portugal 🇵🇹", - description: - "Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.", - version: "0.16.3", - icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png", - online_users: 0, - total_users: 86, - users_active_halfyear: 21, - users_active_month: 7, - open_registrations: true, - linked_instances_count: 4, - require_application: true, - }, { domain: "gtio.io", name: "Go Talk It Out", @@ -273,13 +243,43 @@ export const instance_stats = { version: "0.16.3", icon: "https://gtio.io/pictrs/image/32f393ce-2895-462c-8f9b-25c31d713a05.png", online_users: 1, - total_users: 7, + total_users: 9, users_active_halfyear: 6, users_active_month: 6, open_registrations: true, linked_instances_count: 3, require_application: true, }, + { + domain: "lemmy.pt", + name: "Lemmy Portugal 🇵🇹", + description: + "Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.", + version: "0.16.3", + icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png", + online_users: 2, + total_users: 87, + users_active_halfyear: 20, + users_active_month: 5, + open_registrations: true, + linked_instances_count: 4, + require_application: true, + }, + { + domain: "lemmy.pt", + name: "Lemmy Portugal 🇵🇹", + description: + "Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.", + version: "0.16.3", + icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png", + online_users: 2, + total_users: 87, + users_active_halfyear: 20, + users_active_month: 5, + open_registrations: true, + linked_instances_count: 4, + require_application: true, + }, { domain: "midwest.social", name: "midwest.social", @@ -288,7 +288,7 @@ export const instance_stats = { version: "0.16.3", icon: "https://midwest.social/pictrs/image/Rbal22EuF8.png", online_users: 1, - total_users: 111, + total_users: 113, users_active_halfyear: 26, users_active_month: 5, open_registrations: true, @@ -303,7 +303,7 @@ export const instance_stats = { icon: "https://lemmy.juggler.jp/pictrs/image/v0TGrlnKGS.png", online_users: 1, total_users: 30, - users_active_halfyear: 13, + users_active_halfyear: 12, users_active_month: 4, open_registrations: true, linked_instances_count: 13, @@ -316,9 +316,9 @@ export const instance_stats = { "A space to share and discusss Africa-related content. Karibu.", version: "0.16.2", icon: "https://baraza.africa/pictrs/image/qFpb6BEV2c.png", - online_users: 3, - total_users: 352, - users_active_halfyear: 77, + online_users: 2, + total_users: 354, + users_active_halfyear: 76, users_active_month: 4, open_registrations: true, linked_instances_count: 16, @@ -331,28 +331,13 @@ export const instance_stats = { version: "0.16.3", icon: "https://forum.nobigtech.es/pictrs/image/atwGqrgiHu.png", online_users: 0, - total_users: 8, + total_users: 9, users_active_halfyear: 4, users_active_month: 4, open_registrations: true, linked_instances_count: 3, require_application: true, }, - { - domain: "fapsi.be", - name: "Fapsi", - description: - "This instance is for all the creative. No matter if you're a writer, author or illustrator.", - version: "0.16.3", - icon: "https://fapsi.be/pictrs/image/jUD7V242rr.png", - online_users: 1, - total_users: 150, - users_active_halfyear: 30, - users_active_month: 3, - open_registrations: true, - linked_instances_count: 14, - require_application: false, - }, { domain: "lemmy.cat", name: "Lemmy CAT", @@ -360,13 +345,27 @@ export const instance_stats = { version: "0.16.3", icon: "https://lemmy.cat/pictrs/image/4iwqIfDlgI.png", online_users: 1, - total_users: 73, + total_users: 74, users_active_halfyear: 8, users_active_month: 3, open_registrations: true, - linked_instances_count: 9, + linked_instances_count: 10, require_application: true, }, + { + domain: "elgiebety.pl", + name: "elgiebety", + description: null, + version: "0.15.2", + icon: null, + online_users: 1, + total_users: 4, + users_active_halfyear: 2, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 3, + require_application: false, + }, { domain: "tabinezumi.net", name: "たびねずみのみみ", @@ -392,7 +391,7 @@ export const instance_stats = { users_active_halfyear: 7, users_active_month: 2, open_registrations: true, - linked_instances_count: 6, + linked_instances_count: 7, require_application: true, }, { @@ -431,12 +430,27 @@ export const instance_stats = { "Eine deutschsprachige Instanz für Rollenspieler. Wir bieten einen platz für Rollenspiel, Pen & Paper, Tabletop, TCG und vieles mehr.", version: "0.16.3", icon: "https://lemmy.rollenspiel.monster/pictrs/image/040c05a3-ba7c-4e66-a225-5209dd4893f9.png", - online_users: 2, + online_users: 1, total_users: 3, users_active_halfyear: 2, users_active_month: 2, open_registrations: true, - linked_instances_count: 4, + linked_instances_count: 2, + require_application: false, + }, + { + domain: "fapsi.be", + name: "Fapsi", + description: + "This instance is for all the creative. No matter if you're a writer, author or illustrator.", + version: "0.16.3", + icon: "https://fapsi.be/pictrs/image/jUD7V242rr.png", + online_users: 1, + total_users: 154, + users_active_halfyear: 29, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 15, require_application: false, }, { @@ -445,12 +459,12 @@ export const instance_stats = { description: "Now 50% Prod!", version: "0.16.3", icon: "https://fuckreddit.tryp.digital/pictrs/image/52dc0c6c-71a4-40f9-a326-b51d54c95173.png", - online_users: 3, + online_users: 2, total_users: 5, users_active_halfyear: 2, users_active_month: 2, open_registrations: true, - linked_instances_count: 13, + linked_instances_count: 14, require_application: true, }, { @@ -459,7 +473,7 @@ export const instance_stats = { description: "infopoint virtual", version: "0.16.1", icon: "https://info.prou.be/pictrs/image/82e736ba-6dee-423b-b06b-b47f994ce6b6.png", - online_users: 0, + online_users: 2, total_users: 10, users_active_halfyear: 4, users_active_month: 2, @@ -467,6 +481,20 @@ export const instance_stats = { linked_instances_count: 5, require_application: true, }, + { + domain: "masr.social", + name: "Masr", + description: null, + version: "0.16.3", + icon: null, + online_users: 0, + total_users: 3, + users_active_halfyear: 2, + users_active_month: 2, + open_registrations: true, + linked_instances_count: 3, + require_application: false, + }, { domain: "stammtisch.hallertau.social", name: "Stammtisch", @@ -481,33 +509,33 @@ export const instance_stats = { linked_instances_count: 9, require_application: true, }, + { + domain: "goldandblack.us.to", + name: "goldandblack-lemmy", + description: null, + version: "0.14.3", + icon: "https://goldandblack.us.to/pictrs/image/DDIh7ojBjd.png", + online_users: 0, + total_users: 4, + users_active_halfyear: 2, + users_active_month: 1, + open_registrations: true, + linked_instances_count: 3, + require_application: false, + }, { domain: "lemmy.otakufarms.com", name: "Otaku Farms", description: "Otaku Farms is a network of Fediverse instances created for the sole purpose of letting individuals express themselves.", - version: "0.14.3", + version: "0.16.3", icon: "https://lemmy.otakufarms.com/pictrs/image/ekmeJ0BD8S.png", online_users: 0, - total_users: 3, - users_active_halfyear: 3, - users_active_month: 1, - open_registrations: true, - linked_instances_count: 1, - require_application: false, - }, - { - domain: "elgiebety.pl", - name: "elgiebety", - description: null, - version: "0.15.2", - icon: null, - online_users: 1, total_users: 4, - users_active_halfyear: 1, + users_active_halfyear: 4, users_active_month: 1, open_registrations: true, - linked_instances_count: 3, + linked_instances_count: 5, require_application: false, }, { @@ -531,12 +559,12 @@ export const instance_stats = { description: null, version: "0.16.3", icon: null, - online_users: 2, + online_users: 1, total_users: 40, users_active_halfyear: 2, users_active_month: 1, open_registrations: false, - linked_instances_count: 15, + linked_instances_count: 16, require_application: true, }, { @@ -559,12 +587,12 @@ export const instance_stats = { description: "A Lemmy instance for mesh network denizens.", version: "0.16.3", icon: "https://lemmy.mesh.party/pictrs/image/3653fdce-1fdc-4514-aceb-38c6ba29006c.png", - online_users: 1, - total_users: 8, + online_users: 2, + total_users: 9, users_active_halfyear: 2, users_active_month: 1, open_registrations: true, - linked_instances_count: 4, + linked_instances_count: 5, require_application: false, }, {