LemmyNet/joinlemmy-site
2
0
Fork 0
mirror of https://github.com/LemmyNet/joinlemmy-site.git synced 2024-11-21 11:51:17 +00:00
Code Issues Projects Releases Wiki Activity

separate yarn command for crawl, fix docker build

Browse source
This commit is contained in:
Felix Ableitner 2022-05-03 14:23:37 +02:00
parent 2f6caa3f86
commit cd01171881
7 changed files with 252 additions and 217 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Dockerfile
View file

@ -44,7 +44,6 @@ COPY tsconfig.json \
COPY joinlemmy-translations joinlemmy-translations
COPY lemmy-translations lemmy-translations
COPY lemmy-instance-stats lemmy-instance-stats
COPY src src
# Copy the docs and API

39
crawl.mjs Normal file
View file

@ -0,0 +1,39 @@
import fs from 'fs';
import path from 'path';
import { exit } from 'process';
import { spawnSync } from 'child_process';
const outDir = "src/shared/translations/";
const recommendationsFile = "recommended-instances.json";
const instanceStatsFile = "src/shared/instance_stats.ts";
fs.mkdirSync(outDir, { recursive: true });
// crawl instance stats
try {
const recommended_instances = JSON.parse(fs.readFileSync(recommendationsFile, "utf8"));
var all_recommended = [];
for (var k in recommended_instances) {
if (k != "exclude") {
all_recommended.push(...recommended_instances[k]);
}
}
const run = spawnSync("cargo",
["run", "--", "--start-instances", all_recommended, "--exclude", recommended_instances.exclude], {
cwd: "../lemmy-stats-crawler/",
encoding : 'utf8'
});
//console.log("crawler run output: ", run.output);
const stats = JSON.parse(run.stdout);
let stats2 = {
stats: stats,
recommended: recommended_instances
}
let data = `export const instance_stats = \n `;
data += JSON.stringify(stats2, null, 2) + ";";
fs.writeFileSync(instanceStatsFile, data);
} catch (err) {
console.error(err);
}

4
deploy.sh
View file

@ -3,6 +3,10 @@
# Update all the submodules and translations
./update_submodules.sh
yarn crawl
git add "src/shared/instance_stats.ts"
git commit -m "Crawl instance statistics"
# look for unused translations
for langfile in joinlemmy-translations/translations/*.json; do
lang=$(basename $langfile .json)

35
generate_translations.mjs
View file

@ -1,48 +1,13 @@
import fs from 'fs';
import path from 'path';
import { exit } from 'process';
import { spawnSync } from 'child_process';
const translationDir = "joinlemmy-translations/translations/";
const outDir = "src/shared/translations/";
const translatorsJsonFile = "lemmy-translations/translators.json";
const recommendationsFile = "recommended-instances.json";
const instanceStatsFile = "src/shared/instance_stats.ts";
const newsDir = "src/assets/news";
fs.mkdirSync(outDir, { recursive: true });
// Write the stats file
try {
const recommended_instances = JSON.parse(fs.readFileSync(recommendationsFile, "utf8"));
var all_recommended = [];
for (var k in recommended_instances) {
if (k != "exclude") {
all_recommended.push(...recommended_instances[k]);
}
}
const run = spawnSync("cargo",
["run", "--", "--start-instances", all_recommended, "--exclude", recommended_instances.exclude], {
cwd: "../lemmy-stats-crawler/",
encoding : 'utf8'
});
//console.log("crawler run output: ", run.output);
const stats = JSON.parse(run.stdout);
let stats2 = {
stats: stats,
recommended: recommended_instances
}
let data = `export const instance_stats = \n `;
data += JSON.stringify(stats2, null, 2) + ";";
fs.writeFileSync(instanceStatsFile, data);
} catch (err) {
console.error(err);
}
exit;
// Write the news file
try {
let files = fs.readdirSync(newsDir);

1
package.json
View file

@ -7,6 +7,7 @@
"build:dev": "webpack --mode=development",
"build:prod": "webpack --mode=production",
"clean": "yarn run rimraf dist",
"crawl": "node crawl.mjs",
"lint": "node generate_translations.mjs && tsc --noEmit && eslint --report-unused-disable-directives --ext .js,.ts,.tsx src",
"prebuild:dev": "yarn clean && node generate_translations.mjs",
"prebuild:prod": "yarn clean && node generate_translations.mjs",

1
recommended-instances.json
View file

@ -3,7 +3,6 @@
"de": ["feddit.de"],
"pt": ["lemmy.pt"],
"pt_BR": ["lemmy.pt"],
"pl": ["szmer.info"],
"eu": ["lemmy.eus"],
"ja": ["tabinezumi.net", "lm.korako.me"],
"es": ["forum.nobigtech.es"],

388
src/shared/instance_stats.ts
View file

@ -1,9 +1,9 @@
export const instance_stats = {
stats: {
crawled_instances: 42,
failed_instances: 118,
total_users: 27447,
total_online_users: 355,
crawled_instances: 44,
failed_instances: 61,
total_users: 27594,
total_online_users: 339,
instance_details: [
{
domain: "lemmygrad.ml",
@ -12,12 +12,12 @@ export const instance_stats = {
"A collection of leftist communities, for memes, learning, news, discussion, media, or anything you like.",
version: "0.16.3",
icon: "https://lemmygrad.ml/pictrs/image/gB8yP0oFF5.png",
online_users: 151,
total_users: 6373,
users_active_halfyear: 1177,
users_active_month: 748,
online_users: 123,
total_users: 6403,
users_active_halfyear: 1196,
users_active_month: 674,
open_registrations: true,
linked_instances_count: 20,
linked_instances_count: 21,
require_application: true,
},
{
@ -27,12 +27,12 @@ export const instance_stats = {
"A community of leftist privacy and FOSS enthusiasts, run by Lemmy’s developers",
version: "0.16.3",
icon: "https://lemmy.ml/pictrs/image/bhQ7ELa4oq.webp",
online_users: 131,
total_users: 16719,
online_users: 117,
total_users: 16764,
users_active_halfyear: 1955,
users_active_month: 555,
open_registrations: true,
linked_instances_count: 56,
linked_instances_count: 58,
require_application: true,
},
{
@ -41,12 +41,12 @@ export const instance_stats = {
description: "polskojęzyczna instancja lemmy-iego. ",
version: "0.16.1",
icon: "https://szmer.info/pictrs/image/XcQWr0D2CU.png",
online_users: 16,
total_users: 1209,
users_active_halfyear: 346,
users_active_month: 94,
online_users: 10,
total_users: 1218,
users_active_halfyear: 341,
users_active_month: 85,
open_registrations: true,
linked_instances_count: 8,
linked_instances_count: 9,
require_application: true,
},
{
@ -56,14 +56,28 @@ export const instance_stats = {
"A decentralized federated community of freedom fighting meme farmers ",
version: "0.16.3",
icon: "https://wolfballs.com/pictrs/image/UYZ2wquwlB.png",
online_users: 7,
total_users: 245,
online_users: 14,
total_users: 249,
users_active_halfyear: 128,
users_active_month: 42,
users_active_month: 36,
open_registrations: true,
linked_instances_count: 13,
require_application: false,
},
{
domain: "feddit.de",
name: "Feddit",
description: "Deutschsprachige Lemmy Community",
version: "0.16.3",
icon: "https://feddit.de/pictrs/image/uI7Q7MuePp.png",
online_users: 12,
total_users: 209,
users_active_halfyear: 69,
users_active_month: 34,
open_registrations: true,
linked_instances_count: 21,
require_application: true,
},
{
domain: "community.xmpp.net",
name: "XMPP Community",
@ -71,40 +85,12 @@ export const instance_stats = {
"A community space for projects and users of the Extensible Messaging and Presence Protocol (XMPP).",
version: "0.16.3",
icon: "https://community.xmpp.net/pictrs/image/35c85076-7971-4e2a-b5d1-18ef437366ff.png",
online_users: 6,
online_users: 10,
total_users: 47,
users_active_halfyear: 34,
users_active_month: 34,
open_registrations: true,
linked_instances_count: 7,
require_application: true,
},
{
domain: "feddit.de",
name: "Feddit",
description: "Deutschsprachige Lemmy Community",
version: "0.16.3",
icon: "https://feddit.de/pictrs/image/uI7Q7MuePp.png",
online_users: 3,
total_users: 201,
users_active_halfyear: 68,
users_active_month: 33,
open_registrations: true,
linked_instances_count: 78,
require_application: true,
},
{
domain: "lemmy.perthchat.org",
name: "PerthChat",
description: "The Perth Lemmy Instance",
version: "0.16.3",
icon: "https://lemmy.perthchat.org/pictrs/image/d3728083-9d5f-4311-8c9b-d9dda43f7413.png",
online_users: 0,
total_users: 36,
users_active_halfyear: 26,
users_active_month: 26,
open_registrations: true,
linked_instances_count: 9,
linked_instances_count: 8,
require_application: true,
},
{
@ -114,42 +100,56 @@ export const instance_stats = {
"A general-purpose instance run by a Finn - everyone is welcome here!",
version: "0.16.3",
icon: "https://sopuli.xyz/pictrs/image/AjMQEWabkH.png",
online_users: 2,
total_users: 377,
users_active_halfyear: 83,
users_active_month: 21,
online_users: 4,
total_users: 386,
users_active_halfyear: 85,
users_active_month: 20,
open_registrations: true,
linked_instances_count: 23,
linked_instances_count: 25,
require_application: false,
},
{
domain: "beehaw.org",
name: "Beehaw",
description: "Aspiring to be(e) a safe, friendly and diverse place.",
version: "0.16.3",
icon: "https://beehaw.org/pictrs/image/33fad824-7820-41c0-9c4a-0d7cf831ddd3.png",
online_users: 2,
total_users: 85,
users_active_halfyear: 47,
users_active_month: 18,
open_registrations: true,
linked_instances_count: 4,
require_application: true,
},
{
domain: "slrpnk.net",
name: "SLRPNK",
description: "where solarpunks organize for a better world!",
version: "0.16.3",
icon: "https://slrpnk.net/pictrs/image/9d582815-3984-4996-9b10-b18706bebac1.png",
online_users: 6,
total_users: 31,
users_active_halfyear: 18,
online_users: 2,
total_users: 33,
users_active_halfyear: 20,
users_active_month: 18,
open_registrations: true,
linked_instances_count: 6,
require_application: true,
},
{
domain: "beehaw.org",
name: "Beehaw",
description: "Aspiring to be(e) a safe, friendly and diverse place.",
version: "0.16.3",
icon: "https://beehaw.org/pictrs/image/33fad824-7820-41c0-9c4a-0d7cf831ddd3.png",
online_users: 7,
total_users: 89,
users_active_halfyear: 48,
users_active_month: 17,
open_registrations: true,
linked_instances_count: 5,
require_application: true,
},
{
domain: "collapse.cat",
name: "Col·lapse / Colapso",
description: "Col·lapse, emergència climàtica i temes relacionats",
version: "0.15.1",
icon: "https://collapse.cat/pictrs/image/NUpYQehEdv.png",
online_users: 3,
total_users: 77,
users_active_halfyear: 61,
users_active_month: 17,
open_registrations: true,
linked_instances_count: 11,
require_application: false,
},
{
domain: "exploding-heads.com",
name: "Exploding Heads",
@ -157,27 +157,13 @@ export const instance_stats = {
version: "0.16.3",
icon: "https://exploding-heads.com/pictrs/image/f48abd93-f2d9-4a44-a327-380e063744b1.png",
online_users: 1,
total_users: 16,
users_active_halfyear: 15,
users_active_month: 15,
total_users: 17,
users_active_halfyear: 16,
users_active_month: 16,
open_registrations: true,
linked_instances_count: 19,
require_application: false,
},
{
domain: "collapse.cat",
name: "Col·lapse / Colapso",
description: "Col·lapse, emergència climàtica i temes relacionats",
version: "0.15.1",
icon: "https://collapse.cat/pictrs/image/NUpYQehEdv.png",
online_users: 2,
total_users: 75,
users_active_halfyear: 58,
users_active_month: 14,
open_registrations: true,
linked_instances_count: 11,
require_application: false,
},
{
domain: "mandacaru.caatinga.digital",
name: "Mandacaru",
@ -185,7 +171,7 @@ export const instance_stats = {
version: "0.14.1",
icon: null,
online_users: 0,
total_users: 35,
total_users: 36,
users_active_halfyear: 21,
users_active_month: 12,
open_registrations: true,
@ -199,14 +185,28 @@ export const instance_stats = {
"A canadian-run community, geared towards canadians, but all are welcome!",
version: "0.16.3",
icon: "https://lemmy.ca/pictrs/image/AX0BZv78yT.png",
online_users: 0,
total_users: 311,
users_active_halfyear: 104,
online_users: 3,
total_users: 315,
users_active_halfyear: 102,
users_active_month: 11,
open_registrations: true,
linked_instances_count: 22,
require_application: true,
},
{
domain: "lemmy.perthchat.org",
name: "PerthChat",
description: "The Perth Lemmy Instance",
version: "0.16.3",
icon: "https://lemmy.perthchat.org/pictrs/image/d3728083-9d5f-4311-8c9b-d9dda43f7413.png",
online_users: 0,
total_users: 36,
users_active_halfyear: 26,
users_active_month: 9,
open_registrations: true,
linked_instances_count: 9,
require_application: true,
},
{
domain: "lemmy.eus",
name: "Lemmy.eus",
@ -215,9 +215,9 @@ export const instance_stats = {
version: "0.16.2",
icon: "https://lemmy.eus/pictrs/image/R55fPm9RfM.png",
online_users: 9,
total_users: 475,
users_active_halfyear: 150,
users_active_month: 9,
total_users: 478,
users_active_halfyear: 143,
users_active_month: 8,
open_registrations: true,
linked_instances_count: 10,
require_application: true,
@ -228,44 +228,14 @@ export const instance_stats = {
description: "An instance dedicated to nature and science.",
version: "0.16.3",
icon: "https://mander.xyz/pictrs/image/2fc86ef3-797e-4bcd-bcca-e75cdf0959f9.png",
online_users: 0,
total_users: 70,
online_users: 2,
total_users: 72,
users_active_halfyear: 27,
users_active_month: 8,
open_registrations: true,
linked_instances_count: 23,
require_application: true,
},
{
domain: "lemmy.pt",
name: "Lemmy Portugal 🇵🇹",
description:
"Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.",
version: "0.16.3",
icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png",
online_users: 0,
total_users: 86,
users_active_halfyear: 21,
users_active_month: 7,
open_registrations: true,
linked_instances_count: 4,
require_application: true,
},
{
domain: "lemmy.pt",
name: "Lemmy Portugal 🇵🇹",
description:
"Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.",
version: "0.16.3",
icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png",
online_users: 0,
total_users: 86,
users_active_halfyear: 21,
users_active_month: 7,
open_registrations: true,
linked_instances_count: 4,
require_application: true,
},
{
domain: "gtio.io",
name: "Go Talk It Out",
@ -273,13 +243,43 @@ export const instance_stats = {
version: "0.16.3",
icon: "https://gtio.io/pictrs/image/32f393ce-2895-462c-8f9b-25c31d713a05.png",
online_users: 1,
total_users: 7,
total_users: 9,
users_active_halfyear: 6,
users_active_month: 6,
open_registrations: true,
linked_instances_count: 3,
require_application: true,
},
{
domain: "lemmy.pt",
name: "Lemmy Portugal 🇵🇹",
description:
"Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.",
version: "0.16.3",
icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png",
online_users: 2,
total_users: 87,
users_active_halfyear: 20,
users_active_month: 5,
open_registrations: true,
linked_instances_count: 4,
require_application: true,
},
{
domain: "lemmy.pt",
name: "Lemmy Portugal 🇵🇹",
description:
"Uma instância pública de Lemmy dedicada a Portugal e à Língua Portuguesa.",
version: "0.16.3",
icon: "https://lemmy.pt/pictrs/image/fHkiLTfJNO.png",
online_users: 2,
total_users: 87,
users_active_halfyear: 20,
users_active_month: 5,
open_registrations: true,
linked_instances_count: 4,
require_application: true,
},
{
domain: "midwest.social",
name: "midwest.social",
@ -288,7 +288,7 @@ export const instance_stats = {
version: "0.16.3",
icon: "https://midwest.social/pictrs/image/Rbal22EuF8.png",
online_users: 1,
total_users: 111,
total_users: 113,
users_active_halfyear: 26,
users_active_month: 5,
open_registrations: true,
@ -303,7 +303,7 @@ export const instance_stats = {
icon: "https://lemmy.juggler.jp/pictrs/image/v0TGrlnKGS.png",
online_users: 1,
total_users: 30,
users_active_halfyear: 13,
users_active_halfyear: 12,
users_active_month: 4,
open_registrations: true,
linked_instances_count: 13,
@ -316,9 +316,9 @@ export const instance_stats = {
"A space to share and discusss Africa-related content. Karibu.",
version: "0.16.2",
icon: "https://baraza.africa/pictrs/image/qFpb6BEV2c.png",
online_users: 3,
total_users: 352,
users_active_halfyear: 77,
online_users: 2,
total_users: 354,
users_active_halfyear: 76,
users_active_month: 4,
open_registrations: true,
linked_instances_count: 16,
@ -331,28 +331,13 @@ export const instance_stats = {
version: "0.16.3",
icon: "https://forum.nobigtech.es/pictrs/image/atwGqrgiHu.png",
online_users: 0,
total_users: 8,
total_users: 9,
users_active_halfyear: 4,
users_active_month: 4,
open_registrations: true,
linked_instances_count: 3,
require_application: true,
},
{
domain: "fapsi.be",
name: "Fapsi",
description:
"This instance is for all the creative. No matter if you're a writer, author or illustrator.",
version: "0.16.3",
icon: "https://fapsi.be/pictrs/image/jUD7V242rr.png",
online_users: 1,
total_users: 150,
users_active_halfyear: 30,
users_active_month: 3,
open_registrations: true,
linked_instances_count: 14,
require_application: false,
},
{
domain: "lemmy.cat",
name: "Lemmy CAT",
@ -360,13 +345,27 @@ export const instance_stats = {
version: "0.16.3",
icon: "https://lemmy.cat/pictrs/image/4iwqIfDlgI.png",
online_users: 1,
total_users: 73,
total_users: 74,
users_active_halfyear: 8,
users_active_month: 3,
open_registrations: true,
linked_instances_count: 9,
linked_instances_count: 10,
require_application: true,
},
{
domain: "elgiebety.pl",
name: "elgiebety",
description: null,
version: "0.15.2",
icon: null,
online_users: 1,
total_users: 4,
users_active_halfyear: 2,
users_active_month: 2,
open_registrations: true,
linked_instances_count: 3,
require_application: false,
},
{
domain: "tabinezumi.net",
name: "たびねずみのみみ",
@ -392,7 +391,7 @@ export const instance_stats = {
users_active_halfyear: 7,
users_active_month: 2,
open_registrations: true,
linked_instances_count: 6,
linked_instances_count: 7,
require_application: true,
},
{
@ -431,12 +430,27 @@ export const instance_stats = {
"Eine deutschsprachige Instanz für Rollenspieler. Wir bieten einen platz für Rollenspiel, Pen & Paper, Tabletop, TCG und vieles mehr.",
version: "0.16.3",
icon: "https://lemmy.rollenspiel.monster/pictrs/image/040c05a3-ba7c-4e66-a225-5209dd4893f9.png",
online_users: 2,
online_users: 1,
total_users: 3,
users_active_halfyear: 2,
users_active_month: 2,
open_registrations: true,
linked_instances_count: 4,
linked_instances_count: 2,
require_application: false,
},
{
domain: "fapsi.be",
name: "Fapsi",
description:
"This instance is for all the creative. No matter if you're a writer, author or illustrator.",
version: "0.16.3",
icon: "https://fapsi.be/pictrs/image/jUD7V242rr.png",
online_users: 1,
total_users: 154,
users_active_halfyear: 29,
users_active_month: 2,
open_registrations: true,
linked_instances_count: 15,
require_application: false,
},
{
@ -445,12 +459,12 @@ export const instance_stats = {
description: "Now 50% Prod!",
version: "0.16.3",
icon: "https://fuckreddit.tryp.digital/pictrs/image/52dc0c6c-71a4-40f9-a326-b51d54c95173.png",
online_users: 3,
online_users: 2,
total_users: 5,
users_active_halfyear: 2,
users_active_month: 2,
open_registrations: true,
linked_instances_count: 13,
linked_instances_count: 14,
require_application: true,
},
{
@ -459,7 +473,7 @@ export const instance_stats = {
description: "infopoint virtual",
version: "0.16.1",
icon: "https://info.prou.be/pictrs/image/82e736ba-6dee-423b-b06b-b47f994ce6b6.png",
online_users: 0,
online_users: 2,
total_users: 10,
users_active_halfyear: 4,
users_active_month: 2,
@ -467,6 +481,20 @@ export const instance_stats = {
linked_instances_count: 5,
require_application: true,
},
{
domain: "masr.social",
name: "Masr",
description: null,
version: "0.16.3",
icon: null,
online_users: 0,
total_users: 3,
users_active_halfyear: 2,
users_active_month: 2,
open_registrations: true,
linked_instances_count: 3,
require_application: false,
},
{
domain: "stammtisch.hallertau.social",
name: "Stammtisch",
@ -481,33 +509,33 @@ export const instance_stats = {
linked_instances_count: 9,
require_application: true,
},
{
domain: "goldandblack.us.to",
name: "goldandblack-lemmy",
description: null,
version: "0.14.3",
icon: "https://goldandblack.us.to/pictrs/image/DDIh7ojBjd.png",
online_users: 0,
total_users: 4,
users_active_halfyear: 2,
users_active_month: 1,
open_registrations: true,
linked_instances_count: 3,
require_application: false,
},
{
domain: "lemmy.otakufarms.com",
name: "Otaku Farms",
description:
"Otaku Farms is a network of Fediverse instances created for the sole purpose of letting individuals express themselves.",
version: "0.14.3",
version: "0.16.3",
icon: "https://lemmy.otakufarms.com/pictrs/image/ekmeJ0BD8S.png",
online_users: 0,
total_users: 3,
users_active_halfyear: 3,
users_active_month: 1,
open_registrations: true,
linked_instances_count: 1,
require_application: false,
},
{
domain: "elgiebety.pl",
name: "elgiebety",
description: null,
version: "0.15.2",
icon: null,
online_users: 1,
total_users: 4,
users_active_halfyear: 1,
users_active_halfyear: 4,
users_active_month: 1,
open_registrations: true,
linked_instances_count: 3,
linked_instances_count: 5,
require_application: false,
},
{
@ -531,12 +559,12 @@ export const instance_stats = {
description: null,
version: "0.16.3",
icon: null,
online_users: 2,
online_users: 1,
total_users: 40,
users_active_halfyear: 2,
users_active_month: 1,
open_registrations: false,
linked_instances_count: 15,
linked_instances_count: 16,
require_application: true,
},
{
@ -559,12 +587,12 @@ export const instance_stats = {
description: "A Lemmy instance for mesh network denizens.",
version: "0.16.3",
icon: "https://lemmy.mesh.party/pictrs/image/3653fdce-1fdc-4514-aceb-38c6ba29006c.png",
online_users: 1,
total_users: 8,
online_users: 2,
total_users: 9,
users_active_halfyear: 2,
users_active_month: 1,
open_registrations: true,
linked_instances_count: 4,
linked_instances_count: 5,
require_application: false,
},
{