LemmyNet/joinlemmy-site
2
0
Fork 0
mirror of https://github.com/LemmyNet/joinlemmy-site.git synced 2024-11-26 06:11:21 +00:00
Code Issues Projects Releases Wiki Activity

exclude more data

Browse source
This commit is contained in:
Felix Ableitner 2023-06-29 22:07:46 +02:00
parent 1ac0f2690c
commit 7fd84943c8
2 changed files with 255 additions and 152145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
crawl.mjs
View file

@ -25,6 +25,7 @@ try {
"run", "run",
"--", "--",
"--json", "--json",
"--max-crawl-distance", "0",
"--start-instances", "--start-instances",
all_recommended, all_recommended,
"--exclude-instances", "--exclude-instances",
@ -50,16 +51,29 @@ try {
run.on("close", exitCode => { run.on("close", exitCode => {
var stats = JSON.parse(savedOutput); var stats = JSON.parse(savedOutput);
// Crawl results from all instances include tons of data which needs to be compiled.
// If it is too much data it breaks the build, so we need to exclude as much as possible.
stats.instance_details = stats.instance_details stats.instance_details = stats.instance_details
// Exclude instances with closed registration
.filter( .filter(
i => i =>
i.site_info.site_view.local_site.registration_mode != "closed" i.site_info.site_view.local_site.registration_mode != "closed"
) )
// Exclude instances with few active users
.filter( .filter(
i => i =>
i.site_info.site_view.counts.users_active_month > i.site_info.site_view.counts.users_active_month >
min_monthly_users min_monthly_users
); );
// Exclude unnecessary data
stats.instance_details.forEach(i => {
delete i.site_info.admins;
delete i.site_info.all_languages;
delete i.site_info.discussion_languages;
delete i.site_info.taglines;
delete i.site_info.custom_emojis;
delete i.federated_instances;
});
let stats2 = { let stats2 = {
stats: stats, stats: stats,

152372
src/shared/instance_stats.ts
View file

File diff suppressed because one or more lines are too long