LemmyNet/joinlemmy-site
2
0
Fork 0
mirror of https://github.com/LemmyNet/joinlemmy-site.git synced 2024-11-22 20:31:17 +00:00
Code Issues Projects Releases Wiki Activity

Use jq to filter instance list and avoid js crashes (fixes #216)

Browse source
This commit is contained in:
Felix Ableitner 2023-07-11 12:59:06 +02:00
parent 2062904b40
commit 638fd0df57
1 changed files with 12 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
crawl.mjs
View file

@ -19,16 +19,19 @@ try {
all_recommended.push(...recommended_instances[k]); all_recommended.push(...recommended_instances[k]);
} }
} }
const run = spawn( // Run Rust crawler with given params. Then pipe output directly into jq, to filter
"cargo", // out fields with lots of data which we dont need. This is necessary because otherwise
// Javascript may crash when loading the crawl output.
const run = spawn("sh",
[ [
"run", "-c",
"--", `cargo run -- --json --start-instances ${all_recommended} \
"--json", --exclude-instances ${recommended_instances.exclude} | \
"--start-instances", jq 'del(.instance_details[].federated_instances, \
all_recommended, .instance_details[].site_info.all_languages, \
"--exclude-instances", .instance_details[].site_info.discussion_languages, \
recommended_instances.exclude, .instance_details[].site_info.admins, .instance_details[].site_info.taglines, \
.instance_details[].site_info.custom_emojis)'`
], ],
{ {
cwd: "lemmy-stats-crawler", cwd: "lemmy-stats-crawler",
@ -62,15 +65,6 @@ try {
i => i =>
i.site_info.site_view.counts.users_active_month > min_monthly_users, i.site_info.site_view.counts.users_active_month > min_monthly_users,
); );
// Exclude unnecessary data
stats.instance_details.forEach(i => {
delete i.site_info.admins;
delete i.site_info.all_languages;
delete i.site_info.discussion_languages;
delete i.site_info.taglines;
delete i.site_info.custom_emojis;
delete i.federated_instances;
});
let stats2 = { let stats2 = {
stats: stats, stats: stats,