LemmyNet/joinlemmy-site
2
0
Fork 0
mirror of https://github.com/LemmyNet/joinlemmy-site.git synced 2024-11-22 04:11:15 +00:00
Code Issues Projects Releases Wiki Activity

Use jq to filter instance list and avoid js crashes (fixes #216)

Browse source
This commit is contained in:
Felix Ableitner 2023-07-11 12:59:06 +02:00
parent 2062904b40
commit 638fd0df57
1 changed files with 12 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
crawl.mjs
View file

@ -19,16 +19,19 @@ try {
all_recommended.push(...recommended_instances[k]);
}
}
const run = spawn(
"cargo",
// Run Rust crawler with given params. Then pipe output directly into jq, to filter
// out fields with lots of data which we dont need. This is necessary because otherwise
// Javascript may crash when loading the crawl output.
const run = spawn("sh",
[
"run",
"--",
"--json",
"--start-instances",
all_recommended,
"--exclude-instances",
recommended_instances.exclude,
"-c",
`cargo run -- --json --start-instances ${all_recommended} \
--exclude-instances ${recommended_instances.exclude} | \
jq 'del(.instance_details[].federated_instances, \
.instance_details[].site_info.all_languages, \
.instance_details[].site_info.discussion_languages, \
.instance_details[].site_info.admins, .instance_details[].site_info.taglines, \
.instance_details[].site_info.custom_emojis)'`
],
{
cwd: "lemmy-stats-crawler",
@ -62,15 +65,6 @@ try {
i =>
i.site_info.site_view.counts.users_active_month > min_monthly_users,
);
// Exclude unnecessary data
stats.instance_details.forEach(i => {
delete i.site_info.admins;
delete i.site_info.all_languages;
delete i.site_info.discussion_languages;
delete i.site_info.taglines;
delete i.site_info.custom_emojis;
delete i.federated_instances;
});
let stats2 = {
stats: stats,