LemmyNet/joinlemmy-site
2
0
Fork 0
mirror of https://github.com/LemmyNet/joinlemmy-site.git synced 2024-11-21 20:01:16 +00:00
Code Issues Projects Releases Wiki Activity

Use jq to filter instance list and avoid js crashes (fixes #216) (#217)

Browse source 
* Use jq to filter instance list and avoid js crashes (fixes #216)

* prettier

* prettier 2
This commit is contained in:
Nutomic 2023-07-11 14:13:03 +02:00 committed by GitHub
parent 2062904b40
commit bbee0a29a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 12 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
crawl.mjs
View file

@ -19,16 +19,20 @@ try {
all_recommended.push(...recommended_instances[k]); all_recommended.push(...recommended_instances[k]);
} }
} }
// Run Rust crawler with given params. Then pipe output directly into jq, to filter
// out fields with lots of data which we dont need. This is necessary because otherwise
// Javascript may crash when loading the crawl output.
const run = spawn( const run = spawn(
"cargo", "sh",
[ [
"run", "-c",
"--", `cargo run -- --json --start-instances ${all_recommended} \
"--json", --exclude-instances ${recommended_instances.exclude} | \
"--start-instances", jq 'del(.instance_details[].federated_instances, \
all_recommended, .instance_details[].site_info.all_languages, \
"--exclude-instances", .instance_details[].site_info.discussion_languages, \
recommended_instances.exclude, .instance_details[].site_info.admins, .instance_details[].site_info.taglines, \
.instance_details[].site_info.custom_emojis)'`,
], ],
{ {
cwd: "lemmy-stats-crawler", cwd: "lemmy-stats-crawler",
@ -62,15 +66,6 @@ try {
i => i =>
i.site_info.site_view.counts.users_active_month > min_monthly_users, i.site_info.site_view.counts.users_active_month > min_monthly_users,
); );
// Exclude unnecessary data
stats.instance_details.forEach(i => {
delete i.site_info.admins;
delete i.site_info.all_languages;
delete i.site_info.discussion_languages;
delete i.site_info.taglines;
delete i.site_info.custom_emojis;
delete i.federated_instances;
});
let stats2 = { let stats2 = {
stats: stats, stats: stats,