Output instance info as json
This commit is contained in:
parent
cb4e6e221a
commit
9cbed8f2a9
4 changed files with 38 additions and 3 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
@ -417,6 +417,7 @@ dependencies = [
|
|||
"futures",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
|
|
|
@ -10,3 +10,4 @@ serde = { version = "1.0.123", features = ["derive"] }
|
|||
anyhow = "1.0.38"
|
||||
tokio = { version = "0.2.25", features = ["full"] }
|
||||
futures = "0.3.13"
|
||||
serde_json = "1.0.64"
|
||||
|
|
15
README.md
Normal file
15
README.md
Normal file
|
@ -0,0 +1,15 @@
|
|||
# Lemmy-Stats-Crawler
|
||||
|
||||
Crawls Lemmy instances using nodeinfo and API endpoints, to generate a list of instances and overall details.
|
||||
|
||||
## Usage
|
||||
|
||||
For testing:
|
||||
```
|
||||
cargo run
|
||||
```
|
||||
|
||||
For production (hide debug logs):
|
||||
```
|
||||
cargo run 2>/dev/null
|
||||
```
|
24
src/main.rs
24
src/main.rs
|
@ -1,6 +1,7 @@
|
|||
use anyhow::Error;
|
||||
use lemmy_stats_crawler::federated_instances::GetSiteResponse;
|
||||
use lemmy_stats_crawler::node_info::NodeInfo;
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
struct TotalStats {
|
||||
|
@ -12,20 +13,22 @@ struct TotalStats {
|
|||
pub async fn main() -> Result<(), Error> {
|
||||
let mut pending_instances = vec!["lemmy.ml".to_string()];
|
||||
let mut crawled_instances = vec![];
|
||||
let mut instance_details = vec![];
|
||||
let mut total_stats = TotalStats::default();
|
||||
while let Some(pi) = pending_instances.iter().next() {
|
||||
crawled_instances.push(pi.to_owned());
|
||||
let instance_details = fetch_instance_details(&pi).await.ok();
|
||||
let current_instance_details = fetch_instance_details(&pi).await.ok();
|
||||
pending_instances = pending_instances
|
||||
.iter()
|
||||
.filter(|i| i != &pi)
|
||||
.map(|i| i.to_owned())
|
||||
.collect();
|
||||
|
||||
if let Some(details) = instance_details {
|
||||
if let Some(details) = current_instance_details {
|
||||
instance_details.push(details.to_owned());
|
||||
total_stats.online_users += details.online_users;
|
||||
total_stats.users += details.total_users;
|
||||
// remove all which are in crawled_instances
|
||||
// add all unknown, linked instances to pending
|
||||
for ci in details.linked_instances {
|
||||
if !crawled_instances.contains(&ci) {
|
||||
pending_instances.push(ci);
|
||||
|
@ -33,14 +36,26 @@ pub async fn main() -> Result<(), Error> {
|
|||
}
|
||||
}
|
||||
}
|
||||
instance_details = instance_details
|
||||
.iter()
|
||||
.filter(|i| i.open_registrations)
|
||||
.map(|i| i.to_owned())
|
||||
.collect();
|
||||
instance_details.sort_by(|a, b| b.users_active_halfyear.cmp(&a.users_active_halfyear));
|
||||
print!("{}", serde_json::to_string(&instance_details)?);
|
||||
dbg!(total_stats);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Serialize, Clone)]
|
||||
struct InstanceDetails {
|
||||
domain: String,
|
||||
online_users: i32,
|
||||
total_users: i64,
|
||||
users_active_halfyear: i64,
|
||||
users_active_month: i64,
|
||||
open_registrations: bool,
|
||||
#[serde(skip)]
|
||||
linked_instances: Vec<String>,
|
||||
}
|
||||
|
||||
|
@ -57,6 +72,9 @@ async fn fetch_instance_details(domain: &str) -> Result<InstanceDetails, Error>
|
|||
domain: domain.to_owned(),
|
||||
online_users: site_info.online as i32,
|
||||
total_users: node_info.usage.users.total,
|
||||
users_active_halfyear: node_info.usage.users.active_halfyear,
|
||||
users_active_month: node_info.usage.users.active_month,
|
||||
open_registrations: node_info.open_registrations,
|
||||
linked_instances: site_info
|
||||
.federated_instances
|
||||
.map(|f| f.linked)
|
||||
|
|
Loading…
Reference in a new issue