LemmyNet/lemmy-stats-crawler
2
1
Fork 0
Code Issues 3 Pull requests Projects Releases Wiki Activity

Merge pull request 'Use lemmy structs' (#10) from lemmy-structs into main

Browse source 
Reviewed-on: https://yerbamate.ml/LemmyNet/lemmy-stats-crawler/pulls/10
This commit is contained in:
nutomic 2022-05-13 11:37:45 +00:00
commit 0c3ba08d6c
7 changed files with 2602 additions and 515 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2882
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

17
Cargo.toml
View file

@ -5,12 +5,13 @@ authors = ["Felix Ableitner"]
edition = "2018"
[dependencies]
reqwest = { version = "0.10.10", default-features = false, features = ["json", "rustls-tls"] }
serde = { version = "1.0.123", features = ["derive"] }
anyhow = "1.0.38"
tokio = { version = "0.2.25", features = ["rt-threaded", "macros"] }
futures = "0.3.13"
serde_json = "1.0.64"
clap = "2.33.3"
semver = "1.0.7"
reqwest = { version = "0.11.10", default-features = false, features = ["json", "rustls-tls"] }
serde = { version = "1.0.137", features = ["derive"] }
anyhow = "1.0.57"
tokio = { version = "1.18.1", features = ["macros", "rt-multi-thread"] }
futures = "0.3.21"
serde_json = "1.0.81"
clap = "3.1.15"
semver = "1.0.9"
once_cell = "1.10.0"
lemmy_api_common = "0.16.0"

111
src/crawl.rs
View file

@ -1,9 +1,7 @@
use crate::federated_instances::GetSiteResponse;
use crate::node_info::NodeInfo;
use crate::REQUEST_TIMEOUT;
use anyhow::anyhow;
use anyhow::Error;
use futures::try_join;
use lemmy_api_common::site::GetSiteResponse;
use once_cell::sync::Lazy;
use reqwest::Client;
use semver::Version;
@ -32,16 +30,18 @@ pub async fn crawl(
}
match fetch_instance_details(&current_instance.domain, &min_lemmy_version).await {
Ok(details) => {
instance_details.push(details.to_owned());
for i in details.linked_instances {
let is_in_crawled = crawled_instances.contains(&i);
let is_in_pending = pending_instances.iter().any(|p| p.domain == i);
if let Some(federated) = &details.site_info.federated_instances.as_ref() {
for i in &federated.linked {
let is_in_crawled = crawled_instances.contains(i);
let is_in_pending = pending_instances.iter().any(|p| &p.domain == i);
if !is_in_crawled && !is_in_pending {
let ci = CrawlInstance::new(i, current_instance.depth + 1);
let ci = CrawlInstance::new(i.clone(), current_instance.depth + 1);
pending_instances.push_back(ci);
}
}
}
instance_details.push(details);
}
Err(e) => {
failed_instances += 1;
eprintln!("Failed to crawl {}: {}", current_instance.domain, e)
@ -50,29 +50,22 @@ pub async fn crawl(
}
// Sort by active monthly users descending
instance_details.sort_by_key(|i| i.users_active_month);
instance_details.sort_by_key(|i| {
i.site_info
.site_view
.as_ref()
.map(|s| s.counts.users_active_month)
.unwrap_or(0)
});
instance_details.reverse();
Ok((instance_details, failed_instances))
}
#[derive(Serialize, Clone)]
#[derive(Serialize, Debug)]
pub struct InstanceDetails {
pub domain: String,
pub name: String,
pub description: Option<String>,
pub version: String,
pub icon: Option<String>,
pub online_users: i32,
pub total_users: i64,
pub users_active_halfyear: i64,
pub users_active_month: i64,
pub open_registrations: bool,
pub linked_instances_count: i32,
pub require_application: bool,
// The following fields are only used for aggregation, but not shown in output
#[serde(skip)]
pub linked_instances: Vec<String>,
pub site_info: GetSiteResponse,
}
struct CrawlInstance {
@ -92,75 +85,23 @@ async fn fetch_instance_details(
) -> Result<InstanceDetails, Error> {
let client = Client::default();
let node_info_url = format!("https://{}/nodeinfo/2.0.json", domain);
let node_info_request = client.get(&node_info_url).timeout(REQUEST_TIMEOUT).send();
let site_info_url_v2 = format!("https://{}/api/v2/site", domain);
let site_info_request_v2 = client
.get(&site_info_url_v2)
let site_info_url = format!("https://{}/api/v3/site", domain);
let site_info = client
.get(&site_info_url)
.timeout(REQUEST_TIMEOUT)
.send();
let site_info_url_v3 = format!("https://{}/api/v3/site", domain);
let site_info_request_v3 = client
.get(&site_info_url_v3)
.timeout(REQUEST_TIMEOUT)
.send();
.send()
.await?
.json::<GetSiteResponse>()
.await?;
let (node_info, site_info_v2, site_info_v3) = try_join!(
node_info_request,
site_info_request_v2,
site_info_request_v3
)?;
let node_info: NodeInfo = node_info.json().await?;
if node_info.software.name != "lemmy" {
return Err(anyhow!("not a lemmy instance"));
}
let version = Version::parse(&node_info.software.version)?;
let version = Version::parse(&site_info.version)?;
if &version < min_lemmy_version {
return Err(anyhow!("lemmy version is too old ({})", version));
}
let site_info_v2 = site_info_v2.json::<GetSiteResponse>().await.ok();
let site_info_v3 = site_info_v3.json::<GetSiteResponse>().await.ok();
let mut site_info: GetSiteResponse = if let Some(site_info_v2) = site_info_v2 {
site_info_v2
} else if let Some(site_info_v3) = site_info_v3 {
site_info_v3
} else {
return Err(anyhow!("Failed to read site_info"));
};
if let Some(description) = &site_info.site_view.site.description {
if description.len() > 150 {
site_info.site_view.site.description = None;
}
}
let require_application = site_info
.site_view
.site
.require_application
.unwrap_or(false);
let linked_instances: Vec<String> = site_info
.federated_instances
.map(|f| f.linked)
.unwrap_or_default()
.iter()
.map(|l| l.to_lowercase())
.collect();
Ok(InstanceDetails {
domain: domain.to_owned(),
name: site_info.site_view.site.name,
description: site_info.site_view.site.description,
version: node_info.software.version,
icon: site_info.site_view.site.icon,
online_users: site_info.online as i32,
total_users: node_info.usage.users.total,
users_active_halfyear: node_info.usage.users.active_halfyear,
users_active_month: node_info.usage.users.active_month,
open_registrations: node_info.open_registrations,
linked_instances_count: linked_instances.len() as i32,
require_application,
linked_instances,
site_info,
})
}

28
src/federated_instances.rs
View file

@ -1,28 +0,0 @@
use serde::Deserialize;
#[derive(Deserialize, Debug, Clone)]
pub struct GetSiteResponse {
pub site_view: SiteView,
pub online: usize,
pub federated_instances: Option<FederatedInstances>,
}
#[derive(Deserialize, Debug, Clone)]
pub struct FederatedInstances {
pub linked: Vec<String>,
pub allowed: Option<Vec<String>>,
pub blocked: Option<Vec<String>>,
}
#[derive(Deserialize, Debug, Clone)]
pub struct SiteView {
pub site: Site,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Site {
pub name: String,
pub icon: Option<String>,
pub description: Option<String>,
pub require_application: Option<bool>,
}

2
src/lib.rs
View file

@ -1,8 +1,6 @@
use std::time::Duration;
pub mod crawl;
pub mod federated_instances;
pub mod node_info;
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
pub const DEFAULT_START_INSTANCES: &str = "lemmy.ml";

40
src/main.rs
View file

@ -1,20 +1,20 @@
use anyhow::Error;
use clap::{App, Arg};
use clap::{Arg, Command};
use lemmy_stats_crawler::crawl::{crawl, InstanceDetails};
use lemmy_stats_crawler::{DEFAULT_MAX_CRAWL_DEPTH, DEFAULT_START_INSTANCES, EXCLUDE_INSTANCES};
use serde::Serialize;
#[tokio::main]
pub async fn main() -> Result<(), Error> {
let matches = App::new("Lemmy Stats Crawler")
let matches = Command::new("Lemmy Stats Crawler")
.arg(
Arg::with_name("start-instances")
Arg::new("start-instances")
.long("start-instances")
.takes_value(true),
)
.arg(Arg::with_name("exclude").long("exclude").takes_value(true))
.arg(Arg::new("exclude").long("exclude").takes_value(true))
.arg(
Arg::with_name("max-crawl-depth")
Arg::new("max-crawl-depth")
.long("max-crawl-depth")
.takes_value(true),
)
@ -49,25 +49,43 @@ pub async fn main() -> Result<(), Error> {
struct TotalStats {
crawled_instances: i32,
failed_instances: i32,
online_users: usize,
total_users: i64,
total_online_users: i32,
users_active_day: i64,
users_active_week: i64,
users_active_month: i64,
users_active_halfyear: i64,
instance_details: Vec<InstanceDetails>,
}
fn aggregate(instance_details: Vec<InstanceDetails>, failed_instances: i32) -> TotalStats {
let mut crawled_instances = 0;
let mut online_users = 0;
let mut total_users = 0;
let mut total_online_users = 0;
let mut users_active_day = 0;
let mut users_active_week = 0;
let mut users_active_month = 0;
let mut users_active_halfyear = 0;
let mut crawled_instances = 0;
for i in &instance_details {
crawled_instances += 1;
total_users += i.total_users;
total_online_users += i.online_users;
online_users += i.site_info.online;
if let Some(site_view) = &i.site_info.site_view {
total_users += site_view.counts.users;
users_active_day += site_view.counts.users_active_day;
users_active_week += site_view.counts.users_active_week;
users_active_month += site_view.counts.users_active_month;
users_active_halfyear += site_view.counts.users_active_half_year;
}
}
TotalStats {
crawled_instances,
failed_instances,
online_users,
total_users,
total_online_users,
users_active_day,
users_active_week,
users_active_halfyear,
users_active_month,
instance_details,
}
}

33
src/node_info.rs
View file

@ -1,33 +0,0 @@
use serde::Deserialize;
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct NodeInfo {
pub version: String,
pub software: NodeInfoSoftware,
pub protocols: Vec<String>,
pub usage: NodeInfoUsage,
pub open_registrations: bool,
}
#[derive(Deserialize, Debug)]
pub struct NodeInfoSoftware {
pub name: String,
pub version: String,
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct NodeInfoUsage {
pub users: NodeInfoUsers,
pub local_posts: i64,
pub local_comments: i64,
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct NodeInfoUsers {
pub total: i64,
pub active_halfyear: i64,
pub active_month: i64,
}