From 6dac23b88701c77f4bb1e5268bcf6f28cabf0fbb Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Sat, 14 Mar 2020 22:03:05 +0100 Subject: [PATCH] Implemented basics for post federation, plus a bunch of other stuff --- docker/federation-test/docker-compose.yml | 2 +- server/Cargo.lock | 13 +-- server/Cargo.toml | 3 +- server/src/api/community.rs | 6 +- server/src/api/post.rs | 5 +- server/src/apub/post.rs | 7 +- server/src/apub/puller.rs | 106 +++++++++++++++++----- server/src/lib.rs | 1 + server/src/routes/federation.rs | 2 +- 9 files changed, 106 insertions(+), 39 deletions(-) diff --git a/docker/federation-test/docker-compose.yml b/docker/federation-test/docker-compose.yml index 1a7265e1b..63dbf27d3 100644 --- a/docker/federation-test/docker-compose.yml +++ b/docker/federation-test/docker-compose.yml @@ -58,5 +58,5 @@ services: ports: - "127.0.0.1:8061:80" volumes: - - ./iframely.config.local.js:/iframely/config.local.js:ro + - ../iframely.config.local.js:/iframely/config.local.js:ro restart: always diff --git a/server/Cargo.lock b/server/Cargo.lock index 9360230ea..a645d56e8 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [[package]] name = "activitystreams" -version = "0.5.0-alpha.0" +version = "0.5.0-alpha.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "activitystreams-derive 0.5.0-alpha.0 (registry+https://github.com/rust-lang/crates.io-index)", + "activitystreams-derive 0.5.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", @@ -16,7 +16,7 @@ dependencies = [ [[package]] name = "activitystreams-derive" -version = "0.5.0-alpha.0" +version = "0.5.0-alpha.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1562,7 +1562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "lemmy_server" version = "0.0.1" dependencies = [ - "activitystreams 0.5.0-alpha.0 (registry+https://github.com/rust-lang/crates.io-index)", + "activitystreams 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", "actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "actix-files 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "actix-rt 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1583,6 +1583,7 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "lettre 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "lettre_email 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3281,8 +3282,8 @@ dependencies = [ ] [metadata] -"checksum activitystreams 0.5.0-alpha.0 (registry+https://github.com/rust-lang/crates.io-index)" = "48f1b48fa9d528f1f5fdef4c54622e7d84f1fd2c4fdfb1ef474b424ae89dc462" -"checksum activitystreams-derive 0.5.0-alpha.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab374464a70b290cf771fc15ca2f096feb9ad9f5a7b86564219727e23421d59e" +"checksum activitystreams 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1f5d1db7f182bc74c9a6d2002cb7a5eb99b001ef41ddc8df1c21750ccc3638fa" +"checksum activitystreams-derive 0.5.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f95c948a832a0b7b230b28369bafe79477bb8ebe7306dc97bcaff43832d3cc4d" "checksum actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a4af87564ff659dee8f9981540cac9418c45e910c8072fdedd643a262a38fcaf" "checksum actix-codec 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "09e55f0a5c2ca15795035d90c46bd0e73a5123b72f68f12596d6ba5282051380" "checksum actix-connect 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c95cc9569221e9802bf4c377f6c18b90ef10227d787611decf79fd47d2a8e76c" diff --git a/server/Cargo.toml b/server/Cargo.toml index 2efec9656..6aa9adc23 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -9,7 +9,7 @@ diesel = { version = "1.4.2", features = ["postgres","chrono", "r2d2", "64-colum diesel_migrations = "1.4.0" dotenv = "0.15.0" bcrypt = "0.6.1" -activitystreams = "0.5.0-alpha.0" +activitystreams = "0.5.0-alpha.4" chrono = { version = "0.4.7", features = ["serde"] } failure = "0.1.5" serde_json = { version = "1.0.45", features = ["preserve_order"]} @@ -19,6 +19,7 @@ actix-web = "2.0.0" actix-files = "0.2.1" actix-web-actors = "2.0.0" actix-rt = "1.0.0" +log = "0.4.0" env_logger = "0.7.1" rand = "0.7.3" strum = "0.17.1" diff --git a/server/src/api/community.rs b/server/src/api/community.rs index 0f104c2d6..a07b15d72 100644 --- a/server/src/api/community.rs +++ b/server/src/api/community.rs @@ -34,12 +34,13 @@ pub struct CommunityResponse { pub community: CommunityView, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct ListCommunities { sort: String, page: Option, limit: Option, auth: Option, + local_only: Option, } #[derive(Serialize, Deserialize, Debug)] @@ -342,7 +343,8 @@ impl Perform for Oper { fn perform(&self, conn: &PgConnection) -> Result { let data: &ListCommunities = &self.data; - if Settings::get().federation_enabled { + let local_only = data.local_only.unwrap_or(false); + if Settings::get().federation_enabled && !local_only { return Ok(ListCommunitiesResponse { communities: get_all_communities()?, }); diff --git a/server/src/api/post.rs b/server/src/api/post.rs index e19d4ee96..1602626be 100644 --- a/server/src/api/post.rs +++ b/server/src/api/post.rs @@ -44,9 +44,9 @@ pub struct GetPosts { auth: Option, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct GetPostsResponse { - posts: Vec, + pub posts: Vec, } #[derive(Serialize, Deserialize)] @@ -222,7 +222,6 @@ impl Perform for Oper { let data: &GetPosts = &self.data; if Settings::get().federation_enabled { - dbg!(&data); // TODO: intercept here (but the type is wrong) //get_remote_community_posts(get_posts.community_id.unwrap()) } diff --git a/server/src/apub/post.rs b/server/src/apub/post.rs index 6950e561a..61b5a78da 100644 --- a/server/src/apub/post.rs +++ b/server/src/apub/post.rs @@ -22,9 +22,10 @@ impl PostView { oprops.set_content_xsd_string(body.to_owned())?; } - if let Some(_url) = &self.url { - // TODO: crashes here - //oprops.set_url_xsd_any_uri(url.to_owned())?; + // TODO: hacky code because we get self.url == Some("") + let url = self.url.as_ref(); + if url.is_some() && !url.unwrap().is_empty() { + oprops.set_url_xsd_any_uri(url.unwrap().to_owned())?; } if let Some(u) = self.updated { diff --git a/server/src/apub/puller.rs b/server/src/apub/puller.rs index edadb7bf5..50a6cd69d 100644 --- a/server/src/apub/puller.rs +++ b/server/src/apub/puller.rs @@ -3,21 +3,27 @@ extern crate reqwest; use crate::api::community::{GetCommunityResponse, ListCommunitiesResponse}; use crate::api::post::GetPostsResponse; use crate::db::community_view::CommunityView; +use crate::db::post_view::PostView; +use crate::naive_now; use crate::settings::Settings; use activitystreams::actor::apub::Group; use activitystreams::collection::apub::{OrderedCollection, UnorderedCollection}; +use activitystreams::object::apub::Page; +use activitystreams::object::ObjectBox; use failure::Error; - -// TODO: right now all of the data is requested on demand, for production we will need to store -// things in the local database to not ruin the performance +use log::warn; +use serde::Deserialize; fn fetch_communities_from_instance(domain: &str) -> Result, Error> { // TODO: check nodeinfo to make sure we are dealing with a lemmy instance // -> means we need proper nodeinfo json classes instead of inline generation // TODO: follow pagination (seems like page count is missing?) // TODO: see if there is any standard for discovering remote actors, so we dont have to rely on lemmy apis - let communities_uri = format!("http://{}/api/v1/communities/list?sort=Hot", domain); - let communities1: ListCommunitiesResponse = reqwest::get(&communities_uri)?.json()?; + let communities_uri = format!( + "http://{}/api/v1/communities/list?sort=Hot&local_only=true", + domain + ); + let communities1 = fetch_remote_object::(&communities_uri)?; let mut communities2 = communities1.communities; for c in &mut communities2 { c.name = format_community_name(&c.name, domain); @@ -25,7 +31,6 @@ fn fetch_communities_from_instance(domain: &str) -> Result, E Ok(communities2) } -// TODO: this should be cached or stored in the database fn get_remote_community_uri(identifier: &str) -> String { let x: Vec<&str> = identifier.split('@').collect(); let name = x[0].replace("!", ""); @@ -33,29 +38,84 @@ fn get_remote_community_uri(identifier: &str) -> String { format!("http://{}/federation/c/{}", instance, name) } +fn fetch_remote_object(uri: &str) -> Result +where + Response: for<'de> Deserialize<'de>, +{ + // TODO: should cache responses here when we are in production + // TODO: this function should return a future + let x: Response = reqwest::get(uri)?.json()?; + Ok(x) +} + pub fn get_remote_community_posts(identifier: &str) -> Result { - let community: Group = reqwest::get(&get_remote_community_uri(identifier))?.json()?; + let community = fetch_remote_object::(&get_remote_community_uri(identifier))?; let outbox_uri = &community.ap_actor_props.get_outbox().to_string(); - let outbox: OrderedCollection = reqwest::get(outbox_uri)?.json()?; + let outbox = fetch_remote_object::(outbox_uri)?; let items = outbox.collection_props.get_many_items_object_boxs(); - dbg!(items); - unimplemented!() + + let posts: Vec = items + .unwrap() + .map(|obox: &ObjectBox| { + let page: Page = obox.clone().to_concrete::().unwrap(); + // TODO: need to populate this + PostView { + id: -1, + name: page.object_props.get_name_xsd_string().unwrap().to_string(), + url: None, + body: None, + creator_id: -1, + community_id: -1, + removed: false, + locked: false, + published: naive_now(), + updated: None, + deleted: false, + nsfw: false, + stickied: false, + embed_title: None, + embed_description: None, + embed_html: None, + thumbnail_url: None, + banned: false, + banned_from_community: false, + creator_name: "".to_string(), + creator_avatar: None, + community_name: "".to_string(), + community_removed: false, + community_deleted: false, + community_nsfw: false, + number_of_comments: -1, + score: -1, + upvotes: -1, + downvotes: -1, + hot_rank: -1, + newest_activity_time: naive_now(), + user_id: None, + my_vote: None, + subscribed: None, + read: None, + saved: None, + } + }) + .collect(); + Ok(GetPostsResponse { posts }) } pub fn get_remote_community(identifier: &str) -> Result { - let community: Group = reqwest::get(&get_remote_community_uri(identifier))?.json()?; + let community = fetch_remote_object::(&get_remote_community_uri(identifier))?; let followers_uri = &community .ap_actor_props .get_followers() .unwrap() .to_string(); let outbox_uri = &community.ap_actor_props.get_outbox().to_string(); - let outbox: OrderedCollection = reqwest::get(outbox_uri)?.json()?; - // TODO: this need to be done in get_remote_community_posts() (meaning we need to store the outbox uri?) - let followers: UnorderedCollection = reqwest::get(followers_uri)?.json()?; + let outbox = fetch_remote_object::(outbox_uri)?; + let followers = fetch_remote_object::(followers_uri)?; + // TODO: this is only for testing until we can call that function from GetPosts + // (once string ids are supported) + //dbg!(get_remote_community_posts(identifier)?); - // TODO: looks like a bunch of data is missing from the activitypub response - // TODO: i dont think simple numeric ids are going to work, we probably need something like uuids Ok(GetCommunityResponse { moderators: vec![], admins: vec![], @@ -106,18 +166,20 @@ pub fn get_remote_community(identifier: &str) -> Result Result, Error> { - let instance_list = match Settings::get().federated_instance.clone() { +pub fn get_following_instances() -> Vec { + match Settings::get().federated_instance.clone() { Some(f) => vec![f, Settings::get().hostname.clone()], None => vec![Settings::get().hostname.clone()], - }; - Ok(instance_list) + } } pub fn get_all_communities() -> Result, Error> { let mut communities_list: Vec = vec![]; - for instance in &get_following_instances()? { - communities_list.append(fetch_communities_from_instance(instance)?.as_mut()); + for instance in &get_following_instances() { + match fetch_communities_from_instance(instance) { + Ok(mut c) => communities_list.append(c.as_mut()), + Err(e) => warn!("Failed to fetch instance list from remote instance: {}", e), + }; } Ok(communities_list) } diff --git a/server/src/lib.rs b/server/src/lib.rs index 3336ae4d2..bf3c3c0ac 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -15,6 +15,7 @@ pub extern crate dotenv; pub extern crate jsonwebtoken; pub extern crate lettre; pub extern crate lettre_email; +extern crate log; pub extern crate rand; pub extern crate regex; pub extern crate serde; diff --git a/server/src/routes/federation.rs b/server/src/routes/federation.rs index 0be051eb8..99b4d2c0b 100644 --- a/server/src/routes/federation.rs +++ b/server/src/routes/federation.rs @@ -28,7 +28,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { "/federation/u/{user_name}", web::get().to(apub::user::get_apub_user), ) - // TODO: this is a very quick and dirty implementation for http api calls + // TODO: we should be able to remove this but somehow that breaks the remote community list .route( "/api/v1/communities/list", web::get().to(