Implemented basics for post federation, plus a bunch of other stuff

docker/federation-test/docker-compose.yml

@@ -58,5 +58,5 @@ services:
     ports:
       - "127.0.0.1:8061:80"
     volumes:
-      - ./iframely.config.local.js:/iframely/config.local.js:ro
+      - ../iframely.config.local.js:/iframely/config.local.js:ro
     restart: always

server/Cargo.lock

@@ -2,10 +2,10 @@
 # It is not intended for manual editing.
 [[package]]
 name = "activitystreams"
-version = "0.5.0-alpha.0"
+version = "0.5.0-alpha.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "activitystreams-derive 0.5.0-alpha.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "activitystreams-derive 0.5.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -16,7 +16,7 @@ dependencies = [
 
 [[package]]
 name = "activitystreams-derive"
-version = "0.5.0-alpha.0"
+version = "0.5.0-alpha.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "proc-macro2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1562,7 +1562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 name = "lemmy_server"
 version = "0.0.1"
 dependencies = [
- "activitystreams 0.5.0-alpha.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "activitystreams 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "actix-files 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "actix-rt 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1583,6 +1583,7 @@ dependencies = [
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lettre 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lettre_email 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3281,8 +3282,8 @@ dependencies = [
 ]
 
 [metadata]
-"checksum activitystreams 0.5.0-alpha.0 (registry+https://github.com/rust-lang/crates.io-index)" = "48f1b48fa9d528f1f5fdef4c54622e7d84f1fd2c4fdfb1ef474b424ae89dc462"
-"checksum activitystreams-derive 0.5.0-alpha.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab374464a70b290cf771fc15ca2f096feb9ad9f5a7b86564219727e23421d59e"
+"checksum activitystreams 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1f5d1db7f182bc74c9a6d2002cb7a5eb99b001ef41ddc8df1c21750ccc3638fa"
+"checksum activitystreams-derive 0.5.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f95c948a832a0b7b230b28369bafe79477bb8ebe7306dc97bcaff43832d3cc4d"
 "checksum actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a4af87564ff659dee8f9981540cac9418c45e910c8072fdedd643a262a38fcaf"
 "checksum actix-codec 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "09e55f0a5c2ca15795035d90c46bd0e73a5123b72f68f12596d6ba5282051380"
 "checksum actix-connect 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c95cc9569221e9802bf4c377f6c18b90ef10227d787611decf79fd47d2a8e76c"

server/Cargo.toml

@@ -9,7 +9,7 @@ diesel = { version = "1.4.2", features = ["postgres","chrono", "r2d2", "64-colum
 diesel_migrations = "1.4.0"
 dotenv = "0.15.0"
 bcrypt = "0.6.1"
-activitystreams = "0.5.0-alpha.0"
+activitystreams = "0.5.0-alpha.4"
 chrono = { version = "0.4.7", features = ["serde"] }
 failure = "0.1.5"
 serde_json = { version = "1.0.45", features = ["preserve_order"]}
@@ -19,6 +19,7 @@ actix-web = "2.0.0"
 actix-files = "0.2.1"
 actix-web-actors = "2.0.0"
 actix-rt = "1.0.0"
+log = "0.4.0"
 env_logger = "0.7.1"
 rand = "0.7.3"
 strum = "0.17.1"

server/src/api/community.rs

@@ -34,12 +34,13 @@ pub struct CommunityResponse {
   pub community: CommunityView,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Debug)]
 pub struct ListCommunities {
   sort: String,
   page: Option<i64>,
   limit: Option<i64>,
   auth: Option<String>,
+  local_only: Option<bool>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -342,7 +343,8 @@ impl Perform<ListCommunitiesResponse> for Oper<ListCommunities> {
   fn perform(&self, conn: &PgConnection) -> Result<ListCommunitiesResponse, Error> {
     let data: &ListCommunities = &self.data;
 
-    if Settings::get().federation_enabled {
+    let local_only = data.local_only.unwrap_or(false);
+    if Settings::get().federation_enabled && !local_only {
       return Ok(ListCommunitiesResponse {
         communities: get_all_communities()?,
       });

server/src/api/post.rs

@@ -44,9 +44,9 @@ pub struct GetPosts {
   auth: Option<String>,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Debug)]
 pub struct GetPostsResponse {
-  posts: Vec<PostView>,
+  pub posts: Vec<PostView>,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -222,7 +222,6 @@ impl Perform<GetPostsResponse> for Oper<GetPosts> {
     let data: &GetPosts = &self.data;
 
     if Settings::get().federation_enabled {
-      dbg!(&data);
       // TODO: intercept here (but the type is wrong)
       //get_remote_community_posts(get_posts.community_id.unwrap())
     }

server/src/apub/post.rs

@@ -22,9 +22,10 @@ impl PostView {
       oprops.set_content_xsd_string(body.to_owned())?;
     }
 
-    if let Some(_url) = &self.url {
-      // TODO: crashes here
-      //oprops.set_url_xsd_any_uri(url.to_owned())?;
+    // TODO: hacky code because we get self.url == Some("")
+    let url = self.url.as_ref();
+    if url.is_some() && !url.unwrap().is_empty() {
+      oprops.set_url_xsd_any_uri(url.unwrap().to_owned())?;
     }
 
     if let Some(u) = self.updated {

server/src/apub/puller.rs

@@ -3,21 +3,27 @@ extern crate reqwest;
 use crate::api::community::{GetCommunityResponse, ListCommunitiesResponse};
 use crate::api::post::GetPostsResponse;
 use crate::db::community_view::CommunityView;
+use crate::db::post_view::PostView;
+use crate::naive_now;
 use crate::settings::Settings;
 use activitystreams::actor::apub::Group;
 use activitystreams::collection::apub::{OrderedCollection, UnorderedCollection};
+use activitystreams::object::apub::Page;
+use activitystreams::object::ObjectBox;
 use failure::Error;
-
-// TODO: right now all of the data is requested on demand, for production we will need to store
-//       things in the local database to not ruin the performance
+use log::warn;
+use serde::Deserialize;
 
 fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityView>, Error> {
   // TODO: check nodeinfo to make sure we are dealing with a lemmy instance
   //       -> means we need proper nodeinfo json classes instead of inline generation
   // TODO: follow pagination (seems like page count is missing?)
   // TODO: see if there is any standard for discovering remote actors, so we dont have to rely on lemmy apis
-  let communities_uri = format!("http://{}/api/v1/communities/list?sort=Hot", domain);
-  let communities1: ListCommunitiesResponse = reqwest::get(&communities_uri)?.json()?;
+  let communities_uri = format!(
+    "http://{}/api/v1/communities/list?sort=Hot&local_only=true",
+    domain
+  );
+  let communities1 = fetch_remote_object::<ListCommunitiesResponse>(&communities_uri)?;
   let mut communities2 = communities1.communities;
   for c in &mut communities2 {
     c.name = format_community_name(&c.name, domain);
@@ -25,7 +31,6 @@ fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityView>, E
   Ok(communities2)
 }
 
-// TODO: this should be cached or stored in the database
 fn get_remote_community_uri(identifier: &str) -> String {
   let x: Vec<&str> = identifier.split('@').collect();
   let name = x[0].replace("!", "");
@@ -33,29 +38,84 @@ fn get_remote_community_uri(identifier: &str) -> String {
   format!("http://{}/federation/c/{}", instance, name)
 }
 
+fn fetch_remote_object<Response>(uri: &str) -> Result<Response, Error>
+where
+  Response: for<'de> Deserialize<'de>,
+{
+  // TODO: should cache responses here when we are in production
+  // TODO: this function should return a future
+  let x: Response = reqwest::get(uri)?.json()?;
+  Ok(x)
+}
+
 pub fn get_remote_community_posts(identifier: &str) -> Result<GetPostsResponse, Error> {
-  let community: Group = reqwest::get(&get_remote_community_uri(identifier))?.json()?;
+  let community = fetch_remote_object::<Group>(&get_remote_community_uri(identifier))?;
   let outbox_uri = &community.ap_actor_props.get_outbox().to_string();
-  let outbox: OrderedCollection = reqwest::get(outbox_uri)?.json()?;
+  let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
   let items = outbox.collection_props.get_many_items_object_boxs();
-  dbg!(items);
-  unimplemented!()
+
+  let posts: Vec<PostView> = items
+    .unwrap()
+    .map(|obox: &ObjectBox| {
+      let page: Page = obox.clone().to_concrete::<Page>().unwrap();
+      // TODO: need to populate this
+      PostView {
+        id: -1,
+        name: page.object_props.get_name_xsd_string().unwrap().to_string(),
+        url: None,
+        body: None,
+        creator_id: -1,
+        community_id: -1,
+        removed: false,
+        locked: false,
+        published: naive_now(),
+        updated: None,
+        deleted: false,
+        nsfw: false,
+        stickied: false,
+        embed_title: None,
+        embed_description: None,
+        embed_html: None,
+        thumbnail_url: None,
+        banned: false,
+        banned_from_community: false,
+        creator_name: "".to_string(),
+        creator_avatar: None,
+        community_name: "".to_string(),
+        community_removed: false,
+        community_deleted: false,
+        community_nsfw: false,
+        number_of_comments: -1,
+        score: -1,
+        upvotes: -1,
+        downvotes: -1,
+        hot_rank: -1,
+        newest_activity_time: naive_now(),
+        user_id: None,
+        my_vote: None,
+        subscribed: None,
+        read: None,
+        saved: None,
+      }
+    })
+    .collect();
+  Ok(GetPostsResponse { posts })
 }
 
 pub fn get_remote_community(identifier: &str) -> Result<GetCommunityResponse, failure::Error> {
-  let community: Group = reqwest::get(&get_remote_community_uri(identifier))?.json()?;
+  let community = fetch_remote_object::<Group>(&get_remote_community_uri(identifier))?;
   let followers_uri = &community
     .ap_actor_props
     .get_followers()
     .unwrap()
     .to_string();
   let outbox_uri = &community.ap_actor_props.get_outbox().to_string();
-  let outbox: OrderedCollection = reqwest::get(outbox_uri)?.json()?;
-  // TODO: this need to be done in get_remote_community_posts() (meaning we need to store the outbox uri?)
-  let followers: UnorderedCollection = reqwest::get(followers_uri)?.json()?;
+  let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
+  let followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
+  // TODO: this is only for testing until we can call that function from GetPosts
+  // (once string ids are supported)
+  //dbg!(get_remote_community_posts(identifier)?);
 
-  // TODO: looks like a bunch of data is missing from the activitypub response
-  // TODO: i dont think simple numeric ids are going to work, we probably need something like uuids
   Ok(GetCommunityResponse {
     moderators: vec![],
     admins: vec![],
@@ -106,18 +166,20 @@ pub fn get_remote_community(identifier: &str) -> Result<GetCommunityResponse, fa
   })
 }
 
-pub fn get_following_instances() -> Result<Vec<String>, Error> {
-  let instance_list = match Settings::get().federated_instance.clone() {
+pub fn get_following_instances() -> Vec<String> {
+  match Settings::get().federated_instance.clone() {
     Some(f) => vec![f, Settings::get().hostname.clone()],
     None => vec![Settings::get().hostname.clone()],
-  };
-  Ok(instance_list)
+  }
 }
 
 pub fn get_all_communities() -> Result<Vec<CommunityView>, Error> {
   let mut communities_list: Vec<CommunityView> = vec![];
-  for instance in &get_following_instances()? {
-    communities_list.append(fetch_communities_from_instance(instance)?.as_mut());
+  for instance in &get_following_instances() {
+    match fetch_communities_from_instance(instance) {
+      Ok(mut c) => communities_list.append(c.as_mut()),
+      Err(e) => warn!("Failed to fetch instance list from remote instance: {}", e),
+    };
   }
   Ok(communities_list)
 }

server/src/lib.rs

@@ -15,6 +15,7 @@ pub extern crate dotenv;
 pub extern crate jsonwebtoken;
 pub extern crate lettre;
 pub extern crate lettre_email;
+extern crate log;
 pub extern crate rand;
 pub extern crate regex;
 pub extern crate serde;

server/src/routes/federation.rs

@@ -28,7 +28,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
         "/federation/u/{user_name}",
         web::get().to(apub::user::get_apub_user),
       )
-      // TODO: this is a very quick and dirty implementation for http api calls
+      // TODO: we should be able to remove this but somehow that breaks the remote community list
       .route(
         "/api/v1/communities/list",
         web::get().to(