Share list of communities over apub, some refactoring

server/Cargo.lock

@@ -2,10 +2,10 @@
 # It is not intended for manual editing.
 [[package]]
 name = "activitystreams"
-version = "0.5.0-alpha.10"
+version = "0.5.0-alpha.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "activitystreams-derive 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "activitystreams-derive 0.5.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -16,7 +16,7 @@ dependencies = [
 
 [[package]]
 name = "activitystreams-derive"
-version = "0.5.0-alpha.4"
+version = "0.5.0-alpha.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "proc-macro2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1401,7 +1401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 name = "lemmy_server"
 version = "0.0.1"
 dependencies = [
- "activitystreams 0.5.0-alpha.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "activitystreams 0.5.0-alpha.15 (registry+https://github.com/rust-lang/crates.io-index)",
  "actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "actix-files 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "actix-rt 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2855,8 +2855,8 @@ dependencies = [
 ]
 
 [metadata]
-"checksum activitystreams 0.5.0-alpha.10 (registry+https://github.com/rust-lang/crates.io-index)" = "04827f3390831f772d15ff3171336cc4fbac714f59233d24731beb7c865293a6"
+"checksum activitystreams 0.5.0-alpha.15 (registry+https://github.com/rust-lang/crates.io-index)" = "1ecff5b18578fac8cff5f139009cd0a4f605b6f5eddc91f2b8319bfef76cc2c3"
-"checksum activitystreams-derive 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2bc640808dceb2efac81e6bcb77a7f4e2e76af7fb60e88f966b48123b625d2f"
+"checksum activitystreams-derive 0.5.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d7498811842309cc5b54c123bcf328e33eebe9841037067b1e9b93caa820085d"
 "checksum actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a4af87564ff659dee8f9981540cac9418c45e910c8072fdedd643a262a38fcaf"
 "checksum actix-codec 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "09e55f0a5c2ca15795035d90c46bd0e73a5123b72f68f12596d6ba5282051380"
 "checksum actix-connect 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c95cc9569221e9802bf4c377f6c18b90ef10227d787611decf79fd47d2a8e76c"

server/Cargo.toml

@@ -9,7 +9,7 @@ diesel = { version = "1.4.2", features = ["postgres","chrono", "r2d2", "64-colum
 diesel_migrations = "1.4.0"
 dotenv = "0.15.0"
 bcrypt = "0.6.1"
-activitystreams = "0.5.0-alpha.10"
+activitystreams = "0.5.0-alpha.15"
 chrono = { version = "0.4.7", features = ["serde"] }
 failure = "0.1.5"
 serde_json = { version = "1.0.45", features = ["preserve_order"]}

server/src/api/community.rs

@@ -36,11 +36,11 @@ pub struct CommunityResponse {
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct ListCommunities {
-  sort: String,
+  pub sort: String,
-  page: Option<i64>,
+  pub page: Option<i64>,
-  limit: Option<i64>,
+  pub limit: Option<i64>,
-  auth: Option<String>,
+  pub auth: Option<String>,
-  local_only: Option<bool>,
+  pub local_only: Option<bool>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]

server/src/apub/community.rs

@@ -1,15 +1,19 @@
-use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType};
+use crate::api::community::ListCommunities;
+use crate::api::{Oper, Perform};
+use crate::apub::puller::{fetch_remote_object, format_community_name};
+use crate::apub::{
+  create_apub_response, get_apub_protocol_string, make_apub_endpoint, EndpointType, GroupExt,
+};
 use crate::convert_datetime;
 use crate::db::community::Community;
-use crate::db::community_view::CommunityFollowerView;
+use crate::db::community_view::{CommunityFollowerView, CommunityView};
 use crate::db::establish_unpooled_connection;
 use crate::db::post_view::{PostQueryBuilder, PostView};
+use crate::settings::Settings;
+use activitystreams::actor::properties::ApActorProperties;
 use activitystreams::collection::OrderedCollection;
 use activitystreams::{
-  actor::{properties::ApActorProperties, Group},
+  actor::Group, collection::UnorderedCollection, context, ext::Extensible,
-  collection::UnorderedCollection,
-  context,
-  ext::Extensible,
   object::properties::ObjectProperties,
 };
 use actix_web::body::Body;
@@ -26,12 +30,42 @@ pub struct CommunityQuery {
   community_name: String,
 }
 
-pub async fn get_apub_community(
+pub async fn get_apub_community_list(
-  info: Path<CommunityQuery>,
   db: web::Data<Pool<ConnectionManager<PgConnection>>>,
 ) -> Result<HttpResponse<Body>, Error> {
-  let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?;
+  // TODO: implement pagination
-  let base_url = make_apub_endpoint(EndpointType::Community, &community.name);
+  let query = ListCommunities {
+    sort: "Hot".to_string(),
+    page: None,
+    limit: None,
+    auth: None,
+    local_only: Some(true),
+  };
+  let communities = Oper::new(query)
+    .perform(&db.get().unwrap())
+    .unwrap()
+    .communities
+    .iter()
+    .map(|c| c.as_group())
+    .collect::<Result<Vec<GroupExt>, failure::Error>>()?;
+  let mut collection = UnorderedCollection::default();
+  let oprops: &mut ObjectProperties = collection.as_mut();
+  oprops.set_context_xsd_any_uri(context())?.set_id(format!(
+    "{}://{}/federation/communities",
+    get_apub_protocol_string(),
+    Settings::get().hostname
+  ))?;
+
+  collection
+    .collection_props
+    .set_total_items(communities.len() as u64)?
+    .set_many_items_base_boxes(communities)?;
+  Ok(create_apub_response(&collection))
+}
+
+impl CommunityView {
+  fn as_group(&self) -> Result<GroupExt, Error> {
+    let base_url = make_apub_endpoint(EndpointType::Community, &self.name);
 
     let mut group = Group::default();
     let oprops: &mut ObjectProperties = group.as_mut();
@@ -39,28 +73,83 @@ pub async fn get_apub_community(
     oprops
       .set_context_xsd_any_uri(context())?
       .set_id(base_url.to_owned())?
-    .set_name_xsd_string(community.title.to_owned())?
+      .set_name_xsd_string(self.name.to_owned())?
-    .set_published(convert_datetime(community.published))?
+      .set_published(convert_datetime(self.published))?
       .set_attributed_to_xsd_any_uri(make_apub_endpoint(
         EndpointType::User,
-      &community.creator_id.to_string(),
+        &self.creator_id.to_string(),
       ))?;
 
-  if let Some(u) = community.updated.to_owned() {
+    if let Some(u) = self.updated.to_owned() {
       oprops.set_updated(convert_datetime(u))?;
     }
-  if let Some(d) = community.description {
+    if let Some(d) = self.description.to_owned() {
       oprops.set_summary_xsd_string(d)?;
     }
 
     let mut actor_props = ApActorProperties::default();
 
     actor_props
+      .set_preferred_username(self.title.to_owned())?
       .set_inbox(format!("{}/inbox", &base_url))?
       .set_outbox(format!("{}/outbox", &base_url))?
       .set_followers(format!("{}/followers", &base_url))?;
 
-  Ok(create_apub_response(&group.extend(actor_props)))
+    Ok(group.extend(actor_props))
+  }
+
+  pub fn from_group(group: &GroupExt, domain: &str) -> Result<CommunityView, Error> {
+    let followers_uri = &group.extension.get_followers().unwrap().to_string();
+    let outbox_uri = &group.extension.get_outbox().to_string();
+    let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
+    let followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
+    let oprops = &group.base.object_props;
+    let aprops = &group.extension;
+    Ok(CommunityView {
+      // TODO: we need to merge id and name into a single thing (stuff like @user@instance.com)
+      id: 1337, //community.object_props.get_id()
+      name: format_community_name(&oprops.get_name_xsd_string().unwrap().to_string(), domain),
+      title: aprops.get_preferred_username().unwrap().to_string(),
+      description: oprops.get_summary_xsd_string().map(|s| s.to_string()),
+      category_id: -1,
+      creator_id: -1, //community.object_props.get_attributed_to_xsd_any_uri()
+      removed: false,
+      published: oprops
+        .get_published()
+        .unwrap()
+        .as_ref()
+        .naive_local()
+        .to_owned(),
+      updated: oprops
+        .get_updated()
+        .map(|u| u.as_ref().to_owned().naive_local()),
+      deleted: false,
+      nsfw: false,
+      creator_name: "".to_string(),
+      creator_avatar: None,
+      category_name: "".to_string(),
+      number_of_subscribers: *followers
+        .collection_props
+        .get_total_items()
+        .unwrap()
+        .as_ref() as i64,
+      number_of_posts: *outbox.collection_props.get_total_items().unwrap().as_ref() as i64,
+      number_of_comments: -1,
+      hot_rank: -1,
+      user_id: None,
+      subscribed: None,
+    })
+  }
+}
+
+pub async fn get_apub_community_http(
+  info: Path<CommunityQuery>,
+  db: web::Data<Pool<ConnectionManager<PgConnection>>>,
+) -> Result<HttpResponse<Body>, Error> {
+  let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?;
+  let community_view = CommunityView::read(&&db.get()?, community.id, None)?;
+  let c = community_view.as_group()?;
+  Ok(create_apub_response(&c))
 }
 
 pub async fn get_apub_community_followers(
@@ -107,7 +196,7 @@ pub async fn get_apub_community_outbox(
     .set_id(base_url)?;
   collection
     .collection_props
-    .set_many_items_object_boxs(
+    .set_many_items_base_boxes(
       community_posts
         .iter()
         .map(|c| c.as_page().unwrap())

server/src/apub/mod.rs

@@ -4,10 +4,14 @@ pub mod puller;
 pub mod user;
 use crate::Settings;
 
+use activitystreams::actor::{properties::ApActorProperties, Group};
+use activitystreams::ext::Ext;
 use actix_web::body::Body;
 use actix_web::HttpResponse;
 use url::Url;
 
+type GroupExt = Ext<Group, ApActorProperties>;
+
 fn create_apub_response<T>(json: &T) -> HttpResponse<Body>
 where
   T: serde::ser::Serialize,

server/src/apub/post.rs

@@ -1,6 +1,6 @@
 use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType};
-use crate::convert_datetime;
 use crate::db::post_view::PostView;
+use crate::{convert_datetime, naive_now};
 use activitystreams::{object::properties::ObjectProperties, object::Page};
 use actix_web::body::Body;
 use actix_web::web::Path;
@@ -59,4 +59,53 @@ impl PostView {
 
     Ok(page)
   }
+
+  pub fn from_page(page: &Page) -> Result<PostView, Error> {
+    let oprops = &page.object_props;
+    Ok(PostView {
+      id: -1,
+      name: oprops.get_name_xsd_string().unwrap().to_string(),
+      url: oprops.get_url_xsd_any_uri().map(|u| u.to_string()),
+      body: oprops.get_content_xsd_string().map(|c| c.to_string()),
+      creator_id: -1,
+      community_id: -1,
+      removed: false,
+      locked: false,
+      published: oprops
+        .get_published()
+        .unwrap()
+        .as_ref()
+        .naive_local()
+        .to_owned(),
+      updated: oprops
+        .get_updated()
+        .map(|u| u.as_ref().to_owned().naive_local()),
+      deleted: false,
+      nsfw: false,
+      stickied: false,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
+      banned: false,
+      banned_from_community: false,
+      creator_name: "".to_string(),
+      creator_avatar: None,
+      community_name: "".to_string(),
+      community_removed: false,
+      community_deleted: false,
+      community_nsfw: false,
+      number_of_comments: -1,
+      score: -1,
+      upvotes: -1,
+      downvotes: -1,
+      hot_rank: -1,
+      newest_activity_time: naive_now(),
+      user_id: None,
+      my_vote: None,
+      subscribed: None,
+      read: None,
+      saved: None,
+    })
+  }
 }

server/src/apub/puller.rs

@@ -1,16 +1,13 @@
-use crate::api::community::{GetCommunityResponse, ListCommunitiesResponse};
+use crate::api::community::GetCommunityResponse;
 use crate::api::post::GetPostsResponse;
-use crate::apub::get_apub_protocol_string;
+use crate::apub::{get_apub_protocol_string, GroupExt};
 use crate::db::community_view::CommunityView;
 use crate::db::post_view::PostView;
-use crate::naive_now;
 use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown};
 use crate::settings::Settings;
-use activitystreams::actor::{properties::ApActorProperties, Group};
 use activitystreams::collection::{OrderedCollection, UnorderedCollection};
-use activitystreams::ext::Ext;
-use activitystreams::object::ObjectBox;
 use activitystreams::object::Page;
+use activitystreams::BaseBox;
 use chttp::prelude::*;
 use failure::Error;
 use log::warn;
@@ -28,39 +25,44 @@ fn fetch_node_info(domain: &str) -> Result<NodeInfo, Error> {
 
 fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityView>, Error> {
   let node_info = fetch_node_info(domain)?;
-  if node_info.software.name != "lemmy" {
+
-    return Err(format_err!(
+  if let Some(community_list_url) = node_info.metadata.community_list_url {
+    let collection = fetch_remote_object::<UnorderedCollection>(&community_list_url)?;
+    let object_boxes = collection
+      .collection_props
+      .get_many_items_base_boxes()
+      .unwrap();
+    let communities: Result<Vec<CommunityView>, Error> = object_boxes
+      .map(|c| -> Result<CommunityView, Error> {
+        let group = c.to_owned().to_concrete::<GroupExt>()?;
+        CommunityView::from_group(&group, domain)
+      })
+      .collect();
+    Ok(communities?)
+  } else {
+    Err(format_err!(
       "{} is not a Lemmy instance, federation is not supported",
       domain
-    ));
+    ))
   }
+}
 
-  // TODO: follow pagination (seems like page count is missing?)
+/// Returns a tuple of (username, domain) from an identifier like "main@dev.lemmy.ml"
-  // TODO: see if there is any standard for discovering remote actors, so we dont have to rely on lemmy apis
+fn split_identifier(identifier: &str) -> (String, String) {
-  let communities_uri = format!(
+  let x: Vec<&str> = identifier.split('@').collect();
-    "http://{}/api/v1/communities/list?sort=Hot&local_only=true",
+  (x[0].replace("!", ""), x[1].to_string())
-    domain
-  );
-  let communities1 = fetch_remote_object::<ListCommunitiesResponse>(&communities_uri)?;
-  let mut communities2 = communities1.communities;
-  for c in &mut communities2 {
-    c.name = format_community_name(&c.name, domain);
-  }
-  Ok(communities2)
 }
 
 fn get_remote_community_uri(identifier: &str) -> String {
-  let x: Vec<&str> = identifier.split('@').collect();
+  let (name, domain) = split_identifier(identifier);
-  let name = x[0].replace("!", "");
+  format!("http://{}/federation/c/{}", domain, name)
-  let instance = x[1];
-  format!("http://{}/federation/c/{}", instance, name)
 }
 
-fn fetch_remote_object<Response>(uri: &str) -> Result<Response, Error>
+pub fn fetch_remote_object<Response>(uri: &str) -> Result<Response, Error>
 where
   Response: for<'de> Deserialize<'de>,
 {
-  if Settings::get().federation.tls_enabled && !uri.starts_with("https") {
+  if Settings::get().federation.tls_enabled && !uri.starts_with("https://") {
     return Err(format_err!("Activitypub uri is insecure: {}", uri));
   }
   // TODO: should cache responses here when we are in production
@@ -71,131 +73,32 @@ where
 }
 
 pub fn get_remote_community_posts(identifier: &str) -> Result<GetPostsResponse, Error> {
-  let community =
+  let community = fetch_remote_object::<GroupExt>(&get_remote_community_uri(identifier))?;
-    fetch_remote_object::<Ext<Group, ApActorProperties>>(&get_remote_community_uri(identifier))?;
   let outbox_uri = &community.extension.get_outbox().to_string();
   let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
-  let items = outbox.collection_props.get_many_items_object_boxs();
+  let items = outbox.collection_props.get_many_items_base_boxes();
 
-  let posts: Vec<PostView> = items
+  let posts: Result<Vec<PostView>, Error> = items
     .unwrap()
-    .map(|obox: &ObjectBox| {
+    .map(|obox: &BaseBox| {
-      let page: Page = obox.clone().to_concrete::<Page>().unwrap();
+      let page = obox.clone().to_concrete::<Page>().unwrap();
-      PostView {
+      PostView::from_page(&page)
-        id: -1,
-        name: page.object_props.get_name_xsd_string().unwrap().to_string(),
-        url: page
-          .object_props
-          .get_url_xsd_any_uri()
-          .map(|u| u.to_string()),
-        body: page
-          .object_props
-          .get_content_xsd_string()
-          .map(|c| c.to_string()),
-        creator_id: -1,
-        community_id: -1,
-        removed: false,
-        locked: false,
-        published: page
-          .object_props
-          .get_published()
-          .unwrap()
-          .as_ref()
-          .naive_local()
-          .to_owned(),
-        updated: page
-          .object_props
-          .get_updated()
-          .map(|u| u.as_ref().to_owned().naive_local()),
-        deleted: false,
-        nsfw: false,
-        stickied: false,
-        embed_title: None,
-        embed_description: None,
-        embed_html: None,
-        thumbnail_url: None,
-        banned: false,
-        banned_from_community: false,
-        creator_name: "".to_string(),
-        creator_avatar: None,
-        community_name: "".to_string(),
-        community_removed: false,
-        community_deleted: false,
-        community_nsfw: false,
-        number_of_comments: -1,
-        score: -1,
-        upvotes: -1,
-        downvotes: -1,
-        hot_rank: -1,
-        newest_activity_time: naive_now(),
-        user_id: None,
-        my_vote: None,
-        subscribed: None,
-        read: None,
-        saved: None,
-      }
     })
     .collect();
-  Ok(GetPostsResponse { posts })
+  Ok(GetPostsResponse { posts: posts? })
 }
 
 pub fn get_remote_community(identifier: &str) -> Result<GetCommunityResponse, failure::Error> {
-  let community =
+  let group = fetch_remote_object::<GroupExt>(&get_remote_community_uri(identifier))?;
-    fetch_remote_object::<Ext<Group, ApActorProperties>>(&get_remote_community_uri(identifier))?;
-  let followers_uri = &community.extension.get_followers().unwrap().to_string();
-  let outbox_uri = &community.extension.get_outbox().to_string();
-  let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
-  let followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
   // TODO: this is only for testing until we can call that function from GetPosts
   // (once string ids are supported)
   //dbg!(get_remote_community_posts(identifier)?);
 
+  let (_, domain) = split_identifier(identifier);
   Ok(GetCommunityResponse {
     moderators: vec![],
     admins: vec![],
-    community: CommunityView {
+    community: CommunityView::from_group(&group, &domain)?,
-      // TODO: we need to merge id and name into a single thing (stuff like @user@instance.com)
-      id: 1337, //community.object_props.get_id()
-      name: identifier.to_string(),
-      title: community
-        .as_ref()
-        .get_name_xsd_string()
-        .unwrap()
-        .to_string(),
-      description: community
-        .as_ref()
-        .get_summary_xsd_string()
-        .map(|s| s.to_string()),
-      category_id: -1,
-      creator_id: -1, //community.object_props.get_attributed_to_xsd_any_uri()
-      removed: false,
-      published: community
-        .as_ref()
-        .get_published()
-        .unwrap()
-        .as_ref()
-        .naive_local()
-        .to_owned(),
-      updated: community
-        .as_ref()
-        .get_updated()
-        .map(|u| u.as_ref().to_owned().naive_local()),
-      deleted: false,
-      nsfw: false,
-      creator_name: "".to_string(),
-      creator_avatar: None,
-      category_name: "".to_string(),
-      number_of_subscribers: *followers
-        .collection_props
-        .get_total_items()
-        .unwrap()
-        .as_ref() as i64, // TODO: need to use the same type
-      number_of_posts: *outbox.collection_props.get_total_items().unwrap().as_ref() as i64,
-      number_of_comments: -1,
-      hot_rank: -1,
-      user_id: None,
-      subscribed: None,
-    },
     online: 0,
   })
 }

server/src/routes/federation.rs

@@ -12,9 +12,13 @@ pub fn config(cfg: &mut web::ServiceConfig) {
   if Settings::get().federation.enabled {
     println!("federation enabled, host is {}", Settings::get().hostname);
     cfg
+      .route(
+        "/federation/communities",
+        web::get().to(apub::community::get_apub_community_list),
+      )
       .route(
         "/federation/c/{community_name}",
-        web::get().to(apub::community::get_apub_community),
+        web::get().to(apub::community::get_apub_community_http),
       )
       .route(
         "/federation/c/{community_name}/followers",

server/src/routes/nodeinfo.rs

@@ -60,6 +60,13 @@ async fn node_info(
         local_comments: site_view.number_of_comments,
         open_registrations: site_view.open_registration,
       },
+      metadata: NodeInfoMetadata {
+        community_list_url: Some(format!(
+          "{}://{}/federation/communities",
+          get_apub_protocol_string(),
+          Settings::get().hostname
+        )),
+      },
     })
   })
   .await
@@ -85,6 +92,7 @@ pub struct NodeInfo {
   pub software: NodeInfoSoftware,
   pub protocols: Vec<String>,
   pub usage: NodeInfoUsage,
+  pub metadata: NodeInfoMetadata,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -106,3 +114,8 @@ pub struct NodeInfoUsage {
 pub struct NodeInfoUsers {
   pub total: i64,
 }
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct NodeInfoMetadata {
+  pub community_list_url: Option<String>,
+}