Merge branch 'remove_username_lower_unique' into federation

docker/federation/docker-compose.yml

@@ -24,14 +24,14 @@ services:
       - LEMMY_JWT_SECRET=changeme
       - LEMMY_FRONT_END_DIR=/app/dist
       - LEMMY_FEDERATION__ENABLED=true
-      - LEMMY_FEDERATION__FOLLOWED_INSTANCES=lemmy_beta:8550
       - LEMMY_FEDERATION__TLS_ENABLED=false
+      - LEMMY_FEDERATION__INSTANCE_WHITELIST=lemmy_beta
       - LEMMY_PORT=8540
       - LEMMY_SETUP__ADMIN_USERNAME=lemmy_alpha
       - LEMMY_SETUP__ADMIN_PASSWORD=lemmy
       - LEMMY_SETUP__SITE_NAME=lemmy_alpha
       - RUST_BACKTRACE=1
-      - RUST_LOG=actix_web=debug
+      - RUST_LOG=debug
     restart: always
     depends_on:
       - postgres_alpha
@@ -58,14 +58,14 @@ services:
       - LEMMY_JWT_SECRET=changeme
       - LEMMY_FRONT_END_DIR=/app/dist
       - LEMMY_FEDERATION__ENABLED=true
-      - LEMMY_FEDERATION__FOLLOWED_INSTANCES=lemmy_alpha:8540
       - LEMMY_FEDERATION__TLS_ENABLED=false
+      - LEMMY_FEDERATION__INSTANCE_WHITELIST=lemmy_alpha
       - LEMMY_PORT=8550
       - LEMMY_SETUP__ADMIN_USERNAME=lemmy_beta
       - LEMMY_SETUP__ADMIN_PASSWORD=lemmy
       - LEMMY_SETUP__SITE_NAME=lemmy_beta
       - RUST_BACKTRACE=1
-      - RUST_LOG=actix_web=debug
+      - RUST_LOG=debug
     restart: always
     depends_on:
       - postgres_beta

docs/src/contributing_federation_development.md

@@ -5,17 +5,17 @@
 If you don't have a local clone of the Lemmy repo yet, just run the following command:
 
 ```bash
-git clone https://yerbamate.dev/nutomic/lemmy.git -b federation
+git clone https://yerbamate.dev/LemmyNet/lemmy.git -b federation
 ```
 
 If you already have the Lemmy repo cloned, you need to add a new remote:
 ```bash
-git remote add federation https://yerbamate.dev/nutomic/lemmy.git
+git remote add federation https://yerbamate.dev/LemmyNet/lemmy.git
 git checkout federation
 git pull federation federation
 ```
 
-## Running
+## Running locally
 
 You need to have the following packages installed, the Docker service needs to be running.
 
@@ -31,7 +31,30 @@ cd dev/federation-test
 ```
 
 After the build is finished and the docker-compose setup is running, open [127.0.0.1:8540](http://127.0.0.1:8540) and
-[127.0.0.1:8541](http://127.0.0.1:8541) in your browser to use the test instances. You can login as admin with
+[127.0.0.1:8550](http://127.0.0.1:8550) in your browser to use the test instances. You can login as admin with
-username `lemmy` and password `lemmy`, or create new accounts.
+username `lemmy_alpha` and `lemmy_beta` respectively, with password `lemmy`.
 
-Please get in touch if you want to contribute to this, so we can coordinate things and avoid duplicate work.
+## Running on a server
+
+Note that federation is currently in alpha. Only use it for testing, not on any production server, and be aware
+that you might have to wipe the instance data at one point or another.
+
+Follow the normal installation instructions, either with [Ansible](administration_install_ansible.md) or
+[manually](administration_install_docker.md). Then replace the line `image: dessalines/lemmy:v0.x.x` in 
+`/lemmy/docker-compose.yml` with `image: dessalines/lemmy:federation`. Also add the following in
+`/lemmy/lemmy.hjson`:
+
+```
+    federation: {
+        enabled: true
+        instance_whitelist: example.com
+    }
+```
+
+Afterwards, and whenver you want to update to the latest version, run these commands on the server:
+
+```
+cd /lemmy/
+sudo docker-compose pull
+sudo docker-compose up -d
+```

server/Cargo.lock

@@ -74,7 +74,7 @@ dependencies = [
  "derive_more 0.99.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "trust-dns-proto 0.18.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "trust-dns-resolver 0.18.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -126,7 +126,7 @@ dependencies = [
  "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "h2 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -160,7 +160,7 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bytestring 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1176,7 +1176,7 @@ dependencies = [
  "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1237,7 +1237,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "http"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1245,6 +1245,15 @@ dependencies = [
  "itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "http-signature-normalization"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "httparse"
 version = "1.3.4"
@@ -1314,7 +1323,7 @@ dependencies = [
  "futures-channel 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-io 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1387,6 +1396,7 @@ dependencies = [
  "actix-rt 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "actix-web 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "actix-web-actors 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "base64 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bcrypt 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "comrak 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1398,6 +1408,8 @@ dependencies = [
  "failure 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "hjson 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http-signature-normalization 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "isahc 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonwebtoken 7.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3070,7 +3082,8 @@ dependencies = [
 "checksum hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "21ceb46a83a85e824ef93669c8b390009623863b5c195d1ba747292c0c72f94e"
 "checksum hostname 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867"
 "checksum htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
-"checksum http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b"
+"checksum http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9"
+"checksum http-signature-normalization 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "257835255b5d40c6de712d90e56dc874ca5da2816121e7b9f3cfc7b3a55a5714"
 "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
 "checksum humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
 "checksum ident_case 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"

server/Cargo.toml

@@ -39,3 +39,6 @@ percent-encoding = "2.1.0"
 isahc = "0.9"
 comrak = "0.7"
 openssl = "0.10"
+http = "0.2.1"
+http-signature-normalization = "0.4.1"
+base64 = "0.12.0"

server/config/defaults.hjson

@@ -54,10 +54,10 @@
   federation: {
     # whether to enable activitypub federation. this feature is in alpha, do not enable in production.
     enabled: false
-    # comma seperated list of instances to follow
-    followed_instances: ""
     # whether tls is required for activitypub. only disable this for debugging, never for producion.
     tls_enabled: true
+    # comma seperated list of instances with which federation is allowed
+    instance_whitelist: ""
   }
 #  # email sending configuration
 #  email: {

server/migrations/2020-04-21-123957_remove_unique_user_constraints/down.sql

@@ -0,0 +1,4 @@
+-- The username index
+drop index idx_user_name_lower_actor_id;
+create unique index idx_user_name_lower on user_ (lower(name));
+

server/migrations/2020-04-21-123957_remove_unique_user_constraints/up.sql

@@ -0,0 +1,2 @@
+drop index idx_user_name_lower;
+create unique index idx_user_name_lower_actor_id on user_ (lower(name), lower(actor_id));

server/src/api/community.rs

@@ -1,6 +1,7 @@
 use super::*;
 use crate::apub::activities::follow_community;
-use crate::apub::{gen_keypair_str, make_apub_endpoint, EndpointType};
+use crate::apub::signatures::generate_actor_keypair;
+use crate::apub::{make_apub_endpoint, EndpointType};
 use diesel::PgConnection;
 use std::str::FromStr;
 
@@ -200,7 +201,7 @@ impl Perform<CommunityResponse> for Oper<CreateCommunity> {
     }
 
     // When you create a community, make sure the user becomes a moderator and a follower
-    let (community_public_key, community_private_key) = gen_keypair_str();
+    let keypair = generate_actor_keypair()?;
 
     let community_form = CommunityForm {
       name: data.name.to_owned(),
@@ -214,8 +215,8 @@ impl Perform<CommunityResponse> for Oper<CreateCommunity> {
       updated: None,
       actor_id: make_apub_endpoint(EndpointType::Community, &data.name).to_string(),
       local: true,
-      private_key: Some(community_private_key),
+      private_key: Some(keypair.private_key),
-      public_key: Some(community_public_key),
+      public_key: Some(keypair.public_key),
       last_refreshed_at: None,
       published: None,
     };

server/src/api/site.rs

@@ -1,9 +1,10 @@
 use super::*;
 use crate::api::user::Register;
 use crate::api::{Oper, Perform};
+use crate::apub::fetcher::search_by_apub_id;
 use crate::settings::Settings;
 use diesel::PgConnection;
-use log::info;
+use log::{debug, info};
 use std::str::FromStr;
 
 #[derive(Serialize, Deserialize)]
@@ -14,7 +15,7 @@ pub struct ListCategoriesResponse {
   categories: Vec<Category>,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Debug)]
 pub struct Search {
   q: String,
   type_: String,
@@ -25,13 +26,13 @@ pub struct Search {
   auth: Option<String>,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Debug)]
 pub struct SearchResponse {
-  type_: String,
+  pub type_: String,
-  comments: Vec<CommentView>,
+  pub comments: Vec<CommentView>,
-  posts: Vec<PostView>,
+  pub posts: Vec<PostView>,
-  communities: Vec<CommunityView>,
+  pub communities: Vec<CommunityView>,
-  users: Vec<UserView>,
+  pub users: Vec<UserView>,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -354,6 +355,12 @@ impl Perform<SearchResponse> for Oper<Search> {
   fn perform(&self, conn: &PgConnection) -> Result<SearchResponse, Error> {
     let data: &Search = &self.data;
 
+    dbg!(&data);
+    match search_by_apub_id(&data.q, conn) {
+      Ok(r) => return Ok(r),
+      Err(e) => debug!("Failed to resolve search query as activitypub ID: {}", e),
+    }
+
     let user_id: Option<i32> = match &data.auth {
       Some(auth) => match Claims::decode(&auth) {
         Ok(claims) => {

server/src/api/user.rs

@@ -1,5 +1,6 @@
 use super::*;
-use crate::apub::{gen_keypair_str, make_apub_endpoint, EndpointType};
+use crate::apub::signatures::generate_actor_keypair;
+use crate::apub::{make_apub_endpoint, EndpointType};
 use crate::settings::Settings;
 use crate::{generate_random_string, send_email};
 use bcrypt::verify;
@@ -251,7 +252,7 @@ impl Perform<LoginResponse> for Oper<Register> {
       return Err(APIError::err("admin_already_created").into());
     }
 
-    let (user_public_key, user_private_key) = gen_keypair_str();
+    let keypair = generate_actor_keypair()?;
 
     // Register the new user
     let user_form = UserForm {
@@ -274,8 +275,8 @@ impl Perform<LoginResponse> for Oper<Register> {
       actor_id: make_apub_endpoint(EndpointType::User, &data.username).to_string(),
       bio: None,
       local: true,
-      private_key: Some(user_private_key),
+      private_key: Some(keypair.private_key),
-      public_key: Some(user_public_key),
+      public_key: Some(keypair.public_key),
       last_refreshed_at: None,
     };
 
@@ -295,7 +296,7 @@ impl Perform<LoginResponse> for Oper<Register> {
       }
     };
 
-    let (community_public_key, community_private_key) = gen_keypair_str();
+    let keypair = generate_actor_keypair()?;
 
     // Create the main community if it doesn't exist
     let main_community: Community = match Community::read(&conn, 2) {
@@ -314,8 +315,8 @@ impl Perform<LoginResponse> for Oper<Register> {
           updated: None,
           actor_id: make_apub_endpoint(EndpointType::Community, default_community_name).to_string(),
           local: true,
-          private_key: Some(community_private_key),
+          private_key: Some(keypair.private_key),
-          public_key: Some(community_public_key),
+          public_key: Some(keypair.public_key),
           last_refreshed_at: None,
           published: None,
         };

server/src/apub/activities.rs

@@ -1,5 +1,7 @@
-use crate::apub::{get_apub_protocol_string, get_following_instances};
+use crate::apub::is_apub_id_valid;
+use crate::apub::signatures::sign;
 use crate::db::community::Community;
+use crate::db::community_view::CommunityFollowerView;
 use crate::db::post::Post;
 use crate::db::user::User_;
 use crate::db::Crud;
@@ -11,7 +13,9 @@ use diesel::PgConnection;
 use failure::Error;
 use failure::_core::fmt::Debug;
 use isahc::prelude::*;
+use log::debug;
 use serde::Serialize;
+use url::Url;
 
 fn populate_object_props(
   props: &mut ObjectProperties,
@@ -29,37 +33,48 @@ fn populate_object_props(
   Ok(())
 }
 
-fn send_activity<A>(activity: &A, to: Vec<String>) -> Result<(), Error>
+/// Send an activity to a list of recipients, using the correct headers etc.
+fn send_activity<A>(
+  activity: &A,
+  private_key: &str,
+  sender_id: &str,
+  to: Vec<String>,
+) -> Result<(), Error>
 where
   A: Serialize + Debug,
 {
   let json = serde_json::to_string(&activity)?;
-  println!("sending data {}", json);
+  debug!("Sending activitypub activity {} to {:?}", json, to);
   for t in to {
-    println!("to: {}", t);
+    let to_url = Url::parse(&t)?;
-    let res = Request::post(t)
+    if !is_apub_id_valid(&to_url) {
+      debug!("Not sending activity to {} (invalid or blacklisted)", t);
+      continue;
+    }
+    let request = Request::post(t).header("Host", to_url.domain().unwrap());
+    let signature = sign(&request, private_key, sender_id)?;
+    let res = request
+      .header("Signature", signature)
       .header("Content-Type", "application/json")
       .body(json.to_owned())?
       .send()?;
-    dbg!(res);
+    debug!("Result for activity send: {:?}", res);
   }
   Ok(())
 }
 
-fn get_followers(_community: &Community) -> Vec<String> {
+/// For a given community, returns the inboxes of all followers.
-  // TODO: this is wrong, needs to go to the (non-local) followers of the community
+fn get_follower_inboxes(conn: &PgConnection, community: &Community) -> Result<Vec<String>, Error> {
-  get_following_instances()
+  Ok(
+    CommunityFollowerView::for_community(conn, community.id)?
       .iter()
-    .map(|i| {
+      .filter(|c| !c.user_local)
-      format!(
+      .map(|c| format!("{}/inbox", c.user_actor_id.to_owned()))
-        "{}://{}/federation/inbox",
+      .collect(),
-        get_apub_protocol_string(),
-        i.domain
   )
-    })
-    .collect()
 }
 
+/// Send out information about a newly created post, to the followers of the community.
 pub fn post_create(post: &Post, creator: &User_, conn: &PgConnection) -> Result<(), Error> {
   let page = post.as_page(conn)?;
   let community = Community::read(conn, post.community_id)?;
@@ -73,10 +88,16 @@ pub fn post_create(post: &Post, creator: &User_, conn: &PgConnection) -> Result<
     .create_props
     .set_actor_xsd_any_uri(creator.actor_id.to_owned())?
     .set_object_base_box(page)?;
-  send_activity(&create, get_followers(&community))?;
+  send_activity(
+    &create,
+    &creator.private_key.as_ref().unwrap(),
+    &creator.actor_id,
+    get_follower_inboxes(conn, &community)?,
+  )?;
   Ok(())
 }
 
+/// Send out information about an edited post, to the followers of the community.
 pub fn post_update(post: &Post, creator: &User_, conn: &PgConnection) -> Result<(), Error> {
   let page = post.as_page(conn)?;
   let community = Community::read(conn, post.community_id)?;
@@ -90,10 +111,16 @@ pub fn post_update(post: &Post, creator: &User_, conn: &PgConnection) -> Result<
     .update_props
     .set_actor_xsd_any_uri(creator.actor_id.to_owned())?
     .set_object_base_box(page)?;
-  send_activity(&update, get_followers(&community))?;
+  send_activity(
+    &update,
+    &creator.private_key.as_ref().unwrap(),
+    &creator.actor_id,
+    get_follower_inboxes(conn, &community)?,
+  )?;
   Ok(())
 }
 
+/// As a given local user, send out a follow request to a remote community.
 pub fn follow_community(
   community: &Community,
   user: &User_,
@@ -110,11 +137,23 @@ pub fn follow_community(
     .set_actor_xsd_any_uri(user.actor_id.clone())?
     .set_object_xsd_any_uri(community.actor_id.clone())?;
   let to = format!("{}/inbox", community.actor_id);
-  send_activity(&follow, vec![to])?;
+  send_activity(
+    &follow,
+    &user.private_key.as_ref().unwrap(),
+    &community.actor_id,
+    vec![to],
+  )?;
   Ok(())
 }
 
-pub fn accept_follow(follow: &Follow) -> Result<(), Error> {
+/// As a local community, accept the follow request from a remote user.
+pub fn accept_follow(follow: &Follow, conn: &PgConnection) -> Result<(), Error> {
+  let community_uri = follow
+    .follow_props
+    .get_object_xsd_any_uri()
+    .unwrap()
+    .to_string();
+  let community = Community::read_from_actor_id(conn, &community_uri)?;
   let mut accept = Accept::new();
   accept
     .object_props
@@ -130,14 +169,12 @@ pub fn accept_follow(follow: &Follow) -> Result<(), Error> {
   accept
     .accept_props
     .set_object_base_box(BaseBox::from_concrete(follow.clone())?)?;
-  let to = format!(
+  let to = format!("{}/inbox", community_uri);
-    "{}/inbox",
+  send_activity(
-    follow
+    &accept,
-      .follow_props
+    &community.private_key.unwrap(),
-      .get_actor_xsd_any_uri()
+    &community.actor_id,
-      .unwrap()
+    vec![to],
-      .to_string()
+  )?;
-  );
-  send_activity(&accept, vec![to])?;
   Ok(())
 }

server/src/apub/community.rs

@@ -7,7 +7,6 @@ use crate::db::establish_unpooled_connection;
 use crate::db::post::Post;
 use crate::db::user::User_;
 use crate::db::Crud;
-use crate::settings::Settings;
 use crate::{convert_datetime, naive_now};
 use activitystreams::actor::properties::ApActorProperties;
 use activitystreams::collection::OrderedCollection;
@@ -30,30 +29,8 @@ pub struct CommunityQuery {
   community_name: String,
 }
 
-pub async fn get_apub_community_list(
-  db: web::Data<Pool<ConnectionManager<PgConnection>>>,
-) -> Result<HttpResponse<Body>, Error> {
-  // TODO: implement pagination
-  let communities = Community::list_local(&db.get().unwrap())?
-    .iter()
-    .map(|c| c.as_group(&db.get().unwrap()))
-    .collect::<Result<Vec<GroupExt>, Error>>()?;
-  let mut collection = UnorderedCollection::default();
-  let oprops: &mut ObjectProperties = collection.as_mut();
-  oprops.set_context_xsd_any_uri(context())?.set_id(format!(
-    "{}://{}/federation/communities",
-    get_apub_protocol_string(),
-    Settings::get().hostname
-  ))?;
-
-  collection
-    .collection_props
-    .set_total_items(communities.len() as u64)?
-    .set_many_items_base_boxes(communities)?;
-  Ok(create_apub_response(&collection))
-}
-
 impl Community {
+  // Turn a Lemmy Community into an ActivityPub group that can be sent out over the network.
   fn as_group(&self, conn: &PgConnection) -> Result<GroupExt, Error> {
     let mut group = Group::default();
     let oprops: &mut ObjectProperties = group.as_mut();
@@ -104,6 +81,7 @@ impl Community {
 }
 
 impl CommunityForm {
+  /// Parse an ActivityPub group received from another instance into a Lemmy community.
   pub fn from_group(group: &GroupExt, conn: &PgConnection) -> Result<Self, Error> {
     let oprops = &group.base.base.object_props;
     let aprops = &group.base.extension;
@@ -142,6 +120,7 @@ impl CommunityForm {
   }
 }
 
+/// Return the community json over HTTP.
 pub async fn get_apub_community_http(
   info: Path<CommunityQuery>,
   db: web::Data<Pool<ConnectionManager<PgConnection>>>,
@@ -151,6 +130,7 @@ pub async fn get_apub_community_http(
   Ok(create_apub_response(&c))
 }
 
+/// Returns an empty followers collection, only populating the siz (for privacy).
 pub async fn get_apub_community_followers(
   info: Path<CommunityQuery>,
   db: web::Data<Pool<ConnectionManager<PgConnection>>>,
@@ -173,6 +153,7 @@ pub async fn get_apub_community_followers(
   Ok(create_apub_response(&collection))
 }
 
+/// Returns an UnorderedCollection with the latest posts from the community.
 pub async fn get_apub_community_outbox(
   info: Path<CommunityQuery>,
   db: web::Data<Pool<ConnectionManager<PgConnection>>>,

server/src/apub/community_inbox.rs

@@ -0,0 +1,69 @@
+use crate::apub::activities::accept_follow;
+use crate::apub::fetcher::fetch_remote_user;
+use crate::apub::signatures::verify;
+use crate::db::community::{Community, CommunityFollower, CommunityFollowerForm};
+use crate::db::Followable;
+use activitystreams::activity::Follow;
+use actix_web::{web, HttpRequest, HttpResponse};
+use diesel::r2d2::{ConnectionManager, Pool};
+use diesel::PgConnection;
+use failure::Error;
+use log::debug;
+use serde::Deserialize;
+use url::Url;
+
+#[serde(untagged)]
+#[derive(Deserialize, Debug)]
+pub enum CommunityAcceptedObjects {
+  Follow(Follow),
+}
+
+/// Handler for all incoming activities to community inboxes.
+pub async fn community_inbox(
+  request: HttpRequest,
+  input: web::Json<CommunityAcceptedObjects>,
+  path: web::Path<String>,
+  db: web::Data<Pool<ConnectionManager<PgConnection>>>,
+) -> Result<HttpResponse, Error> {
+  let input = input.into_inner();
+  let conn = &db.get().unwrap();
+  debug!(
+    "Community {} received activity {:?}",
+    &path.into_inner(),
+    &input
+  );
+  match input {
+    CommunityAcceptedObjects::Follow(f) => handle_follow(&f, &request, conn),
+  }
+}
+
+/// Handle a follow request from a remote user, adding it to the local database and returning an
+/// Accept activity.
+fn handle_follow(
+  follow: &Follow,
+  request: &HttpRequest,
+  conn: &PgConnection,
+) -> Result<HttpResponse, Error> {
+  let user_uri = follow
+    .follow_props
+    .get_actor_xsd_any_uri()
+    .unwrap()
+    .to_string();
+  let user = fetch_remote_user(&Url::parse(&user_uri)?, conn)?;
+  verify(&request, &user.public_key.unwrap())?;
+
+  // TODO: make sure this is a local community
+  let community_uri = follow
+    .follow_props
+    .get_object_xsd_any_uri()
+    .unwrap()
+    .to_string();
+  let community = Community::read_from_actor_id(conn, &community_uri)?;
+  let community_follower_form = CommunityFollowerForm {
+    community_id: community.id,
+    user_id: user.id,
+  };
+  CommunityFollower::follow(&conn, &community_follower_form)?;
+  accept_follow(&follow, conn)?;
+  Ok(HttpResponse::Ok().finish())
+}

server/src/apub/fetcher.rs

@@ -1,65 +1,74 @@
+use crate::api::site::SearchResponse;
 use crate::apub::*;
 use crate::db::community::{Community, CommunityForm};
+use crate::db::community_view::CommunityView;
 use crate::db::post::{Post, PostForm};
+use crate::db::post_view::PostView;
 use crate::db::user::{UserForm, User_};
-use crate::db::Crud;
+use crate::db::user_view::UserView;
+use crate::db::{Crud, SearchType};
 use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown};
-use crate::settings::Settings;
+use activitystreams::collection::OrderedCollection;
-use activitystreams::collection::{OrderedCollection, UnorderedCollection};
 use activitystreams::object::Page;
 use activitystreams::BaseBox;
 use diesel::result::Error::NotFound;
 use diesel::PgConnection;
 use failure::Error;
 use isahc::prelude::*;
-use log::warn;
 use serde::Deserialize;
 use std::time::Duration;
 use url::Url;
 
-fn fetch_node_info(instance: &Instance) -> Result<NodeInfo, Error> {
+// Fetch nodeinfo metadata from a remote instance.
+fn _fetch_node_info(domain: &str) -> Result<NodeInfo, Error> {
   let well_known_uri = Url::parse(&format!(
     "{}://{}/.well-known/nodeinfo",
     get_apub_protocol_string(),
-    instance.domain
+    domain
   ))?;
   let well_known = fetch_remote_object::<NodeInfoWellKnown>(&well_known_uri)?;
   Ok(fetch_remote_object::<NodeInfo>(&well_known.links.href)?)
 }
 
-fn fetch_communities_from_instance(
+// TODO: move these to db
-  community_list: &Url,
+fn upsert_community(
+  community_form: &CommunityForm,
   conn: &PgConnection,
-) -> Result<Vec<Community>, Error> {
+) -> Result<Community, Error> {
-  fetch_remote_object::<UnorderedCollection>(community_list)?
+  let existing = Community::read_from_actor_id(conn, &community_form.actor_id);
-    .collection_props
-    .get_many_items_base_boxes()
-    .unwrap()
-    .map(|b| -> Result<CommunityForm, Error> {
-      let group = b.to_owned().to_concrete::<GroupExt>()?;
-      Ok(CommunityForm::from_group(&group, conn)?)
-    })
-    .map(
-      |cf: Result<CommunityForm, Error>| -> Result<Community, Error> {
-        let cf2 = cf?;
-        let existing = Community::read_from_actor_id(conn, &cf2.actor_id);
   match existing {
-          Err(NotFound {}) => Ok(Community::create(conn, &cf2)?),
+    Err(NotFound {}) => Ok(Community::create(conn, &community_form)?),
-          Ok(c) => Ok(Community::update(conn, c.id, &cf2)?),
+    Ok(c) => Ok(Community::update(conn, c.id, &community_form)?),
     Err(e) => Err(Error::from(e)),
   }
-      },
+}
-    )
+fn upsert_user(user_form: &UserForm, conn: &PgConnection) -> Result<User_, Error> {
-    .collect()
+  let existing = User_::read_from_apub_id(conn, &user_form.actor_id);
+  Ok(match existing {
+    Err(NotFound {}) => User_::create(conn, &user_form)?,
+    Ok(u) => User_::update(conn, u.id, &user_form)?,
+    Err(e) => return Err(Error::from(e)),
+  })
 }
 
-// TODO: add an optional param last_updated and only fetch if its too old
+fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result<Post, Error> {
+  let existing = Post::read_from_apub_id(conn, &post_form.ap_id);
+  match existing {
+    Err(NotFound {}) => Ok(Post::create(conn, &post_form)?),
+    Ok(p) => Ok(Post::update(conn, p.id, &post_form)?),
+    Err(e) => Err(Error::from(e)),
+  }
+}
+
+/// Fetch any type of ActivityPub object, handling things like HTTP headers, deserialisation,
+/// timeouts etc.
+/// TODO: add an optional param last_updated and only fetch if its too old
 pub fn fetch_remote_object<Response>(url: &Url) -> Result<Response, Error>
 where
   Response: for<'de> Deserialize<'de>,
 {
-  if Settings::get().federation.tls_enabled && url.scheme() != "https" {
+  if !is_apub_id_valid(&url) {
-    return Err(format_err!("Activitypub uri is insecure: {}", url));
+    return Err(format_err!("Activitypub uri invalid or blocked: {}", url));
   }
   // TODO: this function should return a future
   let timeout = Duration::from_secs(60);
@@ -74,10 +83,50 @@ where
   Ok(res)
 }
 
-fn fetch_remote_community_posts(
+/// The types of ActivityPub objects that can be fetched directly by searching for their ID.
-  community: &Community,
+#[serde(untagged)]
-  conn: &PgConnection,
+#[derive(serde::Deserialize, Debug)]
-) -> Result<Vec<Post>, Error> {
+pub enum SearchAcceptedObjects {
+  Person(Box<PersonExt>),
+  Group(Box<GroupExt>),
+  Page(Box<Page>),
+}
+
+/// Attempt to parse the query as URL, and fetch an ActivityPub object from it.
+///
+/// Some working examples for use with the docker/federation/ setup:
+/// http://lemmy_alpha:8540/federation/c/main
+/// http://lemmy_alpha:8540/federation/u/lemmy_alpha
+/// http://lemmy_alpha:8540/federation/p/3
+pub fn search_by_apub_id(query: &str, conn: &PgConnection) -> Result<SearchResponse, Error> {
+  let query_url = Url::parse(&query)?;
+  let mut response = SearchResponse {
+    type_: SearchType::All.to_string(),
+    comments: vec![],
+    posts: vec![],
+    communities: vec![],
+    users: vec![],
+  };
+  match fetch_remote_object::<SearchAcceptedObjects>(&query_url)? {
+    SearchAcceptedObjects::Person(p) => {
+      let u = upsert_user(&UserForm::from_person(&p)?, conn)?;
+      response.users = vec![UserView::read(conn, u.id)?];
+    }
+    SearchAcceptedObjects::Group(g) => {
+      let c = upsert_community(&CommunityForm::from_group(&g, conn)?, conn)?;
+      fetch_community_outbox(&c, conn)?;
+      response.communities = vec![CommunityView::read(conn, c.id, None)?];
+    }
+    SearchAcceptedObjects::Page(p) => {
+      let p = upsert_post(&PostForm::from_page(&p, conn)?, conn)?;
+      response.posts = vec![PostView::read(conn, p.id, None)?];
+    }
+  }
+  Ok(response)
+}
+
+/// Fetch all posts in the outbox of the given user, and insert them into the database.
+fn fetch_community_outbox(community: &Community, conn: &PgConnection) -> Result<Vec<Post>, Error> {
   let outbox_url = Url::parse(&community.get_outbox_url())?;
   let outbox = fetch_remote_object::<OrderedCollection>(&outbox_url)?;
   let items = outbox.collection_props.get_many_items_base_boxes();
@@ -89,57 +138,21 @@ fn fetch_remote_community_posts(
         let page = obox.clone().to_concrete::<Page>()?;
         PostForm::from_page(&page, conn)
       })
-      .map(|pf: Result<PostForm, Error>| -> Result<Post, Error> {
+      .map(|pf| upsert_post(&pf?, conn))
-        let pf2 = pf?;
-        let existing = Post::read_from_apub_id(conn, &pf2.ap_id);
-        match existing {
-          Err(NotFound {}) => Ok(Post::create(conn, &pf2)?),
-          Ok(p) => Ok(Post::update(conn, p.id, &pf2)?),
-          Err(e) => Err(Error::from(e)),
-        }
-      })
       .collect::<Result<Vec<Post>, Error>>()?,
   )
 }
 
-// TODO: can probably merge these two methods?
+/// Fetch a user, insert/update it in the database and return the user.
 pub fn fetch_remote_user(apub_id: &Url, conn: &PgConnection) -> Result<User_, Error> {
   let person = fetch_remote_object::<PersonExt>(apub_id)?;
   let uf = UserForm::from_person(&person)?;
-  let existing = User_::read_from_apub_id(conn, &uf.actor_id);
+  upsert_user(&uf, conn)
-  Ok(match existing {
-    Err(NotFound {}) => User_::create(conn, &uf)?,
-    Ok(u) => User_::update(conn, u.id, &uf)?,
-    Err(e) => return Err(Error::from(e)),
-  })
 }
+
+/// Fetch a community, insert/update it in the database and return the community.
 pub fn fetch_remote_community(apub_id: &Url, conn: &PgConnection) -> Result<Community, Error> {
   let group = fetch_remote_object::<GroupExt>(apub_id)?;
   let cf = CommunityForm::from_group(&group, conn)?;
-  let existing = Community::read_from_actor_id(conn, &cf.actor_id);
+  upsert_community(&cf, conn)
-  Ok(match existing {
-    Err(NotFound {}) => Community::create(conn, &cf)?,
-    Ok(u) => Community::update(conn, u.id, &cf)?,
-    Err(e) => return Err(Error::from(e)),
-  })
-}
-
-// TODO: in the future, this should only be done when an instance is followed for the first time
-//       after that, we should rely in the inbox, and fetch on demand when needed
-pub fn fetch_all(conn: &PgConnection) -> Result<(), Error> {
-  for instance in &get_following_instances() {
-    let node_info = fetch_node_info(instance)?;
-    if let Some(community_list) = node_info.metadata.community_list_url {
-      let communities = fetch_communities_from_instance(&community_list, conn)?;
-      for c in communities {
-        fetch_remote_community_posts(&c, conn)?;
-      }
-    } else {
-      warn!(
-        "{} is not a Lemmy instance, federation is not supported",
-        instance.domain
-      );
-    }
-  }
-  Ok(())
 }

server/src/apub/inbox.rs

@@ -1,101 +0,0 @@
-use crate::apub::activities::accept_follow;
-use crate::apub::fetcher::fetch_remote_user;
-use crate::db::community::{Community, CommunityFollower, CommunityFollowerForm};
-use crate::db::post::{Post, PostForm};
-use crate::db::Crud;
-use crate::db::Followable;
-use activitystreams::activity::{Accept, Create, Follow, Update};
-use activitystreams::object::Page;
-use actix_web::{web, HttpResponse};
-use diesel::r2d2::{ConnectionManager, Pool};
-use diesel::PgConnection;
-use failure::Error;
-use url::Url;
-
-// TODO: need a proper actor that has this inbox
-
-pub async fn inbox(
-  input: web::Json<AcceptedObjects>,
-  db: web::Data<Pool<ConnectionManager<PgConnection>>>,
-) -> Result<HttpResponse, Error> {
-  // TODO: make sure that things are received in the correct inbox
-  //      (by using seperate handler functions and checking the user/community name in the path)
-  let input = input.into_inner();
-  let conn = &db.get().unwrap();
-  match input {
-    AcceptedObjects::Create(c) => handle_create(&c, conn),
-    AcceptedObjects::Update(u) => handle_update(&u, conn),
-    AcceptedObjects::Follow(f) => handle_follow(&f, conn),
-    AcceptedObjects::Accept(a) => handle_accept(&a, conn),
-  }
-}
-
-fn handle_create(create: &Create, conn: &PgConnection) -> Result<HttpResponse, Error> {
-  let page = create
-    .create_props
-    .get_object_base_box()
-    .to_owned()
-    .unwrap()
-    .to_owned()
-    .to_concrete::<Page>()?;
-  let post = PostForm::from_page(&page, conn)?;
-  Post::create(conn, &post)?;
-  // TODO: send the new post out via websocket
-  Ok(HttpResponse::Ok().finish())
-}
-
-fn handle_update(update: &Update, conn: &PgConnection) -> Result<HttpResponse, Error> {
-  let page = update
-    .update_props
-    .get_object_base_box()
-    .to_owned()
-    .unwrap()
-    .to_owned()
-    .to_concrete::<Page>()?;
-  let post = PostForm::from_page(&page, conn)?;
-  let id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
-  Post::update(conn, id, &post)?;
-  // TODO: send the new post out via websocket
-  Ok(HttpResponse::Ok().finish())
-}
-
-fn handle_follow(follow: &Follow, conn: &PgConnection) -> Result<HttpResponse, Error> {
-  println!("received follow: {:?}", &follow);
-
-  // TODO: make sure this is a local community
-  let community_uri = follow
-    .follow_props
-    .get_object_xsd_any_uri()
-    .unwrap()
-    .to_string();
-  let community = Community::read_from_actor_id(conn, &community_uri)?;
-  let user_uri = follow
-    .follow_props
-    .get_actor_xsd_any_uri()
-    .unwrap()
-    .to_string();
-  let user = fetch_remote_user(&Url::parse(&user_uri)?, conn)?;
-  // TODO: insert ID of the user into follows of the community
-  let community_follower_form = CommunityFollowerForm {
-    community_id: community.id,
-    user_id: user.id,
-  };
-  CommunityFollower::follow(&conn, &community_follower_form)?;
-  accept_follow(&follow)?;
-  Ok(HttpResponse::Ok().finish())
-}
-
-fn handle_accept(accept: &Accept, _conn: &PgConnection) -> Result<HttpResponse, Error> {
-  println!("received accept: {:?}", &accept);
-  // TODO: at this point, indicate to the user that they are following the community
-  Ok(HttpResponse::Ok().finish())
-}
-
-#[serde(untagged)]
-#[derive(serde::Deserialize)]
-pub enum AcceptedObjects {
-  Create(Create),
-  Update(Update),
-  Follow(Follow),
-  Accept(Accept),
-}

server/src/apub/mod.rs

@@ -1,17 +1,18 @@
 pub mod activities;
 pub mod community;
+pub mod community_inbox;
 pub mod fetcher;
-pub mod inbox;
 pub mod post;
 pub mod signatures;
 pub mod user;
+pub mod user_inbox;
 use crate::apub::signatures::PublicKeyExtension;
 use crate::Settings;
 use activitystreams::actor::{properties::ApActorProperties, Group, Person};
 use activitystreams::ext::Ext;
 use actix_web::body::Body;
 use actix_web::HttpResponse;
-use openssl::{pkey::PKey, rsa::Rsa};
+use serde::ser::Serialize;
 use url::Url;
 
 type GroupExt = Ext<Ext<Group, ApActorProperties>, PublicKeyExtension>;
@@ -26,22 +27,22 @@ pub enum EndpointType {
   Comment,
 }
 
-pub struct Instance {
+/// Convert the data to json and turn it into an HTTP Response with the correct ActivityPub
-  domain: String,
+/// headers.
-}
+fn create_apub_response<T>(data: &T) -> HttpResponse<Body>
-
-fn create_apub_response<T>(json: &T) -> HttpResponse<Body>
 where
-  T: serde::ser::Serialize,
+  T: Serialize,
 {
   HttpResponse::Ok()
     .content_type(APUB_JSON_CONTENT_TYPE)
-    .json(json)
+    .json(data)
 }
 
-// TODO: we will probably need to change apub endpoint urls so that html and activity+json content
+/// Generates the ActivityPub ID for a given object type and name.
-//       types are handled at the same endpoint, so that you can copy the url into mastodon search
+///
-//       and have it fetch the object.
+/// TODO: we will probably need to change apub endpoint urls so that html and activity+json content
+///       types are handled at the same endpoint, so that you can copy the url into mastodon search
+///       and have it fetch the object.
 pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url {
   let point = match endpoint_type {
     EndpointType::Community => "c",
@@ -70,35 +71,20 @@ pub fn get_apub_protocol_string() -> &'static str {
   }
 }
 
-pub fn gen_keypair() -> (Vec<u8>, Vec<u8>) {
+// Checks if the ID has a valid format, correct scheme, and is in the whitelist.
-  let rsa = Rsa::generate(2048).expect("sign::gen_keypair: key generation error");
+fn is_apub_id_valid(apub_id: &Url) -> bool {
-  let pkey = PKey::from_rsa(rsa).expect("sign::gen_keypair: parsing error");
+  if apub_id.scheme() != get_apub_protocol_string() {
-  (
+    return false;
-    pkey
+  }
-      .public_key_to_pem()
-      .expect("sign::gen_keypair: public key encoding error"),
-    pkey
-      .private_key_to_pem_pkcs8()
-      .expect("sign::gen_keypair: private key encoding error"),
-  )
-}
 
-pub fn gen_keypair_str() -> (String, String) {
+  let whitelist: Vec<String> = Settings::get()
-  let (public_key, private_key) = gen_keypair();
-  (vec_bytes_to_str(public_key), vec_bytes_to_str(private_key))
-}
-
-fn vec_bytes_to_str(bytes: Vec<u8>) -> String {
-  String::from_utf8_lossy(&bytes).into_owned()
-}
-
-pub fn get_following_instances() -> Vec<Instance> {
-  Settings::get()
     .federation
-    .followed_instances
+    .instance_whitelist
     .split(',')
-    .map(|i| Instance {
+    .map(|d| d.to_string())
-      domain: i.to_string(),
+    .collect();
-    })
+  match apub_id.domain() {
-    .collect()
+    Some(d) => whitelist.contains(&d.to_owned()),
+    None => false,
+  }
 }

server/src/apub/post.rs

@@ -20,6 +20,7 @@ pub struct PostQuery {
   post_id: String,
 }
 
+/// Return the post json over HTTP.
 pub async fn get_apub_post(
   info: Path<PostQuery>,
   db: web::Data<Pool<ConnectionManager<PgConnection>>>,
@@ -30,6 +31,7 @@ pub async fn get_apub_post(
 }
 
 impl Post {
+  // Turn a Lemmy post into an ActivityPub page that can be sent out over the network.
   pub fn as_page(&self, conn: &PgConnection) -> Result<Page, Error> {
     let mut page = Page::default();
     let oprops: &mut ObjectProperties = page.as_mut();
@@ -67,6 +69,7 @@ impl Post {
 }
 
 impl PostForm {
+  /// Parse an ActivityPub page received from another instance into a Lemmy post.
   pub fn from_page(page: &Page, conn: &PgConnection) -> Result<PostForm, Error> {
     let oprops = &page.object_props;
     let creator_id = Url::parse(&oprops.get_attributed_to_xsd_any_uri().unwrap().to_string())?;

server/src/apub/signatures.rs

@@ -1,11 +1,113 @@
-// For this example, we'll use the Extensible trait, the Extension trait, the Actor trait, and
-// the Person type
 use activitystreams::{actor::Actor, ext::Extension};
+use actix_web::HttpRequest;
+use failure::Error;
+use http::request::Builder;
+use http_signature_normalization::Config;
+use log::debug;
+use openssl::hash::MessageDigest;
+use openssl::sign::{Signer, Verifier};
+use openssl::{pkey::PKey, rsa::Rsa};
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+
+lazy_static! {
+  static ref HTTP_SIG_CONFIG: Config = Config::new();
+}
+
+pub struct Keypair {
+  pub private_key: String,
+  pub public_key: String,
+}
+
+/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
+pub fn generate_actor_keypair() -> Result<Keypair, Error> {
+  let rsa = Rsa::generate(2048)?;
+  let pkey = PKey::from_rsa(rsa)?;
+  let public_key = pkey.public_key_to_pem()?;
+  let private_key = pkey.private_key_to_pem_pkcs8()?;
+  Ok(Keypair {
+    private_key: String::from_utf8(private_key)?,
+    public_key: String::from_utf8(public_key)?,
+  })
+}
+
+/// Signs request headers with the given keypair.
+/// TODO: would be nice to pass the sending actor in, instead of raw privatekey/id strings
+pub fn sign(request: &Builder, private_key: &str, sender_id: &str) -> Result<String, Error> {
+  let signing_key_id = format!("{}#main-key", sender_id);
+
+  let headers = request
+    .headers_ref()
+    .unwrap()
+    .iter()
+    .map(|h| -> Result<(String, String), Error> {
+      Ok((h.0.as_str().to_owned(), h.1.to_str()?.to_owned()))
+    })
+    .collect::<Result<BTreeMap<String, String>, Error>>()?;
+
+  let signature_header_value = HTTP_SIG_CONFIG
+    .begin_sign(
+      request.method_ref().unwrap().as_str(),
+      request
+        .uri_ref()
+        .unwrap()
+        .path_and_query()
+        .unwrap()
+        .as_str(),
+      headers,
+    )
+    .sign(signing_key_id, |signing_string| {
+      let private_key = PKey::private_key_from_pem(private_key.as_bytes())?;
+      let mut signer = Signer::new(MessageDigest::sha256(), &private_key).unwrap();
+      signer.update(signing_string.as_bytes()).unwrap();
+      Ok(base64::encode(signer.sign_to_vec()?)) as Result<_, Error>
+    })?
+    .signature_header();
+
+  Ok(signature_header_value)
+}
+
+pub fn verify(request: &HttpRequest, public_key: &str) -> Result<(), Error> {
+  let headers = request
+    .headers()
+    .iter()
+    .map(|h| -> Result<(String, String), Error> {
+      Ok((h.0.as_str().to_owned(), h.1.to_str()?.to_owned()))
+    })
+    .collect::<Result<BTreeMap<String, String>, Error>>()?;
+
+  let verified = HTTP_SIG_CONFIG
+    .begin_verify(
+      request.method().as_str(),
+      request.uri().path_and_query().unwrap().as_str(),
+      headers,
+    )?
+    .verify(|signature, signing_string| -> Result<bool, Error> {
+      debug!(
+        "Verifying with key {}, message {}",
+        &public_key, &signing_string
+      );
+      let public_key = PKey::public_key_from_pem(public_key.as_bytes())?;
+      let mut verifier = Verifier::new(MessageDigest::sha256(), &public_key).unwrap();
+      verifier.update(&signing_string.as_bytes()).unwrap();
+      Ok(verifier.verify(&base64::decode(signature)?)?)
+    })?;
+
+  if verified {
+    debug!("verified signature for {}", &request.uri());
+    Ok(())
+  } else {
+    Err(format_err!(
+      "Invalid signature on request: {}",
+      &request.uri()
+    ))
+  }
+}
 
 // The following is taken from here:
 // https://docs.rs/activitystreams/0.5.0-alpha.17/activitystreams/ext/index.html
 
-#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct PublicKey {
   pub id: String,
@@ -13,7 +115,7 @@ pub struct PublicKey {
   pub public_key_pem: String,
 }
 
-#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct PublicKeyExtension {
   pub public_key: PublicKey,

server/src/apub/user.rs

@@ -22,6 +22,7 @@ pub struct UserQuery {
   user_name: String,
 }
 
+// Turn a Lemmy user into an ActivityPub person and return it as json.
 pub async fn get_apub_user(
   info: Path<UserQuery>,
   db: web::Data<Pool<ConnectionManager<PgConnection>>>,
@@ -64,6 +65,7 @@ pub async fn get_apub_user(
 }
 
 impl UserForm {
+  /// Parse an ActivityPub person received from another instance into a Lemmy user.
   pub fn from_person(person: &PersonExt) -> Result<Self, Error> {
     let oprops = &person.base.base.object_props;
     let aprops = &person.base.extension;

server/src/apub/user_inbox.rs

@@ -0,0 +1,119 @@
+use crate::apub::fetcher::{fetch_remote_community, fetch_remote_user};
+use crate::apub::signatures::verify;
+use crate::db::post::{Post, PostForm};
+use crate::db::Crud;
+use activitystreams::activity::{Accept, Create, Update};
+use activitystreams::object::Page;
+use actix_web::{web, HttpRequest, HttpResponse};
+use diesel::r2d2::{ConnectionManager, Pool};
+use diesel::PgConnection;
+use failure::Error;
+use log::debug;
+use serde::Deserialize;
+use url::Url;
+
+#[serde(untagged)]
+#[derive(Deserialize, Debug)]
+pub enum UserAcceptedObjects {
+  Create(Create),
+  Update(Update),
+  Accept(Accept),
+}
+
+/// Handler for all incoming activities to user inboxes.
+pub async fn user_inbox(
+  request: HttpRequest,
+  input: web::Json<UserAcceptedObjects>,
+  path: web::Path<String>,
+  db: web::Data<Pool<ConnectionManager<PgConnection>>>,
+) -> Result<HttpResponse, Error> {
+  // TODO: would be nice if we could do the signature check here, but we cant access the actor property
+  let input = input.into_inner();
+  let conn = &db.get().unwrap();
+  debug!(
+    "User {} received activity: {:?}",
+    &path.into_inner(),
+    &input
+  );
+
+  match input {
+    UserAcceptedObjects::Create(c) => handle_create(&c, &request, conn),
+    UserAcceptedObjects::Update(u) => handle_update(&u, &request, conn),
+    UserAcceptedObjects::Accept(a) => handle_accept(&a, &request, conn),
+  }
+}
+
+/// Handle create activities and insert them in the database.
+fn handle_create(
+  create: &Create,
+  request: &HttpRequest,
+  conn: &PgConnection,
+) -> Result<HttpResponse, Error> {
+  let community_uri = create
+    .create_props
+    .get_actor_xsd_any_uri()
+    .unwrap()
+    .to_string();
+  // TODO: should do this in a generic way so we dont need to know if its a user or a community
+  let user = fetch_remote_user(&Url::parse(&community_uri)?, conn)?;
+  verify(request, &user.public_key.unwrap())?;
+
+  let page = create
+    .create_props
+    .get_object_base_box()
+    .to_owned()
+    .unwrap()
+    .to_owned()
+    .to_concrete::<Page>()?;
+  let post = PostForm::from_page(&page, conn)?;
+  Post::create(conn, &post)?;
+  // TODO: send the new post out via websocket
+  Ok(HttpResponse::Ok().finish())
+}
+
+/// Handle update activities and insert them in the database.
+fn handle_update(
+  update: &Update,
+  request: &HttpRequest,
+  conn: &PgConnection,
+) -> Result<HttpResponse, Error> {
+  let community_uri = update
+    .update_props
+    .get_actor_xsd_any_uri()
+    .unwrap()
+    .to_string();
+  let user = fetch_remote_user(&Url::parse(&community_uri)?, conn)?;
+  verify(request, &user.public_key.unwrap())?;
+
+  let page = update
+    .update_props
+    .get_object_base_box()
+    .to_owned()
+    .unwrap()
+    .to_owned()
+    .to_concrete::<Page>()?;
+  let post = PostForm::from_page(&page, conn)?;
+  let id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
+  Post::update(conn, id, &post)?;
+  // TODO: send the new post out via websocket
+  Ok(HttpResponse::Ok().finish())
+}
+
+/// Handle accepted follows.
+fn handle_accept(
+  accept: &Accept,
+  request: &HttpRequest,
+  conn: &PgConnection,
+) -> Result<HttpResponse, Error> {
+  let community_uri = accept
+    .accept_props
+    .get_actor_xsd_any_uri()
+    .unwrap()
+    .to_string();
+  let community = fetch_remote_community(&Url::parse(&community_uri)?, conn)?;
+  verify(request, &community.public_key.unwrap())?;
+
+  // TODO: make sure that we actually requested a follow
+  // TODO: at this point, indicate to the user that they are following the community
+  Ok(HttpResponse::Ok().finish())
+}

server/src/db/code_migrations.rs

@@ -4,8 +4,10 @@ use super::community::{Community, CommunityForm};
 use super::post::Post;
 use super::user::{UserForm, User_};
 use super::*;
-use crate::apub::{gen_keypair_str, make_apub_endpoint, EndpointType};
+use crate::apub::signatures::generate_actor_keypair;
+use crate::apub::{make_apub_endpoint, EndpointType};
 use crate::naive_now;
+use failure::Error;
 use log::info;
 
 pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), Error> {
@@ -29,7 +31,7 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
     .load::<User_>(conn)?;
 
   for cuser in &incorrect_users {
-    let (user_public_key, user_private_key) = gen_keypair_str();
+    let keypair = generate_actor_keypair()?;
 
     let form = UserForm {
       name: cuser.name.to_owned(),
@@ -51,8 +53,8 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
       actor_id: make_apub_endpoint(EndpointType::User, &cuser.name).to_string(),
       bio: cuser.bio.to_owned(),
       local: cuser.local,
-      private_key: Some(user_private_key),
+      private_key: Some(keypair.private_key),
-      public_key: Some(user_public_key),
+      public_key: Some(keypair.public_key),
       last_refreshed_at: Some(naive_now()),
     };
 
@@ -76,7 +78,7 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
     .load::<Community>(conn)?;
 
   for ccommunity in &incorrect_communities {
-    let (community_public_key, community_private_key) = gen_keypair_str();
+    let keypair = generate_actor_keypair()?;
 
     let form = CommunityForm {
       name: ccommunity.name.to_owned(),
@@ -90,8 +92,8 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
       updated: None,
       actor_id: make_apub_endpoint(EndpointType::Community, &ccommunity.name).to_string(),
       local: ccommunity.local,
-      private_key: Some(community_private_key),
+      private_key: Some(keypair.private_key),
-      public_key: Some(community_public_key),
+      public_key: Some(keypair.public_key),
       last_refreshed_at: Some(naive_now()),
       published: None,
     };

server/src/main.rs

@@ -7,15 +7,10 @@ use actix_web::*;
 use diesel::r2d2::{ConnectionManager, Pool};
 use diesel::PgConnection;
 use failure::Error;
-use lemmy_server::apub::fetcher::fetch_all;
 use lemmy_server::db::code_migrations::run_advanced_migrations;
 use lemmy_server::routes::{api, federation, feeds, index, nodeinfo, webfinger, websocket};
 use lemmy_server::settings::Settings;
 use lemmy_server::websocket::server::*;
-use log::warn;
-use std::thread;
-use std::thread::sleep;
-use std::time::Duration;
 
 embed_migrations!();
 
@@ -39,16 +34,6 @@ async fn main() -> Result<(), Error> {
   // Set up websocket server
   let server = ChatServer::startup(pool.clone()).start();
 
-  thread::spawn(move || {
-    // some work here
-    sleep(Duration::from_secs(5));
-    println!("Fetching apub data");
-    match fetch_all(&conn) {
-      Ok(_) => {}
-      Err(e) => warn!("Error during apub fetch: {}", e),
-    }
-  });
-
   println!(
     "Starting http server at {}:{}",
     settings.bind, settings.port

server/src/routes/federation.rs

@@ -6,19 +6,14 @@ pub fn config(cfg: &mut web::ServiceConfig) {
   if Settings::get().federation.enabled {
     println!("federation enabled, host is {}", Settings::get().hostname);
     cfg
+      // TODO: check the user/community params for these
       .route(
-        "/federation/communities",
+        "/federation/c/{community_name}/inbox",
-        web::get().to(apub::community::get_apub_community_list),
+        web::post().to(apub::community_inbox::community_inbox),
-      )
-      // TODO: this needs to be moved to the actors (eg /federation/u/{}/inbox)
-      .route("/federation/inbox", web::post().to(apub::inbox::inbox))
-      .route(
-        "/federation/c/{_}/inbox",
-        web::post().to(apub::inbox::inbox),
       )
       .route(
-        "/federation/u/{_}/inbox",
+        "/federation/u/{user_name}/inbox",
-        web::post().to(apub::inbox::inbox),
+        web::post().to(apub::user_inbox::user_inbox),
       )
       .route(
         "/federation/c/{community_name}",
@@ -38,7 +33,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
       )
       .route(
         "/federation/p/{post_id}",
-        web::get().to(apub::user::get_apub_user),
+        web::get().to(apub::post::get_apub_post),
       );
   }
 }

server/src/routes/nodeinfo.rs

@@ -61,13 +61,6 @@ async fn node_info(
         local_comments: site_view.number_of_comments,
         open_registrations: site_view.open_registration,
       },
-      metadata: NodeInfoMetadata {
-        community_list_url: Some(Url::parse(&format!(
-          "{}://{}/federation/communities",
-          get_apub_protocol_string(),
-          Settings::get().hostname
-        ))?),
-      },
     })
   })
   .await
@@ -93,7 +86,6 @@ pub struct NodeInfo {
   pub software: NodeInfoSoftware,
   pub protocols: Vec<String>,
   pub usage: NodeInfoUsage,
-  pub metadata: NodeInfoMetadata,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -115,8 +107,3 @@ pub struct NodeInfoUsage {
 pub struct NodeInfoUsers {
   pub total: i64,
 }
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct NodeInfoMetadata {
-  pub community_list_url: Option<Url>,
-}

server/src/settings.rs

@@ -63,8 +63,8 @@ pub struct Database {
 #[derive(Debug, Deserialize, Clone)]
 pub struct Federation {
   pub enabled: bool,
-  pub followed_instances: String,
   pub tls_enabled: bool,
+  pub instance_whitelist: String,
 }
 
 lazy_static! {