Merge apub, apub_receive crates (fixes #1621)

Cargo.lock

@@ -1726,6 +1726,7 @@ dependencies = [
  "itertools",
  "lazy_static",
  "lemmy_api_common",
+ "lemmy_apub_lib",
  "lemmy_db_queries",
  "lemmy_db_schema",
  "lemmy_db_views",
@@ -1772,51 +1773,6 @@ dependencies = [
  "trybuild",
 ]
 
-[[package]]
-name = "lemmy_apub_receive"
-version = "0.1.0"
-dependencies = [
- "activitystreams",
- "activitystreams-ext",
- "actix",
- "actix-rt",
- "actix-web",
- "anyhow",
- "async-trait",
- "awc",
- "backtrace",
- "base64 0.13.0",
- "bcrypt",
- "chrono",
- "diesel",
- "futures",
- "http",
- "http-signature-normalization-actix",
- "http-signature-normalization-reqwest",
- "itertools",
- "lemmy_api_common",
- "lemmy_apub",
- "lemmy_apub_lib",
- "lemmy_db_queries",
- "lemmy_db_schema",
- "lemmy_db_views",
- "lemmy_db_views_actor",
- "lemmy_utils",
- "lemmy_websocket",
- "log",
- "openssl",
- "percent-encoding",
- "rand 0.8.4",
- "serde",
- "serde_json",
- "sha2",
- "strum",
- "strum_macros",
- "thiserror",
- "tokio",
- "url",
-]
-
 [[package]]
 name = "lemmy_db_queries"
 version = "0.1.0"
@@ -1932,7 +1888,6 @@ dependencies = [
  "lemmy_api_common",
  "lemmy_api_crud",
  "lemmy_apub",
- "lemmy_apub_receive",
  "lemmy_db_queries",
  "lemmy_db_schema",
  "lemmy_db_views",

Cargo.toml

@@ -17,7 +17,6 @@ members = [
     "crates/apub_lib",
     "crates/apub_lib_derive",
     "crates/apub",
-    "crates/apub_receive",
     "crates/utils",
     "crates/db_queries",
     "crates/db_schema",
@@ -32,7 +31,6 @@ members = [
 lemmy_api = { path = "./crates/api" }
 lemmy_api_crud = { path = "./crates/api_crud" }
 lemmy_apub = { path = "./crates/apub" }
-lemmy_apub_receive = { path = "./crates/apub_receive" }
 lemmy_utils = { path = "./crates/utils" }
 lemmy_db_schema = { path = "./crates/db_schema" }
 lemmy_db_queries = { path = "./crates/db_queries" }

ansible/templates/docker-compose.yml

@@ -7,7 +7,7 @@ services:
       - "127.0.0.1:8536:8536"
     restart: always
     environment:
-      - RUST_LOG="warn,lemmy_server=info,lemmy_api=info,lemmy_api_common=info,lemmy_api_crud=info,lemmy_apub=info,lemmy_apub_receive=info,lemmy_db_queries=info,lemmy_db_schema=info,lemmy_db_views=info,lemmy_db_views_actor=info,lemmy_db_views_moderator=info,lemmy_routes=info,lemmy_utils=info,lemmy_websocket=info"
+      - RUST_LOG="warn,lemmy_server=info,lemmy_api=info,lemmy_api_common=info,lemmy_api_crud=info,lemmy_apub=info,lemmy_db_queries=info,lemmy_db_schema=info,lemmy_db_views=info,lemmy_db_views_actor=info,lemmy_db_views_moderator=info,lemmy_routes=info,lemmy_utils=info,lemmy_websocket=info"
     volumes:
       - ./lemmy.hjson:/config/config.hjson:ro
     depends_on:

api_tests/prepare-drone-federation-test.sh

@@ -3,7 +3,7 @@ set -e
 
 export LEMMY_TEST_SEND_SYNC=1
 export RUST_BACKTRACE=1
-export RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_apub_receive=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
+export RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
 
 for INSTANCE in lemmy_alpha lemmy_beta lemmy_gamma lemmy_delta lemmy_epsilon; do
   psql "${LEMMY_DATABASE_URL}/lemmy" -c "DROP DATABASE IF EXISTS $INSTANCE"

crates/apub/Cargo.toml

@@ -10,6 +10,7 @@ doctest = false
 
 [dependencies]
 lemmy_utils = { path = "../utils" }
+lemmy_apub_lib = { path = "../apub_lib" }
 lemmy_db_queries = { path = "../db_queries" }
 lemmy_db_schema = { path = "../db_schema" }
 lemmy_db_views = { path = "../db_views" }
@@ -31,7 +32,6 @@ log = "0.4.14"
 rand = "0.8.4"
 strum = "0.21.0"
 strum_macros = "0.21.1"
-lazy_static = "1.4.0"
 url = { version = "2.2.2", features = ["serde"] }
 percent-encoding = "2.1.0"
 openssl = "0.10.35"
@@ -50,3 +50,5 @@ thiserror = "1.0.26"
 background-jobs = "0.9.0"
 reqwest = { version = "0.11.4", features = ["json"] }
 backtrace = "0.3.60"
+lazy_static = "1.4.0"
+

crates/apub/src/activities/comment/create.rs

@@ -1,10 +1,13 @@
-use crate::activities::{
+use crate::{
+  activities::{
     comment::{get_notif_recipients, send_websocket_message},
     verify_activity,
     verify_person_in_community,
+  },
+  objects::FromApub,
+  NoteExt,
 };
 use activitystreams::{activity::kind::CreateType, base::BaseExt};
-use lemmy_apub::{objects::FromApub, NoteExt};
 use lemmy_apub_lib::{verify_domains_match_opt, ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_schema::source::comment::Comment;
 use lemmy_utils::LemmyError;

crates/apub/src/activities/comment/mod.rs

@@ -1,5 +1,5 @@
+use crate::fetcher::person::get_or_fetch_and_upsert_person;
 use lemmy_api_common::{blocking, comment::CommentResponse, send_local_notifs};
-use lemmy_apub::fetcher::person::get_or_fetch_and_upsert_person;
 use lemmy_db_queries::Crud;
 use lemmy_db_schema::{
   source::{comment::Comment, post::Post},

crates/apub/src/activities/comment/remove.rs

@@ -1,7 +1,7 @@
 use crate::activities::{comment::send_websocket_message, verify_mod_action};
 use activitystreams::activity::kind::RemoveType;
 use lemmy_api_common::blocking;
-use lemmy_apub::{check_is_apub_id_valid, fetcher::objects::get_or_fetch_and_insert_comment};
+use crate::{check_is_apub_id_valid, fetcher::objects::get_or_fetch_and_insert_comment};
 use lemmy_apub_lib::{verify_domains_match, ActivityCommonFields, ActivityHandlerNew, PublicUrl};
 use lemmy_db_queries::source::comment::Comment_;
 use lemmy_db_schema::source::comment::Comment;

crates/apub/src/activities/comment/undo_remove.rs

@@ -4,7 +4,7 @@ use crate::activities::{
 };
 use activitystreams::activity::kind::UndoType;
 use lemmy_api_common::blocking;
-use lemmy_apub::{check_is_apub_id_valid, fetcher::objects::get_or_fetch_and_insert_comment};
+use crate::{check_is_apub_id_valid, fetcher::objects::get_or_fetch_and_insert_comment};
 use lemmy_apub_lib::{verify_domains_match, ActivityCommonFields, ActivityHandlerNew, PublicUrl};
 use lemmy_db_queries::source::comment::Comment_;
 use lemmy_db_schema::source::comment::Comment;

crates/apub/src/activities/comment/update.rs

@@ -1,10 +1,13 @@
-use crate::activities::{
+use crate::{
+  activities::{
     comment::{get_notif_recipients, send_websocket_message},
     verify_activity,
     verify_person_in_community,
+  },
+  objects::FromApub,
+  NoteExt,
 };
 use activitystreams::{activity::kind::UpdateType, base::BaseExt};
-use lemmy_apub::{objects::FromApub, NoteExt};
 use lemmy_apub_lib::{verify_domains_match_opt, ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_schema::source::comment::Comment;
 use lemmy_utils::LemmyError;

crates/apub/src/activities/community/add_mod.rs

@@ -1,15 +1,15 @@
-use crate::activities::{
+use crate::{
+  activities::{
     verify_activity,
     verify_add_remove_moderator_target,
     verify_mod_action,
     verify_person_in_community,
-};
+  },
-use activitystreams::{activity::kind::AddType, base::AnyBase};
-use lemmy_api_common::blocking;
-use lemmy_apub::{
   fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
   CommunityType,
 };
+use activitystreams::{activity::kind::AddType, base::AnyBase};
+use lemmy_api_common::blocking;
 use lemmy_apub_lib::{ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_queries::{source::community::CommunityModerator_, Joinable};
 use lemmy_db_schema::source::community::{CommunityModerator, CommunityModeratorForm};

crates/apub/src/activities/community/announce.rs

@@ -25,9 +25,9 @@ use crate::{
     },
   },
   http::is_activity_already_known,
+  insert_activity,
 };
 use activitystreams::activity::kind::AnnounceType;
-use lemmy_apub::insert_activity;
 use lemmy_apub_lib::{ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_utils::LemmyError;
 use lemmy_websocket::LemmyContext;

crates/apub/src/activities/community/block_user.rs

@@ -1,10 +1,9 @@
-use crate::activities::{verify_activity, verify_mod_action, verify_person_in_community};
+use crate::{
+  activities::{verify_activity, verify_mod_action, verify_person_in_community},
+  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
+};
 use activitystreams::activity::kind::BlockType;
 use lemmy_api_common::blocking;
-use lemmy_apub::fetcher::{
-  community::get_or_fetch_and_upsert_community,
-  person::get_or_fetch_and_upsert_person,
-};
 use lemmy_apub_lib::{ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_queries::{Bannable, Followable};
 use lemmy_db_schema::source::community::{

crates/apub/src/activities/community/undo_block_user.rs

@@ -1,15 +1,14 @@
-use crate::activities::{
+use crate::{
+  activities::{
     community::block_user::BlockUserFromCommunity,
     verify_activity,
     verify_mod_action,
     verify_person_in_community,
+  },
+  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
 };
 use activitystreams::activity::kind::UndoType;
 use lemmy_api_common::blocking;
-use lemmy_apub::fetcher::{
-  community::get_or_fetch_and_upsert_community,
-  person::get_or_fetch_and_upsert_person,
-};
 use lemmy_apub_lib::{ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_queries::Bannable;
 use lemmy_db_schema::source::community::{CommunityPersonBan, CommunityPersonBanForm};

crates/apub/src/activities/community/update.rs

@@ -1,12 +1,15 @@
-use crate::activities::{
+use crate::{
+  activities::{
     community::send_websocket_message,
     verify_activity,
     verify_mod_action,
     verify_person_in_community,
+  },
+  objects::FromApubToForm,
+  GroupExt,
 };
 use activitystreams::activity::kind::UpdateType;
 use lemmy_api_common::blocking;
-use lemmy_apub::{objects::FromApubToForm, GroupExt};
 use lemmy_apub_lib::{ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_queries::{ApubObject, Crud};
 use lemmy_db_schema::source::community::{Community, CommunityForm};

crates/apub/src/activities/deletion/delete.rs

@@ -1,14 +1,12 @@
-use crate::activities::{
+use crate::{
+  activities::{
     comment::send_websocket_message as send_comment_message,
     community::send_websocket_message as send_community_message,
     post::send_websocket_message as send_post_message,
     verify_activity,
     verify_mod_action,
     verify_person_in_community,
-};
+  },
-use activitystreams::activity::kind::DeleteType;
-use lemmy_api_common::blocking;
-use lemmy_apub::{
   fetcher::{
     community::get_or_fetch_and_upsert_community,
     objects::get_or_fetch_and_insert_post_or_comment,
@@ -18,6 +16,8 @@ use lemmy_apub::{
   CommunityType,
   PostOrComment,
 };
+use activitystreams::activity::kind::DeleteType;
+use lemmy_api_common::blocking;
 use lemmy_apub_lib::{verify_urls_match, ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_queries::{
   source::{comment::Comment_, community::Community_, post::Post_},

crates/apub/src/activities/deletion/undo_delete.rs

@@ -1,4 +1,5 @@
-use crate::activities::{
+use crate::{
+  activities::{
     comment::send_websocket_message as send_comment_message,
     community::send_websocket_message as send_community_message,
     deletion::delete::DeletePostCommentOrCommunity,
@@ -6,10 +7,7 @@ use crate::activities::{
     verify_activity,
     verify_mod_action,
     verify_person_in_community,
-};
+  },
-use activitystreams::activity::kind::UndoType;
-use lemmy_api_common::blocking;
-use lemmy_apub::{
   fetcher::{
     community::get_or_fetch_and_upsert_community,
     objects::get_or_fetch_and_insert_post_or_comment,
@@ -18,6 +16,8 @@ use lemmy_apub::{
   CommunityType,
   PostOrComment,
 };
+use activitystreams::activity::kind::UndoType;
+use lemmy_api_common::blocking;
 use lemmy_apub_lib::{verify_urls_match, ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_queries::source::{comment::Comment_, community::Community_, post::Post_};
 use lemmy_db_schema::source::{comment::Comment, community::Community, post::Post};

crates/apub/src/activities/following/accept.rs

@@ -1,10 +1,9 @@
-use crate::activities::{following::follow::FollowCommunity, verify_activity, verify_community};
+use crate::{
+  activities::{following::follow::FollowCommunity, verify_activity, verify_community},
+  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
+};
 use activitystreams::activity::kind::AcceptType;
 use lemmy_api_common::blocking;
-use lemmy_apub::fetcher::{
-  community::get_or_fetch_and_upsert_community,
-  person::get_or_fetch_and_upsert_person,
-};
 use lemmy_apub_lib::{verify_urls_match, ActivityCommonFields, ActivityHandler};
 use lemmy_db_queries::Followable;
 use lemmy_db_schema::source::community::CommunityFollower;

crates/apub/src/activities/following/follow.rs

@@ -1,14 +1,14 @@
-use crate::activities::{verify_activity, verify_person};
+use crate::{
+  activities::{verify_activity, verify_person},
+  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
+  CommunityType,
+};
 use activitystreams::{
   activity::{kind::FollowType, Follow},
   base::{AnyBase, ExtendsExt},
 };
 use anyhow::Context;
 use lemmy_api_common::blocking;
-use lemmy_apub::{
-  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
-  CommunityType,
-};
 use lemmy_apub_lib::{verify_urls_match, ActivityCommonFields, ActivityHandler};
 use lemmy_db_queries::Followable;
 use lemmy_db_schema::source::community::{CommunityFollower, CommunityFollowerForm};

crates/apub/src/activities/following/undo.rs

@@ -1,10 +1,9 @@
-use crate::activities::{following::follow::FollowCommunity, verify_activity, verify_person};
+use crate::{
+  activities::{following::follow::FollowCommunity, verify_activity, verify_person},
+  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
+};
 use activitystreams::activity::kind::UndoType;
 use lemmy_api_common::blocking;
-use lemmy_apub::fetcher::{
-  community::get_or_fetch_and_upsert_community,
-  person::get_or_fetch_and_upsert_person,
-};
 use lemmy_apub_lib::{verify_urls_match, ActivityCommonFields, ActivityHandler};
 use lemmy_db_queries::Followable;
 use lemmy_db_schema::source::community::{CommunityFollower, CommunityFollowerForm};

crates/apub/src/activities/mod.rs

@@ -1 +1,122 @@
+use crate::{
+  check_community_or_site_ban,
+  check_is_apub_id_valid,
+  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
+  generate_moderators_url,
+};
+use anyhow::anyhow;
+use lemmy_api_common::blocking;
+use lemmy_apub_lib::{verify_domains_match, ActivityCommonFields};
+use lemmy_db_queries::ApubObject;
+use lemmy_db_schema::{
+  source::{community::Community, person::Person},
+  DbUrl,
+};
+use lemmy_db_views_actor::community_view::CommunityView;
+use lemmy_utils::LemmyError;
+use lemmy_websocket::LemmyContext;
+use url::Url;
+
+pub mod comment;
+pub mod community;
+pub mod deletion;
+pub mod following;
+pub mod post;
+pub mod private_message;
+pub mod removal;
 pub mod send;
+pub mod voting;
+
+/// Checks that the specified Url actually identifies a Person (by fetching it), and that the person
+/// doesn't have a site ban.
+async fn verify_person(
+  person_id: &Url,
+  context: &LemmyContext,
+  request_counter: &mut i32,
+) -> Result<(), LemmyError> {
+  let person = get_or_fetch_and_upsert_person(person_id, context, request_counter).await?;
+  if person.banned {
+    return Err(anyhow!("Person {} is banned", person_id).into());
+  }
+  Ok(())
+}
+
+/// Fetches the person and community to verify their type, then checks if person is banned from site
+/// or community.
+async fn verify_person_in_community(
+  person_id: &Url,
+  cc: &[Url],
+  context: &LemmyContext,
+  request_counter: &mut i32,
+) -> Result<Community, LemmyError> {
+  let person = get_or_fetch_and_upsert_person(person_id, context, request_counter).await?;
+  let mut cc_iter = cc.iter();
+  let community: Community = loop {
+    if let Some(cid) = cc_iter.next() {
+      if let Ok(c) = get_or_fetch_and_upsert_community(cid, context, request_counter).await {
+        break c;
+      }
+    } else {
+      return Err(anyhow!("No community found in cc").into());
+    }
+  };
+  check_community_or_site_ban(&person, community.id, context.pool()).await?;
+  Ok(community)
+}
+
+/// Simply check that the url actually refers to a valid group.
+async fn verify_community(
+  community_id: &Url,
+  context: &LemmyContext,
+  request_counter: &mut i32,
+) -> Result<(), LemmyError> {
+  get_or_fetch_and_upsert_community(community_id, context, request_counter).await?;
+  Ok(())
+}
+
+fn verify_activity(common: &ActivityCommonFields) -> Result<(), LemmyError> {
+  check_is_apub_id_valid(&common.actor, false)?;
+  verify_domains_match(common.id_unchecked(), &common.actor)?;
+  Ok(())
+}
+
+async fn verify_mod_action(
+  actor_id: &Url,
+  activity_cc: Url,
+  context: &LemmyContext,
+) -> Result<(), LemmyError> {
+  let community = blocking(context.pool(), move |conn| {
+    Community::read_from_apub_id(conn, &activity_cc.into())
+  })
+  .await??;
+
+  if community.local {
+    let actor_id: DbUrl = actor_id.clone().into();
+    let actor = blocking(context.pool(), move |conn| {
+      Person::read_from_apub_id(conn, &actor_id)
+    })
+    .await??;
+
+    // Note: this will also return true for admins in addition to mods, but as we dont know about
+    //       remote admins, it doesnt make any difference.
+    let community_id = community.id;
+    let actor_id = actor.id;
+    let is_mod_or_admin = blocking(context.pool(), move |conn| {
+      CommunityView::is_mod_or_admin(conn, actor_id, community_id)
+    })
+    .await?;
+    if !is_mod_or_admin {
+      return Err(anyhow!("Not a mod").into());
+    }
+  }
+  Ok(())
+}
+
+/// For Add/Remove community moderator activities, check that the target field actually contains
+/// /c/community/moderators. Any different values are unsupported.
+fn verify_add_remove_moderator_target(target: &Url, community: Url) -> Result<(), LemmyError> {
+  if target != &generate_moderators_url(&community.into())?.into_inner() {
+    return Err(anyhow!("Unkown target url").into());
+  }
+  Ok(())
+}

crates/apub/src/activities/post/create.rs

@@ -1,15 +1,11 @@
-use crate::activities::{
+use crate::{
-  post::send_websocket_message,
+  activities::{post::send_websocket_message, verify_activity, verify_person_in_community},
-  verify_activity,
-  verify_person_in_community,
-};
-use activitystreams::{activity::kind::CreateType, base::BaseExt};
-use lemmy_apub::{
   fetcher::person::get_or_fetch_and_upsert_person,
   objects::FromApub,
   ActorType,
   PageExt,
 };
+use activitystreams::{activity::kind::CreateType, base::BaseExt};
 use lemmy_apub_lib::{verify_domains_match_opt, ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_schema::source::post::Post;
 use lemmy_utils::LemmyError;

crates/apub/src/activities/post/update.rs

@@ -1,17 +1,17 @@
-use crate::activities::{
+use crate::{
+  activities::{
     post::send_websocket_message,
     verify_activity,
     verify_mod_action,
     verify_person_in_community,
-};
+  },
-use activitystreams::{activity::kind::UpdateType, base::BaseExt};
-use anyhow::Context;
-use lemmy_api_common::blocking;
-use lemmy_apub::{
   objects::{FromApub, FromApubToForm},
   ActorType,
   PageExt,
 };
+use activitystreams::{activity::kind::UpdateType, base::BaseExt};
+use anyhow::Context;
+use lemmy_api_common::blocking;
 use lemmy_apub_lib::{verify_domains_match_opt, ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_queries::ApubObject;
 use lemmy_db_schema::{

crates/apub/src/activities/private_message/create.rs

@@ -1,6 +1,9 @@
-use crate::activities::{private_message::send_websocket_message, verify_activity, verify_person};
+use crate::{
+  activities::{private_message::send_websocket_message, verify_activity, verify_person},
+  objects::FromApub,
+  NoteExt,
+};
 use activitystreams::{activity::kind::CreateType, base::BaseExt};
-use lemmy_apub::{objects::FromApub, NoteExt};
 use lemmy_apub_lib::{verify_domains_match_opt, ActivityCommonFields, ActivityHandler};
 use lemmy_db_schema::source::private_message::PrivateMessage;
 use lemmy_utils::LemmyError;

crates/apub/src/activities/private_message/update.rs

@@ -1,6 +1,9 @@
-use crate::activities::{private_message::send_websocket_message, verify_activity, verify_person};
+use crate::{
+  activities::{private_message::send_websocket_message, verify_activity, verify_person},
+  objects::FromApub,
+  NoteExt,
+};
 use activitystreams::{activity::kind::UpdateType, base::BaseExt};
-use lemmy_apub::{objects::FromApub, NoteExt};
 use lemmy_apub_lib::{verify_domains_match_opt, ActivityCommonFields, ActivityHandler};
 use lemmy_db_schema::source::private_message::PrivateMessage;
 use lemmy_utils::LemmyError;

crates/apub/src/activities/removal/remove.rs

@@ -1,4 +1,5 @@
-use crate::activities::{
+use crate::{
+  activities::{
     comment::send_websocket_message as send_comment_message,
     community::send_websocket_message as send_community_message,
     post::send_websocket_message as send_post_message,
@@ -6,11 +7,7 @@ use crate::activities::{
     verify_add_remove_moderator_target,
     verify_mod_action,
     verify_person_in_community,
-};
+  },
-use activitystreams::{activity::kind::RemoveType, base::AnyBase};
-use anyhow::anyhow;
-use lemmy_api_common::blocking;
-use lemmy_apub::{
   fetcher::{
     community::get_or_fetch_and_upsert_community,
     objects::get_or_fetch_and_insert_post_or_comment,
@@ -19,6 +16,9 @@ use lemmy_apub::{
   CommunityType,
   PostOrComment,
 };
+use activitystreams::{activity::kind::RemoveType, base::AnyBase};
+use anyhow::anyhow;
+use lemmy_api_common::blocking;
 use lemmy_apub_lib::{ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_queries::{
   source::{comment::Comment_, community::Community_, post::Post_},

crates/apub/src/activities/removal/undo_remove.rs

@@ -1,4 +1,5 @@
-use crate::activities::{
+use crate::{
+  activities::{
     comment::send_websocket_message as send_comment_message,
     community::send_websocket_message as send_community_message,
     post::send_websocket_message as send_post_message,
@@ -6,17 +7,16 @@ use crate::activities::{
     verify_activity,
     verify_mod_action,
     verify_person_in_community,
-};
+  },
-use activitystreams::activity::kind::UndoType;
-use anyhow::anyhow;
-use lemmy_api_common::blocking;
-use lemmy_apub::{
   fetcher::{
     community::get_or_fetch_and_upsert_community,
     objects::get_or_fetch_and_insert_post_or_comment,
   },
   PostOrComment,
 };
+use activitystreams::activity::kind::UndoType;
+use anyhow::anyhow;
+use lemmy_api_common::blocking;
 use lemmy_apub_lib::{ActivityCommonFields, ActivityHandler, PublicUrl};
 use lemmy_db_queries::source::{comment::Comment_, community::Community_, post::Post_};
 use lemmy_db_schema::source::{comment::Comment, community::Community, post::Post};

crates/apub/src/activities/voting/mod.rs

@@ -1,15 +1,15 @@
-use crate::activities::{
+use crate::{
+  activities::{
     comment::send_websocket_message as send_comment_message,
     post::send_websocket_message as send_post_message,
-};
+  },
-use lemmy_api_common::blocking;
-use lemmy_apub::{
   fetcher::{
     objects::get_or_fetch_and_insert_post_or_comment,
     person::get_or_fetch_and_upsert_person,
   },
   PostOrComment,
 };
+use lemmy_api_common::blocking;
 use lemmy_db_queries::Likeable;
 use lemmy_db_schema::source::{
   comment::{Comment, CommentLike, CommentLikeForm},

crates/apub/src/fetcher/community.rs

@@ -1,4 +1,5 @@
 use crate::{
+  activities::community::announce::AnnounceActivity,
   fetcher::{
     fetch::fetch_remote_object,
     is_deleted,
@@ -15,6 +16,7 @@ use activitystreams::{
 use anyhow::Context;
 use diesel::result::Error::NotFound;
 use lemmy_api_common::blocking;
+use lemmy_apub_lib::ActivityHandler;
 use lemmy_db_queries::{source::community::Community_, ApubObject, Joinable};
 use lemmy_db_schema::source::community::{Community, CommunityModerator, CommunityModeratorForm};
 use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView;
@@ -84,7 +86,7 @@ async fn fetch_remote_community(
   // only fetch outbox for new communities, otherwise this can create an infinite loop
   if old_community.is_none() {
     let outbox = group.inner.outbox()?.context(location_info!())?;
-    fetch_community_outbox(context, outbox, &community, request_counter).await?
+    fetch_community_outbox(context, outbox, request_counter).await?
   }
 
   Ok(community)
@@ -143,7 +145,6 @@ async fn update_community_mods(
 async fn fetch_community_outbox(
   context: &LemmyContext,
   outbox: &Url,
-  community: &Community,
   recursion_counter: &mut i32,
 ) -> Result<(), LemmyError> {
   let outbox =
@@ -154,9 +155,12 @@ async fn fetch_community_outbox(
     outbox_activities = outbox_activities[0..20].to_vec();
   }
 
-  for activity in outbox_activities {
+  for announce in outbox_activities {
-    todo!("{:?} {:?} {:?}", activity, community, recursion_counter);
+    // TODO: instead of converting like this, we should create a struct CommunityOutbox with
-    //receive_announce(context, activity, community, recursion_counter).await?;
+    //       AnnounceActivity as inner type, but that gives me stackoverflow
+    let ser = serde_json::to_string(&announce)?;
+    let announce: AnnounceActivity = serde_json::from_str(&ser)?;
+    announce.receive(context, recursion_counter).await?;
   }
 
   Ok(())

crates/apub/src/http/comment.rs

@@ -1,8 +1,10 @@
-use crate::http::{create_apub_response, create_apub_tombstone_response};
+use crate::{
+  http::{create_apub_response, create_apub_tombstone_response},
+  objects::ToApub,
+};
 use actix_web::{body::Body, web, web::Path, HttpResponse};
 use diesel::result::Error::NotFound;
 use lemmy_api_common::blocking;
-use lemmy_apub::objects::ToApub;
 use lemmy_db_queries::Crud;
 use lemmy_db_schema::{source::comment::Comment, CommentId};
 use lemmy_utils::LemmyError;

crates/apub/src/http/community.rs

@@ -1,9 +1,15 @@
-use crate::http::{
+use crate::{
+  extensions::context::lemmy_context,
+  generate_moderators_url,
+  http::{
     create_apub_response,
     create_apub_tombstone_response,
     inbox_enums::GroupInboxActivities,
     payload_to_string,
     receive_activity,
+  },
+  objects::ToApub,
+  ActorType,
 };
 use activitystreams::{
   base::{AnyBase, BaseExt},
@@ -12,12 +18,6 @@ use activitystreams::{
 };
 use actix_web::{body::Body, web, web::Payload, HttpRequest, HttpResponse};
 use lemmy_api_common::blocking;
-use lemmy_apub::{
-  extensions::context::lemmy_context,
-  generate_moderators_url,
-  objects::ToApub,
-  ActorType,
-};
 use lemmy_db_queries::source::{activity::Activity_, community::Community_};
 use lemmy_db_schema::source::{activity::Activity, community::Community};
 use lemmy_db_views_actor::{

crates/apub/src/http/mod.rs

@@ -1,4 +1,11 @@
-use crate::http::inbox_enums::SharedInboxActivities;
+use crate::{
+  check_is_apub_id_valid,
+  extensions::signatures::verify_signature,
+  fetcher::get_or_fetch_and_upsert_actor,
+  http::inbox_enums::SharedInboxActivities,
+  insert_activity,
+  APUB_JSON_CONTENT_TYPE,
+};
 use actix_web::{
   body::Body,
   web,
@@ -10,13 +17,6 @@ use anyhow::{anyhow, Context};
 use futures::StreamExt;
 use http::StatusCode;
 use lemmy_api_common::blocking;
-use lemmy_apub::{
-  check_is_apub_id_valid,
-  extensions::signatures::verify_signature,
-  fetcher::get_or_fetch_and_upsert_actor,
-  insert_activity,
-  APUB_JSON_CONTENT_TYPE,
-};
 use lemmy_apub_lib::ActivityHandler;
 use lemmy_db_queries::{source::activity::Activity_, DbPool};
 use lemmy_db_schema::source::activity::Activity;

crates/apub/src/http/person.rs

@@ -1,9 +1,14 @@
-use crate::http::{
+use crate::{
+  extensions::context::lemmy_context,
+  http::{
     create_apub_response,
     create_apub_tombstone_response,
     inbox_enums::PersonInboxActivities,
     payload_to_string,
     receive_activity,
+  },
+  objects::ToApub,
+  ActorType,
 };
 use activitystreams::{
   base::BaseExt,
@@ -11,7 +16,6 @@ use activitystreams::{
 };
 use actix_web::{body::Body, web, web::Payload, HttpRequest, HttpResponse};
 use lemmy_api_common::blocking;
-use lemmy_apub::{extensions::context::lemmy_context, objects::ToApub, ActorType};
 use lemmy_db_queries::source::person::Person_;
 use lemmy_db_schema::source::person::Person;
 use lemmy_utils::LemmyError;

crates/apub/src/http/post.rs

@@ -1,8 +1,10 @@
-use crate::http::{create_apub_response, create_apub_tombstone_response};
+use crate::{
+  http::{create_apub_response, create_apub_tombstone_response},
+  objects::ToApub,
+};
 use actix_web::{body::Body, web, HttpResponse};
 use diesel::result::Error::NotFound;
 use lemmy_api_common::blocking;
-use lemmy_apub::objects::ToApub;
 use lemmy_db_queries::Crud;
 use lemmy_db_schema::{source::post::Post, PostId};
 use lemmy_utils::LemmyError;

crates/apub/src/http/routes.rs

@@ -1,4 +1,5 @@
-use crate::http::{
+use crate::{
+  http::{
     comment::get_apub_comment,
     community::{
       community_inbox,
@@ -12,10 +13,11 @@ use crate::http::{
     person::{get_apub_person_http, get_apub_person_inbox, get_apub_person_outbox, person_inbox},
     post::get_apub_post,
     shared_inbox,
+  },
+  APUB_JSON_CONTENT_TYPE,
 };
 use actix_web::*;
 use http_signature_normalization_actix::digest::middleware::VerifyDigest;
-use lemmy_apub::APUB_JSON_CONTENT_TYPE;
 use lemmy_utils::settings::structs::Settings;
 use sha2::{Digest, Sha256};
 

crates/apub/src/lib.rs

@@ -5,6 +5,7 @@ pub mod activities;
 pub mod activity_queue;
 pub mod extensions;
 pub mod fetcher;
+pub mod http;
 pub mod objects;
 
 use crate::{

crates/apub_receive/Cargo.toml

@@ -1,46 +0,0 @@
-[package]
-name = "lemmy_apub_receive"
-version = "0.1.0"
-edition = "2018"
-
-[dependencies]
-lemmy_utils = { path = "../utils" }
-lemmy_apub_lib = { path = "../apub_lib" }
-lemmy_apub = { path = "../apub" }
-lemmy_db_queries = { path = "../db_queries" }
-lemmy_db_schema = { path = "../db_schema" }
-lemmy_db_views = { path = "../db_views" }
-lemmy_db_views_actor = { path = "../db_views_actor" }
-lemmy_api_common = { path = "../api_common" }
-lemmy_websocket = { path = "../websocket" }
-diesel = "1.4.7"
-activitystreams = "0.7.0-alpha.11"
-activitystreams-ext = "0.1.0-alpha.2"
-bcrypt = "0.10.0"
-chrono = { version = "0.4.19", features = ["serde"] }
-serde_json = { version = "1.0.64", features = ["preserve_order"] }
-serde = { version = "1.0.126", features = ["derive"] }
-actix = "0.12.0"
-actix-web = { version = "4.0.0-beta.8", default-features = false }
-actix-rt = { version = "2.2.0", default-features = false }
-awc = { version = "3.0.0-beta.7", default-features = false }
-log = "0.4.14"
-rand = "0.8.4"
-strum = "0.21.0"
-strum_macros = "0.21.1"
-url = { version = "2.2.2", features = ["serde"] }
-percent-encoding = "2.1.0"
-openssl = "0.10.35"
-http = "0.2.4"
-http-signature-normalization-actix = { version = "0.5.0-beta.6", default-features = false, features = ["sha-2"] }
-http-signature-normalization-reqwest = { version = "0.2.0", default-features = false, features = ["sha-2"] }
-base64 = "0.13.0"
-tokio = "1.8.0"
-futures = "0.3.15"
-itertools = "0.10.1"
-sha2 = "0.9.5"
-async-trait = "0.1.50"
-anyhow = "1.0.41"
-thiserror = "1.0.26"
-backtrace = "0.3.60"
-

crates/apub_receive/src/activities/mod.rs

@@ -1,121 +0,0 @@
-use anyhow::anyhow;
-use lemmy_api_common::blocking;
-use lemmy_apub::{
-  check_community_or_site_ban,
-  check_is_apub_id_valid,
-  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
-  generate_moderators_url,
-};
-use lemmy_apub_lib::{verify_domains_match, ActivityCommonFields};
-use lemmy_db_queries::ApubObject;
-use lemmy_db_schema::{
-  source::{community::Community, person::Person},
-  DbUrl,
-};
-use lemmy_db_views_actor::community_view::CommunityView;
-use lemmy_utils::LemmyError;
-use lemmy_websocket::LemmyContext;
-use url::Url;
-
-pub mod comment;
-pub mod community;
-pub mod deletion;
-pub mod following;
-pub mod post;
-pub mod private_message;
-pub mod removal;
-pub mod voting;
-
-/// Checks that the specified Url actually identifies a Person (by fetching it), and that the person
-/// doesn't have a site ban.
-async fn verify_person(
-  person_id: &Url,
-  context: &LemmyContext,
-  request_counter: &mut i32,
-) -> Result<(), LemmyError> {
-  let person = get_or_fetch_and_upsert_person(person_id, context, request_counter).await?;
-  if person.banned {
-    return Err(anyhow!("Person {} is banned", person_id).into());
-  }
-  Ok(())
-}
-
-/// Fetches the person and community to verify their type, then checks if person is banned from site
-/// or community.
-async fn verify_person_in_community(
-  person_id: &Url,
-  cc: &[Url],
-  context: &LemmyContext,
-  request_counter: &mut i32,
-) -> Result<Community, LemmyError> {
-  let person = get_or_fetch_and_upsert_person(person_id, context, request_counter).await?;
-  let mut cc_iter = cc.iter();
-  let community: Community = loop {
-    if let Some(cid) = cc_iter.next() {
-      if let Ok(c) = get_or_fetch_and_upsert_community(cid, context, request_counter).await {
-        break c;
-      }
-    } else {
-      return Err(anyhow!("No community found in cc").into());
-    }
-  };
-  check_community_or_site_ban(&person, community.id, context.pool()).await?;
-  Ok(community)
-}
-
-/// Simply check that the url actually refers to a valid group.
-async fn verify_community(
-  community_id: &Url,
-  context: &LemmyContext,
-  request_counter: &mut i32,
-) -> Result<(), LemmyError> {
-  get_or_fetch_and_upsert_community(community_id, context, request_counter).await?;
-  Ok(())
-}
-
-fn verify_activity(common: &ActivityCommonFields) -> Result<(), LemmyError> {
-  check_is_apub_id_valid(&common.actor, false)?;
-  verify_domains_match(common.id_unchecked(), &common.actor)?;
-  Ok(())
-}
-
-async fn verify_mod_action(
-  actor_id: &Url,
-  activity_cc: Url,
-  context: &LemmyContext,
-) -> Result<(), LemmyError> {
-  let community = blocking(context.pool(), move |conn| {
-    Community::read_from_apub_id(conn, &activity_cc.into())
-  })
-  .await??;
-
-  if community.local {
-    let actor_id: DbUrl = actor_id.clone().into();
-    let actor = blocking(context.pool(), move |conn| {
-      Person::read_from_apub_id(conn, &actor_id)
-    })
-    .await??;
-
-    // Note: this will also return true for admins in addition to mods, but as we dont know about
-    //       remote admins, it doesnt make any difference.
-    let community_id = community.id;
-    let actor_id = actor.id;
-    let is_mod_or_admin = blocking(context.pool(), move |conn| {
-      CommunityView::is_mod_or_admin(conn, actor_id, community_id)
-    })
-    .await?;
-    if !is_mod_or_admin {
-      return Err(anyhow!("Not a mod").into());
-    }
-  }
-  Ok(())
-}
-
-/// For Add/Remove community moderator activities, check that the target field actually contains
-/// /c/community/moderators. Any different values are unsupported.
-fn verify_add_remove_moderator_target(target: &Url, community: Url) -> Result<(), LemmyError> {
-  if target != &generate_moderators_url(&community.into())?.into_inner() {
-    return Err(anyhow!("Unkown target url").into());
-  }
-  Ok(())
-}

crates/apub_receive/src/lib.rs

@@ -1,2 +0,0 @@
-mod activities;
-pub mod http;

docker/dev/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - "8536:8536"
     restart: always
     environment:
-      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_apub_receive=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
+      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
     volumes:
       - ../lemmy.hjson:/config/config.hjson
     depends_on: 

docker/federation/docker-compose.yml

@@ -43,7 +43,7 @@ services:
     environment:
       - LEMMY_TEST_SEND_SYNC=1
       - RUST_BACKTRACE=1
-      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_apub_receive=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
+      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
     depends_on:
       - postgres_alpha
     ports: 
@@ -72,7 +72,7 @@ services:
     environment:
       - LEMMY_TEST_SEND_SYNC=1
       - RUST_BACKTRACE=1
-      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_apub_receive=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
+      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
     depends_on:
       - postgres_beta
     ports: 
@@ -101,7 +101,7 @@ services:
     environment:
       - LEMMY_TEST_SEND_SYNC=1
       - RUST_BACKTRACE=1
-      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_apub_receive=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
+      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
     depends_on:
       - postgres_gamma
     ports: 
@@ -131,7 +131,7 @@ services:
     environment:
       - LEMMY_TEST_SEND_SYNC=1
       - RUST_BACKTRACE=1
-      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_apub_receive=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
+      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
     depends_on:
       - postgres_delta
     ports: 
@@ -161,7 +161,7 @@ services:
     environment:
       - LEMMY_TEST_SEND_SYNC=1
       - RUST_BACKTRACE=1
-      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_apub_receive=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
+      - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_queries=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
     depends_on:
       - postgres_epsilon
     ports: 

docker/prod/docker-compose.yml

@@ -17,7 +17,7 @@ services:
       - "127.0.0.1:8536:8536"
     restart: always
     environment:
-      - RUST_LOG="warn,lemmy_server=info,lemmy_api=info,lemmy_api_common=info,lemmy_api_crud=info,lemmy_apub=info,lemmy_apub_receive=info,lemmy_db_queries=info,lemmy_db_schema=info,lemmy_db_views=info,lemmy_db_views_actor=info,lemmy_db_views_moderator=info,lemmy_routes=info,lemmy_utils=info,lemmy_websocket=info"
+      - RUST_LOG="warn,lemmy_server=info,lemmy_api=info,lemmy_api_common=info,lemmy_api_crud=info,lemmy_apub=info,lemmy_db_queries=info,lemmy_db_schema=info,lemmy_db_views=info,lemmy_db_views_actor=info,lemmy_db_views_moderator=info,lemmy_routes=info,lemmy_utils=info,lemmy_websocket=info"
     volumes:
       - ./lemmy.hjson:/config/config.hjson
     depends_on:

src/main.rs

@@ -91,7 +91,7 @@ async fn main() -> Result<(), LemmyError> {
       .app_data(Data::new(context))
       // The routes
       .configure(|cfg| api_routes::config(cfg, &rate_limiter))
-      .configure(lemmy_apub_receive::http::routes::config)
+      .configure(lemmy_apub::http::routes::config)
       .configure(feeds::config)
       .configure(|cfg| images::config(cfg, &rate_limiter))
       .configure(nodeinfo::config)