Merge pull request 'Adding unique ap_ids. Fixes #1100' (#90) from unique_ap_ids into activity-sender

Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/90

server/lemmy_db/src/activity.rs

@@ -113,7 +113,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_862362".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,

server/lemmy_db/src/comment.rs

@@ -39,13 +39,13 @@ pub struct CommentForm {
   pub published: Option<chrono::NaiveDateTime>,
   pub updated: Option<chrono::NaiveDateTime>,
   pub deleted: Option<bool>,
-  pub ap_id: String,
+  pub ap_id: Option<String>,
   pub local: bool,
 }
 
 impl CommentForm {
   pub fn get_ap_id(&self) -> Result<Url, ParseError> {
-    Url::parse(&self.ap_id)
+    Url::parse(&self.ap_id.as_ref().unwrap_or(&"not_a_url".to_string()))
   }
 }
 
@@ -61,7 +61,6 @@ impl Crud<CommentForm> for Comment {
   }
 
   fn create(conn: &PgConnection, comment_form: &CommentForm) -> Result<Self, Error> {
-    println!("creating {}", &comment_form.ap_id);
     use crate::schema::comment::dsl::*;
     insert_into(comment)
       .values(comment_form)
@@ -164,17 +163,13 @@ impl Comment {
   }
 
   pub fn upsert(conn: &PgConnection, comment_form: &CommentForm) -> Result<Self, Error> {
-    println!("Comment::upsert() (entered into function)");
+    use crate::schema::comment::dsl::*;
-    let existing = Self::read_from_apub_id(conn, &comment_form.ap_id);
+    insert_into(comment)
-    println!("Comment::upsert() (checked if comment exists)");
+      .values(comment_form)
-    let x = match existing {
+      .on_conflict(ap_id)
-      Err(NotFound {}) => Ok(Self::create(conn, &comment_form)?),
+      .do_update()
-      // both the old and new comment should be identical so no need to do an update here
+      .set(comment_form)
-      Ok(p) => Ok(Self::read(conn, p.id)?),
+      .get_result::<Self>(conn)
-      Err(e) => Err(e),
-    };
-    println!("Comment::upsert() (after match statement)");
-    x
   }
 }
 
@@ -278,7 +273,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_283687".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -298,7 +293,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme_928738972".into(),
+      actor_id: None,
       local: true,
       private_key: None,
       public_key: None,
@@ -326,7 +321,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
       published: None,
     };
@@ -343,7 +338,7 @@ mod tests {
       parent_id: None,
       published: None,
       updated: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
     };
 
@@ -360,7 +355,7 @@ mod tests {
       parent_id: None,
       published: inserted_comment.published,
       updated: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: inserted_comment.ap_id.to_owned(),
       local: true,
     };
 
@@ -374,7 +369,7 @@ mod tests {
       read: None,
       published: None,
       updated: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
     };
 

server/lemmy_db/src/comment_view.rs

@@ -517,7 +517,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_92873982".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -537,7 +537,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme_7625376".into(),
+      actor_id: None,
       local: true,
       private_key: None,
       public_key: None,
@@ -565,7 +565,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
       published: None,
     };
@@ -582,7 +582,7 @@ mod tests {
       read: None,
       published: None,
       updated: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
     };
 
@@ -627,7 +627,7 @@ mod tests {
       my_vote: None,
       subscribed: None,
       saved: None,
-      ap_id: "http://fake.com".to_string(),
+      ap_id: inserted_comment.ap_id.to_owned(),
       local: true,
       community_actor_id: inserted_community.actor_id.to_owned(),
       community_local: true,
@@ -665,7 +665,7 @@ mod tests {
       my_vote: Some(1),
       subscribed: Some(false),
       saved: Some(false),
-      ap_id: "http://fake.com".to_string(),
+      ap_id: inserted_comment.ap_id.to_owned(),
       local: true,
       community_actor_id: inserted_community.actor_id.to_owned(),
       community_local: true,

server/lemmy_db/src/community.rs

@@ -45,7 +45,7 @@ pub struct CommunityForm {
   pub updated: Option<chrono::NaiveDateTime>,
   pub deleted: Option<bool>,
   pub nsfw: bool,
-  pub actor_id: String,
+  pub actor_id: Option<String>,
   pub local: bool,
   pub private_key: Option<String>,
   pub public_key: Option<String>,
@@ -330,7 +330,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_8266238".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -350,7 +350,7 @@ mod tests {
       removed: None,
       deleted: None,
       updated: None,
-      actor_id: "changeme_7625376".into(),
+      actor_id: None,
       local: true,
       private_key: None,
       public_key: None,

server/lemmy_db/src/moderator.rs

@@ -426,7 +426,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_829398".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -454,7 +454,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_82982738".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -474,7 +474,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme_283687".into(),
+      actor_id: None,
       local: true,
       private_key: None,
       public_key: None,
@@ -502,7 +502,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
       published: None,
     };
@@ -519,7 +519,7 @@ mod tests {
       parent_id: None,
       published: None,
       updated: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
     };
 

server/lemmy_db/src/password_reset_request.rs

@@ -103,7 +103,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_8292378".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,

server/lemmy_db/src/post.rs

@@ -53,13 +53,13 @@ pub struct PostForm {
   pub embed_description: Option<String>,
   pub embed_html: Option<String>,
   pub thumbnail_url: Option<String>,
-  pub ap_id: String,
+  pub ap_id: Option<String>,
   pub local: bool,
 }
 
 impl PostForm {
   pub fn get_ap_id(&self) -> Result<Url, ParseError> {
-    Url::parse(&self.ap_id)
+    Url::parse(&self.ap_id.as_ref().unwrap_or(&"not_a_url".to_string()))
   }
 }
 
@@ -183,7 +183,9 @@ impl Post {
     use crate::schema::post::dsl::*;
     insert_into(post)
       .values(post_form)
-      .on_conflict_do_nothing()
+      .on_conflict(ap_id)
+      .do_update()
+      .set(post_form)
       .get_result::<Self>(conn)
   }
 }
@@ -357,7 +359,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_8292683678".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -377,7 +379,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme_8223262378".into(),
+      actor_id: None,
       local: true,
       private_key: None,
       public_key: None,
@@ -405,7 +407,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
       published: None,
     };
@@ -430,7 +432,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: inserted_post.ap_id.to_owned(),
       local: true,
     };
 

server/lemmy_db/src/post_view.rs

@@ -423,7 +423,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_8282738268".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -443,7 +443,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme_2763".into(),
+      actor_id: None,
       local: true,
       private_key: None,
       public_key: None,
@@ -471,7 +471,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
       published: None,
     };
@@ -555,7 +555,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "http://fake.com".to_string(),
+      ap_id: inserted_post.ap_id.to_owned(),
       local: true,
       creator_actor_id: inserted_user.actor_id.to_owned(),
       creator_local: true,
@@ -604,7 +604,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "http://fake.com".to_string(),
+      ap_id: inserted_post.ap_id.to_owned(),
       local: true,
       creator_actor_id: inserted_user.actor_id.to_owned(),
       creator_local: true,

server/lemmy_db/src/private_message.rs

@@ -27,7 +27,7 @@ pub struct PrivateMessageForm {
   pub read: Option<bool>,
   pub published: Option<chrono::NaiveDateTime>,
   pub updated: Option<chrono::NaiveDateTime>,
-  pub ap_id: String,
+  pub ap_id: Option<String>,
   pub local: bool,
 }
 
@@ -119,6 +119,17 @@ impl PrivateMessage {
     .set(read.eq(true))
     .get_results::<Self>(conn)
   }
+
+  // TODO use this
+  pub fn upsert(conn: &PgConnection, private_message_form: &PrivateMessageForm) -> Result<Self, Error> {
+    use crate::schema::private_message::dsl::*;
+    insert_into(private_message)
+      .values(private_message_form)
+      .on_conflict(ap_id)
+      .do_update()
+      .set(private_message_form)
+      .get_result::<Self>(conn)
+  }
 }
 
 #[cfg(test)]
@@ -153,7 +164,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_6723878".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -181,7 +192,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_287263876".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -199,7 +210,7 @@ mod tests {
       read: None,
       published: None,
       updated: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
     };
 
@@ -214,7 +225,7 @@ mod tests {
       read: false,
       updated: None,
       published: inserted_private_message.published,
-      ap_id: "http://fake.com".into(),
+      ap_id: inserted_private_message.ap_id.to_owned(),
       local: true,
     };
 

server/lemmy_db/src/user.rs

@@ -57,7 +57,7 @@ pub struct UserForm {
   pub show_avatars: bool,
   pub send_notifications_to_email: bool,
   pub matrix_user_id: Option<String>,
-  pub actor_id: String,
+  pub actor_id: Option<String>,
   pub bio: Option<String>,
   pub local: bool,
   pub private_key: Option<String>,
@@ -189,7 +189,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_9826382637".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,

server/lemmy_db/src/user_mention.rs

@@ -106,7 +106,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_628763".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -134,7 +134,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme_927389278".into(),
+      actor_id: None,
       bio: None,
       local: true,
       private_key: None,
@@ -154,7 +154,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme_876238".into(),
+      actor_id: None,
       local: true,
       private_key: None,
       public_key: None,
@@ -182,7 +182,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
       published: None,
     };
@@ -199,7 +199,7 @@ mod tests {
       parent_id: None,
       published: None,
       updated: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
     };
 

server/migrations/2020-08-25-132005_add_unique_ap_ids/down.sql

@@ -0,0 +1,27 @@
+-- Drop the uniques
+alter table private_message drop constraint idx_private_message_ap_id;
+alter table post drop constraint idx_post_ap_id;
+alter table comment drop constraint idx_comment_ap_id;
+alter table user_ drop constraint idx_user_actor_id;
+alter table community drop constraint idx_community_actor_id;
+
+alter table private_message alter column ap_id set not null;
+alter table private_message alter column ap_id set default 'http://fake.com';
+
+alter table post alter column ap_id set not null;
+alter table post alter column ap_id set default 'http://fake.com';
+
+alter table comment alter column ap_id set not null;
+alter table comment alter column ap_id set default 'http://fake.com';
+
+update private_message
+set ap_id = 'http://fake.com'
+where ap_id like 'changeme_%';
+
+update post
+set ap_id = 'http://fake.com'
+where ap_id like 'changeme_%';
+
+update comment
+set ap_id = 'http://fake.com'
+where ap_id like 'changeme_%';

server/migrations/2020-08-25-132005_add_unique_ap_ids/up.sql

@@ -0,0 +1,56 @@
+-- Add unique ap_id for private_message, comment, and post
+
+-- Need to delete the possible dupes for ones that don't start with the fake one
+delete from private_message a using (
+  select min(id) as id, ap_id
+    from private_message 
+    group by ap_id having count(*) > 1
+) b
+where a.ap_id = b.ap_id 
+and a.id <> b.id;
+
+delete from post a using (
+  select min(id) as id, ap_id
+    from post 
+    group by ap_id having count(*) > 1
+) b
+where a.ap_id = b.ap_id 
+and a.id <> b.id;
+
+delete from comment a using (
+  select min(id) as id, ap_id
+    from comment 
+    group by ap_id having count(*) > 1
+) b
+where a.ap_id = b.ap_id 
+and a.id <> b.id;
+
+-- Replacing the current default on the columns, to the unique one
+update private_message 
+set ap_id = generate_unique_changeme()
+where ap_id = 'http://fake.com';
+
+update post 
+set ap_id = generate_unique_changeme()
+where ap_id = 'http://fake.com';
+
+update comment 
+set ap_id = generate_unique_changeme()
+where ap_id = 'http://fake.com';
+
+-- Add the unique indexes
+alter table private_message alter column ap_id set not null;
+alter table private_message alter column ap_id set default generate_unique_changeme();
+
+alter table post alter column ap_id set not null;
+alter table post alter column ap_id set default generate_unique_changeme();
+
+alter table comment alter column ap_id set not null;
+alter table comment alter column ap_id set default generate_unique_changeme();
+
+-- Add the uniques, for user_ and community too
+alter table private_message add constraint idx_private_message_ap_id unique (ap_id);
+alter table post add constraint idx_post_ap_id unique (ap_id);
+alter table comment add constraint idx_comment_ap_id unique (ap_id);
+alter table user_ add constraint idx_user_actor_id unique (actor_id);
+alter table community add constraint idx_community_actor_id unique (actor_id);

server/src/api/comment.rs

@@ -146,7 +146,7 @@ impl Perform for CreateComment {
       read: None,
       published: None,
       updated: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
     };
 

server/src/api/community.rs

@@ -274,7 +274,7 @@ impl Perform for CreateCommunity {
       deleted: None,
       nsfw: data.nsfw,
       updated: None,
-      actor_id,
+      actor_id: Some(actor_id),
       local: true,
       private_key: Some(keypair.private_key),
       public_key: Some(keypair.public_key),
@@ -368,7 +368,7 @@ impl Perform for EditCommunity {
       deleted: Some(read_community.deleted),
       nsfw: data.nsfw,
       updated: Some(naive_now()),
-      actor_id: read_community.actor_id,
+      actor_id: Some(read_community.actor_id),
       local: read_community.local,
       private_key: read_community.private_key,
       public_key: read_community.public_key,

server/src/api/post.rs

@@ -187,7 +187,7 @@ impl Perform for CreatePost {
       embed_description: iframely_description,
       embed_html: iframely_html,
       thumbnail_url: pictrs_thumbnail,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
       published: None,
     };
@@ -518,7 +518,7 @@ impl Perform for EditPost {
       embed_description: iframely_description,
       embed_html: iframely_html,
       thumbnail_url: pictrs_thumbnail,
-      ap_id: orig_post.ap_id,
+      ap_id: Some(orig_post.ap_id),
       local: orig_post.local,
       published: None,
     };

server/src/api/user.rs

@@ -410,7 +410,7 @@ impl Perform for Register {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: make_apub_endpoint(EndpointType::User, &data.username).to_string(),
+      actor_id: Some(make_apub_endpoint(EndpointType::User, &data.username).to_string()),
       bio: None,
       local: true,
       private_key: Some(user_keypair.private_key),
@@ -441,37 +441,38 @@ impl Perform for Register {
     let main_community_keypair = generate_actor_keypair()?;
 
     // Create the main community if it doesn't exist
-    let main_community = match blocking(context.pool(), move |conn| Community::read(conn, 2))
+    let main_community =
-      .await?
+      match blocking(context.pool(), move |conn| Community::read(conn, 2)).await? {
-    {
+        Ok(c) => c,
-      Ok(c) => c,
+        Err(_e) => {
-      Err(_e) => {
+          let default_community_name = "main";
-        let default_community_name = "main";
+          let community_form = CommunityForm {
-        let community_form = CommunityForm {
+            name: default_community_name.to_string(),
-          name: default_community_name.to_string(),
+            title: "The Default Community".to_string(),
-          title: "The Default Community".to_string(),
+            description: Some("The Default Community".to_string()),
-          description: Some("The Default Community".to_string()),
+            category_id: 1,
-          category_id: 1,
+            nsfw: false,
-          nsfw: false,
+            creator_id: inserted_user.id,
-          creator_id: inserted_user.id,
+            removed: None,
-          removed: None,
+            deleted: None,
-          deleted: None,
+            updated: None,
-          updated: None,
+            actor_id: Some(
-          actor_id: make_apub_endpoint(EndpointType::Community, default_community_name).to_string(),
+              make_apub_endpoint(EndpointType::Community, default_community_name).to_string(),
-          local: true,
+            ),
-          private_key: Some(main_community_keypair.private_key),
+            local: true,
-          public_key: Some(main_community_keypair.public_key),
+            private_key: Some(main_community_keypair.private_key),
-          last_refreshed_at: None,
+            public_key: Some(main_community_keypair.public_key),
-          published: None,
+            last_refreshed_at: None,
-          icon: None,
+            published: None,
-          banner: None,
+            icon: None,
-        };
+            banner: None,
-        blocking(context.pool(), move |conn| {
+          };
-          Community::create(conn, &community_form)
+          blocking(context.pool(), move |conn| {
-        })
+            Community::create(conn, &community_form)
-        .await??
+          })
-      }
+          .await??
-    };
+        }
+      };
 
     // Sign them up for main community no matter what
     let community_follower_form = CommunityFollowerForm {
@@ -643,7 +644,7 @@ impl Perform for SaveUserSettings {
       lang: data.lang.to_owned(),
       show_avatars: data.show_avatars,
       send_notifications_to_email: data.send_notifications_to_email,
-      actor_id: read_user.actor_id,
+      actor_id: Some(read_user.actor_id),
       bio,
       local: read_user.local,
       private_key: read_user.private_key,
@@ -1218,7 +1219,7 @@ impl Perform for CreatePrivateMessage {
       deleted: None,
       read: None,
       updated: None,
-      ap_id: "http://fake.com".into(),
+      ap_id: None,
       local: true,
       published: None,
     };

server/src/apub/comment.rs

@@ -192,7 +192,7 @@ impl FromApub for CommentForm {
       published: note.published().map(|u| u.to_owned().naive_local()),
       updated: note.updated().map(|u| u.to_owned().naive_local()),
       deleted: None,
-      ap_id: check_actor_domain(note, expected_domain)?,
+      ap_id: Some(check_actor_domain(note, expected_domain)?),
       local: false,
     })
   }

server/src/apub/community.rs

@@ -403,7 +403,7 @@ impl FromApub for CommunityForm {
       updated: group.inner.updated().map(|u| u.to_owned().naive_local()),
       deleted: None,
       nsfw: group.ext_one.sensitive,
-      actor_id: check_actor_domain(group, expected_domain)?,
+      actor_id: Some(check_actor_domain(group, expected_domain)?),
       local: false,
       private_key: None,
       public_key: Some(group.ext_two.to_owned().public_key.public_key_pem),

server/src/apub/fetcher.rs

@@ -337,7 +337,7 @@ async fn fetch_remote_community(
   for o in outbox_items {
     let page = PageExt::from_any_base(o)?.context(location_info!())?;
     let post = PostForm::from_apub(&page, context, None).await?;
-    let post_ap_id = post.ap_id.clone();
+    let post_ap_id = post.ap_id.as_ref().context(location_info!())?.clone();
     // Check whether the post already exists in the local db
     let existing = blocking(context.pool(), move |conn| {
       Post::read_from_apub_id(conn, &post_ap_id)

server/src/apub/inbox/activities/delete.rs

@@ -78,7 +78,7 @@ async fn receive_delete_post(
     embed_description: post.embed_description,
     embed_html: post.embed_html,
     thumbnail_url: post.thumbnail_url,
-    ap_id: post.ap_id,
+    ap_id: Some(post.ap_id),
     local: post.local,
     published: None,
   };
@@ -131,7 +131,7 @@ async fn receive_delete_comment(
     read: None,
     published: None,
     updated: Some(naive_now()),
-    ap_id: comment.ap_id,
+    ap_id: Some(comment.ap_id),
     local: comment.local,
   };
   let comment_id = comment.id;
@@ -175,7 +175,8 @@ async fn receive_delete_community(
 
   let community_actor_id = CommunityForm::from_apub(&group, context, Some(user.actor_id()?))
     .await?
-    .actor_id;
+    .actor_id
+    .context(location_info!())?;
 
   let community = blocking(context.pool(), move |conn| {
     Community::read_from_actor_id(conn, &community_actor_id)
@@ -193,7 +194,7 @@ async fn receive_delete_community(
     updated: Some(naive_now()),
     deleted: Some(true),
     nsfw: community.nsfw,
-    actor_id: community.actor_id,
+    actor_id: Some(community.actor_id),
     local: community.local,
     private_key: community.private_key,
     public_key: community.public_key,

server/src/apub/inbox/activities/remove.rs

@@ -85,7 +85,7 @@ async fn receive_remove_post(
     embed_description: post.embed_description,
     embed_html: post.embed_html,
     thumbnail_url: post.thumbnail_url,
-    ap_id: post.ap_id,
+    ap_id: Some(post.ap_id),
     local: post.local,
     published: None,
   };
@@ -138,7 +138,7 @@ async fn receive_remove_comment(
     read: None,
     published: None,
     updated: Some(naive_now()),
-    ap_id: comment.ap_id,
+    ap_id: Some(comment.ap_id),
     local: comment.local,
   };
   let comment_id = comment.id;
@@ -182,7 +182,8 @@ async fn receive_remove_community(
 
   let community_actor_id = CommunityForm::from_apub(&group, context, Some(mod_.actor_id()?))
     .await?
-    .actor_id;
+    .actor_id
+    .context(location_info!())?;
 
   let community = blocking(context.pool(), move |conn| {
     Community::read_from_actor_id(conn, &community_actor_id)
@@ -200,7 +201,7 @@ async fn receive_remove_community(
     updated: Some(naive_now()),
     deleted: None,
     nsfw: community.nsfw,
-    actor_id: community.actor_id,
+    actor_id: Some(community.actor_id),
     local: community.local,
     private_key: community.private_key,
     public_key: community.public_key,

server/src/apub/inbox/activities/undo.rs

@@ -175,7 +175,7 @@ async fn receive_undo_delete_comment(
     read: None,
     published: None,
     updated: Some(naive_now()),
-    ap_id: comment.ap_id,
+    ap_id: Some(comment.ap_id),
     local: comment.local,
   };
   let comment_id = comment.id;
@@ -234,7 +234,7 @@ async fn receive_undo_remove_comment(
     read: None,
     published: None,
     updated: Some(naive_now()),
-    ap_id: comment.ap_id,
+    ap_id: Some(comment.ap_id),
     local: comment.local,
   };
   let comment_id = comment.id;
@@ -299,7 +299,7 @@ async fn receive_undo_delete_post(
     embed_description: post.embed_description,
     embed_html: post.embed_html,
     thumbnail_url: post.thumbnail_url,
-    ap_id: post.ap_id,
+    ap_id: Some(post.ap_id),
     local: post.local,
     published: None,
   };
@@ -359,7 +359,7 @@ async fn receive_undo_remove_post(
     embed_description: post.embed_description,
     embed_html: post.embed_html,
     thumbnail_url: post.thumbnail_url,
-    ap_id: post.ap_id,
+    ap_id: Some(post.ap_id),
     local: post.local,
     published: None,
   };
@@ -399,7 +399,8 @@ async fn receive_undo_delete_community(
 
   let community_actor_id = CommunityForm::from_apub(&group, context, Some(user.actor_id()?))
     .await?
-    .actor_id;
+    .actor_id
+    .context(location_info!())?;
 
   let community = blocking(context.pool(), move |conn| {
     Community::read_from_actor_id(conn, &community_actor_id)
@@ -417,7 +418,7 @@ async fn receive_undo_delete_community(
     updated: Some(naive_now()),
     deleted: Some(false),
     nsfw: community.nsfw,
-    actor_id: community.actor_id,
+    actor_id: Some(community.actor_id),
     local: community.local,
     private_key: community.private_key,
     public_key: community.public_key,
@@ -464,7 +465,8 @@ async fn receive_undo_remove_community(
 
   let community_actor_id = CommunityForm::from_apub(&group, context, Some(mod_.actor_id()?))
     .await?
-    .actor_id;
+    .actor_id
+    .context(location_info!())?;
 
   let community = blocking(context.pool(), move |conn| {
     Community::read_from_actor_id(conn, &community_actor_id)
@@ -482,7 +484,7 @@ async fn receive_undo_remove_community(
     updated: Some(naive_now()),
     deleted: None,
     nsfw: community.nsfw,
-    actor_id: community.actor_id,
+    actor_id: Some(community.actor_id),
     local: community.local,
     private_key: community.private_key,
     public_key: community.public_key,

server/src/apub/inbox/user_inbox.rs

@@ -175,7 +175,11 @@ async fn receive_update_private_message(
   let domain = Some(update.id_unchecked().context(location_info!())?.to_owned());
   let private_message_form = PrivateMessageForm::from_apub(&note, context, domain).await?;
 
-  let private_message_ap_id = private_message_form.ap_id.clone();
+  let private_message_ap_id = private_message_form
+    .ap_id
+    .as_ref()
+    .context(location_info!())?
+    .clone();
   let private_message = blocking(&context.pool(), move |conn| {
     PrivateMessage::read_from_apub_id(conn, &private_message_ap_id)
   })
@@ -224,7 +228,7 @@ async fn receive_delete_private_message(
   let domain = Some(delete.id_unchecked().context(location_info!())?.to_owned());
   let private_message_form = PrivateMessageForm::from_apub(&note, context, domain).await?;
 
-  let private_message_ap_id = private_message_form.ap_id;
+  let private_message_ap_id = private_message_form.ap_id.context(location_info!())?;
   let private_message = blocking(&context.pool(), move |conn| {
     PrivateMessage::read_from_apub_id(conn, &private_message_ap_id)
   })
@@ -236,7 +240,7 @@ async fn receive_delete_private_message(
     creator_id: private_message.creator_id,
     deleted: Some(true),
     read: None,
-    ap_id: private_message.ap_id,
+    ap_id: Some(private_message.ap_id),
     local: private_message.local,
     published: None,
     updated: Some(naive_now()),
@@ -287,7 +291,11 @@ async fn receive_undo_delete_private_message(
   let domain = Some(undo.id_unchecked().context(location_info!())?.to_owned());
   let private_message = PrivateMessageForm::from_apub(&note, context, domain).await?;
 
-  let private_message_ap_id = private_message.ap_id.clone();
+  let private_message_ap_id = private_message
+    .ap_id
+    .as_ref()
+    .context(location_info!())?
+    .clone();
   let private_message_id = blocking(&context.pool(), move |conn| {
     PrivateMessage::read_from_apub_id(conn, &private_message_ap_id).map(|pm| pm.id)
   })

server/src/apub/post.rs

@@ -289,7 +289,7 @@ impl FromApub for PostForm {
       embed_description: embed.description,
       embed_html: embed.html,
       thumbnail_url,
-      ap_id: check_actor_domain(page, expected_domain)?,
+      ap_id: Some(check_actor_domain(page, expected_domain)?),
       local: false,
     })
   }

server/src/apub/private_message.rs

@@ -111,7 +111,7 @@ impl FromApub for PrivateMessageForm {
       updated: note.updated().map(|u| u.to_owned().naive_local()),
       deleted: None,
       read: None,
-      ap_id: check_actor_domain(note, expected_domain)?,
+      ap_id: Some(check_actor_domain(note, expected_domain)?),
       local: false,
     })
   }

server/src/apub/user.rs

@@ -269,7 +269,7 @@ impl FromApub for UserForm {
       show_avatars: false,
       send_notifications_to_email: false,
       matrix_user_id: None,
-      actor_id: check_actor_domain(person, expected_domain)?,
+      actor_id: Some(check_actor_domain(person, expected_domain)?),
       bio,
       local: false,
       private_key: None,

server/src/code_migrations.rs

@@ -67,7 +67,7 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> {
       lang: cuser.lang.to_owned(),
       show_avatars: cuser.show_avatars,
       send_notifications_to_email: cuser.send_notifications_to_email,
-      actor_id: make_apub_endpoint(EndpointType::User, &cuser.name).to_string(),
+      actor_id: Some(make_apub_endpoint(EndpointType::User, &cuser.name).to_string()),
       bio: cuser.bio.to_owned(),
       local: cuser.local,
       private_key: Some(keypair.private_key),
@@ -111,7 +111,7 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> {
       deleted: None,
       nsfw: ccommunity.nsfw,
       updated: None,
-      actor_id: make_apub_endpoint(EndpointType::Community, &ccommunity.name).to_string(),
+      actor_id: Some(make_apub_endpoint(EndpointType::Community, &ccommunity.name).to_string()),
       local: ccommunity.local,
       private_key: Some(keypair.private_key),
       public_key: Some(keypair.public_key),
@@ -138,7 +138,7 @@ fn post_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> {
 
   // Update the ap_id
   let incorrect_posts = post
-    .filter(ap_id.eq("http://fake.com"))
+    .filter(ap_id.eq("changeme_%"))
     .filter(local.eq(true))
     .load::<Post>(conn)?;
 
@@ -163,7 +163,7 @@ fn comment_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> {
 
   // Update the ap_id
   let incorrect_comments = comment
-    .filter(ap_id.eq("http://fake.com"))
+    .filter(ap_id.eq("changeme_%"))
     .filter(local.eq(true))
     .load::<Comment>(conn)?;
 
@@ -188,7 +188,7 @@ fn private_message_updates_2020_05_05(conn: &PgConnection) -> Result<(), LemmyEr
 
   // Update the ap_id
   let incorrect_pms = private_message
-    .filter(ap_id.eq("http://fake.com"))
+    .filter(ap_id.eq("changeme_%"))
     .filter(local.eq(true))
     .load::<PrivateMessage>(conn)?;
 

ui/src/api_tests/comment.spec.ts

@@ -1,3 +1,4 @@
+jest.setTimeout(120000);
 import {
   alpha,
   beta,
@@ -19,6 +20,7 @@ import {
   createCommunity,
   registerUser,
   API,
+  delay,
 } from './shared';
 
 import { PostResponse } from 'lemmy-js-client';
@@ -30,6 +32,7 @@ beforeAll(async () => {
   await followBeta(alpha);
   await followBeta(gamma);
   let search = await searchForBetaCommunity(alpha);
+  await delay(10000);
   postRes = await createPost(
     alpha,
     search.communities.filter(c => c.local == false)[0].id
@@ -47,6 +50,7 @@ test('Create a comment', async () => {
   expect(commentRes.comment.community_local).toBe(false);
   expect(commentRes.comment.creator_local).toBe(true);
   expect(commentRes.comment.score).toBe(1);
+  await delay();
 
   // Make sure that comment is liked on beta
   let searchBeta = await searchComment(beta, commentRes.comment);
@@ -64,12 +68,14 @@ test('Create a comment in a non-existent post', async () => {
 
 test('Update a comment', async () => {
   let commentRes = await createComment(alpha, postRes.post.id);
+  await delay();
   let updateCommentRes = await updateComment(alpha, commentRes.comment.id);
   expect(updateCommentRes.comment.content).toBe(
     'A jest test federated comment update'
   );
   expect(updateCommentRes.comment.community_local).toBe(false);
   expect(updateCommentRes.comment.creator_local).toBe(true);
+  await delay();
 
   // Make sure that post is updated on beta
   let searchBeta = await searchComment(beta, commentRes.comment);
@@ -79,23 +85,21 @@ test('Update a comment', async () => {
 
 test('Delete a comment', async () => {
   let commentRes = await createComment(alpha, postRes.post.id);
+  await delay();
+
   let deleteCommentRes = await deleteComment(
     alpha,
     true,
     commentRes.comment.id
   );
   expect(deleteCommentRes.comment.deleted).toBe(true);
+  await delay();
 
-  // Make sure that comment is deleted on beta
+  // Make sure that comment is undefined on beta
-  // The search doesnt work below, because it returns a tombstone / http::gone
+  let searchBeta = await searchComment(beta, commentRes.comment);
-  // let searchBeta = await searchComment(beta, commentRes.comment);
+  let betaComment = searchBeta.comments[0];
-  // console.log(searchBeta);
+  expect(betaComment).toBeUndefined();
-  // let betaComment = searchBeta.comments[0];
+  await delay();
-  // Create a fake post, just to get the previous new post id
-  let createdBetaPostJustToGetId = await createPost(beta, 2);
-  let betaPost = await getPost(beta, createdBetaPostJustToGetId.post.id - 1);
-  let betaComment = betaPost.comments[0];
-  expect(betaComment.deleted).toBe(true);
 
   let undeleteCommentRes = await deleteComment(
     alpha,
@@ -103,6 +107,7 @@ test('Delete a comment', async () => {
     commentRes.comment.id
   );
   expect(undeleteCommentRes.comment.deleted).toBe(false);
+  await delay();
 
   // Make sure that comment is undeleted on beta
   let searchBeta2 = await searchComment(beta, commentRes.comment);
@@ -112,6 +117,7 @@ test('Delete a comment', async () => {
 
 test('Remove a comment from admin and community on the same instance', async () => {
   let commentRes = await createComment(alpha, postRes.post.id);
+  await delay();
 
   // Get the id for beta
   let betaCommentId = (await searchComment(beta, commentRes.comment))
@@ -120,6 +126,7 @@ test('Remove a comment from admin and community on the same instance', async ()
   // The beta admin removes it (the community lives on beta)
   let removeCommentRes = await removeComment(beta, true, betaCommentId);
   expect(removeCommentRes.comment.removed).toBe(true);
+  await delay();
 
   // Make sure that comment is removed on alpha (it gets pushed since an admin from beta removed it)
   let refetchedPost = await getPost(alpha, postRes.post.id);
@@ -127,6 +134,7 @@ test('Remove a comment from admin and community on the same instance', async ()
 
   let unremoveCommentRes = await removeComment(beta, false, betaCommentId);
   expect(unremoveCommentRes.comment.removed).toBe(false);
+  await delay();
 
   // Make sure that comment is unremoved on beta
   let refetchedPost2 = await getPost(alpha, postRes.post.id);
@@ -142,15 +150,19 @@ test('Remove a comment from admin and community on different instance', async ()
 
   // New alpha user creates a community, post, and comment.
   let newCommunity = await createCommunity(newAlphaApi);
+  await delay();
   let newPost = await createPost(newAlphaApi, newCommunity.community.id);
+  await delay();
   let commentRes = await createComment(newAlphaApi, newPost.post.id);
   expect(commentRes.comment.content).toBeDefined();
+  await delay();
 
   // Beta searches that to cache it, then removes it
   let searchBeta = await searchComment(beta, commentRes.comment);
   let betaComment = searchBeta.comments[0];
   let removeCommentRes = await removeComment(beta, true, betaComment.id);
   expect(removeCommentRes.comment.removed).toBe(true);
+  await delay();
 
   // Make sure its not removed on alpha
   let refetchedPost = await getPost(newAlphaApi, newPost.post.id);
@@ -159,8 +171,10 @@ test('Remove a comment from admin and community on different instance', async ()
 
 test('Unlike a comment', async () => {
   let commentRes = await createComment(alpha, postRes.post.id);
+  await delay();
   let unlike = await likeComment(alpha, 0, commentRes.comment);
   expect(unlike.comment.score).toBe(0);
+  await delay();
 
   // Make sure that post is unliked on beta
   let searchBeta = await searchComment(beta, commentRes.comment);
@@ -173,6 +187,7 @@ test('Unlike a comment', async () => {
 
 test('Federated comment like', async () => {
   let commentRes = await createComment(alpha, postRes.post.id);
+  await delay();
 
   // Find the comment on beta
   let searchBeta = await searchComment(beta, commentRes.comment);
@@ -180,6 +195,7 @@ test('Federated comment like', async () => {
 
   let like = await likeComment(beta, 1, betaComment);
   expect(like.comment.score).toBe(2);
+  await delay();
 
   // Get the post from alpha, check the likes
   let post = await getPost(alpha, postRes.post.id);
@@ -189,6 +205,7 @@ test('Federated comment like', async () => {
 test('Reply to a comment', async () => {
   // Create a comment on alpha, find it on beta
   let commentRes = await createComment(alpha, postRes.post.id);
+  await delay();
   let searchBeta = await searchComment(beta, commentRes.comment);
   let betaComment = searchBeta.comments[0];
 
@@ -201,6 +218,7 @@ test('Reply to a comment', async () => {
   expect(replyRes.comment.creator_local).toBe(true);
   expect(replyRes.comment.parent_id).toBe(betaComment.id);
   expect(replyRes.comment.score).toBe(1);
+  await delay();
 
   // Make sure that comment is seen on alpha
   // TODO not sure why, but a searchComment back to alpha, for the ap_id of betas
@@ -219,6 +237,7 @@ test('Mention beta', async () => {
   // Create a mention on alpha
   let mentionContent = 'A test mention of @lemmy_beta@lemmy-beta:8550';
   let commentRes = await createComment(alpha, postRes.post.id);
+  await delay();
   let mentionRes = await createComment(
     alpha,
     postRes.post.id,
@@ -229,6 +248,7 @@ test('Mention beta', async () => {
   expect(mentionRes.comment.community_local).toBe(false);
   expect(mentionRes.comment.creator_local).toBe(true);
   expect(mentionRes.comment.score).toBe(1);
+  await delay();
 
   let mentionsRes = await getMentions(beta);
   expect(mentionsRes.mentions[0].content).toBeDefined();
@@ -239,6 +259,7 @@ test('Mention beta', async () => {
 
 test('Comment Search', async () => {
   let commentRes = await createComment(alpha, postRes.post.id);
+  await delay();
   let searchBeta = await searchComment(beta, commentRes.comment);
   expect(searchBeta.comments[0].ap_id).toBe(commentRes.comment.ap_id);
 });
@@ -247,6 +268,7 @@ test('A and G subscribe to B (center) A posts, G mentions B, it gets announced t
   // Create a local post
   let alphaPost = await createPost(alpha, 2);
   expect(alphaPost.post.community_local).toBe(true);
+  await delay();
 
   // Make sure gamma sees it
   let search = await searchPost(gamma, alphaPost.post);
@@ -264,6 +286,7 @@ test('A and G subscribe to B (center) A posts, G mentions B, it gets announced t
   expect(commentRes.comment.community_local).toBe(false);
   expect(commentRes.comment.creator_local).toBe(true);
   expect(commentRes.comment.score).toBe(1);
+  await delay();
 
   // Make sure alpha sees it
   let alphaPost2 = await getPost(alpha, alphaPost.post.id);
@@ -291,6 +314,7 @@ test('Fetch in_reply_tos: A is unsubbed from B, B makes a post, and some embedde
   // B creates a post, and two comments, should be invisible to A
   let postRes = await createPost(beta, 2);
   expect(postRes.post.name).toBeDefined();
+  await delay();
 
   let parentCommentContent = 'An invisible top level comment from beta';
   let parentCommentRes = await createComment(
@@ -300,6 +324,7 @@ test('Fetch in_reply_tos: A is unsubbed from B, B makes a post, and some embedde
     parentCommentContent
   );
   expect(parentCommentRes.comment.content).toBe(parentCommentContent);
+  await delay();
 
   // B creates a comment, then a child one of that.
   let childCommentContent = 'An invisible child comment from beta';
@@ -310,11 +335,13 @@ test('Fetch in_reply_tos: A is unsubbed from B, B makes a post, and some embedde
     childCommentContent
   );
   expect(childCommentRes.comment.content).toBe(childCommentContent);
+  await delay();
 
   // Follow beta again
   let follow = await followBeta(alpha);
   expect(follow.community.local).toBe(false);
   expect(follow.community.name).toBe('main');
+  await delay();
 
   // An update to the child comment on beta, should push the post, parent, and child to alpha now
   let updatedCommentContent = 'An update child comment from beta';
@@ -324,10 +351,14 @@ test('Fetch in_reply_tos: A is unsubbed from B, B makes a post, and some embedde
     updatedCommentContent
   );
   expect(updateRes.comment.content).toBe(updatedCommentContent);
+  await delay();
 
   // Get the post from alpha
-  let createFakeAlphaPostToGetId = await createPost(alpha, 2);
+  let search = await searchPost(alpha, postRes.post);
-  let alphaPost = await getPost(alpha, createFakeAlphaPostToGetId.post.id - 1);
+  let alphaPostB = search.posts[0];
+  await delay();
+
+  let alphaPost = await getPost(alpha, alphaPostB.id);
   expect(alphaPost.post.name).toBeDefined();
   expect(alphaPost.comments[1].content).toBe(parentCommentContent);
   expect(alphaPost.comments[0].content).toBe(updatedCommentContent);

ui/src/api_tests/community.spec.ts

@@ -6,6 +6,7 @@ import {
   createCommunity,
   deleteCommunity,
   removeCommunity,
+  delay,
 } from './shared';
 
 beforeAll(async () => {
@@ -24,12 +25,14 @@ test('Create community', async () => {
 
 test('Delete community', async () => {
   let communityRes = await createCommunity(beta);
+  await delay();
   let deleteCommunityRes = await deleteCommunity(
     beta,
     true,
     communityRes.community.id
   );
   expect(deleteCommunityRes.community.deleted).toBe(true);
+  await delay();
 
   // Make sure it got deleted on A
   let search = await searchForBetaCommunity(alpha);
@@ -44,6 +47,7 @@ test('Delete community', async () => {
     communityRes.community.id
   );
   expect(undeleteCommunityRes.community.deleted).toBe(false);
+  await delay();
 
   // Make sure it got undeleted on A
   let search2 = await searchForBetaCommunity(alpha);
@@ -54,6 +58,7 @@ test('Delete community', async () => {
 
 test('Remove community', async () => {
   let communityRes = await createCommunity(beta);
+  await delay();
   let removeCommunityRes = await removeCommunity(
     beta,
     true,
@@ -66,6 +71,7 @@ test('Remove community', async () => {
   let communityA = search.communities[0];
   // TODO this fails currently, because no updates are pushed
   // expect(communityA.removed).toBe(true);
+  await delay();
 
   // unremove
   let unremoveCommunityRes = await removeCommunity(
@@ -74,6 +80,7 @@ test('Remove community', async () => {
     communityRes.community.id
   );
   expect(unremoveCommunityRes.community.removed).toBe(false);
+  await delay();
 
   // Make sure it got unremoved on A
   let search2 = await searchForBetaCommunity(alpha);

ui/src/api_tests/follow.spec.ts

@@ -5,6 +5,7 @@ import {
   followCommunity,
   checkFollowedCommunities,
   unfollowRemotes,
+  delay,
 } from './shared';
 
 beforeAll(async () => {
@@ -22,6 +23,7 @@ test('Follow federated community', async () => {
   // Make sure the follow response went through
   expect(follow.community.local).toBe(false);
   expect(follow.community.name).toBe('main');
+  await delay();
 
   // Check it from local
   let followCheck = await checkFollowedCommunities(alpha);
@@ -33,6 +35,7 @@ test('Follow federated community', async () => {
   // Test an unfollow
   let unfollow = await followCommunity(alpha, false, remoteCommunityId);
   expect(unfollow.community.local).toBe(false);
+  await delay();
 
   // Make sure you are unsubbed locally
   let unfollowCheck = await checkFollowedCommunities(alpha);

ui/src/api_tests/post.spec.ts

@@ -1,3 +1,4 @@
+jest.setTimeout(120000);
 import {
   alpha,
   beta,
@@ -18,6 +19,7 @@ import {
   removePost,
   getPost,
   unfollowRemotes,
+  delay,
 } from './shared';
 
 beforeAll(async () => {
@@ -26,6 +28,7 @@ beforeAll(async () => {
   await followBeta(gamma);
   await followBeta(delta);
   await followBeta(epsilon);
+  await delay(10000);
 });
 
 afterAll(async () => {
@@ -37,11 +40,13 @@ afterAll(async () => {
 
 test('Create a post', async () => {
   let search = await searchForBetaCommunity(alpha);
+  await delay();
   let postRes = await createPost(alpha, search.communities[0].id);
   expect(postRes.post).toBeDefined();
   expect(postRes.post.community_local).toBe(false);
   expect(postRes.post.creator_local).toBe(true);
   expect(postRes.post.score).toBe(1);
+  await delay();
 
   // Make sure that post is liked on beta
   let searchBeta = await searchPost(beta, postRes.post);
@@ -69,12 +74,15 @@ test('Create a post in a non-existent community', async () => {
 test('Unlike a post', async () => {
   let search = await searchForBetaCommunity(alpha);
   let postRes = await createPost(alpha, search.communities[0].id);
+  await delay();
   let unlike = await likePost(alpha, 0, postRes.post);
   expect(unlike.post.score).toBe(0);
+  await delay();
 
   // Try to unlike it again, make sure it stays at 0
   let unlike2 = await likePost(alpha, 0, postRes.post);
   expect(unlike2.post.score).toBe(0);
+  await delay();
 
   // Make sure that post is unliked on beta
   let searchBeta = await searchPost(beta, postRes.post);
@@ -89,12 +97,14 @@ test('Unlike a post', async () => {
 test('Update a post', async () => {
   let search = await searchForBetaCommunity(alpha);
   let postRes = await createPost(alpha, search.communities[0].id);
+  await delay();
 
   let updatedName = 'A jest test federated post, updated';
   let updatedPost = await updatePost(alpha, postRes.post);
   expect(updatedPost.post.name).toBe(updatedName);
   expect(updatedPost.post.community_local).toBe(false);
   expect(updatedPost.post.creator_local).toBe(true);
+  await delay();
 
   // Make sure that post is updated on beta
   let searchBeta = await searchPost(beta, postRes.post);
@@ -102,6 +112,7 @@ test('Update a post', async () => {
   expect(betaPost.community_local).toBe(true);
   expect(betaPost.creator_local).toBe(false);
   expect(betaPost.name).toBe(updatedName);
+  await delay();
 
   // Make sure lemmy beta cannot update the post
   let updatedPostBeta = await updatePost(beta, betaPost);
@@ -111,9 +122,11 @@ test('Update a post', async () => {
 test('Sticky a post', async () => {
   let search = await searchForBetaCommunity(alpha);
   let postRes = await createPost(alpha, search.communities[0].id);
+  await delay();
 
   let stickiedPostRes = await stickyPost(alpha, true, postRes.post);
   expect(stickiedPostRes.post.stickied).toBe(true);
+  await delay();
 
   // Make sure that post is stickied on beta
   let searchBeta = await searchPost(beta, postRes.post);
@@ -125,6 +138,7 @@ test('Sticky a post', async () => {
   // Unsticky a post
   let unstickiedPost = await stickyPost(alpha, false, postRes.post);
   expect(unstickiedPost.post.stickied).toBe(false);
+  await delay();
 
   // Make sure that post is unstickied on beta
   let searchBeta2 = await searchPost(beta, postRes.post);
@@ -137,6 +151,7 @@ test('Sticky a post', async () => {
   let searchGamma = await searchPost(gamma, postRes.post);
   let gammaPost = searchGamma.posts[0];
   let gammaTrySticky = await stickyPost(gamma, true, gammaPost);
+  await delay();
   let searchBeta3 = await searchPost(beta, postRes.post);
   let betaPost3 = searchBeta3.posts[0];
   expect(gammaTrySticky.post.stickied).toBe(true);
@@ -145,10 +160,13 @@ test('Sticky a post', async () => {
 
 test('Lock a post', async () => {
   let search = await searchForBetaCommunity(alpha);
+  await delay();
   let postRes = await createPost(alpha, search.communities[0].id);
+  await delay();
 
   let lockedPostRes = await lockPost(alpha, true, postRes.post);
   expect(lockedPostRes.post.locked).toBe(true);
+  await delay();
 
   // Make sure that post is locked on beta
   let searchBeta = await searchPost(beta, postRes.post);
@@ -160,14 +178,17 @@ test('Lock a post', async () => {
   // Try to make a new comment there, on alpha
   let comment = await createComment(alpha, postRes.post.id);
   expect(comment['error']).toBe('locked');
+  await delay();
 
   // Try to create a new comment, on beta
   let commentBeta = await createComment(beta, betaPost.id);
   expect(commentBeta['error']).toBe('locked');
+  await delay();
 
   // Unlock a post
   let unlockedPost = await lockPost(alpha, false, postRes.post);
   expect(unlockedPost.post.locked).toBe(false);
+  await delay();
 
   // Make sure that post is unlocked on beta
   let searchBeta2 = await searchPost(beta, postRes.post);
@@ -180,68 +201,84 @@ test('Lock a post', async () => {
 test('Delete a post', async () => {
   let search = await searchForBetaCommunity(alpha);
   let postRes = await createPost(alpha, search.communities[0].id);
+  await delay();
 
   let deletedPost = await deletePost(alpha, true, postRes.post);
   expect(deletedPost.post.deleted).toBe(true);
+  await delay();
 
   // Make sure lemmy beta sees post is deleted
-  let createFakeBetaPostToGetId = (await createPost(beta, 2)).post.id - 1;
+  let searchBeta = await searchPost(beta, postRes.post);
-  let betaPost = await getPost(beta, createFakeBetaPostToGetId);
+  let betaPost = searchBeta.posts[0];
-  expect(betaPost.post.deleted).toBe(true);
+  // This will be undefined because of the tombstone
+  expect(betaPost).toBeUndefined();
+  await delay();
 
   // Undelete
   let undeletedPost = await deletePost(alpha, false, postRes.post);
   expect(undeletedPost.post.deleted).toBe(false);
+  await delay();
 
   // Make sure lemmy beta sees post is undeleted
-  let betaPost2 = await getPost(beta, createFakeBetaPostToGetId);
+  let searchBeta2 = await searchPost(beta, postRes.post);
-  expect(betaPost2.post.deleted).toBe(false);
+  let betaPost2 = searchBeta2.posts[0];
+  expect(betaPost2.deleted).toBe(false);
 
   // Make sure lemmy beta cannot delete the post
-  let deletedPostBeta = await deletePost(beta, true, betaPost2.post);
+  let deletedPostBeta = await deletePost(beta, true, betaPost2);
   expect(deletedPostBeta).toStrictEqual({ error: 'no_post_edit_allowed' });
 });
 
 test('Remove a post from admin and community on different instance', async () => {
   let search = await searchForBetaCommunity(alpha);
   let postRes = await createPost(alpha, search.communities[0].id);
+  await delay();
 
   let removedPost = await removePost(alpha, true, postRes.post);
   expect(removedPost.post.removed).toBe(true);
+  await delay();
 
   // Make sure lemmy beta sees post is NOT removed
-  let createFakeBetaPostToGetId = (await createPost(beta, 2)).post.id - 1;
+  let searchBeta = await searchPost(beta, postRes.post);
-  let betaPost = await getPost(beta, createFakeBetaPostToGetId);
+  let betaPost = searchBeta.posts[0];
-  expect(betaPost.post.removed).toBe(false);
+  expect(betaPost.removed).toBe(false);
+  await delay();
 
   // Undelete
   let undeletedPost = await removePost(alpha, false, postRes.post);
   expect(undeletedPost.post.removed).toBe(false);
+  await delay();
 
   // Make sure lemmy beta sees post is undeleted
-  let betaPost2 = await getPost(beta, createFakeBetaPostToGetId);
+  let searchBeta2 = await searchPost(beta, postRes.post);
-  expect(betaPost2.post.removed).toBe(false);
+  let betaPost2 = searchBeta2.posts[0];
+  expect(betaPost2.removed).toBe(false);
 });
 
 test('Remove a post from admin and community on same instance', async () => {
   let search = await searchForBetaCommunity(alpha);
   let postRes = await createPost(alpha, search.communities[0].id);
+  await delay();
 
   // Get the id for beta
-  let createFakeBetaPostToGetId = (await createPost(beta, 2)).post.id - 1;
+  let searchBeta = await searchPost(beta, postRes.post);
-  let betaPost = await getPost(beta, createFakeBetaPostToGetId);
+  let betaPost = searchBeta.posts[0];
+  await delay();
 
   // The beta admin removes it (the community lives on beta)
-  let removePostRes = await removePost(beta, true, betaPost.post);
+  let removePostRes = await removePost(beta, true, betaPost);
   expect(removePostRes.post.removed).toBe(true);
+  await delay();
 
   // Make sure lemmy alpha sees post is removed
   let alphaPost = await getPost(alpha, postRes.post.id);
   expect(alphaPost.post.removed).toBe(true);
+  await delay();
 
   // Undelete
-  let undeletedPost = await removePost(beta, false, betaPost.post);
+  let undeletedPost = await removePost(beta, false, betaPost);
   expect(undeletedPost.post.removed).toBe(false);
+  await delay();
 
   // Make sure lemmy alpha sees post is undeleted
   let alphaPost2 = await getPost(alpha, postRes.post.id);
@@ -250,7 +287,9 @@ test('Remove a post from admin and community on same instance', async () => {
 
 test('Search for a post', async () => {
   let search = await searchForBetaCommunity(alpha);
+  await delay();
   let postRes = await createPost(alpha, search.communities[0].id);
+  await delay();
   let searchBeta = await searchPost(beta, postRes.post);
 
   expect(searchBeta.posts[0].name).toBeDefined();
@@ -259,6 +298,7 @@ test('Search for a post', async () => {
 test('A and G subscribe to B (center) A posts, it gets announced to G', async () => {
   let search = await searchForBetaCommunity(alpha);
   let postRes = await createPost(alpha, search.communities[0].id);
+  await delay();
 
   let search2 = await searchPost(gamma, postRes.post);
   expect(search2.posts[0].name).toBeDefined();

ui/src/api_tests/private_message.spec.ts

@@ -8,13 +8,16 @@ import {
   listPrivateMessages,
   deletePrivateMessage,
   unfollowRemotes,
+  delay,
 } from './shared';
 
 let recipient_id: number;
 
 beforeAll(async () => {
   await setupLogins();
-  recipient_id = (await followBeta(alpha)).community.creator_id;
+  let follow = await followBeta(alpha);
+  await delay(10000);
+  recipient_id = follow.community.creator_id;
 });
 
 afterAll(async () => {
@@ -27,6 +30,7 @@ test('Create a private message', async () => {
   expect(pmRes.message.local).toBe(true);
   expect(pmRes.message.creator_local).toBe(true);
   expect(pmRes.message.recipient_local).toBe(false);
+  await delay();
 
   let betaPms = await listPrivateMessages(beta);
   expect(betaPms.messages[0].content).toBeDefined();
@@ -41,6 +45,7 @@ test('Update a private message', async () => {
   let pmRes = await createPrivateMessage(alpha, recipient_id);
   let pmUpdated = await updatePrivateMessage(alpha, pmRes.message.id);
   expect(pmUpdated.message.content).toBe(updatedContent);
+  await delay();
 
   let betaPms = await listPrivateMessages(beta);
   expect(betaPms.messages[0].content).toBe(updatedContent);
@@ -48,15 +53,18 @@ test('Update a private message', async () => {
 
 test('Delete a private message', async () => {
   let pmRes = await createPrivateMessage(alpha, recipient_id);
+  await delay();
   let betaPms1 = await listPrivateMessages(beta);
   let deletedPmRes = await deletePrivateMessage(alpha, true, pmRes.message.id);
   expect(deletedPmRes.message.deleted).toBe(true);
+  await delay();
 
   // The GetPrivateMessages filters out deleted,
   // even though they are in the actual database.
   // no reason to show them
   let betaPms2 = await listPrivateMessages(beta);
   expect(betaPms2.messages.length).toBe(betaPms1.messages.length - 1);
+  await delay();
 
   // Undelete
   let undeletedPmRes = await deletePrivateMessage(
@@ -65,6 +73,7 @@ test('Delete a private message', async () => {
     pmRes.message.id
   );
   expect(undeletedPmRes.message.deleted).toBe(false);
+  await delay();
 
   let betaPms3 = await listPrivateMessages(beta);
   expect(betaPms3.messages.length).toBe(betaPms1.messages.length);

ui/src/api_tests/shared.ts

@@ -198,7 +198,7 @@ export async function searchPost(
 ): Promise<SearchResponse> {
   let form: SearchForm = {
     q: post.ap_id,
-    type_: SearchType.All,
+    type_: SearchType.Posts,
     sort: SortType.TopAll,
   };
   return api.client.search(form);
@@ -220,7 +220,7 @@ export async function searchComment(
 ): Promise<SearchResponse> {
   let form: SearchForm = {
     q: comment.ap_id,
-    type_: SearchType.All,
+    type_: SearchType.Comments,
     sort: SortType.TopAll,
   };
   return api.client.search(form);
@@ -233,7 +233,7 @@ export async function searchForBetaCommunity(
   // Use short-hand search url
   let form: SearchForm = {
     q: '!main@lemmy-beta:8550',
-    type_: SearchType.All,
+    type_: SearchType.Communities,
     sort: SortType.TopAll,
   };
   return api.client.search(form);
@@ -247,7 +247,7 @@ export async function searchForUser(
   // Use short-hand search url
   let form: SearchForm = {
     q: apShortname,
-    type_: SearchType.All,
+    type_: SearchType.Users,
     sort: SortType.TopAll,
   };
   return api.client.search(form);
@@ -524,6 +524,11 @@ export async function followBeta(api: API): Promise<CommunityResponse> {
   }
 }
 
+export const delay = (millis: number = 1500) =>
+  new Promise((resolve, _reject) => {
+    setTimeout(_ => resolve(), millis);
+  });
+
 export function wrapper(form: any): string {
   return JSON.stringify(form);
 }