Some fixes to federation.

- Advanced code migrations now disable then re-enable triggers.
  Brings run time down to < 15 seconds, no need to thread them.
- Changing ap_ids and actor_ids in migrations to a fake url,
  so it doesn't break XsdAnyUri in activitystreams.

docker/federation/docker-compose.yml

@@ -18,7 +18,6 @@ services:
       - lemmy_gamma
       - pictrs_gamma
       - iframely
-    restart: "always"
 
   lemmy_alpha:
     image: lemmy-federation:latest
@@ -36,7 +35,6 @@ services:
       - LEMMY_SETUP__SITE_NAME=lemmy_alpha
       - RUST_BACKTRACE=1
       - RUST_LOG=debug
-    restart: always
     depends_on:
       - postgres_alpha
   postgres_alpha:
@@ -47,13 +45,11 @@ services:
       - POSTGRES_DB=lemmy
     volumes:
       - ./volumes/postgres_alpha:/var/lib/postgresql/data
-    restart: always
   pictrs_alpha:
     image: asonix/pictrs:v0.1.13-r0
     user: 991:991
     volumes:
       - ./volumes/pictrs_alpha:/mnt
-    restart: always
 
   lemmy_beta:
     image: lemmy-federation:latest
@@ -71,7 +67,6 @@ services:
       - LEMMY_SETUP__SITE_NAME=lemmy_beta
       - RUST_BACKTRACE=1
       - RUST_LOG=debug
-    restart: always
     depends_on:
       - postgres_beta
   postgres_beta:
@@ -82,13 +77,11 @@ services:
       - POSTGRES_DB=lemmy
     volumes:
       - ./volumes/postgres_beta:/var/lib/postgresql/data
-    restart: always
   pictrs_beta:
     image: asonix/pictrs:v0.1.13-r0
     user: 991:991
     volumes:
       - ./volumes/pictrs_beta:/mnt
-    restart: always
 
   lemmy_gamma:
     image: lemmy-federation:latest
@@ -106,7 +99,6 @@ services:
       - LEMMY_SETUP__SITE_NAME=lemmy_gamma
       - RUST_BACKTRACE=1
       - RUST_LOG=debug
-    restart: always
     depends_on:
       - postgres_gamma
   postgres_gamma:
@@ -117,16 +109,13 @@ services:
       - POSTGRES_DB=lemmy
     volumes:
       - ./volumes/postgres_gamma:/var/lib/postgresql/data
-    restart: always
   pictrs_gamma:
     image: asonix/pictrs:v0.1.13-r0
     user: 991:991
     volumes:
       - ./volumes/pictrs_gamma:/mnt
-    restart: always
 
   iframely:
     image: dogbin/iframely:latest
     volumes:
       - ../iframely.config.local.js:/iframely/config.local.js:ro
-    restart: always

server/migrations/2020-03-26-192410_add_activitypub_tables/up.sql

@@ -15,7 +15,7 @@ create unique index idx_activity_unique_apid on activity ((data ->> 'id'::text))
 -- Add federation columns to the two actor tables
 alter table user_ 
 -- TODO uniqueness constraints should be added on these 3 columns later
-add column actor_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
+add column actor_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
 add column bio text, -- not on community, already has description
 add column local boolean not null default true,
 add column private_key text, -- These need to be generated from code
@@ -25,7 +25,7 @@ add column last_refreshed_at timestamp not null default now() -- Used to re-fetc
 
 -- Community
 alter table community 
-add column actor_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
+add column actor_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
 add column local boolean not null default true,
 add column private_key text, -- These need to be generated from code
 add column public_key text,

server/migrations/2020-04-03-194936_add_activitypub_for_posts_and_comments/up.sql

@@ -2,13 +2,13 @@
 
 alter table post
 -- TODO uniqueness constraints should be added on these 3 columns later
-add column ap_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
+add column ap_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
 add column local boolean not null default true
 ;
 
 alter table comment
 -- TODO uniqueness constraints should be added on these 3 columns later
-add column ap_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
+add column ap_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
 add column local boolean not null default true
 ;
 

server/migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql

@@ -2,7 +2,7 @@
 drop view user_view cascade;
 
 alter table user_ 
-add column fedi_name varchar(40) not null default 'changeme';
+add column fedi_name varchar(40) not null default 'http://fake.com';
 
 alter table user_
 add constraint user__name_fedi_name_key unique (name, fedi_name);

server/migrations/2020-05-05-210233_add_activitypub_for_private_messages/up.sql

@@ -1,5 +1,5 @@
 alter table private_message
-add column ap_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
+add column ap_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
 add column local boolean not null default true
 ;
 

server/src/api/comment.rs

@@ -140,7 +140,7 @@ impl Perform for Oper<CreateComment> {
       read: None,
       published: None,
       updated: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };
 

server/src/api/post.rs

@@ -172,7 +172,7 @@ impl Perform for Oper<CreatePost> {
       embed_description: iframely_description,
       embed_html: iframely_html,
       thumbnail_url: pictrs_thumbnail,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
       published: None,
     };

server/src/api/user.rs

@@ -1107,7 +1107,7 @@ impl Perform for Oper<CreatePrivateMessage> {
       deleted: None,
       read: None,
       updated: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
       published: None,
     };

server/src/db/activity.rs

@@ -101,7 +101,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,

server/src/db/code_migrations.rs

@@ -16,12 +16,11 @@ use failure::Error;
 use log::info;
 
 pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), Error> {
-  user_updates_2020_04_02(conn)?;
+  user_updates_2020_04_02(&conn)?;
-  community_updates_2020_04_02(conn)?;
+  community_updates_2020_04_02(&conn)?;
-  post_updates_2020_04_03(conn)?;
+  post_updates_2020_04_03(&conn)?;
-  comment_updates_2020_04_03(conn)?;
+  comment_updates_2020_04_03(&conn)?;
-  private_message_updates_2020_05_05(conn)?;
+  private_message_updates_2020_05_05(&conn)?;
-
   Ok(())
 }
 
@@ -32,10 +31,12 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
 
   // Update the actor_id, private_key, and public_key, last_refreshed_at
   let incorrect_users = user_
-    .filter(actor_id.eq("changeme"))
+    .filter(actor_id.eq("http://fake.com"))
     .filter(local.eq(true))
     .load::<User_>(conn)?;
 
+  sql_query("alter table user_ disable trigger refresh_user").execute(conn)?;
+
   for cuser in &incorrect_users {
     let keypair = generate_actor_keypair()?;
 
@@ -67,6 +68,8 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
     User_::update(&conn, cuser.id, &form)?;
   }
 
+  sql_query("alter table user_ enable trigger refresh_user").execute(conn)?;
+
   info!("{} user rows updated.", incorrect_users.len());
 
   Ok(())
@@ -79,10 +82,12 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
 
   // Update the actor_id, private_key, and public_key, last_refreshed_at
   let incorrect_communities = community
-    .filter(actor_id.eq("changeme"))
+    .filter(actor_id.eq("http://fake.com"))
     .filter(local.eq(true))
     .load::<Community>(conn)?;
 
+  sql_query("alter table community disable trigger refresh_community").execute(conn)?;
+
   for ccommunity in &incorrect_communities {
     let keypair = generate_actor_keypair()?;
 
@@ -107,6 +112,8 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
     Community::update(&conn, ccommunity.id, &form)?;
   }
 
+  sql_query("alter table community enable trigger refresh_community").execute(conn)?;
+
   info!("{} community rows updated.", incorrect_communities.len());
 
   Ok(())
@@ -119,16 +126,20 @@ fn post_updates_2020_04_03(conn: &PgConnection) -> Result<(), Error> {
 
   // Update the ap_id
   let incorrect_posts = post
-    .filter(ap_id.eq("changeme"))
+    .filter(ap_id.eq("http://fake.com"))
     .filter(local.eq(true))
     .load::<Post>(conn)?;
 
+  sql_query("alter table post disable trigger refresh_post").execute(conn)?;
+
   for cpost in &incorrect_posts {
     Post::update_ap_id(&conn, cpost.id)?;
   }
 
   info!("{} post rows updated.", incorrect_posts.len());
 
+  sql_query("alter table post enable trigger refresh_post").execute(conn)?;
+
   Ok(())
 }
 
@@ -139,14 +150,18 @@ fn comment_updates_2020_04_03(conn: &PgConnection) -> Result<(), Error> {
 
   // Update the ap_id
   let incorrect_comments = comment
-    .filter(ap_id.eq("changeme"))
+    .filter(ap_id.eq("http://fake.com"))
     .filter(local.eq(true))
     .load::<Comment>(conn)?;
 
+  sql_query("alter table comment disable trigger refresh_comment").execute(conn)?;
+
   for ccomment in &incorrect_comments {
     Comment::update_ap_id(&conn, ccomment.id)?;
   }
 
+  sql_query("alter table comment enable trigger refresh_comment").execute(conn)?;
+
   info!("{} comment rows updated.", incorrect_comments.len());
 
   Ok(())
@@ -159,14 +174,18 @@ fn private_message_updates_2020_05_05(conn: &PgConnection) -> Result<(), Error>
 
   // Update the ap_id
   let incorrect_pms = private_message
-    .filter(ap_id.eq("changeme"))
+    .filter(ap_id.eq("http://fake.com"))
     .filter(local.eq(true))
     .load::<PrivateMessage>(conn)?;
 
+  sql_query("alter table private_message disable trigger refresh_private_message").execute(conn)?;
+
   for cpm in &incorrect_pms {
     PrivateMessage::update_ap_id(&conn, cpm.id)?;
   }
 
+  sql_query("alter table private_message enable trigger refresh_private_message").execute(conn)?;
+
   info!("{} private message rows updated.", incorrect_pms.len());
 
   Ok(())

server/src/db/comment.rs

@@ -229,7 +229,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -249,7 +249,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       local: true,
       private_key: None,
       public_key: None,
@@ -275,7 +275,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
       published: None,
     };
@@ -292,7 +292,7 @@ mod tests {
       parent_id: None,
       published: None,
       updated: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };
 
@@ -309,7 +309,7 @@ mod tests {
       parent_id: None,
       published: inserted_comment.published,
       updated: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };
 
@@ -323,7 +323,7 @@ mod tests {
       read: None,
       published: None,
       updated: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };
 

server/src/db/comment_view.rs

@@ -481,7 +481,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -501,7 +501,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       local: true,
       private_key: None,
       public_key: None,
@@ -527,7 +527,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
       published: None,
     };
@@ -544,7 +544,7 @@ mod tests {
       read: None,
       published: None,
       updated: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };
 
@@ -584,7 +584,7 @@ mod tests {
       my_vote: None,
       subscribed: None,
       saved: None,
-      ap_id: "changeme".to_string(),
+      ap_id: "http://fake.com".to_string(),
       local: true,
       community_actor_id: inserted_community.actor_id.to_owned(),
       community_local: true,
@@ -617,7 +617,7 @@ mod tests {
       my_vote: Some(1),
       subscribed: None,
       saved: None,
-      ap_id: "changeme".to_string(),
+      ap_id: "http://fake.com".to_string(),
       local: true,
       community_actor_id: inserted_community.actor_id.to_owned(),
       community_local: true,

server/src/db/community.rs

@@ -256,7 +256,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -276,7 +276,7 @@ mod tests {
       removed: None,
       deleted: None,
       updated: None,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       local: true,
       private_key: None,
       public_key: None,
@@ -298,7 +298,7 @@ mod tests {
       deleted: false,
       published: inserted_community.published,
       updated: None,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       local: true,
       private_key: None,
       public_key: None,

server/src/db/moderator.rs

@@ -465,7 +465,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -492,7 +492,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -512,7 +512,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       local: true,
       private_key: None,
       public_key: None,
@@ -538,7 +538,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
       published: None,
     };
@@ -555,7 +555,7 @@ mod tests {
       parent_id: None,
       published: None,
       updated: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };
 

server/src/db/password_reset_request.rs

@@ -105,7 +105,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,

server/src/db/post.rs

@@ -268,7 +268,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -288,7 +288,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       local: true,
       private_key: None,
       public_key: None,
@@ -314,7 +314,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
       published: None,
     };
@@ -339,7 +339,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };
 

server/src/db/post_view.rs

@@ -395,7 +395,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -415,7 +415,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       local: true,
       private_key: None,
       public_key: None,
@@ -441,7 +441,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
       published: None,
     };
@@ -508,7 +508,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "changeme".to_string(),
+      ap_id: "http://fake.com".to_string(),
       local: true,
       creator_actor_id: inserted_user.actor_id.to_owned(),
       creator_local: true,
@@ -553,7 +553,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "changeme".to_string(),
+      ap_id: "http://fake.com".to_string(),
       local: true,
       creator_actor_id: inserted_user.actor_id.to_owned(),
       creator_local: true,

server/src/db/private_message.rs

@@ -113,7 +113,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -140,7 +140,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -158,7 +158,7 @@ mod tests {
       read: None,
       published: None,
       updated: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };
 
@@ -173,7 +173,7 @@ mod tests {
       read: false,
       updated: None,
       published: inserted_private_message.published,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };
 

server/src/db/user.rs

@@ -237,7 +237,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -266,7 +266,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,

server/src/db/user_mention.rs

@@ -81,7 +81,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -108,7 +108,7 @@ mod tests {
       lang: "browser".into(),
       show_avatars: true,
       send_notifications_to_email: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       bio: None,
       local: true,
       private_key: None,
@@ -128,7 +128,7 @@ mod tests {
       deleted: None,
       updated: None,
       nsfw: false,
-      actor_id: "changeme".into(),
+      actor_id: "http://fake.com".into(),
       local: true,
       private_key: None,
       public_key: None,
@@ -154,7 +154,7 @@ mod tests {
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
       published: None,
     };
@@ -171,7 +171,7 @@ mod tests {
       parent_id: None,
       published: None,
       updated: None,
-      ap_id: "changeme".into(),
+      ap_id: "http://fake.com".into(),
       local: true,
     };