Some fixes to federation.

- Advanced code migrations now disable then re-enable triggers.
  Brings run time down to < 15 seconds, no need to thread them.
- Changing ap_ids and actor_ids in migrations to a fake url,
  so it doesn't break XsdAnyUri in activitystreams.
This commit is contained in:
Dessalines 2020-06-26 21:12:41 -04:00
parent 7d1c6e9a40
commit 86dc50f9f0
20 changed files with 81 additions and 73 deletions

View File

@ -18,7 +18,6 @@ services:
- lemmy_gamma
- pictrs_gamma
- iframely
restart: "always"
lemmy_alpha:
image: lemmy-federation:latest
@ -36,7 +35,6 @@ services:
- LEMMY_SETUP__SITE_NAME=lemmy_alpha
- RUST_BACKTRACE=1
- RUST_LOG=debug
restart: always
depends_on:
- postgres_alpha
postgres_alpha:
@ -47,13 +45,11 @@ services:
- POSTGRES_DB=lemmy
volumes:
- ./volumes/postgres_alpha:/var/lib/postgresql/data
restart: always
pictrs_alpha:
image: asonix/pictrs:v0.1.13-r0
user: 991:991
volumes:
- ./volumes/pictrs_alpha:/mnt
restart: always
lemmy_beta:
image: lemmy-federation:latest
@ -71,7 +67,6 @@ services:
- LEMMY_SETUP__SITE_NAME=lemmy_beta
- RUST_BACKTRACE=1
- RUST_LOG=debug
restart: always
depends_on:
- postgres_beta
postgres_beta:
@ -82,13 +77,11 @@ services:
- POSTGRES_DB=lemmy
volumes:
- ./volumes/postgres_beta:/var/lib/postgresql/data
restart: always
pictrs_beta:
image: asonix/pictrs:v0.1.13-r0
user: 991:991
volumes:
- ./volumes/pictrs_beta:/mnt
restart: always
lemmy_gamma:
image: lemmy-federation:latest
@ -106,7 +99,6 @@ services:
- LEMMY_SETUP__SITE_NAME=lemmy_gamma
- RUST_BACKTRACE=1
- RUST_LOG=debug
restart: always
depends_on:
- postgres_gamma
postgres_gamma:
@ -117,16 +109,13 @@ services:
- POSTGRES_DB=lemmy
volumes:
- ./volumes/postgres_gamma:/var/lib/postgresql/data
restart: always
pictrs_gamma:
image: asonix/pictrs:v0.1.13-r0
user: 991:991
volumes:
- ./volumes/pictrs_gamma:/mnt
restart: always
iframely:
image: dogbin/iframely:latest
volumes:
- ../iframely.config.local.js:/iframely/config.local.js:ro
restart: always

View File

@ -15,7 +15,7 @@ create unique index idx_activity_unique_apid on activity ((data ->> 'id'::text))
-- Add federation columns to the two actor tables
alter table user_
-- TODO uniqueness constraints should be added on these 3 columns later
add column actor_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
add column actor_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
add column bio text, -- not on community, already has description
add column local boolean not null default true,
add column private_key text, -- These need to be generated from code
@ -25,7 +25,7 @@ add column last_refreshed_at timestamp not null default now() -- Used to re-fetc
-- Community
alter table community
add column actor_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
add column actor_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
add column local boolean not null default true,
add column private_key text, -- These need to be generated from code
add column public_key text,

View File

@ -2,13 +2,13 @@
alter table post
-- TODO uniqueness constraints should be added on these 3 columns later
add column ap_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
add column ap_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
add column local boolean not null default true
;
alter table comment
-- TODO uniqueness constraints should be added on these 3 columns later
add column ap_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
add column ap_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
add column local boolean not null default true
;

View File

@ -2,7 +2,7 @@
drop view user_view cascade;
alter table user_
add column fedi_name varchar(40) not null default 'changeme';
add column fedi_name varchar(40) not null default 'http://fake.com';
alter table user_
add constraint user__name_fedi_name_key unique (name, fedi_name);

View File

@ -1,5 +1,5 @@
alter table private_message
add column ap_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
add column ap_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local
add column local boolean not null default true
;

View File

@ -140,7 +140,7 @@ impl Perform for Oper<CreateComment> {
read: None,
published: None,
updated: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};

View File

@ -172,7 +172,7 @@ impl Perform for Oper<CreatePost> {
embed_description: iframely_description,
embed_html: iframely_html,
thumbnail_url: pictrs_thumbnail,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
published: None,
};

View File

@ -1107,7 +1107,7 @@ impl Perform for Oper<CreatePrivateMessage> {
deleted: None,
read: None,
updated: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
published: None,
};

View File

@ -101,7 +101,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,

View File

@ -16,12 +16,11 @@ use failure::Error;
use log::info;
pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), Error> {
user_updates_2020_04_02(conn)?;
community_updates_2020_04_02(conn)?;
post_updates_2020_04_03(conn)?;
comment_updates_2020_04_03(conn)?;
private_message_updates_2020_05_05(conn)?;
user_updates_2020_04_02(&conn)?;
community_updates_2020_04_02(&conn)?;
post_updates_2020_04_03(&conn)?;
comment_updates_2020_04_03(&conn)?;
private_message_updates_2020_05_05(&conn)?;
Ok(())
}
@ -32,10 +31,12 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
// Update the actor_id, private_key, and public_key, last_refreshed_at
let incorrect_users = user_
.filter(actor_id.eq("changeme"))
.filter(actor_id.eq("http://fake.com"))
.filter(local.eq(true))
.load::<User_>(conn)?;
sql_query("alter table user_ disable trigger refresh_user").execute(conn)?;
for cuser in &incorrect_users {
let keypair = generate_actor_keypair()?;
@ -67,6 +68,8 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
User_::update(&conn, cuser.id, &form)?;
}
sql_query("alter table user_ enable trigger refresh_user").execute(conn)?;
info!("{} user rows updated.", incorrect_users.len());
Ok(())
@ -79,10 +82,12 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
// Update the actor_id, private_key, and public_key, last_refreshed_at
let incorrect_communities = community
.filter(actor_id.eq("changeme"))
.filter(actor_id.eq("http://fake.com"))
.filter(local.eq(true))
.load::<Community>(conn)?;
sql_query("alter table community disable trigger refresh_community").execute(conn)?;
for ccommunity in &incorrect_communities {
let keypair = generate_actor_keypair()?;
@ -107,6 +112,8 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
Community::update(&conn, ccommunity.id, &form)?;
}
sql_query("alter table community enable trigger refresh_community").execute(conn)?;
info!("{} community rows updated.", incorrect_communities.len());
Ok(())
@ -119,16 +126,20 @@ fn post_updates_2020_04_03(conn: &PgConnection) -> Result<(), Error> {
// Update the ap_id
let incorrect_posts = post
.filter(ap_id.eq("changeme"))
.filter(ap_id.eq("http://fake.com"))
.filter(local.eq(true))
.load::<Post>(conn)?;
sql_query("alter table post disable trigger refresh_post").execute(conn)?;
for cpost in &incorrect_posts {
Post::update_ap_id(&conn, cpost.id)?;
}
info!("{} post rows updated.", incorrect_posts.len());
sql_query("alter table post enable trigger refresh_post").execute(conn)?;
Ok(())
}
@ -139,14 +150,18 @@ fn comment_updates_2020_04_03(conn: &PgConnection) -> Result<(), Error> {
// Update the ap_id
let incorrect_comments = comment
.filter(ap_id.eq("changeme"))
.filter(ap_id.eq("http://fake.com"))
.filter(local.eq(true))
.load::<Comment>(conn)?;
sql_query("alter table comment disable trigger refresh_comment").execute(conn)?;
for ccomment in &incorrect_comments {
Comment::update_ap_id(&conn, ccomment.id)?;
}
sql_query("alter table comment enable trigger refresh_comment").execute(conn)?;
info!("{} comment rows updated.", incorrect_comments.len());
Ok(())
@ -159,14 +174,18 @@ fn private_message_updates_2020_05_05(conn: &PgConnection) -> Result<(), Error>
// Update the ap_id
let incorrect_pms = private_message
.filter(ap_id.eq("changeme"))
.filter(ap_id.eq("http://fake.com"))
.filter(local.eq(true))
.load::<PrivateMessage>(conn)?;
sql_query("alter table private_message disable trigger refresh_private_message").execute(conn)?;
for cpm in &incorrect_pms {
PrivateMessage::update_ap_id(&conn, cpm.id)?;
}
sql_query("alter table private_message enable trigger refresh_private_message").execute(conn)?;
info!("{} private message rows updated.", incorrect_pms.len());
Ok(())

View File

@ -229,7 +229,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -249,7 +249,7 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
local: true,
private_key: None,
public_key: None,
@ -275,7 +275,7 @@ mod tests {
embed_description: None,
embed_html: None,
thumbnail_url: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
published: None,
};
@ -292,7 +292,7 @@ mod tests {
parent_id: None,
published: None,
updated: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};
@ -309,7 +309,7 @@ mod tests {
parent_id: None,
published: inserted_comment.published,
updated: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};
@ -323,7 +323,7 @@ mod tests {
read: None,
published: None,
updated: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};

View File

@ -481,7 +481,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -501,7 +501,7 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
local: true,
private_key: None,
public_key: None,
@ -527,7 +527,7 @@ mod tests {
embed_description: None,
embed_html: None,
thumbnail_url: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
published: None,
};
@ -544,7 +544,7 @@ mod tests {
read: None,
published: None,
updated: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};
@ -584,7 +584,7 @@ mod tests {
my_vote: None,
subscribed: None,
saved: None,
ap_id: "changeme".to_string(),
ap_id: "http://fake.com".to_string(),
local: true,
community_actor_id: inserted_community.actor_id.to_owned(),
community_local: true,
@ -617,7 +617,7 @@ mod tests {
my_vote: Some(1),
subscribed: None,
saved: None,
ap_id: "changeme".to_string(),
ap_id: "http://fake.com".to_string(),
local: true,
community_actor_id: inserted_community.actor_id.to_owned(),
community_local: true,

View File

@ -256,7 +256,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -276,7 +276,7 @@ mod tests {
removed: None,
deleted: None,
updated: None,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
local: true,
private_key: None,
public_key: None,
@ -298,7 +298,7 @@ mod tests {
deleted: false,
published: inserted_community.published,
updated: None,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
local: true,
private_key: None,
public_key: None,

View File

@ -465,7 +465,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -492,7 +492,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -512,7 +512,7 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
local: true,
private_key: None,
public_key: None,
@ -538,7 +538,7 @@ mod tests {
embed_description: None,
embed_html: None,
thumbnail_url: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
published: None,
};
@ -555,7 +555,7 @@ mod tests {
parent_id: None,
published: None,
updated: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};

View File

@ -105,7 +105,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,

View File

@ -268,7 +268,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -288,7 +288,7 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
local: true,
private_key: None,
public_key: None,
@ -314,7 +314,7 @@ mod tests {
embed_description: None,
embed_html: None,
thumbnail_url: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
published: None,
};
@ -339,7 +339,7 @@ mod tests {
embed_description: None,
embed_html: None,
thumbnail_url: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};

View File

@ -395,7 +395,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -415,7 +415,7 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
local: true,
private_key: None,
public_key: None,
@ -441,7 +441,7 @@ mod tests {
embed_description: None,
embed_html: None,
thumbnail_url: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
published: None,
};
@ -508,7 +508,7 @@ mod tests {
embed_description: None,
embed_html: None,
thumbnail_url: None,
ap_id: "changeme".to_string(),
ap_id: "http://fake.com".to_string(),
local: true,
creator_actor_id: inserted_user.actor_id.to_owned(),
creator_local: true,
@ -553,7 +553,7 @@ mod tests {
embed_description: None,
embed_html: None,
thumbnail_url: None,
ap_id: "changeme".to_string(),
ap_id: "http://fake.com".to_string(),
local: true,
creator_actor_id: inserted_user.actor_id.to_owned(),
creator_local: true,

View File

@ -113,7 +113,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -140,7 +140,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -158,7 +158,7 @@ mod tests {
read: None,
published: None,
updated: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};
@ -173,7 +173,7 @@ mod tests {
read: false,
updated: None,
published: inserted_private_message.published,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};

View File

@ -237,7 +237,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -266,7 +266,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,

View File

@ -81,7 +81,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -108,7 +108,7 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
bio: None,
local: true,
private_key: None,
@ -128,7 +128,7 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
actor_id: "http://fake.com".into(),
local: true,
private_key: None,
public_key: None,
@ -154,7 +154,7 @@ mod tests {
embed_description: None,
embed_html: None,
thumbnail_url: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
published: None,
};
@ -171,7 +171,7 @@ mod tests {
parent_id: None,
published: None,
updated: None,
ap_id: "changeme".into(),
ap_id: "http://fake.com".into(),
local: true,
};