Federation DB Changes.

- Creating an activity table.
- Adding some federation-related columns to the user_ and community
  tables.
- Generating the actor_id and keys in code, updating the tables.
This commit is contained in:
Dessalines 2020-04-03 00:12:05 -04:00
parent 32b0275257
commit 9197b39ed6
22 changed files with 643 additions and 235 deletions

1
server/Cargo.lock generated vendored
View file

@ -1545,6 +1545,7 @@ dependencies = [
"lettre",
"lettre_email",
"log",
"openssl",
"percent-encoding",
"rand 0.7.3",
"regex 1.3.6",

1
server/Cargo.toml vendored
View file

@ -38,3 +38,4 @@ url = "2.1.1"
percent-encoding = "2.1.0"
isahc = "0.9"
comrak = "0.7"
openssl = "0.10"

View file

@ -0,0 +1,16 @@
drop table activity;
alter table user_
drop column actor_id,
drop column private_key,
drop column public_key,
drop column bio,
drop column local,
drop column last_refreshed_at;
alter table community
drop column actor_id,
drop column private_key,
drop column public_key,
drop column local,
drop column last_refreshed_at;

View file

@ -0,0 +1,36 @@
-- The Activitypub activity table
-- All user actions must create a row here.
create table activity (
id serial primary key,
user_id int references user_ on update cascade on delete cascade not null, -- Ensures that the user is set up here.
data jsonb not null,
local boolean not null default true,
published timestamp not null default now(),
updated timestamp
);
-- Making sure that id is unique
create unique index idx_activity_unique_apid on activity ((data ->> 'id'::text));
-- Add federation columns to the two actor tables
alter table user_
-- TODO uniqueness constraints should be added on these 3 columns later
add column actor_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
add column bio text, -- not on community, already has description
add column local boolean not null default true,
add column private_key text, -- These need to be generated from code
add column public_key text,
add column last_refreshed_at timestamp not null default now() -- Used to re-fetch federated actor periodically
;
-- Community
alter table community
add column actor_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local
add column local boolean not null default true,
add column private_key text, -- These need to be generated from code
add column public_key text,
add column last_refreshed_at timestamp not null default now() -- Used to re-fetch federated actor periodically
;
-- Don't worry about rebuilding the views right now.

View file

@ -1,5 +1,6 @@
use super::*;
use crate::apub::puller::{get_all_communities, get_remote_community};
use crate::apub::{gen_keypair_str, make_apub_endpoint, EndpointType};
use crate::settings::Settings;
use diesel::PgConnection;
use std::str::FromStr;
@ -208,6 +209,8 @@ impl Perform<CommunityResponse> for Oper<CreateCommunity> {
}
// When you create a community, make sure the user becomes a moderator and a follower
let (community_public_key, community_private_key) = gen_keypair_str();
let community_form = CommunityForm {
name: data.name.to_owned(),
title: data.title.to_owned(),
@ -218,6 +221,11 @@ impl Perform<CommunityResponse> for Oper<CreateCommunity> {
deleted: None,
nsfw: data.nsfw,
updated: None,
actor_id: make_apub_endpoint(EndpointType::Community, &data.name).to_string(),
local: true,
private_key: Some(community_private_key),
public_key: Some(community_public_key),
last_refreshed_at: None,
};
let inserted_community = match Community::create(&conn, &community_form) {
@ -298,6 +306,8 @@ impl Perform<CommunityResponse> for Oper<EditCommunity> {
return Err(APIError::err("no_community_edit_allowed").into());
}
let read_community = Community::read(&conn, data.edit_id)?;
let community_form = CommunityForm {
name: data.name.to_owned(),
title: data.title.to_owned(),
@ -308,6 +318,11 @@ impl Perform<CommunityResponse> for Oper<EditCommunity> {
deleted: data.deleted.to_owned(),
nsfw: data.nsfw,
updated: Some(naive_now()),
actor_id: read_community.actor_id,
local: read_community.local,
private_key: read_community.private_key,
public_key: read_community.public_key,
last_refreshed_at: None,
};
let _updated_community = match Community::update(&conn, data.edit_id, &community_form) {
@ -571,6 +586,11 @@ impl Perform<GetCommunityResponse> for Oper<TransferCommunity> {
deleted: None,
nsfw: read_community.nsfw,
updated: Some(naive_now()),
actor_id: read_community.actor_id,
local: read_community.local,
private_key: read_community.private_key,
public_key: read_community.public_key,
last_refreshed_at: None,
};
let _updated_community = match Community::update(&conn, data.community_id, &community_form) {

View file

@ -1,4 +1,5 @@
use super::*;
use crate::apub::{gen_keypair_str, make_apub_endpoint, EndpointType};
use crate::settings::Settings;
use crate::{generate_random_string, send_email};
use bcrypt::verify;
@ -250,6 +251,8 @@ impl Perform<LoginResponse> for Oper<Register> {
return Err(APIError::err("admin_already_created").into());
}
let (user_public_key, user_private_key) = gen_keypair_str();
// Register the new user
let user_form = UserForm {
name: data.username.to_owned(),
@ -269,6 +272,12 @@ impl Perform<LoginResponse> for Oper<Register> {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: make_apub_endpoint(EndpointType::User, &data.username).to_string(),
bio: None,
local: true,
private_key: Some(user_private_key),
public_key: Some(user_public_key),
last_refreshed_at: None,
};
// Create the user
@ -287,12 +296,15 @@ impl Perform<LoginResponse> for Oper<Register> {
}
};
let (community_public_key, community_private_key) = gen_keypair_str();
// Create the main community if it doesn't exist
let main_community: Community = match Community::read(&conn, 2) {
Ok(c) => c,
Err(_e) => {
let default_community_name = "main";
let community_form = CommunityForm {
name: "main".to_string(),
name: default_community_name.to_string(),
title: "The Default Community".to_string(),
description: Some("The Default Community".to_string()),
category_id: 1,
@ -301,6 +313,11 @@ impl Perform<LoginResponse> for Oper<Register> {
removed: None,
deleted: None,
updated: None,
actor_id: make_apub_endpoint(EndpointType::Community, default_community_name).to_string(),
local: true,
private_key: Some(community_private_key),
public_key: Some(community_public_key),
last_refreshed_at: None,
};
Community::create(&conn, &community_form).unwrap()
}
@ -403,6 +420,12 @@ impl Perform<LoginResponse> for Oper<SaveUserSettings> {
lang: data.lang.to_owned(),
show_avatars: data.show_avatars,
send_notifications_to_email: data.send_notifications_to_email,
actor_id: read_user.actor_id,
bio: read_user.bio,
local: read_user.local,
private_key: read_user.private_key,
public_key: read_user.public_key,
last_refreshed_at: None,
};
let updated_user = match User_::update(&conn, user_id, &user_form) {
@ -561,6 +584,12 @@ impl Perform<AddAdminResponse> for Oper<AddAdmin> {
lang: read_user.lang,
show_avatars: read_user.show_avatars,
send_notifications_to_email: read_user.send_notifications_to_email,
actor_id: read_user.actor_id,
bio: read_user.bio,
local: read_user.local,
private_key: read_user.private_key,
public_key: read_user.public_key,
last_refreshed_at: None,
};
match User_::update(&conn, data.user_id, &user_form) {
@ -624,6 +653,12 @@ impl Perform<BanUserResponse> for Oper<BanUser> {
lang: read_user.lang,
show_avatars: read_user.show_avatars,
send_notifications_to_email: read_user.send_notifications_to_email,
actor_id: read_user.actor_id,
bio: read_user.bio,
local: read_user.local,
private_key: read_user.private_key,
public_key: read_user.public_key,
last_refreshed_at: None,
};
match User_::update(&conn, data.user_id, &user_form) {

View file

@ -3,6 +3,7 @@ pub mod post;
pub mod puller;
pub mod user;
use crate::Settings;
use openssl::{pkey::PKey, rsa::Rsa};
use actix_web::body::Body;
use actix_web::HttpResponse;
@ -17,13 +18,13 @@ where
.json(json)
}
enum EndpointType {
pub enum EndpointType {
Community,
User,
Post,
}
fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url {
pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url {
let point = match endpoint_type {
EndpointType::Community => "c",
EndpointType::User => "u",
@ -47,3 +48,25 @@ pub fn get_apub_protocol_string() -> &'static str {
"http"
}
}
pub fn gen_keypair() -> (Vec<u8>, Vec<u8>) {
let rsa = Rsa::generate(2048).expect("sign::gen_keypair: key generation error");
let pkey = PKey::from_rsa(rsa).expect("sign::gen_keypair: parsing error");
(
pkey
.public_key_to_pem()
.expect("sign::gen_keypair: public key encoding error"),
pkey
.private_key_to_pem_pkcs8()
.expect("sign::gen_keypair: private key encoding error"),
)
}
pub fn gen_keypair_str() -> (String, String) {
let (public_key, private_key) = gen_keypair();
(vec_bytes_to_str(public_key), vec_bytes_to_str(private_key))
}
fn vec_bytes_to_str(bytes: Vec<u8>) -> String {
String::from_utf8_lossy(&bytes).into_owned()
}

View file

@ -0,0 +1,101 @@
// This is for db migrations that require code
use super::community::{Community, CommunityForm};
use super::user::{UserForm, User_};
use super::*;
use crate::apub::{gen_keypair_str, make_apub_endpoint, EndpointType};
use crate::naive_now;
use log::info;
pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), Error> {
user_updates_2020_04_02(conn)?;
community_updates_2020_04_02(conn)?;
Ok(())
}
fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
use crate::schema::user_::dsl::*;
info!("Running user_updates_2020_04_02");
// Update the actor_id, private_key, and public_key, last_refreshed_at
let incorrect_users = user_
.filter(actor_id.eq("changeme"))
.filter(local.eq(true))
.load::<User_>(conn)?;
for cuser in &incorrect_users {
let (user_public_key, user_private_key) = gen_keypair_str();
let form = UserForm {
name: cuser.name.to_owned(),
fedi_name: cuser.fedi_name.to_owned(),
email: cuser.email.to_owned(),
matrix_user_id: cuser.matrix_user_id.to_owned(),
avatar: cuser.avatar.to_owned(),
password_encrypted: cuser.password_encrypted.to_owned(),
preferred_username: cuser.preferred_username.to_owned(),
updated: None,
admin: cuser.admin,
banned: cuser.banned,
show_nsfw: cuser.show_nsfw,
theme: cuser.theme.to_owned(),
default_sort_type: cuser.default_sort_type,
default_listing_type: cuser.default_listing_type,
lang: cuser.lang.to_owned(),
show_avatars: cuser.show_avatars,
send_notifications_to_email: cuser.send_notifications_to_email,
actor_id: make_apub_endpoint(EndpointType::User, &cuser.name).to_string(),
bio: cuser.bio.to_owned(),
local: cuser.local,
private_key: Some(user_private_key),
public_key: Some(user_public_key),
last_refreshed_at: Some(naive_now()),
};
User_::update(&conn, cuser.id, &form)?;
}
info!("{} user rows updated.", incorrect_users.len());
Ok(())
}
fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
use crate::schema::community::dsl::*;
info!("Running community_updates_2020_04_02");
// Update the actor_id, private_key, and public_key, last_refreshed_at
let incorrect_communities = community
.filter(actor_id.eq("changeme"))
.filter(local.eq(true))
.load::<Community>(conn)?;
for ccommunity in &incorrect_communities {
let (community_public_key, community_private_key) = gen_keypair_str();
let form = CommunityForm {
name: ccommunity.name.to_owned(),
title: ccommunity.title.to_owned(),
description: ccommunity.description.to_owned(),
category_id: ccommunity.category_id,
creator_id: ccommunity.creator_id,
removed: None,
deleted: None,
nsfw: ccommunity.nsfw,
updated: None,
actor_id: make_apub_endpoint(EndpointType::Community, &ccommunity.name).to_string(),
local: ccommunity.local,
private_key: Some(community_private_key),
public_key: Some(community_public_key),
last_refreshed_at: Some(naive_now()),
};
Community::update(&conn, ccommunity.id, &form)?;
}
info!("{} community rows updated.", incorrect_communities.len());
Ok(())
}

View file

@ -186,6 +186,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -200,6 +206,11 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

View file

@ -450,6 +450,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -464,6 +470,11 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

View file

@ -15,6 +15,11 @@ pub struct Community {
pub updated: Option<chrono::NaiveDateTime>,
pub deleted: bool,
pub nsfw: bool,
pub actor_id: String,
pub local: bool,
pub private_key: Option<String>,
pub public_key: Option<String>,
pub last_refreshed_at: chrono::NaiveDateTime,
}
#[derive(Insertable, AsChangeset, Clone, Serialize, Deserialize)]
@ -29,6 +34,11 @@ pub struct CommunityForm {
pub updated: Option<chrono::NaiveDateTime>,
pub deleted: Option<bool>,
pub nsfw: bool,
pub actor_id: String,
pub local: bool,
pub private_key: Option<String>,
pub public_key: Option<String>,
pub last_refreshed_at: Option<chrono::NaiveDateTime>,
}
impl Crud<CommunityForm> for Community {
@ -232,6 +242,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -246,6 +262,11 @@ mod tests {
removed: None,
deleted: None,
updated: None,
actor_id: "changeme".into(),
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();
@ -262,6 +283,11 @@ mod tests {
deleted: false,
published: inserted_community.published,
updated: None,
actor_id: "changeme".into(),
local: true,
private_key: None,
public_key: None,
last_refreshed_at: inserted_community.published,
};
let community_follower_form = CommunityFollowerForm {

View file

@ -5,6 +5,7 @@ use diesel::*;
use serde::{Deserialize, Serialize};
pub mod category;
pub mod code_migrations;
pub mod comment;
pub mod comment_view;
pub mod community;

View file

@ -454,6 +454,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_mod = User_::create(&conn, &new_mod).unwrap();
@ -476,6 +482,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -490,6 +502,11 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

View file

@ -104,6 +104,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();

View file

@ -207,6 +207,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -221,6 +227,11 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

View file

@ -375,6 +375,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -389,6 +395,11 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

View file

@ -81,6 +81,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_creator = User_::create(&conn, &creator_form).unwrap();
@ -103,6 +109,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_recipient = User_::create(&conn, &recipient_form).unwrap();

View file

@ -27,6 +27,12 @@ pub struct User_ {
pub show_avatars: bool,
pub send_notifications_to_email: bool,
pub matrix_user_id: Option<String>,
pub actor_id: String,
pub bio: Option<String>,
pub local: bool,
pub private_key: Option<String>,
pub public_key: Option<String>,
pub last_refreshed_at: chrono::NaiveDateTime,
}
#[derive(Insertable, AsChangeset, Clone)]
@ -49,6 +55,12 @@ pub struct UserForm {
pub show_avatars: bool,
pub send_notifications_to_email: bool,
pub matrix_user_id: Option<String>,
pub actor_id: String,
pub bio: Option<String>,
pub local: bool,
pub private_key: Option<String>,
pub public_key: Option<String>,
pub last_refreshed_at: Option<chrono::NaiveDateTime>,
}
impl Crud<UserForm> for User_ {
@ -78,6 +90,7 @@ impl User_ {
Self::create(&conn, &edited_user)
}
// TODO do more individual updates like these
pub fn update_password(
conn: &PgConnection,
user_id: i32,
@ -202,6 +215,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -226,6 +245,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: inserted_user.published,
};
let read_user = User_::read(&conn, inserted_user.id).unwrap();

View file

@ -80,6 +80,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -102,6 +108,12 @@ mod tests {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: "changeme".into(),
bio: None,
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_recipient = User_::create(&conn, &recipient_form).unwrap();
@ -116,6 +128,11 @@ mod tests {
deleted: None,
updated: None,
nsfw: false,
actor_id: "changeme".into(),
local: true,
private_key: None,
public_key: None,
last_refreshed_at: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

View file

@ -17,6 +17,7 @@ pub extern crate jsonwebtoken;
pub extern crate lettre;
pub extern crate lettre_email;
extern crate log;
pub extern crate openssl;
pub extern crate rand;
pub extern crate regex;
pub extern crate rss;

View file

@ -6,6 +6,7 @@ use actix::prelude::*;
use actix_web::*;
use diesel::r2d2::{ConnectionManager, Pool};
use diesel::PgConnection;
use lemmy_server::db::code_migrations::run_advanced_migrations;
use lemmy_server::routes::{api, federation, feeds, index, nodeinfo, webfinger, websocket};
use lemmy_server::settings::Settings;
use lemmy_server::websocket::server::*;
@ -28,6 +29,7 @@ async fn main() -> io::Result<()> {
// Run the migrations from code
let conn = pool.get().unwrap();
embedded_migrations::run(&conn).unwrap();
run_advanced_migrations(&conn).unwrap();
// Set up websocket server
let server = ChatServer::startup(pool.clone()).start();

View file

@ -1,3 +1,14 @@
table! {
activity (id) {
id -> Int4,
user_id -> Int4,
data -> Jsonb,
local -> Bool,
published -> Timestamp,
updated -> Nullable<Timestamp>,
}
}
table! {
category (id) {
id -> Int4,
@ -53,6 +64,11 @@ table! {
updated -> Nullable<Timestamp>,
deleted -> Bool,
nsfw -> Bool,
actor_id -> Varchar,
local -> Bool,
private_key -> Nullable<Text>,
public_key -> Nullable<Text>,
last_refreshed_at -> Timestamp,
}
}
@ -290,6 +306,12 @@ table! {
show_avatars -> Bool,
send_notifications_to_email -> Bool,
matrix_user_id -> Nullable<Text>,
actor_id -> Varchar,
bio -> Nullable<Text>,
local -> Bool,
private_key -> Nullable<Text>,
public_key -> Nullable<Text>,
last_refreshed_at -> Timestamp,
}
}
@ -311,6 +333,7 @@ table! {
}
}
joinable!(activity -> user_ (user_id));
joinable!(comment -> post (post_id));
joinable!(comment -> user_ (creator_id));
joinable!(comment_like -> comment (comment_id));
@ -353,6 +376,7 @@ joinable!(user_mention -> comment (comment_id));
joinable!(user_mention -> user_ (recipient_id));
allow_tables_to_appear_in_same_query!(
activity,
category,
comment,
comment_like,