From 1f66d047bc775c133501ddf6884cb8c3b3a9b8c4 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 24 Apr 2020 10:04:36 -0400 Subject: [PATCH 1/7] Some fed fixes. --- server/src/api/comment.rs | 4 +- server/src/api/community.rs | 2 +- server/src/api/mod.rs | 4 +- server/src/api/user.rs | 14 +-- server/src/apub/activities.rs | 29 ++---- server/src/apub/community.rs | 47 +++------- server/src/apub/community_inbox.rs | 45 +++++---- server/src/apub/fetcher.rs | 145 +++++++++++++++++------------ server/src/apub/mod.rs | 64 +++++++++++-- server/src/apub/post.rs | 28 ++---- server/src/apub/signatures.rs | 12 +-- server/src/apub/user.rs | 22 +---- server/src/apub/user_inbox.rs | 45 +++++---- server/src/db/community.rs | 2 +- server/src/db/user.rs | 4 +- server/src/routes/feeds.rs | 4 +- server/src/routes/nodeinfo.rs | 2 +- server/src/routes/webfinger.rs | 4 +- server/src/routes/websocket.rs | 1 - 19 files changed, 244 insertions(+), 234 deletions(-) diff --git a/server/src/api/comment.rs b/server/src/api/comment.rs index 0c6d8e89f..eb67d8f20 100644 --- a/server/src/api/comment.rs +++ b/server/src/api/comment.rs @@ -122,7 +122,7 @@ impl Perform for Oper { let extracted_usernames = extract_usernames(&comment_form.content); for username_mention in &extracted_usernames { - if let Ok(mention_user) = User_::read_from_name(&conn, (*username_mention).to_string()) { + if let Ok(mention_user) = User_::read_from_name(&conn, username_mention) { // You can't mention yourself // At some point, make it so you can't tag the parent creator either // This can cause two notifications, one for reply and the other for mention @@ -334,7 +334,7 @@ impl Perform for Oper { let extracted_usernames = extract_usernames(&comment_form.content); for username_mention in &extracted_usernames { - let mention_user = User_::read_from_name(&conn, (*username_mention).to_string()); + let mention_user = User_::read_from_name(&conn, username_mention); if mention_user.is_ok() { let mention_user_id = mention_user?.id; diff --git a/server/src/api/community.rs b/server/src/api/community.rs index 01bfc40b0..ace5b353e 100644 --- a/server/src/api/community.rs +++ b/server/src/api/community.rs @@ -139,7 +139,7 @@ impl Perform for Oper { None => { match Community::read_from_name( &conn, - data.name.to_owned().unwrap_or_else(|| "main".to_string()), + &data.name.to_owned().unwrap_or_else(|| "main".to_string()), ) { Ok(community) => community, Err(_e) => return Err(APIError::err("couldnt_find_community").into()), diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 8711e0b50..04d690014 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -68,8 +68,8 @@ pub struct Oper { data: T, } -impl Oper { - pub fn new(data: T) -> Oper { +impl Oper { + pub fn new(data: Data) -> Oper { Oper { data } } } diff --git a/server/src/api/user.rs b/server/src/api/user.rs index 0dfca4173..ff2760a5c 100644 --- a/server/src/api/user.rs +++ b/server/src/api/user.rs @@ -261,7 +261,7 @@ impl Perform for Oper { return Err(APIError::err("admin_already_created").into()); } - let keypair = generate_actor_keypair()?; + let user_keypair = generate_actor_keypair()?; // Register the new user let user_form = UserForm { @@ -284,8 +284,8 @@ impl Perform for Oper { actor_id: make_apub_endpoint(EndpointType::User, &data.username).to_string(), bio: None, local: true, - private_key: Some(keypair.private_key), - public_key: Some(keypair.public_key), + private_key: Some(user_keypair.private_key), + public_key: Some(user_keypair.public_key), last_refreshed_at: None, }; @@ -305,7 +305,7 @@ impl Perform for Oper { } }; - let keypair = generate_actor_keypair()?; + let main_community_keypair = generate_actor_keypair()?; // Create the main community if it doesn't exist let main_community: Community = match Community::read(&conn, 2) { @@ -324,8 +324,8 @@ impl Perform for Oper { updated: None, actor_id: make_apub_endpoint(EndpointType::Community, default_community_name).to_string(), local: true, - private_key: Some(keypair.private_key), - public_key: Some(keypair.public_key), + private_key: Some(main_community_keypair.private_key), + public_key: Some(main_community_keypair.public_key), last_refreshed_at: None, published: None, }; @@ -504,7 +504,7 @@ impl Perform for Oper { None => { match User_::read_from_name( &conn, - data + &data .username .to_owned() .unwrap_or_else(|| "admin".to_string()), diff --git a/server/src/apub/activities.rs b/server/src/apub/activities.rs index e5980e293..c842bc3cf 100644 --- a/server/src/apub/activities.rs +++ b/server/src/apub/activities.rs @@ -1,21 +1,4 @@ -use crate::apub::is_apub_id_valid; -use crate::apub::signatures::sign; -use crate::db::community::Community; -use crate::db::community_view::CommunityFollowerView; -use crate::db::post::Post; -use crate::db::user::User_; -use crate::db::Crud; -use activitystreams::activity::{Accept, Create, Follow, Update}; -use activitystreams::object::properties::ObjectProperties; -use activitystreams::BaseBox; -use activitystreams::{context, public}; -use diesel::PgConnection; -use failure::Error; -use failure::_core::fmt::Debug; -use isahc::prelude::*; -use log::debug; -use serde::Serialize; -use url::Url; +use super::*; fn populate_object_props( props: &mut ObjectProperties, @@ -45,6 +28,8 @@ where { let json = serde_json::to_string(&activity)?; debug!("Sending activitypub activity {} to {:?}", json, to); + // TODO it needs to expand, the to field needs to expand and dedup the followers urls + // The inbox is determined by first retrieving the target actor's JSON-LD representation and then looking up the inbox property. If a recipient is a Collection or OrderedCollection, then the server MUST dereference the collection (with the user's credentials) and discover inboxes for each item in the collection. Servers MUST limit the number of layers of indirections through collections which will be performed, which MAY be one. for t in to { let to_url = Url::parse(&t)?; if !is_apub_id_valid(&to_url) { @@ -136,6 +121,7 @@ pub fn follow_community( .follow_props .set_actor_xsd_any_uri(user.actor_id.clone())? .set_object_xsd_any_uri(community.actor_id.clone())?; + // TODO this is incorrect, the to field should not be the inbox, but the followers url let to = format!("{}/inbox", community.actor_id); send_activity( &follow, @@ -153,6 +139,7 @@ pub fn accept_follow(follow: &Follow, conn: &PgConnection) -> Result<(), Error> .get_object_xsd_any_uri() .unwrap() .to_string(); + let actor_uri = follow.follow_props.get_actor_xsd_any_uri().unwrap().to_string(); let community = Community::read_from_actor_id(conn, &community_uri)?; let mut accept = Accept::new(); accept @@ -164,12 +151,14 @@ pub fn accept_follow(follow: &Follow, conn: &PgConnection) -> Result<(), Error> .follow_props .get_actor_xsd_any_uri() .unwrap() - .to_string(), + .to_string() )?; accept .accept_props + .set_actor_xsd_any_uri(community.actor_id.clone())? .set_object_base_box(BaseBox::from_concrete(follow.clone())?)?; - let to = format!("{}/inbox", community_uri); + // TODO this is incorrect, the to field should not be the inbox, but the followers url + let to = format!("{}/inbox", actor_uri); send_activity( &accept, &community.private_key.unwrap(), diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index a49357a86..fe31527fd 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -1,34 +1,11 @@ -use crate::apub::fetcher::{fetch_remote_object, fetch_remote_user}; -use crate::apub::signatures::PublicKey; -use crate::apub::*; -use crate::db::community::{Community, CommunityForm}; -use crate::db::community_view::CommunityFollowerView; -use crate::db::establish_unpooled_connection; -use crate::db::post::Post; -use crate::db::user::User_; -use crate::db::Crud; -use crate::{convert_datetime, naive_now}; -use activitystreams::actor::properties::ApActorProperties; -use activitystreams::collection::OrderedCollection; -use activitystreams::{ - actor::Group, collection::UnorderedCollection, context, ext::Extensible, - object::properties::ObjectProperties, -}; -use actix_web::body::Body; -use actix_web::web::Path; -use actix_web::HttpResponse; -use actix_web::{web, Result}; -use diesel::r2d2::{ConnectionManager, Pool}; -use diesel::PgConnection; -use failure::Error; -use serde::Deserialize; -use url::Url; +use super::*; #[derive(Deserialize)] pub struct CommunityQuery { community_name: String, } +// TODO turn these as_group, as_page, into apub trait... something like to_apub impl Community { // Turn a Lemmy Community into an ActivityPub group that can be sent out over the network. fn as_group(&self, conn: &PgConnection) -> Result { @@ -91,8 +68,8 @@ impl CommunityForm { let outbox_uri = Url::parse(&aprops.get_outbox().to_string())?; let _outbox = fetch_remote_object::(&outbox_uri)?; let _followers = fetch_remote_object::(&followers_uri)?; - let apub_id = Url::parse(&oprops.get_attributed_to_xsd_any_uri().unwrap().to_string())?; - let creator = fetch_remote_user(&apub_id, conn)?; + let apub_id = &oprops.get_attributed_to_xsd_any_uri().unwrap().to_string(); + let creator = get_or_fetch_and_upsert_remote_user(&apub_id, conn)?; Ok(CommunityForm { name: oprops.get_name_xsd_string().unwrap().to_string(), @@ -123,19 +100,22 @@ impl CommunityForm { /// Return the community json over HTTP. pub async fn get_apub_community_http( info: Path, - db: web::Data>>, + db: DbPoolParam, + chat_server: ChatServerParam, ) -> Result, Error> { - let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?; + let community = Community::read_from_name(&&db.get()?, &info.community_name)?; let c = community.as_group(&db.get().unwrap())?; Ok(create_apub_response(&c)) } /// Returns an empty followers collection, only populating the siz (for privacy). +// TODO this needs to return the actual followers, and the to: field needs this pub async fn get_apub_community_followers( info: Path, - db: web::Data>>, + db: DbPoolParam, + chat_server: ChatServerParam, ) -> Result, Error> { - let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?; + let community = Community::read_from_name(&&db.get()?, &info.community_name)?; let connection = establish_unpooled_connection(); //As we are an object, we validated that the community id was valid @@ -156,9 +136,10 @@ pub async fn get_apub_community_followers( /// Returns an UnorderedCollection with the latest posts from the community. pub async fn get_apub_community_outbox( info: Path, - db: web::Data>>, + db: DbPoolParam, + chat_server: ChatServerParam, ) -> Result, Error> { - let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?; + let community = Community::read_from_name(&&db.get()?, &info.community_name)?; let conn = establish_unpooled_connection(); //As we are an object, we validated that the community id was valid diff --git a/server/src/apub/community_inbox.rs b/server/src/apub/community_inbox.rs index e7fc856e8..473f35f4c 100644 --- a/server/src/apub/community_inbox.rs +++ b/server/src/apub/community_inbox.rs @@ -1,16 +1,4 @@ -use crate::apub::activities::accept_follow; -use crate::apub::fetcher::fetch_remote_user; -use crate::apub::signatures::verify; -use crate::db::community::{Community, CommunityFollower, CommunityFollowerForm}; -use crate::db::Followable; -use activitystreams::activity::Follow; -use actix_web::{web, HttpRequest, HttpResponse}; -use diesel::r2d2::{ConnectionManager, Pool}; -use diesel::PgConnection; -use failure::Error; -use log::debug; -use serde::Deserialize; -use url::Url; +use super::*; #[serde(untagged)] #[derive(Deserialize, Debug)] @@ -23,17 +11,18 @@ pub async fn community_inbox( request: HttpRequest, input: web::Json, path: web::Path, - db: web::Data>>, + db: DbPoolParam, + chat_server: ChatServerParam, ) -> Result { let input = input.into_inner(); - let conn = &db.get().unwrap(); + let community_name = path.into_inner(); debug!( "Community {} received activity {:?}", - &path.into_inner(), + &community_name, &input ); match input { - CommunityAcceptedObjects::Follow(f) => handle_follow(&f, &request, conn), + CommunityAcceptedObjects::Follow(f) => handle_follow(&f, &request, &community_name, db, chat_server), } } @@ -42,28 +31,36 @@ pub async fn community_inbox( fn handle_follow( follow: &Follow, request: &HttpRequest, - conn: &PgConnection, + community_name: &str, + db: DbPoolParam, + chat_server: ChatServerParam, ) -> Result { let user_uri = follow .follow_props .get_actor_xsd_any_uri() .unwrap() .to_string(); - let user = fetch_remote_user(&Url::parse(&user_uri)?, conn)?; - verify(&request, &user.public_key.unwrap())?; - - // TODO: make sure this is a local community let community_uri = follow .follow_props .get_object_xsd_any_uri() .unwrap() .to_string(); - let community = Community::read_from_actor_id(conn, &community_uri)?; + + let conn = db.get()?; + + let user = get_or_fetch_and_upsert_remote_user(&user_uri, &conn)?; + let community = Community::read_from_name(&conn, &community_name)?; + + verify(&request, &user.public_key.unwrap())?; + let community_follower_form = CommunityFollowerForm { community_id: community.id, user_id: user.id, }; + + // This will fail if they're already a follower CommunityFollower::follow(&conn, &community_follower_form)?; - accept_follow(&follow, conn)?; + + accept_follow(&follow, &conn)?; Ok(HttpResponse::Ok().finish()) } diff --git a/server/src/apub/fetcher.rs b/server/src/apub/fetcher.rs index 71453a2a4..b4af70c96 100644 --- a/server/src/apub/fetcher.rs +++ b/server/src/apub/fetcher.rs @@ -1,23 +1,4 @@ -use crate::api::site::SearchResponse; -use crate::apub::*; -use crate::db::community::{Community, CommunityForm}; -use crate::db::community_view::CommunityView; -use crate::db::post::{Post, PostForm}; -use crate::db::post_view::PostView; -use crate::db::user::{UserForm, User_}; -use crate::db::user_view::UserView; -use crate::db::{Crud, SearchType}; -use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown}; -use activitystreams::collection::OrderedCollection; -use activitystreams::object::Page; -use activitystreams::BaseBox; -use diesel::result::Error::NotFound; -use diesel::PgConnection; -use failure::Error; -use isahc::prelude::*; -use serde::Deserialize; -use std::time::Duration; -use url::Url; +use super::*; // Fetch nodeinfo metadata from a remote instance. fn _fetch_node_info(domain: &str) -> Result { @@ -30,26 +11,27 @@ fn _fetch_node_info(domain: &str) -> Result { Ok(fetch_remote_object::(&well_known.links.href)?) } -// TODO: move these to db -fn upsert_community( - community_form: &CommunityForm, - conn: &PgConnection, -) -> Result { - let existing = Community::read_from_actor_id(conn, &community_form.actor_id); - match existing { - Err(NotFound {}) => Ok(Community::create(conn, &community_form)?), - Ok(c) => Ok(Community::update(conn, c.id, &community_form)?), - Err(e) => Err(Error::from(e)), - } -} -fn upsert_user(user_form: &UserForm, conn: &PgConnection) -> Result { - let existing = User_::read_from_apub_id(conn, &user_form.actor_id); - Ok(match existing { - Err(NotFound {}) => User_::create(conn, &user_form)?, - Ok(u) => User_::update(conn, u.id, &user_form)?, - Err(e) => return Err(Error::from(e)), - }) -} +// // TODO: move these to db +// // TODO use the last_refreshed_at +// fn upsert_community( +// community_form: &CommunityForm, +// conn: &PgConnection, +// ) -> Result { +// let existing = Community::read_from_actor_id(conn, &community_form.actor_id); +// match existing { +// Err(NotFound {}) => Ok(Community::create(conn, &community_form)?), +// Ok(c) => Ok(Community::update(conn, c.id, &community_form)?), +// Err(e) => Err(Error::from(e)), +// } +// } +// fn upsert_user(user_form: &UserForm, conn: &PgConnection) -> Result { +// let existing = User_::read_from_actor_id(conn, &user_form.actor_id); +// Ok(match existing { +// Err(NotFound {}) => User_::create(conn, &user_form)?, +// Ok(u) => User_::update(conn, u.id, &user_form)?, +// Err(e) => return Err(Error::from(e)), +// }) +// } fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result { let existing = Post::read_from_apub_id(conn, &post_form.ap_id); @@ -89,7 +71,7 @@ where pub enum SearchAcceptedObjects { Person(Box), Group(Box), - Page(Box), + // Page(Box), } /// Attempt to parse the query as URL, and fetch an ActivityPub object from it. @@ -109,22 +91,27 @@ pub fn search_by_apub_id(query: &str, conn: &PgConnection) -> Result(&query_url)? { SearchAcceptedObjects::Person(p) => { - let u = upsert_user(&UserForm::from_person(&p)?, conn)?; - response.users = vec![UserView::read(conn, u.id)?]; + let user = get_or_fetch_and_upsert_remote_user(query, &conn)?; + response.users = vec![UserView::read(conn, user.id)?]; } SearchAcceptedObjects::Group(g) => { - let c = upsert_community(&CommunityForm::from_group(&g, conn)?, conn)?; - fetch_community_outbox(&c, conn)?; - response.communities = vec![CommunityView::read(conn, c.id, None)?]; - } - SearchAcceptedObjects::Page(p) => { - let p = upsert_post(&PostForm::from_page(&p, conn)?, conn)?; - response.posts = vec![PostView::read(conn, p.id, None)?]; + let community = get_or_fetch_and_upsert_remote_community(query, &conn)?; + // fetch_community_outbox(&c, conn)?; + response.communities = vec![CommunityView::read(conn, community.id, None)?]; } + // SearchAcceptedObjects::Page(p) => { + // let p = upsert_post(&PostForm::from_page(&p, conn)?, conn)?; + // response.posts = vec![PostView::read(conn, p.id, None)?]; + // } } Ok(response) } +// TODO It should not be fetching data from a community outbox. +// All posts, comments, comment likes, etc should be posts to our community_inbox +// The only data we should be periodically fetching (if it hasn't been fetched in the last day +// maybe), is community and user actors +// and user actors /// Fetch all posts in the outbox of the given user, and insert them into the database. fn fetch_community_outbox(community: &Community, conn: &PgConnection) -> Result, Error> { let outbox_url = Url::parse(&community.get_outbox_url())?; @@ -143,16 +130,54 @@ fn fetch_community_outbox(community: &Community, conn: &PgConnection) -> Result< ) } -/// Fetch a user, insert/update it in the database and return the user. -pub fn fetch_remote_user(apub_id: &Url, conn: &PgConnection) -> Result { - let person = fetch_remote_object::(apub_id)?; - let uf = UserForm::from_person(&person)?; - upsert_user(&uf, conn) +/// Check if a remote user exists, create if not found, if its too old update it.Fetch a user, insert/update it in the database and return the user. +pub fn get_or_fetch_and_upsert_remote_user(apub_id: &str, conn: &PgConnection) -> Result { + match User_::read_from_actor_id(&conn, &apub_id) { + Ok(u) => { + // If its older than a day, re-fetch it + // TODO the less than needs to be tested + if u.last_refreshed_at.lt(&(naive_now() - chrono::Duration::days(1))) { + debug!("Fetching and updating from remote user: {}", apub_id); + let person = fetch_remote_object::(&Url::parse(apub_id)?)?; + let uf = UserForm::from_person(&person)?; + uf.last_refreshed_at = Some(naive_now()); + Ok(User_::update(&conn, u.id, &uf)?) + } else { + Ok(u) + } + }, + Err(NotFound {}) => { + debug!("Fetching and creating remote user: {}", apub_id); + let person = fetch_remote_object::(&Url::parse(apub_id)?)?; + let uf = UserForm::from_person(&person)?; + Ok(User_::create(conn, &uf)?) + } + Err(e) => Err(Error::from(e)), + } } -/// Fetch a community, insert/update it in the database and return the community. -pub fn fetch_remote_community(apub_id: &Url, conn: &PgConnection) -> Result { - let group = fetch_remote_object::(apub_id)?; - let cf = CommunityForm::from_group(&group, conn)?; - upsert_community(&cf, conn) +/// Check if a remote community exists, create if not found, if its too old update it.Fetch a community, insert/update it in the database and return the community. +pub fn get_or_fetch_and_upsert_remote_community(apub_id: &str, conn: &PgConnection) -> Result { + match Community::read_from_actor_id(&conn, &apub_id) { + Ok(c) => { + // If its older than a day, re-fetch it + // TODO the less than needs to be tested + if c.last_refreshed_at.lt(&(naive_now() - chrono::Duration::days(1))) { + debug!("Fetching and updating from remote community: {}", apub_id); + let group = fetch_remote_object::(&Url::parse(apub_id)?)?; + let cf = CommunityForm::from_group(&group, conn)?; + cf.last_refreshed_at = Some(naive_now()); + Ok(Community::update(&conn, c.id, &cf)?) + } else { + Ok(c) + } + }, + Err(NotFound {}) => { + debug!("Fetching and creating remote community: {}", apub_id); + let group = fetch_remote_object::(&Url::parse(apub_id)?)?; + let cf = CommunityForm::from_group(&group, conn)?; + Ok(Community::create(conn, &cf)?) + } + Err(e) => Err(Error::from(e)), + } } diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs index 846458977..08fd97561 100644 --- a/server/src/apub/mod.rs +++ b/server/src/apub/mod.rs @@ -6,14 +6,66 @@ pub mod post; pub mod signatures; pub mod user; pub mod user_inbox; -use crate::apub::signatures::PublicKeyExtension; -use crate::Settings; -use activitystreams::actor::{properties::ApActorProperties, Group, Person}; -use activitystreams::ext::Ext; + +use activitystreams::{ + context, public, BaseBox, + actor::{ + Actor, + Person, + Group, + properties::ApActorProperties, + }, + activity::{Accept, Create, Follow, Update}, + object::{ + Page, + properties::ObjectProperties, + }, + ext::{ + Ext, + Extensible, + Extension, + }, + collection::{ + UnorderedCollection, + OrderedCollection, + }, +}; use actix_web::body::Body; -use actix_web::HttpResponse; -use serde::ser::Serialize; +use actix_web::{web, Result, HttpRequest, HttpResponse}; +use actix_web::web::Path; use url::Url; +use failure::Error; +use failure::_core::fmt::Debug; +use log::debug; +use isahc::prelude::*; +use diesel::result::Error::NotFound; +use diesel::PgConnection; +use http::request::Builder; +use http_signature_normalization::Config; +use openssl::hash::MessageDigest; +use openssl::sign::{Signer, Verifier}; +use openssl::{pkey::PKey, rsa::Rsa}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::time::Duration; + +use crate::routes::{DbPoolParam, ChatServerParam}; +use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown}; +use crate::{convert_datetime, naive_now, Settings}; +use crate::db::community::{Community, CommunityForm, CommunityFollower, CommunityFollowerForm}; +use crate::db::community_view::{CommunityFollowerView, CommunityView}; +use crate::db::post::{Post, PostForm}; +use crate::db::post_view::PostView; +use crate::db::user::{UserForm, User_}; +use crate::db::user_view::UserView; +// TODO check on unpooled connection +use crate::db::{Crud, Followable, SearchType, establish_unpooled_connection}; +use crate::api::site::SearchResponse; + +use signatures::{PublicKey, PublicKeyExtension, sign}; +use activities::accept_follow; +use signatures::verify; +use fetcher::{fetch_remote_object, get_or_fetch_and_upsert_remote_user, get_or_fetch_and_upsert_remote_community}; type GroupExt = Ext, PublicKeyExtension>; type PersonExt = Ext, PublicKeyExtension>; diff --git a/server/src/apub/post.rs b/server/src/apub/post.rs index edae92d06..b56eeab64 100644 --- a/server/src/apub/post.rs +++ b/server/src/apub/post.rs @@ -1,19 +1,4 @@ -use crate::apub::create_apub_response; -use crate::apub::fetcher::{fetch_remote_community, fetch_remote_user}; -use crate::convert_datetime; -use crate::db::community::Community; -use crate::db::post::{Post, PostForm}; -use crate::db::user::User_; -use crate::db::Crud; -use activitystreams::{context, object::properties::ObjectProperties, object::Page}; -use actix_web::body::Body; -use actix_web::web::Path; -use actix_web::{web, HttpResponse}; -use diesel::r2d2::{ConnectionManager, Pool}; -use diesel::PgConnection; -use failure::Error; -use serde::Deserialize; -use url::Url; +use super::*; #[derive(Deserialize)] pub struct PostQuery { @@ -23,7 +8,8 @@ pub struct PostQuery { /// Return the post json over HTTP. pub async fn get_apub_post( info: Path, - db: web::Data>>, + db: DbPoolParam, + chat_server: ChatServerParam, ) -> Result, Error> { let id = info.post_id.parse::()?; let post = Post::read(&&db.get()?, id)?; @@ -72,10 +58,10 @@ impl PostForm { /// Parse an ActivityPub page received from another instance into a Lemmy post. pub fn from_page(page: &Page, conn: &PgConnection) -> Result { let oprops = &page.object_props; - let creator_id = Url::parse(&oprops.get_attributed_to_xsd_any_uri().unwrap().to_string())?; - let creator = fetch_remote_user(&creator_id, conn)?; - let community_id = Url::parse(&oprops.get_to_xsd_any_uri().unwrap().to_string())?; - let community = fetch_remote_community(&community_id, conn)?; + let creator_actor_id = &oprops.get_attributed_to_xsd_any_uri().unwrap().to_string(); + let creator = get_or_fetch_and_upsert_remote_user(&creator_actor_id, &conn)?; + let community_actor_id = &oprops.get_to_xsd_any_uri().unwrap().to_string(); + let community = get_or_fetch_and_upsert_remote_community(&community_actor_id, &conn)?; Ok(PostForm { name: oprops.get_summary_xsd_string().unwrap().to_string(), diff --git a/server/src/apub/signatures.rs b/server/src/apub/signatures.rs index 40b3c738e..cf064603b 100644 --- a/server/src/apub/signatures.rs +++ b/server/src/apub/signatures.rs @@ -1,14 +1,4 @@ -use activitystreams::{actor::Actor, ext::Extension}; -use actix_web::HttpRequest; -use failure::Error; -use http::request::Builder; -use http_signature_normalization::Config; -use log::debug; -use openssl::hash::MessageDigest; -use openssl::sign::{Signer, Verifier}; -use openssl::{pkey::PKey, rsa::Rsa}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use super::*; lazy_static! { static ref HTTP_SIG_CONFIG: Config = Config::new(); diff --git a/server/src/apub/user.rs b/server/src/apub/user.rs index b5a819114..acf72221f 100644 --- a/server/src/apub/user.rs +++ b/server/src/apub/user.rs @@ -1,21 +1,4 @@ -use crate::apub::signatures::PublicKey; -use crate::apub::{create_apub_response, PersonExt}; -use crate::db::user::{UserForm, User_}; -use crate::{convert_datetime, naive_now}; -use activitystreams::{ - actor::{properties::ApActorProperties, Person}, - context, - ext::Extensible, - object::properties::ObjectProperties, -}; -use actix_web::body::Body; -use actix_web::web::Path; -use actix_web::HttpResponse; -use actix_web::{web, Result}; -use diesel::r2d2::{ConnectionManager, Pool}; -use diesel::PgConnection; -use failure::Error; -use serde::Deserialize; +use super::*; #[derive(Deserialize)] pub struct UserQuery { @@ -25,7 +8,8 @@ pub struct UserQuery { // Turn a Lemmy user into an ActivityPub person and return it as json. pub async fn get_apub_user( info: Path, - db: web::Data>>, + db: DbPoolParam, + chat_server: ChatServerParam, ) -> Result, Error> { let user = User_::find_by_email_or_username(&&db.get()?, &info.user_name)?; diff --git a/server/src/apub/user_inbox.rs b/server/src/apub/user_inbox.rs index f9faa0f09..97cdeece7 100644 --- a/server/src/apub/user_inbox.rs +++ b/server/src/apub/user_inbox.rs @@ -1,16 +1,4 @@ -use crate::apub::fetcher::{fetch_remote_community, fetch_remote_user}; -use crate::apub::signatures::verify; -use crate::db::post::{Post, PostForm}; -use crate::db::Crud; -use activitystreams::activity::{Accept, Create, Update}; -use activitystreams::object::Page; -use actix_web::{web, HttpRequest, HttpResponse}; -use diesel::r2d2::{ConnectionManager, Pool}; -use diesel::PgConnection; -use failure::Error; -use log::debug; -use serde::Deserialize; -use url::Url; +use super::*; #[serde(untagged)] #[derive(Deserialize, Debug)] @@ -25,21 +13,23 @@ pub async fn user_inbox( request: HttpRequest, input: web::Json, path: web::Path, - db: web::Data>>, + db: DbPoolParam, + chat_server: ChatServerParam, ) -> Result { // TODO: would be nice if we could do the signature check here, but we cant access the actor property let input = input.into_inner(); let conn = &db.get().unwrap(); + let username = path.into_inner(); debug!( "User {} received activity: {:?}", - &path.into_inner(), + &username, &input ); match input { - UserAcceptedObjects::Create(c) => handle_create(&c, &request, conn), - UserAcceptedObjects::Update(u) => handle_update(&u, &request, conn), - UserAcceptedObjects::Accept(a) => handle_accept(&a, &request, conn), + UserAcceptedObjects::Create(c) => handle_create(&c, &request, &username, &conn), + UserAcceptedObjects::Update(u) => handle_update(&u, &request, &username, &conn), + UserAcceptedObjects::Accept(a) => handle_accept(&a, &request, &username, &conn), } } @@ -47,8 +37,11 @@ pub async fn user_inbox( fn handle_create( create: &Create, request: &HttpRequest, + username: &str, conn: &PgConnection, ) -> Result { + // TODO before this even gets named, because we don't know what type of object it is, we need + // to parse this out let community_uri = create .create_props .get_actor_xsd_any_uri() @@ -75,6 +68,7 @@ fn handle_create( fn handle_update( update: &Update, request: &HttpRequest, + username: &str, conn: &PgConnection, ) -> Result { let community_uri = update @@ -103,6 +97,7 @@ fn handle_update( fn handle_accept( accept: &Accept, request: &HttpRequest, + username: &str, conn: &PgConnection, ) -> Result { let community_uri = accept @@ -110,9 +105,21 @@ fn handle_accept( .get_actor_xsd_any_uri() .unwrap() .to_string(); - let community = fetch_remote_community(&Url::parse(&community_uri)?, conn)?; + + let community = get_or_fetch_and_upsert_remote_community(&community_uri, conn)?; verify(request, &community.public_key.unwrap())?; + let user = User_::read_from_name(&conn, username)?; + + // Now you need to add this to the community follower + let community_follower_form = CommunityFollowerForm { + community_id: community.id, + user_id: user.id, + }; + + // This will fail if they're already a follower + CommunityFollower::follow(&conn, &community_follower_form)?; + // TODO: make sure that we actually requested a follow // TODO: at this point, indicate to the user that they are following the community Ok(HttpResponse::Ok().finish()) diff --git a/server/src/db/community.rs b/server/src/db/community.rs index ca2fc120a..301fce032 100644 --- a/server/src/db/community.rs +++ b/server/src/db/community.rs @@ -73,7 +73,7 @@ impl Crud for Community { } impl Community { - pub fn read_from_name(conn: &PgConnection, community_name: String) -> Result { + pub fn read_from_name(conn: &PgConnection, community_name: &str) -> Result { use crate::schema::community::dsl::*; community .filter(name.eq(community_name)) diff --git a/server/src/db/user.rs b/server/src/db/user.rs index 3a079f091..065b2c578 100644 --- a/server/src/db/user.rs +++ b/server/src/db/user.rs @@ -104,7 +104,7 @@ impl User_ { .get_result::(conn) } - pub fn read_from_name(conn: &PgConnection, from_user_name: String) -> Result { + pub fn read_from_name(conn: &PgConnection, from_user_name: &str) -> Result { user_.filter(name.eq(from_user_name)).first::(conn) } @@ -120,7 +120,7 @@ impl User_ { .get_result::(conn) } - pub fn read_from_apub_id(conn: &PgConnection, object_id: &str) -> Result { + pub fn read_from_actor_id(conn: &PgConnection, object_id: &str) -> Result { use crate::schema::user_::dsl::*; user_.filter(actor_id.eq(object_id)).first::(conn) } diff --git a/server/src/routes/feeds.rs b/server/src/routes/feeds.rs index 815953c55..6e28a7f77 100644 --- a/server/src/routes/feeds.rs +++ b/server/src/routes/feeds.rs @@ -27,7 +27,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { async fn get_all_feed( info: web::Query, - db: web::Data>>, + db: DbPoolParam, ) -> Result { let res = web::block(move || { let conn = db.get()?; @@ -144,7 +144,7 @@ fn get_feed_community( community_name: String, ) -> Result { let site_view = SiteView::read(&conn)?; - let community = Community::read_from_name(&conn, community_name)?; + let community = Community::read_from_name(&conn, &community_name)?; let community_url = community.get_url(); let posts = PostQueryBuilder::create(&conn) diff --git a/server/src/routes/nodeinfo.rs b/server/src/routes/nodeinfo.rs index 32507c6b6..5004bc932 100644 --- a/server/src/routes/nodeinfo.rs +++ b/server/src/routes/nodeinfo.rs @@ -21,7 +21,7 @@ async fn node_info_well_known() -> Result, failure::Error> { } async fn node_info( - db: web::Data>>, + db: DbPoolParam, ) -> Result { let res = web::block(move || { let conn = db.get()?; diff --git a/server/src/routes/webfinger.rs b/server/src/routes/webfinger.rs index 0f30acaf2..7bdce2067 100644 --- a/server/src/routes/webfinger.rs +++ b/server/src/routes/webfinger.rs @@ -31,7 +31,7 @@ lazy_static! { /// https://radical.town/.well-known/webfinger?resource=acct:felix@radical.town async fn get_webfinger_response( info: Query, - db: web::Data>>, + db: DbPoolParam, ) -> Result { let res = web::block(move || { let conn = db.get()?; @@ -46,7 +46,7 @@ async fn get_webfinger_response( }; // Make sure the requested community exists. - let community = match Community::read_from_name(&conn, community_name.to_string()) { + let community = match Community::read_from_name(&conn, &community_name) { Ok(o) => o, Err(_) => return Err(format_err!("not_found")), }; diff --git a/server/src/routes/websocket.rs b/server/src/routes/websocket.rs index 48b7d08fb..a95ff9ee8 100644 --- a/server/src/routes/websocket.rs +++ b/server/src/routes/websocket.rs @@ -32,7 +32,6 @@ struct WSSession { /// Client must send ping at least once per 10 seconds (CLIENT_TIMEOUT), /// otherwise we drop connection. hb: Instant, - // db: Pool>, } impl Actor for WSSession { From 035532512afe934ef9661e3e9d77cbd59fe4edb1 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 24 Apr 2020 18:30:31 +0200 Subject: [PATCH 2/7] Implement integration test for federation --- .dockerignore | 1 + .gitignore | 1 + docker/federation-test/run-tests.sh | 23 ++++ docker/federation/docker-compose.yml | 3 +- docker/federation/run-federation-test.bash | 2 +- ui/src/api_tests/api.spec.ts | 132 +++++++++------------ 6 files changed, 86 insertions(+), 76 deletions(-) create mode 100755 docker/federation-test/run-tests.sh diff --git a/.dockerignore b/.dockerignore index 4f186bcde..255caf67f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ ui/node_modules server/target docker/dev/volumes docker/federation/volumes +docker/federation-test/volumes .git ansible diff --git a/.gitignore b/.gitignore index 5e9fd40d6..236a729eb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ ansible/passwords/ docker/lemmy_mine.hjson docker/dev/env_deploy.sh docker/federation/volumes +docker/federation-test/volumes docker/dev/volumes # local build files diff --git a/docker/federation-test/run-tests.sh b/docker/federation-test/run-tests.sh new file mode 100755 index 000000000..43e2f9093 --- /dev/null +++ b/docker/federation-test/run-tests.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +pushd ../../server/ +cargo build +popd + +sudo docker build ../../ --file ../federation/Dockerfile --tag lemmy-federation:latest + +sudo docker-compose --file ../federation/docker-compose.yml --project-directory . up -d + +# TODO: need to wait until the instances are initialised + +pushd ../../ui +yarn +echo "Waiting for Lemmy to start..." +while [[ "$(curl -s -o /dev/null -w '%{http_code}' 'localhost:8540/api/v1/site')" != "200" ]]; do sleep 5; done +yarn api-test || true +popd + +sudo docker-compose --file ../federation/docker-compose.yml --project-directory . down + +sudo rm -r volumes/ diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index 4f23cdcc2..496e7fa60 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -7,7 +7,8 @@ services: - "8540:8540" - "8550:8550" volumes: - - ./nginx.conf:/etc/nginx/nginx.conf + # Hack to make this work from both docker/federation/ and docker/federation-test/ + - ../federation/nginx.conf:/etc/nginx/nginx.conf depends_on: - lemmy_alpha - pictshare_alpha diff --git a/docker/federation/run-federation-test.bash b/docker/federation/run-federation-test.bash index 2c8b681ce..8486648b3 100755 --- a/docker/federation/run-federation-test.bash +++ b/docker/federation/run-federation-test.bash @@ -12,6 +12,6 @@ pushd ../../server/ || exit cargo build popd || exit -sudo docker build ../../ -f Dockerfile -t lemmy-federation:latest +sudo docker build ../../ --file Dockerfile -t lemmy-federation:latest sudo docker-compose up diff --git a/ui/src/api_tests/api.spec.ts b/ui/src/api_tests/api.spec.ts index 5db9ee647..07e12ecfe 100644 --- a/ui/src/api_tests/api.spec.ts +++ b/ui/src/api_tests/api.spec.ts @@ -3,83 +3,67 @@ import fetch from 'node-fetch'; import { LoginForm, LoginResponse, - GetPostsForm, - GetPostsResponse, - CommentForm, - CommentResponse, - ListingType, - SortType, + PostForm, + PostResponse, + SearchResponse, } from '../interfaces'; -let baseUrl = 'https://test.lemmy.ml'; -let apiUrl = `${baseUrl}/api/v1`; -let auth: string; +let lemmyAlphaUrl = 'http://localhost:8540'; +let lemmyBetaUrl = 'http://localhost:8550'; +let lemmyAlphaApiUrl = `${lemmyAlphaUrl}/api/v1`; +let lemmyBetaApiUrl = `${lemmyBetaUrl}/api/v1`; +let lemmyAlphaAuth: string; -beforeAll(async () => { - console.log('Logging in as test user.'); - let form: LoginForm = { - username_or_email: 'tester', - password: 'tester', - }; +// Workaround for tests being run before beforeAll() is finished +// https://github.com/facebook/jest/issues/9527#issuecomment-592406108 +describe('main', () => { + beforeAll(async () => { + console.log('Logging in as lemmy_alpha'); + let form: LoginForm = { + username_or_email: 'lemmy_alpha', + password: 'lemmy', + }; - let res: LoginResponse = await fetch(`${apiUrl}/user/login`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: wrapper(form), - }).then(d => d.json()); + let res: LoginResponse = await fetch(`${lemmyAlphaApiUrl}/user/login`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: wrapper(form), + }).then(d => d.json()); - auth = res.jwt; + lemmyAlphaAuth = res.jwt; + }); + + test('Create test post on alpha and fetch it on beta', async () => { + let name = 'A jest test post'; + let postForm: PostForm = { + name, + auth: lemmyAlphaAuth, + community_id: 2, + creator_id: 2, + nsfw: false, + }; + + let createResponse: PostResponse = await fetch(`${lemmyAlphaApiUrl}/post`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: wrapper(postForm), + }).then(d => d.json()); + expect(createResponse.post.name).toBe(name); + + let searchUrl = `${lemmyBetaApiUrl}/search?q=${createResponse.post.ap_id}&type_=All&sort=TopAll`; + let searchResponse: SearchResponse = await fetch(searchUrl, { + method: 'GET', + }).then(d => d.json()); + + // TODO: check more fields + expect(searchResponse.posts[0].name).toBe(name); + }); + + function wrapper(form: any): string { + return JSON.stringify(form); + } }); - -test('Get test user posts', async () => { - let form: GetPostsForm = { - type_: ListingType[ListingType.All], - sort: SortType[SortType.TopAll], - auth, - }; - - let res: GetPostsResponse = await fetch( - `${apiUrl}/post/list?type_=${form.type_}&sort=${form.sort}&auth=${auth}` - ).then(d => d.json()); - - // console.debug(res); - - expect(res.posts[0].id).toBe(2); -}); - -test('Create test comment', async () => { - let content = 'A jest test comment'; - let form: CommentForm = { - post_id: 2, - content, - auth, - }; - - let res: CommentResponse = await fetch(`${apiUrl}/comment`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: wrapper(form), - }).then(d => d.json()); - - expect(res.comment.content).toBe(content); -}); - -test('adds 1 + 2 to equal 3', () => { - let sum = (a: number, b: number) => a + b; - expect(sum(1, 2)).toBe(3); -}); - -test(`Get ${baseUrl} nodeinfo href`, async () => { - let url = `${baseUrl}/.well-known/nodeinfo`; - let href = `${baseUrl}/nodeinfo/2.0.json`; - let res = await fetch(url).then(d => d.json()); - expect(res.links.href).toBe(href); -}); - -function wrapper(form: any): string { - return JSON.stringify(form); -} From 0a0250d819ec237ebfba6208beac75d8de254b93 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 24 Apr 2020 15:55:54 -0400 Subject: [PATCH 3/7] Some more cleanup. --- server/src/apub/activities.rs | 14 +-- server/src/apub/community.rs | 93 +++++++++---------- server/src/apub/community_inbox.rs | 12 +-- server/src/apub/fetcher.rs | 138 +++++++++++++---------------- server/src/apub/mod.rs | 88 ++++++++++-------- server/src/apub/post.rs | 11 ++- server/src/apub/user.rs | 92 ++++++++++--------- server/src/apub/user_inbox.rs | 29 +++--- server/src/routes/federation.rs | 11 +-- 9 files changed, 251 insertions(+), 237 deletions(-) diff --git a/server/src/apub/activities.rs b/server/src/apub/activities.rs index c842bc3cf..24631a352 100644 --- a/server/src/apub/activities.rs +++ b/server/src/apub/activities.rs @@ -29,7 +29,7 @@ where let json = serde_json::to_string(&activity)?; debug!("Sending activitypub activity {} to {:?}", json, to); // TODO it needs to expand, the to field needs to expand and dedup the followers urls - // The inbox is determined by first retrieving the target actor's JSON-LD representation and then looking up the inbox property. If a recipient is a Collection or OrderedCollection, then the server MUST dereference the collection (with the user's credentials) and discover inboxes for each item in the collection. Servers MUST limit the number of layers of indirections through collections which will be performed, which MAY be one. + // The inbox is determined by first retrieving the target actor's JSON-LD representation and then looking up the inbox property. If a recipient is a Collection or OrderedCollection, then the server MUST dereference the collection (with the user's credentials) and discover inboxes for each item in the collection. Servers MUST limit the number of layers of indirections through collections which will be performed, which MAY be one. for t in to { let to_url = Url::parse(&t)?; if !is_apub_id_valid(&to_url) { @@ -61,7 +61,7 @@ fn get_follower_inboxes(conn: &PgConnection, community: &Community) -> Result Result<(), Error> { - let page = post.as_page(conn)?; + let page = post.to_apub(conn)?; let community = Community::read(conn, post.community_id)?; let mut create = Create::new(); populate_object_props( @@ -84,7 +84,7 @@ pub fn post_create(post: &Post, creator: &User_, conn: &PgConnection) -> Result< /// Send out information about an edited post, to the followers of the community. pub fn post_update(post: &Post, creator: &User_, conn: &PgConnection) -> Result<(), Error> { - let page = post.as_page(conn)?; + let page = post.to_apub(conn)?; let community = Community::read(conn, post.community_id)?; let mut update = Update::new(); populate_object_props( @@ -139,7 +139,11 @@ pub fn accept_follow(follow: &Follow, conn: &PgConnection) -> Result<(), Error> .get_object_xsd_any_uri() .unwrap() .to_string(); - let actor_uri = follow.follow_props.get_actor_xsd_any_uri().unwrap().to_string(); + let actor_uri = follow + .follow_props + .get_actor_xsd_any_uri() + .unwrap() + .to_string(); let community = Community::read_from_actor_id(conn, &community_uri)?; let mut accept = Accept::new(); accept @@ -151,7 +155,7 @@ pub fn accept_follow(follow: &Follow, conn: &PgConnection) -> Result<(), Error> .follow_props .get_actor_xsd_any_uri() .unwrap() - .to_string() + .to_string(), )?; accept .accept_props diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index fe31527fd..920c41ad3 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -5,10 +5,9 @@ pub struct CommunityQuery { community_name: String, } -// TODO turn these as_group, as_page, into apub trait... something like to_apub -impl Community { +impl ToApub for Community { // Turn a Lemmy Community into an ActivityPub group that can be sent out over the network. - fn as_group(&self, conn: &PgConnection) -> Result { + fn to_apub(&self, conn: &PgConnection) -> Result { let mut group = Group::default(); let oprops: &mut ObjectProperties = group.as_mut(); @@ -45,29 +44,26 @@ impl Community { Ok(group.extend(actor_props).extend(public_key.to_ext())) } +} - pub fn get_followers_url(&self) -> String { - format!("{}/followers", &self.actor_id) - } - pub fn get_inbox_url(&self) -> String { - format!("{}/inbox", &self.actor_id) - } - pub fn get_outbox_url(&self) -> String { - format!("{}/outbox", &self.actor_id) +impl ActorType for Community { + fn actor_id(&self) -> String { + self.actor_id.to_owned() } } -impl CommunityForm { +impl FromApub for CommunityForm { /// Parse an ActivityPub group received from another instance into a Lemmy community. - pub fn from_group(group: &GroupExt, conn: &PgConnection) -> Result { + fn from_apub(group: &GroupExt, conn: &PgConnection) -> Result { let oprops = &group.base.base.object_props; let aprops = &group.base.extension; let public_key: &PublicKey = &group.extension.public_key; - let followers_uri = Url::parse(&aprops.get_followers().unwrap().to_string())?; - let outbox_uri = Url::parse(&aprops.get_outbox().to_string())?; - let _outbox = fetch_remote_object::(&outbox_uri)?; - let _followers = fetch_remote_object::(&followers_uri)?; + let _followers_uri = Url::parse(&aprops.get_followers().unwrap().to_string())?; + let _outbox_uri = Url::parse(&aprops.get_outbox().to_string())?; + // TODO don't do extra fetching here + // let _outbox = fetch_remote_object::(&outbox_uri)?; + // let _followers = fetch_remote_object::(&followers_uri)?; let apub_id = &oprops.get_attributed_to_xsd_any_uri().unwrap().to_string(); let creator = get_or_fetch_and_upsert_remote_user(&apub_id, conn)?; @@ -101,10 +97,9 @@ impl CommunityForm { pub async fn get_apub_community_http( info: Path, db: DbPoolParam, - chat_server: ChatServerParam, ) -> Result, Error> { let community = Community::read_from_name(&&db.get()?, &info.community_name)?; - let c = community.as_group(&db.get().unwrap())?; + let c = community.to_apub(&db.get().unwrap())?; Ok(create_apub_response(&c)) } @@ -113,14 +108,13 @@ pub async fn get_apub_community_http( pub async fn get_apub_community_followers( info: Path, db: DbPoolParam, - chat_server: ChatServerParam, ) -> Result, Error> { let community = Community::read_from_name(&&db.get()?, &info.community_name)?; - let connection = establish_unpooled_connection(); + let conn = db.get()?; + //As we are an object, we validated that the community id was valid - let community_followers = - CommunityFollowerView::for_community(&connection, community.id).unwrap(); + let community_followers = CommunityFollowerView::for_community(&conn, community.id).unwrap(); let mut collection = UnorderedCollection::default(); let oprops: &mut ObjectProperties = collection.as_mut(); @@ -133,32 +127,33 @@ pub async fn get_apub_community_followers( Ok(create_apub_response(&collection)) } -/// Returns an UnorderedCollection with the latest posts from the community. -pub async fn get_apub_community_outbox( - info: Path, - db: DbPoolParam, - chat_server: ChatServerParam, -) -> Result, Error> { - let community = Community::read_from_name(&&db.get()?, &info.community_name)?; +// TODO should not be doing this +// Returns an UnorderedCollection with the latest posts from the community. +//pub async fn get_apub_community_outbox( +// info: Path, +// db: DbPoolParam, +// chat_server: ChatServerParam, +//) -> Result, Error> { +// let community = Community::read_from_name(&&db.get()?, &info.community_name)?; - let conn = establish_unpooled_connection(); - //As we are an object, we validated that the community id was valid - let community_posts: Vec = Post::list_for_community(&conn, community.id)?; +// let conn = establish_unpooled_connection(); +// //As we are an object, we validated that the community id was valid +// let community_posts: Vec = Post::list_for_community(&conn, community.id)?; - let mut collection = OrderedCollection::default(); - let oprops: &mut ObjectProperties = collection.as_mut(); - oprops - .set_context_xsd_any_uri(context())? - .set_id(community.actor_id)?; - collection - .collection_props - .set_many_items_base_boxes( - community_posts - .iter() - .map(|c| c.as_page(&conn).unwrap()) - .collect(), - )? - .set_total_items(community_posts.len() as u64)?; +// let mut collection = OrderedCollection::default(); +// let oprops: &mut ObjectProperties = collection.as_mut(); +// oprops +// .set_context_xsd_any_uri(context())? +// .set_id(community.actor_id)?; +// collection +// .collection_props +// .set_many_items_base_boxes( +// community_posts +// .iter() +// .map(|c| c.as_page(&conn).unwrap()) +// .collect(), +// )? +// .set_total_items(community_posts.len() as u64)?; - Ok(create_apub_response(&collection)) -} +// Ok(create_apub_response(&collection)) +//} diff --git a/server/src/apub/community_inbox.rs b/server/src/apub/community_inbox.rs index 473f35f4c..af6d39e12 100644 --- a/server/src/apub/community_inbox.rs +++ b/server/src/apub/community_inbox.rs @@ -6,6 +6,7 @@ pub enum CommunityAcceptedObjects { Follow(Follow), } +// TODO Consolidate community and user inboxes into a single shared one /// Handler for all incoming activities to community inboxes. pub async fn community_inbox( request: HttpRequest, @@ -18,11 +19,12 @@ pub async fn community_inbox( let community_name = path.into_inner(); debug!( "Community {} received activity {:?}", - &community_name, - &input + &community_name, &input ); match input { - CommunityAcceptedObjects::Follow(f) => handle_follow(&f, &request, &community_name, db, chat_server), + CommunityAcceptedObjects::Follow(f) => { + handle_follow(&f, &request, &community_name, db, chat_server) + } } } @@ -33,14 +35,14 @@ fn handle_follow( request: &HttpRequest, community_name: &str, db: DbPoolParam, - chat_server: ChatServerParam, + _chat_server: ChatServerParam, ) -> Result { let user_uri = follow .follow_props .get_actor_xsd_any_uri() .unwrap() .to_string(); - let community_uri = follow + let _community_uri = follow .follow_props .get_object_xsd_any_uri() .unwrap() diff --git a/server/src/apub/fetcher.rs b/server/src/apub/fetcher.rs index b4af70c96..b8aaeaaf9 100644 --- a/server/src/apub/fetcher.rs +++ b/server/src/apub/fetcher.rs @@ -11,40 +11,8 @@ fn _fetch_node_info(domain: &str) -> Result { Ok(fetch_remote_object::(&well_known.links.href)?) } -// // TODO: move these to db -// // TODO use the last_refreshed_at -// fn upsert_community( -// community_form: &CommunityForm, -// conn: &PgConnection, -// ) -> Result { -// let existing = Community::read_from_actor_id(conn, &community_form.actor_id); -// match existing { -// Err(NotFound {}) => Ok(Community::create(conn, &community_form)?), -// Ok(c) => Ok(Community::update(conn, c.id, &community_form)?), -// Err(e) => Err(Error::from(e)), -// } -// } -// fn upsert_user(user_form: &UserForm, conn: &PgConnection) -> Result { -// let existing = User_::read_from_actor_id(conn, &user_form.actor_id); -// Ok(match existing { -// Err(NotFound {}) => User_::create(conn, &user_form)?, -// Ok(u) => User_::update(conn, u.id, &user_form)?, -// Err(e) => return Err(Error::from(e)), -// }) -// } - -fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result { - let existing = Post::read_from_apub_id(conn, &post_form.ap_id); - match existing { - Err(NotFound {}) => Ok(Post::create(conn, &post_form)?), - Ok(p) => Ok(Post::update(conn, p.id, &post_form)?), - Err(e) => Err(Error::from(e)), - } -} - /// Fetch any type of ActivityPub object, handling things like HTTP headers, deserialisation, /// timeouts etc. -/// TODO: add an optional param last_updated and only fetch if its too old pub fn fetch_remote_object(url: &Url) -> Result where Response: for<'de> Deserialize<'de>, @@ -71,15 +39,14 @@ where pub enum SearchAcceptedObjects { Person(Box), Group(Box), - // Page(Box), } /// Attempt to parse the query as URL, and fetch an ActivityPub object from it. /// /// Some working examples for use with the docker/federation/ setup: -/// http://lemmy_alpha:8540/federation/c/main -/// http://lemmy_alpha:8540/federation/u/lemmy_alpha -/// http://lemmy_alpha:8540/federation/p/3 +/// http://lemmy_alpha:8540/c/main +/// http://lemmy_alpha:8540/u/lemmy_alpha +/// http://lemmy_alpha:8540/p/3 pub fn search_by_apub_id(query: &str, conn: &PgConnection) -> Result { let query_url = Url::parse(&query)?; let mut response = SearchResponse { @@ -91,65 +58,47 @@ pub fn search_by_apub_id(query: &str, conn: &PgConnection) -> Result(&query_url)? { SearchAcceptedObjects::Person(p) => { - let user = get_or_fetch_and_upsert_remote_user(query, &conn)?; + let user_uri = p.base.base.object_props.get_id().unwrap().to_string(); + let user = get_or_fetch_and_upsert_remote_user(&user_uri, &conn)?; response.users = vec![UserView::read(conn, user.id)?]; } SearchAcceptedObjects::Group(g) => { - let community = get_or_fetch_and_upsert_remote_community(query, &conn)?; + let community_uri = g.base.base.object_props.get_id().unwrap().to_string(); + let community = get_or_fetch_and_upsert_remote_community(&community_uri, &conn)?; + // TODO Maybe at some point in the future, fetch all the history of a community // fetch_community_outbox(&c, conn)?; response.communities = vec![CommunityView::read(conn, community.id, None)?]; } - // SearchAcceptedObjects::Page(p) => { - // let p = upsert_post(&PostForm::from_page(&p, conn)?, conn)?; - // response.posts = vec![PostView::read(conn, p.id, None)?]; - // } } Ok(response) } -// TODO It should not be fetching data from a community outbox. -// All posts, comments, comment likes, etc should be posts to our community_inbox -// The only data we should be periodically fetching (if it hasn't been fetched in the last day -// maybe), is community and user actors -// and user actors -/// Fetch all posts in the outbox of the given user, and insert them into the database. -fn fetch_community_outbox(community: &Community, conn: &PgConnection) -> Result, Error> { - let outbox_url = Url::parse(&community.get_outbox_url())?; - let outbox = fetch_remote_object::(&outbox_url)?; - let items = outbox.collection_props.get_many_items_base_boxes(); - - Ok( - items - .unwrap() - .map(|obox: &BaseBox| -> Result { - let page = obox.clone().to_concrete::()?; - PostForm::from_page(&page, conn) - }) - .map(|pf| upsert_post(&pf?, conn)) - .collect::, Error>>()?, - ) -} - /// Check if a remote user exists, create if not found, if its too old update it.Fetch a user, insert/update it in the database and return the user. -pub fn get_or_fetch_and_upsert_remote_user(apub_id: &str, conn: &PgConnection) -> Result { +pub fn get_or_fetch_and_upsert_remote_user( + apub_id: &str, + conn: &PgConnection, +) -> Result { match User_::read_from_actor_id(&conn, &apub_id) { Ok(u) => { // If its older than a day, re-fetch it // TODO the less than needs to be tested - if u.last_refreshed_at.lt(&(naive_now() - chrono::Duration::days(1))) { + if u + .last_refreshed_at + .lt(&(naive_now() - chrono::Duration::days(1))) + { debug!("Fetching and updating from remote user: {}", apub_id); let person = fetch_remote_object::(&Url::parse(apub_id)?)?; - let uf = UserForm::from_person(&person)?; + let mut uf = UserForm::from_apub(&person, &conn)?; uf.last_refreshed_at = Some(naive_now()); Ok(User_::update(&conn, u.id, &uf)?) } else { Ok(u) } - }, + } Err(NotFound {}) => { debug!("Fetching and creating remote user: {}", apub_id); let person = fetch_remote_object::(&Url::parse(apub_id)?)?; - let uf = UserForm::from_person(&person)?; + let uf = UserForm::from_apub(&person, &conn)?; Ok(User_::create(conn, &uf)?) } Err(e) => Err(Error::from(e)), @@ -157,27 +106,66 @@ pub fn get_or_fetch_and_upsert_remote_user(apub_id: &str, conn: &PgConnection) - } /// Check if a remote community exists, create if not found, if its too old update it.Fetch a community, insert/update it in the database and return the community. -pub fn get_or_fetch_and_upsert_remote_community(apub_id: &str, conn: &PgConnection) -> Result { +pub fn get_or_fetch_and_upsert_remote_community( + apub_id: &str, + conn: &PgConnection, +) -> Result { match Community::read_from_actor_id(&conn, &apub_id) { Ok(c) => { // If its older than a day, re-fetch it // TODO the less than needs to be tested - if c.last_refreshed_at.lt(&(naive_now() - chrono::Duration::days(1))) { + if c + .last_refreshed_at + .lt(&(naive_now() - chrono::Duration::days(1))) + { debug!("Fetching and updating from remote community: {}", apub_id); let group = fetch_remote_object::(&Url::parse(apub_id)?)?; - let cf = CommunityForm::from_group(&group, conn)?; + let mut cf = CommunityForm::from_apub(&group, conn)?; cf.last_refreshed_at = Some(naive_now()); Ok(Community::update(&conn, c.id, &cf)?) } else { Ok(c) } - }, + } Err(NotFound {}) => { debug!("Fetching and creating remote community: {}", apub_id); let group = fetch_remote_object::(&Url::parse(apub_id)?)?; - let cf = CommunityForm::from_group(&group, conn)?; + let cf = CommunityForm::from_apub(&group, conn)?; Ok(Community::create(conn, &cf)?) } Err(e) => Err(Error::from(e)), } } + +// TODO Maybe add post, comment searching / caching at a later time +// fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result { +// let existing = Post::read_from_apub_id(conn, &post_form.ap_id); +// match existing { +// Err(NotFound {}) => Ok(Post::create(conn, &post_form)?), +// Ok(p) => Ok(Post::update(conn, p.id, &post_form)?), +// Err(e) => Err(Error::from(e)), +// } +// } + +// TODO It should not be fetching data from a community outbox. +// All posts, comments, comment likes, etc should be posts to our community_inbox +// The only data we should be periodically fetching (if it hasn't been fetched in the last day +// maybe), is community and user actors +// and user actors +// Fetch all posts in the outbox of the given user, and insert them into the database. +// fn fetch_community_outbox(community: &Community, conn: &PgConnection) -> Result, Error> { +// let outbox_url = Url::parse(&community.get_outbox_url())?; +// let outbox = fetch_remote_object::(&outbox_url)?; +// let items = outbox.collection_props.get_many_items_base_boxes(); + +// Ok( +// items +// .unwrap() +// .map(|obox: &BaseBox| -> Result { +// let page = obox.clone().to_concrete::()?; +// PostForm::from_page(&page, conn) +// }) +// .map(|pf| upsert_post(&pf?, conn)) +// .collect::, Error>>()?, +// ) +// } diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs index 08fd97561..d691e8390 100644 --- a/server/src/apub/mod.rs +++ b/server/src/apub/mod.rs @@ -8,64 +8,48 @@ pub mod user; pub mod user_inbox; use activitystreams::{ - context, public, BaseBox, - actor::{ - Actor, - Person, - Group, - properties::ApActorProperties, - }, activity::{Accept, Create, Follow, Update}, - object::{ - Page, - properties::ObjectProperties, - }, - ext::{ - Ext, - Extensible, - Extension, - }, - collection::{ - UnorderedCollection, - OrderedCollection, - }, + actor::{properties::ApActorProperties, Actor, Group, Person}, + collection::UnorderedCollection, + context, + ext::{Ext, Extensible, Extension}, + object::{properties::ObjectProperties, Page}, + public, BaseBox, }; use actix_web::body::Body; -use actix_web::{web, Result, HttpRequest, HttpResponse}; use actix_web::web::Path; -use url::Url; -use failure::Error; -use failure::_core::fmt::Debug; -use log::debug; -use isahc::prelude::*; +use actix_web::{web, HttpRequest, HttpResponse, Result}; use diesel::result::Error::NotFound; use diesel::PgConnection; +use failure::Error; +use failure::_core::fmt::Debug; use http::request::Builder; use http_signature_normalization::Config; +use isahc::prelude::*; +use log::debug; use openssl::hash::MessageDigest; use openssl::sign::{Signer, Verifier}; use openssl::{pkey::PKey, rsa::Rsa}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::time::Duration; +use url::Url; -use crate::routes::{DbPoolParam, ChatServerParam}; -use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown}; -use crate::{convert_datetime, naive_now, Settings}; -use crate::db::community::{Community, CommunityForm, CommunityFollower, CommunityFollowerForm}; +use crate::api::site::SearchResponse; +use crate::db::community::{Community, CommunityFollower, CommunityFollowerForm, CommunityForm}; use crate::db::community_view::{CommunityFollowerView, CommunityView}; use crate::db::post::{Post, PostForm}; -use crate::db::post_view::PostView; use crate::db::user::{UserForm, User_}; use crate::db::user_view::UserView; -// TODO check on unpooled connection -use crate::db::{Crud, Followable, SearchType, establish_unpooled_connection}; -use crate::api::site::SearchResponse; +use crate::db::{Crud, Followable, SearchType}; +use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown}; +use crate::routes::{ChatServerParam, DbPoolParam}; +use crate::{convert_datetime, naive_now, Settings}; -use signatures::{PublicKey, PublicKeyExtension, sign}; use activities::accept_follow; +use fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user}; use signatures::verify; -use fetcher::{fetch_remote_object, get_or_fetch_and_upsert_remote_user, get_or_fetch_and_upsert_remote_community}; +use signatures::{sign, PublicKey, PublicKeyExtension}; type GroupExt = Ext, PublicKeyExtension>; type PersonExt = Ext, PublicKeyExtension>; @@ -140,3 +124,35 @@ fn is_apub_id_valid(apub_id: &Url) -> bool { None => false, } } + +// TODO Not sure good names for these +pub trait ToApub { + fn to_apub(&self, conn: &PgConnection) -> Result; +} + +pub trait FromApub { + fn from_apub(apub: &ApubType, conn: &PgConnection) -> Result + where + Self: Sized; +} + +pub trait ActorType { + fn actor_id(&self) -> String; + + fn get_inbox_url(&self) -> String { + format!("{}/inbox", &self.actor_id()) + } + fn get_outbox_url(&self) -> String { + format!("{}/outbox", &self.actor_id()) + } + + fn get_followers_url(&self) -> String { + format!("{}/followers", &self.actor_id()) + } + fn get_following_url(&self) -> String { + format!("{}/following", &self.actor_id()) + } + fn get_liked_url(&self) -> String { + format!("{}/liked", &self.actor_id()) + } +} diff --git a/server/src/apub/post.rs b/server/src/apub/post.rs index b56eeab64..7ad4394d8 100644 --- a/server/src/apub/post.rs +++ b/server/src/apub/post.rs @@ -9,16 +9,15 @@ pub struct PostQuery { pub async fn get_apub_post( info: Path, db: DbPoolParam, - chat_server: ChatServerParam, ) -> Result, Error> { let id = info.post_id.parse::()?; let post = Post::read(&&db.get()?, id)?; - Ok(create_apub_response(&post.as_page(&db.get().unwrap())?)) + Ok(create_apub_response(&post.to_apub(&db.get().unwrap())?)) } -impl Post { +impl ToApub for Post { // Turn a Lemmy post into an ActivityPub page that can be sent out over the network. - pub fn as_page(&self, conn: &PgConnection) -> Result { + fn to_apub(&self, conn: &PgConnection) -> Result { let mut page = Page::default(); let oprops: &mut ObjectProperties = page.as_mut(); let creator = User_::read(conn, self.creator_id)?; @@ -54,9 +53,9 @@ impl Post { } } -impl PostForm { +impl FromApub for PostForm { /// Parse an ActivityPub page received from another instance into a Lemmy post. - pub fn from_page(page: &Page, conn: &PgConnection) -> Result { + fn from_apub(page: &Page, conn: &PgConnection) -> Result { let oprops = &page.object_props; let creator_actor_id = &oprops.get_attributed_to_xsd_any_uri().unwrap().to_string(); let creator = get_or_fetch_and_upsert_remote_user(&creator_actor_id, &conn)?; diff --git a/server/src/apub/user.rs b/server/src/apub/user.rs index acf72221f..03492f70a 100644 --- a/server/src/apub/user.rs +++ b/server/src/apub/user.rs @@ -5,52 +5,54 @@ pub struct UserQuery { user_name: String, } -// Turn a Lemmy user into an ActivityPub person and return it as json. -pub async fn get_apub_user( - info: Path, - db: DbPoolParam, - chat_server: ChatServerParam, -) -> Result, Error> { - let user = User_::find_by_email_or_username(&&db.get()?, &info.user_name)?; +impl ToApub for User_ { + // Turn a Lemmy Community into an ActivityPub group that can be sent out over the network. + fn to_apub(&self, _conn: &PgConnection) -> Result { + // TODO go through all these to_string and to_owned() + let mut person = Person::default(); + let oprops: &mut ObjectProperties = person.as_mut(); + oprops + .set_context_xsd_any_uri(context())? + .set_id(self.actor_id.to_string())? + .set_name_xsd_string(self.name.to_owned())? + .set_published(convert_datetime(self.published))?; - let mut person = Person::default(); - let oprops: &mut ObjectProperties = person.as_mut(); - oprops - .set_context_xsd_any_uri(context())? - .set_id(user.actor_id.to_string())? - .set_name_xsd_string(user.name.to_owned())? - .set_published(convert_datetime(user.published))?; + if let Some(u) = self.updated { + oprops.set_updated(convert_datetime(u))?; + } - if let Some(u) = user.updated { - oprops.set_updated(convert_datetime(u))?; + if let Some(i) = &self.preferred_username { + oprops.set_name_xsd_string(i.to_owned())?; + } + + let mut actor_props = ApActorProperties::default(); + + actor_props + .set_inbox(self.get_inbox_url())? + .set_outbox(self.get_outbox_url())? + .set_followers(self.get_followers_url())? + .set_following(self.get_following_url())? + .set_liked(self.get_liked_url())?; + + let public_key = PublicKey { + id: format!("{}#main-key", self.actor_id), + owner: self.actor_id.to_owned(), + public_key_pem: self.public_key.to_owned().unwrap(), + }; + + Ok(person.extend(actor_props).extend(public_key.to_ext())) } - - if let Some(i) = &user.preferred_username { - oprops.set_name_xsd_string(i.to_owned())?; - } - - let mut actor_props = ApActorProperties::default(); - - actor_props - .set_inbox(format!("{}/inbox", &user.actor_id))? - .set_outbox(format!("{}/outbox", &user.actor_id))? - .set_following(format!("{}/following", &user.actor_id))? - .set_liked(format!("{}/liked", &user.actor_id))?; - - let public_key = PublicKey { - id: format!("{}#main-key", user.actor_id), - owner: user.actor_id.to_owned(), - public_key_pem: user.public_key.unwrap(), - }; - - Ok(create_apub_response( - &person.extend(actor_props).extend(public_key.to_ext()), - )) } -impl UserForm { +impl ActorType for User_ { + fn actor_id(&self) -> String { + self.actor_id.to_owned() + } +} + +impl FromApub for UserForm { /// Parse an ActivityPub person received from another instance into a Lemmy user. - pub fn from_person(person: &PersonExt) -> Result { + fn from_apub(person: &PersonExt, _conn: &PgConnection) -> Result { let oprops = &person.base.base.object_props; let aprops = &person.base.extension; let public_key: &PublicKey = &person.extension.public_key; @@ -83,3 +85,13 @@ impl UserForm { }) } } + +/// Return the user json over HTTP. +pub async fn get_apub_user_http( + info: Path, + db: DbPoolParam, +) -> Result, Error> { + let user = User_::find_by_email_or_username(&&db.get()?, &info.user_name)?; + let u = user.to_apub(&db.get().unwrap())?; + Ok(create_apub_response(&u)) +} diff --git a/server/src/apub/user_inbox.rs b/server/src/apub/user_inbox.rs index 97cdeece7..251a221c6 100644 --- a/server/src/apub/user_inbox.rs +++ b/server/src/apub/user_inbox.rs @@ -14,17 +14,13 @@ pub async fn user_inbox( input: web::Json, path: web::Path, db: DbPoolParam, - chat_server: ChatServerParam, + _chat_server: ChatServerParam, ) -> Result { // TODO: would be nice if we could do the signature check here, but we cant access the actor property let input = input.into_inner(); let conn = &db.get().unwrap(); let username = path.into_inner(); - debug!( - "User {} received activity: {:?}", - &username, - &input - ); + debug!("User {} received activity: {:?}", &username, &input); match input { UserAcceptedObjects::Create(c) => handle_create(&c, &request, &username, &conn), @@ -37,18 +33,18 @@ pub async fn user_inbox( fn handle_create( create: &Create, request: &HttpRequest, - username: &str, + _username: &str, conn: &PgConnection, ) -> Result { // TODO before this even gets named, because we don't know what type of object it is, we need // to parse this out - let community_uri = create + let user_uri = create .create_props .get_actor_xsd_any_uri() .unwrap() .to_string(); - // TODO: should do this in a generic way so we dont need to know if its a user or a community - let user = fetch_remote_user(&Url::parse(&community_uri)?, conn)?; + + let user = get_or_fetch_and_upsert_remote_user(&user_uri, &conn)?; verify(request, &user.public_key.unwrap())?; let page = create @@ -58,7 +54,7 @@ fn handle_create( .unwrap() .to_owned() .to_concrete::()?; - let post = PostForm::from_page(&page, conn)?; + let post = PostForm::from_apub(&page, conn)?; Post::create(conn, &post)?; // TODO: send the new post out via websocket Ok(HttpResponse::Ok().finish()) @@ -68,15 +64,16 @@ fn handle_create( fn handle_update( update: &Update, request: &HttpRequest, - username: &str, + _username: &str, conn: &PgConnection, ) -> Result { - let community_uri = update + let user_uri = update .update_props .get_actor_xsd_any_uri() .unwrap() .to_string(); - let user = fetch_remote_user(&Url::parse(&community_uri)?, conn)?; + + let user = get_or_fetch_and_upsert_remote_user(&user_uri, &conn)?; verify(request, &user.public_key.unwrap())?; let page = update @@ -86,7 +83,7 @@ fn handle_update( .unwrap() .to_owned() .to_concrete::()?; - let post = PostForm::from_page(&page, conn)?; + let post = PostForm::from_apub(&page, conn)?; let id = Post::read_from_apub_id(conn, &post.ap_id)?.id; Post::update(conn, id, &post)?; // TODO: send the new post out via websocket @@ -105,7 +102,7 @@ fn handle_accept( .get_actor_xsd_any_uri() .unwrap() .to_string(); - + let community = get_or_fetch_and_upsert_remote_community(&community_uri, conn)?; verify(request, &community.public_key.unwrap())?; diff --git a/server/src/routes/federation.rs b/server/src/routes/federation.rs index eeae1fa26..bab88ca39 100644 --- a/server/src/routes/federation.rs +++ b/server/src/routes/federation.rs @@ -21,11 +21,12 @@ pub fn config(cfg: &mut web::ServiceConfig) { "/c/{community_name}/followers", web::get().to(get_apub_community_followers), ) - .route( - "/c/{community_name}/outbox", - web::get().to(get_apub_community_outbox), - ) - .route("/u/{user_name}", web::get().to(get_apub_user)) + // TODO This is only useful for history which we aren't doing right now + // .route( + // "/c/{community_name}/outbox", + // web::get().to(get_apub_community_outbox), + // ) + .route("/u/{user_name}", web::get().to(get_apub_user_http)) .route("/post/{post_id}", web::get().to(get_apub_post)), ) // Inboxes dont work with the header guard for some reason. From a44cdc4bf4f346332bc081b76d3cee753c00291b Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 24 Apr 2020 17:30:27 -0400 Subject: [PATCH 4/7] Use an associated type instead of Generic. --- server/src/apub/community.rs | 8 ++++++-- server/src/apub/mod.rs | 10 ++++++---- server/src/apub/post.rs | 8 ++++++-- server/src/apub/user.rs | 7 +++++-- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index 920c41ad3..05e004eee 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -5,7 +5,9 @@ pub struct CommunityQuery { community_name: String, } -impl ToApub for Community { +impl ToApub for Community { + type Response = GroupExt; + // Turn a Lemmy Community into an ActivityPub group that can be sent out over the network. fn to_apub(&self, conn: &PgConnection) -> Result { let mut group = Group::default(); @@ -52,7 +54,9 @@ impl ActorType for Community { } } -impl FromApub for CommunityForm { +impl FromApub for CommunityForm { + type ApubType = GroupExt; + /// Parse an ActivityPub group received from another instance into a Lemmy community. fn from_apub(group: &GroupExt, conn: &PgConnection) -> Result { let oprops = &group.base.base.object_props; diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs index d691e8390..057929688 100644 --- a/server/src/apub/mod.rs +++ b/server/src/apub/mod.rs @@ -126,12 +126,14 @@ fn is_apub_id_valid(apub_id: &Url) -> bool { } // TODO Not sure good names for these -pub trait ToApub { - fn to_apub(&self, conn: &PgConnection) -> Result; +pub trait ToApub { + type Response; + fn to_apub(&self, conn: &PgConnection) -> Result; } -pub trait FromApub { - fn from_apub(apub: &ApubType, conn: &PgConnection) -> Result +pub trait FromApub { + type ApubType; + fn from_apub(apub: &Self::ApubType, conn: &PgConnection) -> Result where Self: Sized; } diff --git a/server/src/apub/post.rs b/server/src/apub/post.rs index 7ad4394d8..51ba861ef 100644 --- a/server/src/apub/post.rs +++ b/server/src/apub/post.rs @@ -15,7 +15,9 @@ pub async fn get_apub_post( Ok(create_apub_response(&post.to_apub(&db.get().unwrap())?)) } -impl ToApub for Post { +impl ToApub for Post { + type Response = Page; + // Turn a Lemmy post into an ActivityPub page that can be sent out over the network. fn to_apub(&self, conn: &PgConnection) -> Result { let mut page = Page::default(); @@ -53,7 +55,9 @@ impl ToApub for Post { } } -impl FromApub for PostForm { +impl FromApub for PostForm { + type ApubType = Page; + /// Parse an ActivityPub page received from another instance into a Lemmy post. fn from_apub(page: &Page, conn: &PgConnection) -> Result { let oprops = &page.object_props; diff --git a/server/src/apub/user.rs b/server/src/apub/user.rs index 03492f70a..274c70a94 100644 --- a/server/src/apub/user.rs +++ b/server/src/apub/user.rs @@ -5,7 +5,9 @@ pub struct UserQuery { user_name: String, } -impl ToApub for User_ { +impl ToApub for User_ { + type Response = PersonExt; + // Turn a Lemmy Community into an ActivityPub group that can be sent out over the network. fn to_apub(&self, _conn: &PgConnection) -> Result { // TODO go through all these to_string and to_owned() @@ -50,7 +52,8 @@ impl ActorType for User_ { } } -impl FromApub for UserForm { +impl FromApub for UserForm { + type ApubType = PersonExt; /// Parse an ActivityPub person received from another instance into a Lemmy user. fn from_apub(person: &PersonExt, _conn: &PgConnection) -> Result { let oprops = &person.base.base.object_props; From 4e8dcb27039f96c5b2a32a2e70d2e19c29d4c0ac Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 24 Apr 2020 22:02:12 -0400 Subject: [PATCH 5/7] Removing run-tests TODO. --- docker/federation-test/run-tests.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/federation-test/run-tests.sh b/docker/federation-test/run-tests.sh index 43e2f9093..4206f060f 100755 --- a/docker/federation-test/run-tests.sh +++ b/docker/federation-test/run-tests.sh @@ -9,8 +9,6 @@ sudo docker build ../../ --file ../federation/Dockerfile --tag lemmy-federation: sudo docker-compose --file ../federation/docker-compose.yml --project-directory . up -d -# TODO: need to wait until the instances are initialised - pushd ../../ui yarn echo "Waiting for Lemmy to start..." From 122bb3857a0d1c60fe8c9fbf80c2191b864c356d Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 24 Apr 2020 22:34:51 -0400 Subject: [PATCH 6/7] Adding get_public_key_ext() to ActorType trait. --- server/src/apub/community.rs | 12 +++++------- server/src/apub/community_inbox.rs | 4 ++-- server/src/apub/mod.rs | 11 +++++++++++ server/src/apub/user.rs | 12 +++++------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index 05e004eee..e74a5fd14 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -38,13 +38,7 @@ impl ToApub for Community { .set_outbox(self.get_outbox_url())? .set_followers(self.get_followers_url())?; - let public_key = PublicKey { - id: format!("{}#main-key", self.actor_id), - owner: self.actor_id.to_owned(), - public_key_pem: self.public_key.to_owned().unwrap(), - }; - - Ok(group.extend(actor_props).extend(public_key.to_ext())) + Ok(group.extend(actor_props).extend(self.get_public_key_ext())) } } @@ -52,6 +46,10 @@ impl ActorType for Community { fn actor_id(&self) -> String { self.actor_id.to_owned() } + + fn public_key(&self) -> String { + self.public_key.to_owned().unwrap() + } } impl FromApub for CommunityForm { diff --git a/server/src/apub/community_inbox.rs b/server/src/apub/community_inbox.rs index af6d39e12..6931cdf1a 100644 --- a/server/src/apub/community_inbox.rs +++ b/server/src/apub/community_inbox.rs @@ -60,8 +60,8 @@ fn handle_follow( user_id: user.id, }; - // This will fail if they're already a follower - CommunityFollower::follow(&conn, &community_follower_form)?; + // This will fail if they're already a follower, but ignore the error. + CommunityFollower::follow(&conn, &community_follower_form).ok(); accept_follow(&follow, &conn)?; Ok(HttpResponse::Ok().finish()) diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs index 057929688..9c02d1071 100644 --- a/server/src/apub/mod.rs +++ b/server/src/apub/mod.rs @@ -141,6 +141,8 @@ pub trait FromApub { pub trait ActorType { fn actor_id(&self) -> String; + fn public_key(&self) -> String; + fn get_inbox_url(&self) -> String { format!("{}/inbox", &self.actor_id()) } @@ -157,4 +159,13 @@ pub trait ActorType { fn get_liked_url(&self) -> String { format!("{}/liked", &self.actor_id()) } + + fn get_public_key_ext(&self) -> PublicKeyExtension { + PublicKey { + id: format!("{}#main-key", self.actor_id()), + owner: self.actor_id(), + public_key_pem: self.public_key(), + } + .to_ext() + } } diff --git a/server/src/apub/user.rs b/server/src/apub/user.rs index 274c70a94..88238b5d4 100644 --- a/server/src/apub/user.rs +++ b/server/src/apub/user.rs @@ -36,13 +36,7 @@ impl ToApub for User_ { .set_following(self.get_following_url())? .set_liked(self.get_liked_url())?; - let public_key = PublicKey { - id: format!("{}#main-key", self.actor_id), - owner: self.actor_id.to_owned(), - public_key_pem: self.public_key.to_owned().unwrap(), - }; - - Ok(person.extend(actor_props).extend(public_key.to_ext())) + Ok(person.extend(actor_props).extend(self.get_public_key_ext())) } } @@ -50,6 +44,10 @@ impl ActorType for User_ { fn actor_id(&self) -> String { self.actor_id.to_owned() } + + fn public_key(&self) -> String { + self.public_key.to_owned().unwrap() + } } impl FromApub for UserForm { From c16fb9d00abf2a959620d9a65781e0e56659217f Mon Sep 17 00:00:00 2001 From: Dessalines Date: Sat, 25 Apr 2020 11:49:15 -0400 Subject: [PATCH 7/7] Adding back in post fetching. --- server/src/apub/fetcher.rs | 22 +++++++++++++--------- server/src/apub/mod.rs | 1 + 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/server/src/apub/fetcher.rs b/server/src/apub/fetcher.rs index b8aaeaaf9..e07e410ba 100644 --- a/server/src/apub/fetcher.rs +++ b/server/src/apub/fetcher.rs @@ -39,6 +39,7 @@ where pub enum SearchAcceptedObjects { Person(Box), Group(Box), + Page(Box), } /// Attempt to parse the query as URL, and fetch an ActivityPub object from it. @@ -69,6 +70,10 @@ pub fn search_by_apub_id(query: &str, conn: &PgConnection) -> Result { + let p = upsert_post(&PostForm::from_apub(&p, conn)?, conn)?; + response.posts = vec![PostView::read(conn, p.id, None)?]; + } } Ok(response) } @@ -137,15 +142,14 @@ pub fn get_or_fetch_and_upsert_remote_community( } } -// TODO Maybe add post, comment searching / caching at a later time -// fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result { -// let existing = Post::read_from_apub_id(conn, &post_form.ap_id); -// match existing { -// Err(NotFound {}) => Ok(Post::create(conn, &post_form)?), -// Ok(p) => Ok(Post::update(conn, p.id, &post_form)?), -// Err(e) => Err(Error::from(e)), -// } -// } +fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result { + let existing = Post::read_from_apub_id(conn, &post_form.ap_id); + match existing { + Err(NotFound {}) => Ok(Post::create(conn, &post_form)?), + Ok(p) => Ok(Post::update(conn, p.id, &post_form)?), + Err(e) => Err(Error::from(e)), + } +} // TODO It should not be fetching data from a community outbox. // All posts, comments, comment likes, etc should be posts to our community_inbox diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs index 9c02d1071..671f8c5ac 100644 --- a/server/src/apub/mod.rs +++ b/server/src/apub/mod.rs @@ -39,6 +39,7 @@ use crate::api::site::SearchResponse; use crate::db::community::{Community, CommunityFollower, CommunityFollowerForm, CommunityForm}; use crate::db::community_view::{CommunityFollowerView, CommunityView}; use crate::db::post::{Post, PostForm}; +use crate::db::post_view::PostView; use crate::db::user::{UserForm, User_}; use crate::db::user_view::UserView; use crate::db::{Crud, Followable, SearchType};