From 61c560c12c0edb56d006ba2c6ab07013567ffab8 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 7 Apr 2020 23:02:32 +0200 Subject: [PATCH] Get users federated --- server/src/apub/community.rs | 36 +++++++++++++++++-------------- server/src/apub/mod.rs | 3 ++- server/src/apub/post.rs | 25 ++++++++++++--------- server/src/apub/puller.rs | 42 +++++++++++++++++++++++++----------- server/src/apub/user.rs | 41 ++++++++++++++++++++++++++++++++--- server/src/db/community.rs | 4 ++-- server/src/db/user.rs | 7 +++++- 7 files changed, 112 insertions(+), 46 deletions(-) diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index eb98a6f6..83895196 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -1,11 +1,13 @@ -use crate::apub::puller::fetch_remote_object; +use crate::apub::puller::{fetch_remote_object, fetch_remote_user}; use crate::apub::*; -use crate::convert_datetime; use crate::db::community::{Community, CommunityForm}; use crate::db::community_view::CommunityFollowerView; use crate::db::establish_unpooled_connection; use crate::db::post::Post; +use crate::db::user::User_; +use crate::db::Crud; use crate::settings::Settings; +use crate::{convert_datetime, naive_now}; use activitystreams::actor::properties::ApActorProperties; use activitystreams::collection::OrderedCollection; use activitystreams::{ @@ -30,9 +32,9 @@ pub async fn get_apub_community_list( db: web::Data>>, ) -> Result, Error> { // TODO: implement pagination - let communities = Community::list(&db.get().unwrap())? + let communities = Community::list_local(&db.get().unwrap())? .iter() - .map(|c| c.as_group()) + .map(|c| c.as_group(&db.get().unwrap())) .collect::, Error>>()?; let mut collection = UnorderedCollection::default(); let oprops: &mut ObjectProperties = collection.as_mut(); @@ -50,21 +52,19 @@ pub async fn get_apub_community_list( } impl Community { - fn as_group(&self) -> Result { + fn as_group(&self, conn: &PgConnection) -> Result { let base_url = make_apub_endpoint(EndpointType::Community, &self.name); let mut group = Group::default(); let oprops: &mut ObjectProperties = group.as_mut(); + let creator = User_::read(conn, self.creator_id)?; oprops .set_context_xsd_any_uri(context())? .set_id(base_url.to_owned())? .set_name_xsd_string(self.name.to_owned())? .set_published(convert_datetime(self.published))? - .set_attributed_to_xsd_any_uri(make_apub_endpoint( - EndpointType::User, - &self.creator_id.to_string(), - ))?; + .set_attributed_to_xsd_any_uri(make_apub_endpoint(EndpointType::User, &creator.name))?; if let Some(u) = self.updated.to_owned() { oprops.set_updated(convert_datetime(u))?; @@ -86,19 +86,23 @@ impl Community { } impl CommunityForm { - pub fn from_group(group: &GroupExt) -> Result { + pub fn from_group(group: &GroupExt, conn: &PgConnection) -> Result { let followers_uri = &group.extension.get_followers().unwrap().to_string(); let outbox_uri = &group.extension.get_outbox().to_string(); let _outbox = fetch_remote_object::(outbox_uri)?; let _followers = fetch_remote_object::(followers_uri)?; let oprops = &group.base.object_props; let aprops = &group.extension; + let creator = fetch_remote_user( + &oprops.get_attributed_to_xsd_any_uri().unwrap().to_string(), + conn, + )?; Ok(CommunityForm { name: oprops.get_name_xsd_string().unwrap().to_string(), title: aprops.get_preferred_username().unwrap().to_string(), description: oprops.get_summary_xsd_string().map(|s| s.to_string()), category_id: 1, - creator_id: 2, //community.object_props.get_attributed_to_xsd_any_uri() + creator_id: creator.id, removed: None, published: oprops .get_published() @@ -112,7 +116,7 @@ impl CommunityForm { local: false, private_key: None, public_key: None, - last_refreshed_at: None, + last_refreshed_at: Some(naive_now()), }) } } @@ -122,7 +126,7 @@ pub async fn get_apub_community_http( db: web::Data>>, ) -> Result, Error> { let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?; - let c = community.as_group()?; + let c = community.as_group(&db.get().unwrap())?; Ok(create_apub_response(&c)) } @@ -156,9 +160,9 @@ pub async fn get_apub_community_outbox( let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?; let base_url = make_apub_endpoint(EndpointType::Community, &community.name); - let connection = establish_unpooled_connection(); + let conn = establish_unpooled_connection(); //As we are an object, we validated that the community id was valid - let community_posts: Vec = Post::list_for_community(&connection, community.id)?; + let community_posts: Vec = Post::list_for_community(&conn, community.id)?; let mut collection = OrderedCollection::default(); let oprops: &mut ObjectProperties = collection.as_mut(); @@ -170,7 +174,7 @@ pub async fn get_apub_community_outbox( .set_many_items_base_boxes( community_posts .iter() - .map(|c| c.as_page().unwrap()) + .map(|c| c.as_page(&conn).unwrap()) .collect(), )? .set_total_items(community_posts.len() as u64)?; diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs index 7de8f7b1..cbfbe735 100644 --- a/server/src/apub/mod.rs +++ b/server/src/apub/mod.rs @@ -5,13 +5,14 @@ pub mod user; use crate::Settings; use openssl::{pkey::PKey, rsa::Rsa}; -use activitystreams::actor::{properties::ApActorProperties, Group}; +use activitystreams::actor::{properties::ApActorProperties, Group, Person}; use activitystreams::ext::Ext; use actix_web::body::Body; use actix_web::HttpResponse; use url::Url; type GroupExt = Ext; +type PersonExt = Ext; static APUB_JSON_CONTENT_TYPE: &str = "application/activity+json"; diff --git a/server/src/apub/post.rs b/server/src/apub/post.rs index 4c788418..2db5d8ec 100644 --- a/server/src/apub/post.rs +++ b/server/src/apub/post.rs @@ -1,7 +1,10 @@ +use crate::apub::puller::fetch_remote_user; use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType}; use crate::convert_datetime; use crate::db::post::{Post, PostForm}; -use activitystreams::{object::properties::ObjectProperties, object::Page}; +use crate::db::user::User_; +use crate::db::Crud; +use activitystreams::{context, object::properties::ObjectProperties, object::Page}; use actix_web::body::Body; use actix_web::web::Path; use actix_web::{web, HttpResponse}; @@ -21,25 +24,23 @@ pub async fn get_apub_post( ) -> Result, Error> { let id = info.post_id.parse::()?; let post = Post::read(&&db.get()?, id)?; - Ok(create_apub_response(&post.as_page()?)) + Ok(create_apub_response(&post.as_page(&db.get().unwrap())?)) } impl Post { - pub fn as_page(&self) -> Result { + pub fn as_page(&self, conn: &PgConnection) -> Result { let base_url = make_apub_endpoint(EndpointType::Post, &self.id.to_string()); let mut page = Page::default(); let oprops: &mut ObjectProperties = page.as_mut(); + let creator = User_::read(conn, self.creator_id)?; oprops // Not needed when the Post is embedded in a collection (like for community outbox) - //.set_context_xsd_any_uri(context())? + .set_context_xsd_any_uri(context())? .set_id(base_url)? .set_name_xsd_string(self.name.to_owned())? .set_published(convert_datetime(self.published))? - .set_attributed_to_xsd_any_uri(make_apub_endpoint( - EndpointType::User, - &self.creator_id.to_string(), - ))?; + .set_attributed_to_xsd_any_uri(make_apub_endpoint(EndpointType::User, &creator.name))?; if let Some(body) = &self.body { oprops.set_content_xsd_string(body.to_owned())?; @@ -61,13 +62,17 @@ impl Post { } impl PostForm { - pub fn from_page(page: &Page) -> Result { + pub fn from_page(page: &Page, conn: &PgConnection) -> Result { let oprops = &page.object_props; + let creator = fetch_remote_user( + &oprops.get_attributed_to_xsd_any_uri().unwrap().to_string(), + conn, + )?; Ok(PostForm { name: oprops.get_name_xsd_string().unwrap().to_string(), url: oprops.get_url_xsd_any_uri().map(|u| u.to_string()), body: oprops.get_content_xsd_string().map(|c| c.to_string()), - creator_id: 2, + creator_id: creator.id, community_id: -1, removed: None, locked: None, diff --git a/server/src/apub/puller.rs b/server/src/apub/puller.rs index 9796faf4..5b73dd73 100644 --- a/server/src/apub/puller.rs +++ b/server/src/apub/puller.rs @@ -1,6 +1,7 @@ use crate::apub::*; use crate::db::community::{Community, CommunityForm}; use crate::db::post::{Post, PostForm}; +use crate::db::user::{UserForm, User_}; use crate::db::Crud; use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown}; use crate::settings::Settings; @@ -23,7 +24,10 @@ fn fetch_node_info(domain: &str) -> Result { Ok(fetch_remote_object::(&well_known.links.href)?) } -fn fetch_communities_from_instance(domain: &str) -> Result, Error> { +fn fetch_communities_from_instance( + domain: &str, + conn: &PgConnection, +) -> Result, Error> { let node_info = fetch_node_info(domain)?; if let Some(community_list_url) = node_info.metadata.community_list_url { @@ -35,7 +39,7 @@ fn fetch_communities_from_instance(domain: &str) -> Result, E let communities: Result, Error> = object_boxes .map(|c| { let group = c.to_owned().to_concrete::()?; - CommunityForm::from_group(&group) + CommunityForm::from_group(&group, conn) }) .collect(); Ok(communities?) @@ -47,6 +51,7 @@ fn fetch_communities_from_instance(domain: &str) -> Result, E } } +// TODO: add an optional param last_updated and only fetch if its too old pub fn fetch_remote_object(uri: &str) -> Result where Response: for<'de> Deserialize<'de>, @@ -68,7 +73,11 @@ where Ok(res) } -fn fetch_remote_community_posts(instance: &str, community: &str) -> Result, Error> { +fn fetch_remote_community_posts( + instance: &str, + community: &str, + conn: &PgConnection, +) -> Result, Error> { let endpoint = format!("http://{}/federation/c/{}", instance, community); let community = fetch_remote_object::(&endpoint)?; let outbox_uri = &community.extension.get_outbox().to_string(); @@ -79,17 +88,28 @@ fn fetch_remote_community_posts(instance: &str, community: &str) -> Result().unwrap(); - PostForm::from_page(&page) + PostForm::from_page(&page, conn) }) .collect::, Error>>()?; Ok(posts) } +pub fn fetch_remote_user(apub_id: &str, conn: &PgConnection) -> Result { + let person = fetch_remote_object::(apub_id)?; + let uf = UserForm::from_person(&person)?; + let existing = User_::read_from_apub_id(conn, &uf.actor_id); + Ok(match existing { + // TODO: should make sure that this is actually a `NotFound` error + Err(_) => User_::create(conn, &uf)?, + Ok(u) => User_::update(conn, u.id, &uf)?, + }) +} + // TODO: in the future, this should only be done when an instance is followed for the first time // after that, we should rely in the inbox, and fetch on demand when needed pub fn fetch_all(conn: &PgConnection) -> Result<(), Error> { for instance in &get_following_instances() { - let communities = fetch_communities_from_instance(instance)?; + let communities = fetch_communities_from_instance(instance, conn)?; for community in &communities { let existing = Community::read_from_actor_id(conn, &community.actor_id); @@ -98,19 +118,15 @@ pub fn fetch_all(conn: &PgConnection) -> Result<(), Error> { Err(_) => Community::create(conn, community)?.id, Ok(c) => Community::update(conn, c.id, community)?.id, }; - let mut posts = fetch_remote_community_posts(instance, &community.name)?; + let mut posts = fetch_remote_community_posts(instance, &community.name, conn)?; for post_ in &mut posts { post_.community_id = community_id; let existing = Post::read_from_apub_id(conn, &post_.ap_id); match existing { // TODO: should make sure that this is actually a `NotFound` error - Err(_) => { - Post::create(conn, post_)?; - } - Ok(p) => { - Post::update(conn, p.id, post_)?; - } - } + Err(_) => Post::create(conn, post_)?, + Ok(p) => Post::update(conn, p.id, post_)?, + }; } } } diff --git a/server/src/apub/user.rs b/server/src/apub/user.rs index 6e196bb2..be02922f 100644 --- a/server/src/apub/user.rs +++ b/server/src/apub/user.rs @@ -1,6 +1,6 @@ -use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType}; -use crate::convert_datetime; -use crate::db::user::User_; +use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType, PersonExt}; +use crate::db::user::{UserForm, User_}; +use crate::{convert_datetime, naive_now}; use activitystreams::{ actor::{properties::ApActorProperties, Person}, context, @@ -25,6 +25,7 @@ pub async fn get_apub_user( info: Path, db: web::Data>>, ) -> Result, Error> { + dbg!(&info.user_name); let user = User_::find_by_email_or_username(&&db.get()?, &info.user_name)?; let base_url = make_apub_endpoint(EndpointType::User, &user.name); @@ -33,6 +34,7 @@ pub async fn get_apub_user( oprops .set_context_xsd_any_uri(context())? .set_id(base_url.to_string())? + .set_name_xsd_string(user.name.to_owned())? .set_published(convert_datetime(user.published))?; if let Some(u) = user.updated { @@ -53,3 +55,36 @@ pub async fn get_apub_user( Ok(create_apub_response(&person.extend(actor_props))) } + +impl UserForm { + pub fn from_person(person: &PersonExt) -> Result { + let oprops = &person.base.object_props; + let aprops = &person.extension; + Ok(UserForm { + name: oprops.get_name_xsd_string().unwrap().to_string(), + preferred_username: aprops.get_preferred_username().map(|u| u.to_string()), + password_encrypted: "".to_string(), + admin: false, + banned: false, + email: None, + avatar: None, + updated: oprops + .get_updated() + .map(|u| u.as_ref().to_owned().naive_local()), + show_nsfw: false, + theme: "".to_string(), + default_sort_type: 0, + default_listing_type: 0, + lang: "".to_string(), + show_avatars: false, + send_notifications_to_email: false, + matrix_user_id: None, + actor_id: oprops.get_id().unwrap().to_string(), + bio: oprops.get_summary_xsd_string().map(|s| s.to_string()), + local: false, + private_key: None, + public_key: None, + last_refreshed_at: Some(naive_now()), + }) + } +} diff --git a/server/src/db/community.rs b/server/src/db/community.rs index 1b81ac8f..ca2fc120 100644 --- a/server/src/db/community.rs +++ b/server/src/db/community.rs @@ -87,9 +87,9 @@ impl Community { .first::(conn) } - pub fn list(conn: &PgConnection) -> Result, Error> { + pub fn list_local(conn: &PgConnection) -> Result, Error> { use crate::schema::community::dsl::*; - community.load::(conn) + community.filter(local.eq(true)).load::(conn) } pub fn get_url(&self) -> String { diff --git a/server/src/db/user.rs b/server/src/db/user.rs index 7b10d874..c7b1b2d3 100644 --- a/server/src/db/user.rs +++ b/server/src/db/user.rs @@ -34,7 +34,7 @@ pub struct User_ { pub last_refreshed_at: chrono::NaiveDateTime, } -#[derive(Insertable, AsChangeset, Clone)] +#[derive(Insertable, AsChangeset, Clone, Debug)] #[table_name = "user_"] pub struct UserForm { pub name: String, @@ -119,6 +119,11 @@ impl User_ { .set(banned.eq(ban)) .get_result::(conn) } + + pub fn read_from_apub_id(conn: &PgConnection, object_id: &str) -> Result { + use crate::schema::user_::dsl::*; + user_.filter(actor_id.eq(object_id)).first::(conn) + } } #[derive(Debug, Serialize, Deserialize)]