grishka/lemmy
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Get users federated

Browse source
This commit is contained in:
Felix Ableitner 2020-04-07 23:02:32 +02:00
parent eab4edc246
commit d3138f1dc9
7 changed files with 112 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
server/src/apub/community.rs
View file

@ -1,11 +1,13 @@
use crate::apub::puller::fetch_remote_object; use crate::apub::puller::{fetch_remote_object, fetch_remote_user};
use crate::apub::*; use crate::apub::*;
use crate::convert_datetime;
use crate::db::community::{Community, CommunityForm}; use crate::db::community::{Community, CommunityForm};
use crate::db::community_view::CommunityFollowerView; use crate::db::community_view::CommunityFollowerView;
use crate::db::establish_unpooled_connection; use crate::db::establish_unpooled_connection;
use crate::db::post::Post; use crate::db::post::Post;
use crate::db::user::User_;
use crate::db::Crud;
use crate::settings::Settings; use crate::settings::Settings;
use crate::{convert_datetime, naive_now};
use activitystreams::actor::properties::ApActorProperties; use activitystreams::actor::properties::ApActorProperties;
use activitystreams::collection::OrderedCollection; use activitystreams::collection::OrderedCollection;
use activitystreams::{ use activitystreams::{
@ -30,9 +32,9 @@ pub async fn get_apub_community_list(
db: web::Data<Pool<ConnectionManager<PgConnection>>>, db: web::Data<Pool<ConnectionManager<PgConnection>>>,
) -> Result<HttpResponse<Body>, Error> { ) -> Result<HttpResponse<Body>, Error> {
// TODO: implement pagination // TODO: implement pagination
let communities = Community::list(&db.get().unwrap())? let communities = Community::list_local(&db.get().unwrap())?
.iter() .iter()
.map(|c| c.as_group()) .map(|c| c.as_group(&db.get().unwrap()))
.collect::<Result<Vec<GroupExt>, Error>>()?; .collect::<Result<Vec<GroupExt>, Error>>()?;
let mut collection = UnorderedCollection::default(); let mut collection = UnorderedCollection::default();
let oprops: &mut ObjectProperties = collection.as_mut(); let oprops: &mut ObjectProperties = collection.as_mut();
@ -50,21 +52,19 @@ pub async fn get_apub_community_list(
} }
impl Community { impl Community {
fn as_group(&self) -> Result<GroupExt, Error> { fn as_group(&self, conn: &PgConnection) -> Result<GroupExt, Error> {
let base_url = make_apub_endpoint(EndpointType::Community, &self.name); let base_url = make_apub_endpoint(EndpointType::Community, &self.name);
let mut group = Group::default(); let mut group = Group::default();
let oprops: &mut ObjectProperties = group.as_mut(); let oprops: &mut ObjectProperties = group.as_mut();
let creator = User_::read(conn, self.creator_id)?;
oprops oprops
.set_context_xsd_any_uri(context())? .set_context_xsd_any_uri(context())?
.set_id(base_url.to_owned())? .set_id(base_url.to_owned())?
.set_name_xsd_string(self.name.to_owned())? .set_name_xsd_string(self.name.to_owned())?
.set_published(convert_datetime(self.published))? .set_published(convert_datetime(self.published))?
.set_attributed_to_xsd_any_uri(make_apub_endpoint( .set_attributed_to_xsd_any_uri(make_apub_endpoint(EndpointType::User, &creator.name))?;
EndpointType::User,
&self.creator_id.to_string(),
))?;
if let Some(u) = self.updated.to_owned() { if let Some(u) = self.updated.to_owned() {
oprops.set_updated(convert_datetime(u))?; oprops.set_updated(convert_datetime(u))?;
@ -86,19 +86,23 @@ impl Community {
} }
impl CommunityForm { impl CommunityForm {
pub fn from_group(group: &GroupExt) -> Result<Self, Error> { pub fn from_group(group: &GroupExt, conn: &PgConnection) -> Result<Self, Error> {
let followers_uri = &group.extension.get_followers().unwrap().to_string(); let followers_uri = &group.extension.get_followers().unwrap().to_string();
let outbox_uri = &group.extension.get_outbox().to_string(); let outbox_uri = &group.extension.get_outbox().to_string();
let _outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?; let _outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
let _followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?; let _followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
let oprops = &group.base.object_props; let oprops = &group.base.object_props;
let aprops = &group.extension; let aprops = &group.extension;
let creator = fetch_remote_user(
&oprops.get_attributed_to_xsd_any_uri().unwrap().to_string(),
conn,
)?;
Ok(CommunityForm { Ok(CommunityForm {
name: oprops.get_name_xsd_string().unwrap().to_string(), name: oprops.get_name_xsd_string().unwrap().to_string(),
title: aprops.get_preferred_username().unwrap().to_string(), title: aprops.get_preferred_username().unwrap().to_string(),
description: oprops.get_summary_xsd_string().map(|s| s.to_string()), description: oprops.get_summary_xsd_string().map(|s| s.to_string()),
category_id: 1, category_id: 1,
creator_id: 2, //community.object_props.get_attributed_to_xsd_any_uri() creator_id: creator.id,
removed: None, removed: None,
published: oprops published: oprops
.get_published() .get_published()
@ -112,7 +116,7 @@ impl CommunityForm {
local: false, local: false,
private_key: None, private_key: None,
public_key: None, public_key: None,
last_refreshed_at: None, last_refreshed_at: Some(naive_now()),
}) })
} }
} }
@ -122,7 +126,7 @@ pub async fn get_apub_community_http(
db: web::Data<Pool<ConnectionManager<PgConnection>>>, db: web::Data<Pool<ConnectionManager<PgConnection>>>,
) -> Result<HttpResponse<Body>, Error> { ) -> Result<HttpResponse<Body>, Error> {
let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?; let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?;
let c = community.as_group()?; let c = community.as_group(&db.get().unwrap())?;
Ok(create_apub_response(&c)) Ok(create_apub_response(&c))
} }
@ -156,9 +160,9 @@ pub async fn get_apub_community_outbox(
let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?; let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?;
let base_url = make_apub_endpoint(EndpointType::Community, &community.name); let base_url = make_apub_endpoint(EndpointType::Community, &community.name);
let connection = establish_unpooled_connection(); let conn = establish_unpooled_connection();
//As we are an object, we validated that the community id was valid //As we are an object, we validated that the community id was valid
let community_posts: Vec<Post> = Post::list_for_community(&connection, community.id)?; let community_posts: Vec<Post> = Post::list_for_community(&conn, community.id)?;
let mut collection = OrderedCollection::default(); let mut collection = OrderedCollection::default();
let oprops: &mut ObjectProperties = collection.as_mut(); let oprops: &mut ObjectProperties = collection.as_mut();
@ -170,7 +174,7 @@ pub async fn get_apub_community_outbox(
.set_many_items_base_boxes( .set_many_items_base_boxes(
community_posts community_posts
.iter() .iter()
.map(|c| c.as_page().unwrap()) .map(|c| c.as_page(&conn).unwrap())
.collect(), .collect(),
)? )?
.set_total_items(community_posts.len() as u64)?; .set_total_items(community_posts.len() as u64)?;

3
server/src/apub/mod.rs
View file

@ -5,13 +5,14 @@ pub mod user;
use crate::Settings; use crate::Settings;
use openssl::{pkey::PKey, rsa::Rsa}; use openssl::{pkey::PKey, rsa::Rsa};
use activitystreams::actor::{properties::ApActorProperties, Group}; use activitystreams::actor::{properties::ApActorProperties, Group, Person};
use activitystreams::ext::Ext; use activitystreams::ext::Ext;
use actix_web::body::Body; use actix_web::body::Body;
use actix_web::HttpResponse; use actix_web::HttpResponse;
use url::Url; use url::Url;
type GroupExt = Ext<Group, ApActorProperties>; type GroupExt = Ext<Group, ApActorProperties>;
type PersonExt = Ext<Person, ApActorProperties>;
static APUB_JSON_CONTENT_TYPE: &str = "application/activity+json"; static APUB_JSON_CONTENT_TYPE: &str = "application/activity+json";

25
server/src/apub/post.rs
View file

@ -1,7 +1,10 @@
use crate::apub::puller::fetch_remote_user;
use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType}; use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType};
use crate::convert_datetime; use crate::convert_datetime;
use crate::db::post::{Post, PostForm}; use crate::db::post::{Post, PostForm};
use activitystreams::{object::properties::ObjectProperties, object::Page}; use crate::db::user::User_;
use crate::db::Crud;
use activitystreams::{context, object::properties::ObjectProperties, object::Page};
use actix_web::body::Body; use actix_web::body::Body;
use actix_web::web::Path; use actix_web::web::Path;
use actix_web::{web, HttpResponse}; use actix_web::{web, HttpResponse};
@ -21,25 +24,23 @@ pub async fn get_apub_post(
) -> Result<HttpResponse<Body>, Error> { ) -> Result<HttpResponse<Body>, Error> {
let id = info.post_id.parse::<i32>()?; let id = info.post_id.parse::<i32>()?;
let post = Post::read(&&db.get()?, id)?; let post = Post::read(&&db.get()?, id)?;
Ok(create_apub_response(&post.as_page()?)) Ok(create_apub_response(&post.as_page(&db.get().unwrap())?))
} }
impl Post { impl Post {
pub fn as_page(&self) -> Result<Page, Error> { pub fn as_page(&self, conn: &PgConnection) -> Result<Page, Error> {
let base_url = make_apub_endpoint(EndpointType::Post, &self.id.to_string()); let base_url = make_apub_endpoint(EndpointType::Post, &self.id.to_string());
let mut page = Page::default(); let mut page = Page::default();
let oprops: &mut ObjectProperties = page.as_mut(); let oprops: &mut ObjectProperties = page.as_mut();
let creator = User_::read(conn, self.creator_id)?;
oprops oprops
// Not needed when the Post is embedded in a collection (like for community outbox) // Not needed when the Post is embedded in a collection (like for community outbox)
//.set_context_xsd_any_uri(context())? .set_context_xsd_any_uri(context())?
.set_id(base_url)? .set_id(base_url)?
.set_name_xsd_string(self.name.to_owned())? .set_name_xsd_string(self.name.to_owned())?
.set_published(convert_datetime(self.published))? .set_published(convert_datetime(self.published))?
.set_attributed_to_xsd_any_uri(make_apub_endpoint( .set_attributed_to_xsd_any_uri(make_apub_endpoint(EndpointType::User, &creator.name))?;
EndpointType::User,
&self.creator_id.to_string(),
))?;
if let Some(body) = &self.body { if let Some(body) = &self.body {
oprops.set_content_xsd_string(body.to_owned())?; oprops.set_content_xsd_string(body.to_owned())?;
@ -61,13 +62,17 @@ impl Post {
} }
impl PostForm { impl PostForm {
pub fn from_page(page: &Page) -> Result<PostForm, Error> { pub fn from_page(page: &Page, conn: &PgConnection) -> Result<PostForm, Error> {
let oprops = &page.object_props; let oprops = &page.object_props;
let creator = fetch_remote_user(
&oprops.get_attributed_to_xsd_any_uri().unwrap().to_string(),
conn,
)?;
Ok(PostForm { Ok(PostForm {
name: oprops.get_name_xsd_string().unwrap().to_string(), name: oprops.get_name_xsd_string().unwrap().to_string(),
url: oprops.get_url_xsd_any_uri().map(|u| u.to_string()), url: oprops.get_url_xsd_any_uri().map(|u| u.to_string()),
body: oprops.get_content_xsd_string().map(|c| c.to_string()), body: oprops.get_content_xsd_string().map(|c| c.to_string()),
creator_id: 2, creator_id: creator.id,
community_id: -1, community_id: -1,
removed: None, removed: None,
locked: None, locked: None,

42
server/src/apub/puller.rs
View file

@ -1,6 +1,7 @@
use crate::apub::*; use crate::apub::*;
use crate::db::community::{Community, CommunityForm}; use crate::db::community::{Community, CommunityForm};
use crate::db::post::{Post, PostForm}; use crate::db::post::{Post, PostForm};
use crate::db::user::{UserForm, User_};
use crate::db::Crud; use crate::db::Crud;
use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown}; use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown};
use crate::settings::Settings; use crate::settings::Settings;
@ -23,7 +24,10 @@ fn fetch_node_info(domain: &str) -> Result<NodeInfo, Error> {
Ok(fetch_remote_object::<NodeInfo>(&well_known.links.href)?) Ok(fetch_remote_object::<NodeInfo>(&well_known.links.href)?)
} }
fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityForm>, Error> { fn fetch_communities_from_instance(
domain: &str,
conn: &PgConnection,
) -> Result<Vec<CommunityForm>, Error> {
let node_info = fetch_node_info(domain)?; let node_info = fetch_node_info(domain)?;
if let Some(community_list_url) = node_info.metadata.community_list_url { if let Some(community_list_url) = node_info.metadata.community_list_url {
@ -35,7 +39,7 @@ fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityForm>, E
let communities: Result<Vec<CommunityForm>, Error> = object_boxes let communities: Result<Vec<CommunityForm>, Error> = object_boxes
.map(|c| { .map(|c| {
let group = c.to_owned().to_concrete::<GroupExt>()?; let group = c.to_owned().to_concrete::<GroupExt>()?;
CommunityForm::from_group(&group) CommunityForm::from_group(&group, conn)
}) })
.collect(); .collect();
Ok(communities?) Ok(communities?)
@ -47,6 +51,7 @@ fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityForm>, E
} }
} }
// TODO: add an optional param last_updated and only fetch if its too old
pub fn fetch_remote_object<Response>(uri: &str) -> Result<Response, Error> pub fn fetch_remote_object<Response>(uri: &str) -> Result<Response, Error>
where where
Response: for<'de> Deserialize<'de>, Response: for<'de> Deserialize<'de>,
@ -68,7 +73,11 @@ where
Ok(res) Ok(res)
} }
fn fetch_remote_community_posts(instance: &str, community: &str) -> Result<Vec<PostForm>, Error> { fn fetch_remote_community_posts(
instance: &str,
community: &str,
conn: &PgConnection,
) -> Result<Vec<PostForm>, Error> {
let endpoint = format!("http://{}/federation/c/{}", instance, community); let endpoint = format!("http://{}/federation/c/{}", instance, community);
let community = fetch_remote_object::<GroupExt>(&endpoint)?; let community = fetch_remote_object::<GroupExt>(&endpoint)?;
let outbox_uri = &community.extension.get_outbox().to_string(); let outbox_uri = &community.extension.get_outbox().to_string();
@ -79,17 +88,28 @@ fn fetch_remote_community_posts(instance: &str, community: &str) -> Result<Vec<P
.unwrap() .unwrap()
.map(|obox: &BaseBox| { .map(|obox: &BaseBox| {
let page = obox.clone().to_concrete::<Page>().unwrap(); let page = obox.clone().to_concrete::<Page>().unwrap();
PostForm::from_page(&page) PostForm::from_page(&page, conn)
}) })
.collect::<Result<Vec<PostForm>, Error>>()?; .collect::<Result<Vec<PostForm>, Error>>()?;
Ok(posts) Ok(posts)
} }
pub fn fetch_remote_user(apub_id: &str, conn: &PgConnection) -> Result<User_, Error> {
let person = fetch_remote_object::<PersonExt>(apub_id)?;
let uf = UserForm::from_person(&person)?;
let existing = User_::read_from_apub_id(conn, &uf.actor_id);
Ok(match existing {
// TODO: should make sure that this is actually a `NotFound` error
Err(_) => User_::create(conn, &uf)?,
Ok(u) => User_::update(conn, u.id, &uf)?,
})
}
// TODO: in the future, this should only be done when an instance is followed for the first time // TODO: in the future, this should only be done when an instance is followed for the first time
// after that, we should rely in the inbox, and fetch on demand when needed // after that, we should rely in the inbox, and fetch on demand when needed
pub fn fetch_all(conn: &PgConnection) -> Result<(), Error> { pub fn fetch_all(conn: &PgConnection) -> Result<(), Error> {
for instance in &get_following_instances() { for instance in &get_following_instances() {
let communities = fetch_communities_from_instance(instance)?; let communities = fetch_communities_from_instance(instance, conn)?;
for community in &communities { for community in &communities {
let existing = Community::read_from_actor_id(conn, &community.actor_id); let existing = Community::read_from_actor_id(conn, &community.actor_id);
@ -98,19 +118,15 @@ pub fn fetch_all(conn: &PgConnection) -> Result<(), Error> {
Err(_) => Community::create(conn, community)?.id, Err(_) => Community::create(conn, community)?.id,
Ok(c) => Community::update(conn, c.id, community)?.id, Ok(c) => Community::update(conn, c.id, community)?.id,
}; };
let mut posts = fetch_remote_community_posts(instance, &community.name)?; let mut posts = fetch_remote_community_posts(instance, &community.name, conn)?;
for post_ in &mut posts { for post_ in &mut posts {
post_.community_id = community_id; post_.community_id = community_id;
let existing = Post::read_from_apub_id(conn, &post_.ap_id); let existing = Post::read_from_apub_id(conn, &post_.ap_id);
match existing { match existing {
// TODO: should make sure that this is actually a `NotFound` error // TODO: should make sure that this is actually a `NotFound` error
Err(_) => { Err(_) => Post::create(conn, post_)?,
Post::create(conn, post_)?; Ok(p) => Post::update(conn, p.id, post_)?,
} };
Ok(p) => {
Post::update(conn, p.id, post_)?;
}
}
} }
} }
} }

41
server/src/apub/user.rs
View file

@ -1,6 +1,6 @@
use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType}; use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType, PersonExt};
use crate::convert_datetime; use crate::db::user::{UserForm, User_};
use crate::db::user::User_; use crate::{convert_datetime, naive_now};
use activitystreams::{ use activitystreams::{
actor::{properties::ApActorProperties, Person}, actor::{properties::ApActorProperties, Person},
context, context,
@ -25,6 +25,7 @@ pub async fn get_apub_user(
info: Path<UserQuery>, info: Path<UserQuery>,
db: web::Data<Pool<ConnectionManager<PgConnection>>>, db: web::Data<Pool<ConnectionManager<PgConnection>>>,
) -> Result<HttpResponse<Body>, Error> { ) -> Result<HttpResponse<Body>, Error> {
dbg!(&info.user_name);
let user = User_::find_by_email_or_username(&&db.get()?, &info.user_name)?; let user = User_::find_by_email_or_username(&&db.get()?, &info.user_name)?;
let base_url = make_apub_endpoint(EndpointType::User, &user.name); let base_url = make_apub_endpoint(EndpointType::User, &user.name);
@ -33,6 +34,7 @@ pub async fn get_apub_user(
oprops oprops
.set_context_xsd_any_uri(context())? .set_context_xsd_any_uri(context())?
.set_id(base_url.to_string())? .set_id(base_url.to_string())?
.set_name_xsd_string(user.name.to_owned())?
.set_published(convert_datetime(user.published))?; .set_published(convert_datetime(user.published))?;
if let Some(u) = user.updated { if let Some(u) = user.updated {
@ -53,3 +55,36 @@ pub async fn get_apub_user(
Ok(create_apub_response(&person.extend(actor_props))) Ok(create_apub_response(&person.extend(actor_props)))
} }
impl UserForm {
pub fn from_person(person: &PersonExt) -> Result<Self, Error> {
let oprops = &person.base.object_props;
let aprops = &person.extension;
Ok(UserForm {
name: oprops.get_name_xsd_string().unwrap().to_string(),
preferred_username: aprops.get_preferred_username().map(|u| u.to_string()),
password_encrypted: "".to_string(),
admin: false,
banned: false,
email: None,
avatar: None,
updated: oprops
.get_updated()
.map(|u| u.as_ref().to_owned().naive_local()),
show_nsfw: false,
theme: "".to_string(),
default_sort_type: 0,
default_listing_type: 0,
lang: "".to_string(),
show_avatars: false,
send_notifications_to_email: false,
matrix_user_id: None,
actor_id: oprops.get_id().unwrap().to_string(),
bio: oprops.get_summary_xsd_string().map(|s| s.to_string()),
local: false,
private_key: None,
public_key: None,
last_refreshed_at: Some(naive_now()),
})
}
}

4
server/src/db/community.rs
View file

@ -87,9 +87,9 @@ impl Community {
.first::<Self>(conn) .first::<Self>(conn)
} }
pub fn list(conn: &PgConnection) -> Result<Vec<Self>, Error> { pub fn list_local(conn: &PgConnection) -> Result<Vec<Self>, Error> {
use crate::schema::community::dsl::*; use crate::schema::community::dsl::*;
community.load::<Community>(conn) community.filter(local.eq(true)).load::<Community>(conn)
} }
pub fn get_url(&self) -> String { pub fn get_url(&self) -> String {

7
server/src/db/user.rs
View file

@ -34,7 +34,7 @@ pub struct User_ {
pub last_refreshed_at: chrono::NaiveDateTime, pub last_refreshed_at: chrono::NaiveDateTime,
} }
#[derive(Insertable, AsChangeset, Clone)] #[derive(Insertable, AsChangeset, Clone, Debug)]
#[table_name = "user_"] #[table_name = "user_"]
pub struct UserForm { pub struct UserForm {
pub name: String, pub name: String,
@ -119,6 +119,11 @@ impl User_ {
.set(banned.eq(ban)) .set(banned.eq(ban))
.get_result::<Self>(conn) .get_result::<Self>(conn)
} }
pub fn read_from_apub_id(conn: &PgConnection, object_id: &str) -> Result<Self, Error> {
use crate::schema::user_::dsl::*;
user_.filter(actor_id.eq(object_id)).first::<Self>(conn)
}
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]