grishka/lemmy
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Store remote communities/posts in db, federate posts!

Browse source
This commit is contained in:
Felix Ableitner 2020-04-07 18:47:19 +02:00
parent 3b18808434
commit cca3a52856
16 changed files with 172 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
server/src/api/community.rs
View file

@ -1,9 +1,8 @@
use super::*;
use crate::apub::puller::{fetch_all_communities, fetch_remote_community};
use crate::apub::{gen_keypair_str, make_apub_endpoint, EndpointType};
use crate::settings::Settings;
use crate::apub::{format_community_name, gen_keypair_str, make_apub_endpoint, EndpointType};
use diesel::PgConnection;
use std::str::FromStr;
use url::Url;
#[derive(Serialize, Deserialize)]
pub struct GetCommunity {
@ -41,7 +40,6 @@ pub struct ListCommunities {
pub page: Option<i64>,
pub limit: Option<i64>,
pub auth: Option<String>,
pub local_only: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug)]
@ -121,13 +119,6 @@ impl Perform<GetCommunityResponse> for Oper<GetCommunity> {
fn perform(&self, conn: &PgConnection) -> Result<GetCommunityResponse, Error> {
let data: &GetCommunity = &self.data;
if data.name.is_some()
&& Settings::get().federation.enabled
&& data.name.as_ref().unwrap().contains('@')
{
return fetch_remote_community(data.name.as_ref().unwrap());
}
let user_id: Option<i32> = match &data.auth {
Some(auth) => match Claims::decode(&auth) {
Ok(claims) => {
@ -139,25 +130,25 @@ impl Perform<GetCommunityResponse> for Oper<GetCommunity> {
None => None,
};
let community_id = match data.id {
Some(id) => id,
let community = match data.id {
Some(id) => Community::read(&conn, id)?,
None => {
match Community::read_from_name(
&conn,
data.name.to_owned().unwrap_or_else(|| "main".to_string()),
) {
Ok(community) => community.id,
Ok(community) => community,
Err(_e) => return Err(APIError::err("couldnt_find_community").into()),
}
}
};
let community_view = match CommunityView::read(&conn, community_id, user_id) {
let mut community_view = match CommunityView::read(&conn, community.id, user_id) {
Ok(community) => community,
Err(_e) => return Err(APIError::err("couldnt_find_community").into()),
};
let moderators = match CommunityModeratorView::for_community(&conn, community_id) {
let moderators = match CommunityModeratorView::for_community(&conn, community.id) {
Ok(moderators) => moderators,
Err(_e) => return Err(APIError::err("couldnt_find_community").into()),
};
@ -168,6 +159,12 @@ impl Perform<GetCommunityResponse> for Oper<GetCommunity> {
let creator_user = admins.remove(creator_index);
admins.insert(0, creator_user);
if !community.local {
let domain = Url::parse(&community.actor_id)?;
community_view.name =
format_community_name(&community_view.name.to_string(), domain.host_str().unwrap());
}
// Return the jwt
Ok(GetCommunityResponse {
community: community_view,
@ -226,6 +223,7 @@ impl Perform<CommunityResponse> for Oper<CreateCommunity> {
private_key: Some(community_private_key),
public_key: Some(community_public_key),
last_refreshed_at: None,
published: None,
};
let inserted_community = match Community::create(&conn, &community_form) {
@ -323,6 +321,7 @@ impl Perform<CommunityResponse> for Oper<EditCommunity> {
private_key: read_community.private_key,
public_key: read_community.public_key,
last_refreshed_at: None,
published: None,
};
let _updated_community = match Community::update(&conn, data.edit_id, &community_form) {
@ -358,13 +357,6 @@ impl Perform<ListCommunitiesResponse> for Oper<ListCommunities> {
fn perform(&self, conn: &PgConnection) -> Result<ListCommunitiesResponse, Error> {
let data: &ListCommunities = &self.data;
let local_only = data.local_only.unwrap_or(false);
if Settings::get().federation.enabled && !local_only {
return Ok(ListCommunitiesResponse {
communities: fetch_all_communities()?,
});
}
let user_claims: Option<Claims> = match &data.auth {
Some(auth) => match Claims::decode(&auth) {
Ok(claims) => Some(claims.claims),
@ -591,6 +583,7 @@ impl Perform<GetCommunityResponse> for Oper<TransferCommunity> {
private_key: read_community.private_key,
public_key: read_community.public_key,
last_refreshed_at: None,
published: None,
};
let _updated_community = match Community::update(&conn, data.community_id, &community_form) {

8
server/src/api/post.rs
View file

@ -1,5 +1,4 @@
use super::*;
use crate::settings::Settings;
use diesel::PgConnection;
use std::str::FromStr;
@ -133,6 +132,7 @@ impl Perform<PostResponse> for Oper<CreatePost> {
thumbnail_url: pictshare_thumbnail,
ap_id: "changeme".into(),
local: true,
published: None,
};
let inserted_post = match Post::create(&conn, &post_form) {
@ -228,11 +228,6 @@ impl Perform<GetPostsResponse> for Oper<GetPosts> {
fn perform(&self, conn: &PgConnection) -> Result<GetPostsResponse, Error> {
let data: &GetPosts = &self.data;
if Settings::get().federation.enabled {
// TODO: intercept here (but the type is wrong)
//get_remote_community_posts(get_posts.community_id.unwrap())
}
let user_claims: Option<Claims> = match &data.auth {
Some(auth) => match Claims::decode(&auth) {
Ok(claims) => Some(claims.claims),
@ -398,6 +393,7 @@ impl Perform<PostResponse> for Oper<EditPost> {
thumbnail_url: pictshare_thumbnail,
ap_id: read_post.ap_id,
local: read_post.local,
published: None,
};
let _updated_post = match Post::update(&conn, data.edit_id, &post_form) {

1
server/src/api/user.rs
View file

@ -317,6 +317,7 @@ impl Perform<LoginResponse> for Oper<Register> {
private_key: Some(community_private_key),
public_key: Some(community_public_key),
last_refreshed_at: None,
published: None,
};
Community::create(&conn, &community_form).unwrap()
}

49
server/src/apub/community.rs
View file

@ -1,8 +1,8 @@
use crate::apub::puller::fetch_remote_object;
use crate::apub::*;
use crate::convert_datetime;
use crate::db::community::Community;
use crate::db::community_view::{CommunityFollowerView, CommunityView};
use crate::db::community::{Community, CommunityForm};
use crate::db::community_view::CommunityFollowerView;
use crate::db::establish_unpooled_connection;
use crate::db::post::Post;
use crate::settings::Settings;
@ -85,47 +85,34 @@ impl Community {
}
}
impl CommunityView {
pub fn from_group(group: &GroupExt, domain: &str) -> Result<CommunityView, Error> {
impl CommunityForm {
pub fn from_group(group: &GroupExt) -> Result<Self, Error> {
let followers_uri = &group.extension.get_followers().unwrap().to_string();
let outbox_uri = &group.extension.get_outbox().to_string();
let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
let followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
let _outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
let _followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
let oprops = &group.base.object_props;
let aprops = &group.extension;
Ok(CommunityView {
// TODO: we need to merge id and name into a single thing (stuff like @user@instance.com)
id: 1337, //community.object_props.get_id()
name: format_community_name(&oprops.get_name_xsd_string().unwrap().to_string(), domain),
Ok(CommunityForm {
name: oprops.get_name_xsd_string().unwrap().to_string(),
title: aprops.get_preferred_username().unwrap().to_string(),
description: oprops.get_summary_xsd_string().map(|s| s.to_string()),
category_id: -1,
creator_id: -1, //community.object_props.get_attributed_to_xsd_any_uri()
removed: false,
category_id: 1,
creator_id: 2, //community.object_props.get_attributed_to_xsd_any_uri()
removed: None,
published: oprops
.get_published()
.unwrap()
.as_ref()
.naive_local()
.to_owned(),
.map(|u| u.as_ref().to_owned().naive_local()),
updated: oprops
.get_updated()
.map(|u| u.as_ref().to_owned().naive_local()),
deleted: false,
deleted: None,
nsfw: false,
creator_name: "".to_string(),
creator_avatar: None,
category_name: "".to_string(),
number_of_subscribers: *followers
.collection_props
.get_total_items()
.unwrap()
.as_ref() as i64,
number_of_posts: *outbox.collection_props.get_total_items().unwrap().as_ref() as i64,
number_of_comments: -1,
hot_rank: -1,
user_id: None,
subscribed: None,
actor_id: oprops.get_id().unwrap().to_string(),
local: false,
private_key: None,
public_key: None,
last_refreshed_at: None,
})
}
}

11
server/src/apub/mod.rs
View file

@ -99,14 +99,3 @@ pub fn get_following_instances() -> Vec<&'static str> {
.split(',')
.collect()
}
/// Returns a tuple of (username, domain) from an identifier like "main@dev.lemmy.ml"
fn split_identifier(identifier: &str) -> (String, String) {
let x: Vec<&str> = identifier.split('@').collect();
(x[0].replace("!", ""), x[1].to_string())
}
fn get_remote_community_uri(identifier: &str) -> String {
let (name, domain) = split_identifier(identifier);
format!("http://{}/federation/c/{}", domain, name)
}

48
server/src/apub/post.rs
View file

@ -1,7 +1,6 @@
use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType};
use crate::db::post::Post;
use crate::db::post_view::PostView;
use crate::{convert_datetime, naive_now};
use crate::convert_datetime;
use crate::db::post::{Post, PostForm};
use activitystreams::{object::properties::ObjectProperties, object::Page};
use actix_web::body::Body;
use actix_web::web::Path;
@ -61,53 +60,32 @@ impl Post {
}
}
impl PostView {
pub fn from_page(page: &Page) -> Result<PostView, Error> {
impl PostForm {
pub fn from_page(page: &Page) -> Result<PostForm, Error> {
let oprops = &page.object_props;
Ok(PostView {
id: -1,
Ok(PostForm {
name: oprops.get_name_xsd_string().unwrap().to_string(),
url: oprops.get_url_xsd_any_uri().map(|u| u.to_string()),
body: oprops.get_content_xsd_string().map(|c| c.to_string()),
creator_id: -1,
creator_id: 2,
community_id: -1,
removed: false,
locked: false,
removed: None,
locked: None,
published: oprops
.get_published()
.unwrap()
.as_ref()
.naive_local()
.to_owned(),
.map(|u| u.as_ref().to_owned().naive_local()),
updated: oprops
.get_updated()
.map(|u| u.as_ref().to_owned().naive_local()),
deleted: false,
deleted: None,
nsfw: false,
stickied: false,
stickied: None,
embed_title: None,
embed_description: None,
embed_html: None,
thumbnail_url: None,
banned: false,
banned_from_community: false,
creator_name: "".to_string(),
creator_avatar: None,
community_name: "".to_string(),
community_removed: false,
community_deleted: false,
community_nsfw: false,
number_of_comments: -1,
score: -1,
upvotes: -1,
downvotes: -1,
hot_rank: -1,
newest_activity_time: naive_now(),
user_id: None,
my_vote: None,
subscribed: None,
read: None,
saved: None,
ap_id: oprops.get_id().unwrap().to_string(),
local: false,
})
}
}

78
server/src/apub/puller.rs
View file

@ -1,16 +1,15 @@
use crate::api::community::GetCommunityResponse;
use crate::api::post::GetPostsResponse;
use crate::apub::*;
use crate::db::community_view::CommunityView;
use crate::db::post_view::PostView;
use crate::db::community::{Community, CommunityForm};
use crate::db::post::{Post, PostForm};
use crate::db::Crud;
use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown};
use crate::settings::Settings;
use activitystreams::collection::{OrderedCollection, UnorderedCollection};
use activitystreams::object::Page;
use activitystreams::BaseBox;
use diesel::PgConnection;
use failure::Error;
use isahc::prelude::*;
use log::warn;
use serde::Deserialize;
use std::time::Duration;
@ -24,7 +23,7 @@ fn fetch_node_info(domain: &str) -> Result<NodeInfo, Error> {
Ok(fetch_remote_object::<NodeInfo>(&well_known.links.href)?)
}
fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityView>, Error> {
fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityForm>, Error> {
let node_info = fetch_node_info(domain)?;
if let Some(community_list_url) = node_info.metadata.community_list_url {
@ -33,10 +32,10 @@ fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityView>, E
.collection_props
.get_many_items_base_boxes()
.unwrap();
let communities: Result<Vec<CommunityView>, Error> = object_boxes
.map(|c| -> Result<CommunityView, Error> {
let communities: Result<Vec<CommunityForm>, Error> = object_boxes
.map(|c| {
let group = c.to_owned().to_concrete::<GroupExt>()?;
CommunityView::from_group(&group, domain)
CommunityForm::from_group(&group)
})
.collect();
Ok(communities?)
@ -69,44 +68,51 @@ where
Ok(res)
}
pub fn fetch_remote_community_posts(identifier: &str) -> Result<GetPostsResponse, Error> {
let community = fetch_remote_object::<GroupExt>(&get_remote_community_uri(identifier))?;
fn fetch_remote_community_posts(instance: &str, community: &str) -> Result<Vec<PostForm>, Error> {
let endpoint = format!("http://{}/federation/c/{}", instance, community);
let community = fetch_remote_object::<GroupExt>(&endpoint)?;
let outbox_uri = &community.extension.get_outbox().to_string();
let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
let items = outbox.collection_props.get_many_items_base_boxes();
let posts: Result<Vec<PostView>, Error> = items
let posts = items
.unwrap()
.map(|obox: &BaseBox| {
let page = obox.clone().to_concrete::<Page>().unwrap();
PostView::from_page(&page)
PostForm::from_page(&page)
})
.collect();
Ok(GetPostsResponse { posts: posts? })
.collect::<Result<Vec<PostForm>, Error>>()?;
Ok(posts)
}
pub fn fetch_remote_community(identifier: &str) -> Result<GetCommunityResponse, failure::Error> {
let group = fetch_remote_object::<GroupExt>(&get_remote_community_uri(identifier))?;
// TODO: this is only for testing until we can call that function from GetPosts
// (once string ids are supported)
//dbg!(get_remote_community_posts(identifier)?);
let (_, domain) = split_identifier(identifier);
Ok(GetCommunityResponse {
moderators: vec![],
admins: vec![],
community: CommunityView::from_group(&group, &domain)?,
online: 0,
})
}
pub fn fetch_all_communities() -> Result<Vec<CommunityView>, Error> {
let mut communities_list: Vec<CommunityView> = vec![];
// TODO: in the future, this should only be done when an instance is followed for the first time
// after that, we should rely in the inbox, and fetch on demand when needed
pub fn fetch_all(conn: &PgConnection) -> Result<(), Error> {
for instance in &get_following_instances() {
match fetch_communities_from_instance(instance) {
Ok(mut c) => communities_list.append(c.as_mut()),
Err(e) => warn!("Failed to fetch instance list from remote instance: {}", e),
let communities = fetch_communities_from_instance(instance)?;
for community in &communities {
let existing = Community::read_from_actor_id(conn, &community.actor_id);
let community_id = match existing {
// TODO: should make sure that this is actually a `NotFound` error
Err(_) => Community::create(conn, community)?.id,
Ok(c) => Community::update(conn, c.id, community)?.id,
};
let mut posts = fetch_remote_community_posts(instance, &community.name)?;
for post_ in &mut posts {
post_.community_id = community_id;
let existing = Post::read_from_apub_id(conn, &post_.ap_id);
match existing {
// TODO: should make sure that this is actually a `NotFound` error
Err(_) => {
Post::create(conn, post_)?;
}
Ok(communities_list)
Ok(p) => {
Post::update(conn, p.id, post_)?;
}
}
}
}
}
Ok(())
}

1
server/src/db/code_migrations.rs
View file

@ -93,6 +93,7 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
private_key: Some(community_private_key),
public_key: Some(community_public_key),
last_refreshed_at: Some(naive_now()),
published: None,
};
Community::update(&conn, ccommunity.id, &form)?;

2
server/src/db/comment.rs
View file

@ -247,6 +247,7 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
published: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();
@ -269,6 +270,7 @@ mod tests {
thumbnail_url: None,
ap_id: "changeme".into(),
local: true,
published: None,
};
let inserted_post = Post::create(&conn, &new_post).unwrap();

2
server/src/db/comment_view.rs
View file

@ -474,6 +474,7 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
published: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();
@ -496,6 +497,7 @@ mod tests {
thumbnail_url: None,
ap_id: "changeme".into(),
local: true,
published: None,
};
let inserted_post = Post::create(&conn, &new_post).unwrap();

11
server/src/db/community.rs
View file

@ -22,7 +22,7 @@ pub struct Community {
pub last_refreshed_at: chrono::NaiveDateTime,
}
#[derive(Insertable, AsChangeset, Clone, Serialize, Deserialize)]
#[derive(Insertable, AsChangeset, Clone, Serialize, Deserialize, Debug)]
#[table_name = "community"]
pub struct CommunityForm {
pub name: String,
@ -31,6 +31,7 @@ pub struct CommunityForm {
pub category_id: i32,
pub creator_id: i32,
pub removed: Option<bool>,
pub published: Option<chrono::NaiveDateTime>,
pub updated: Option<chrono::NaiveDateTime>,
pub deleted: Option<bool>,
pub nsfw: bool,
@ -79,6 +80,13 @@ impl Community {
.first::<Self>(conn)
}
pub fn read_from_actor_id(conn: &PgConnection, community_id: &str) -> Result<Self, Error> {
use crate::schema::community::dsl::*;
community
.filter(actor_id.eq(community_id))
.first::<Self>(conn)
}
pub fn list(conn: &PgConnection) -> Result<Vec<Self>, Error> {
use crate::schema::community::dsl::*;
community.load::<Community>(conn)
@ -271,6 +279,7 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
published: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

2
server/src/db/moderator.rs
View file

@ -505,6 +505,7 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
published: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();
@ -527,6 +528,7 @@ mod tests {
thumbnail_url: None,
ap_id: "changeme".into(),
local: true,
published: None,
};
let inserted_post = Post::create(&conn, &new_post).unwrap();

12
server/src/db/post.rs
View file

@ -27,7 +27,7 @@ pub struct Post {
pub local: bool,
}
#[derive(Insertable, AsChangeset, Clone)]
#[derive(Insertable, AsChangeset, Clone, Debug)]
#[table_name = "post"]
pub struct PostForm {
pub name: String,
@ -37,6 +37,7 @@ pub struct PostForm {
pub community_id: i32,
pub removed: Option<bool>,
pub locked: Option<bool>,
pub published: Option<chrono::NaiveDateTime>,
pub updated: Option<chrono::NaiveDateTime>,
pub deleted: Option<bool>,
pub nsfw: bool,
@ -65,6 +66,11 @@ impl Post {
.load::<Self>(conn)
}
pub fn read_from_apub_id(conn: &PgConnection, object_id: &str) -> Result<Self, Error> {
use crate::schema::post::dsl::*;
post.filter(ap_id.eq(object_id)).first::<Self>(conn)
}
pub fn update_ap_id(conn: &PgConnection, post_id: i32) -> Result<Self, Error> {
use crate::schema::post::dsl::*;
@ -105,11 +111,13 @@ impl Crud<PostForm> for Post {
fn create(conn: &PgConnection, new_post: &PostForm) -> Result<Self, Error> {
use crate::schema::post::dsl::*;
dbg!(&new_post);
insert_into(post).values(new_post).get_result::<Self>(conn)
}
fn update(conn: &PgConnection, post_id: i32, new_post: &PostForm) -> Result<Self, Error> {
use crate::schema::post::dsl::*;
dbg!(&new_post);
diesel::update(post.find(post_id))
.set(new_post)
.get_result::<Self>(conn)
@ -280,6 +288,7 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
published: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();
@ -302,6 +311,7 @@ mod tests {
thumbnail_url: None,
ap_id: "changeme".into(),
local: true,
published: None,
};
let inserted_post = Post::create(&conn, &new_post).unwrap();

2
server/src/db/post_view.rs
View file

@ -399,6 +399,7 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
published: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();
@ -421,6 +422,7 @@ mod tests {
thumbnail_url: None,
ap_id: "changeme".into(),
local: true,
published: None,
};
let inserted_post = Post::create(&conn, &new_post).unwrap();

2
server/src/db/user_mention.rs
View file

@ -131,6 +131,7 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
published: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();
@ -153,6 +154,7 @@ mod tests {
thumbnail_url: None,
ap_id: "changeme".into(),
local: true,
published: None,
};
let inserted_post = Post::create(&conn, &new_post).unwrap();

25
server/src/main.rs
View file

@ -6,16 +6,21 @@ use actix::prelude::*;
use actix_web::*;
use diesel::r2d2::{ConnectionManager, Pool};
use diesel::PgConnection;
use failure::Error;
use lemmy_server::apub::puller::fetch_all;
use lemmy_server::db::code_migrations::run_advanced_migrations;
use lemmy_server::routes::{api, federation, feeds, index, nodeinfo, webfinger, websocket};
use lemmy_server::settings::Settings;
use lemmy_server::websocket::server::*;
use std::io;
use log::warn;
use std::thread;
use std::thread::sleep;
use std::time::Duration;
embed_migrations!();
#[actix_rt::main]
async fn main() -> io::Result<()> {
async fn main() -> Result<(), Error> {
env_logger::init();
let settings = Settings::get();
@ -34,12 +39,25 @@ async fn main() -> io::Result<()> {
// Set up websocket server
let server = ChatServer::startup(pool.clone()).start();
// TODO: its probably failing because the other instance is not up yet
// need to make a new thread and wait a bit before fetching
thread::spawn(move || {
// some work here
sleep(Duration::from_secs(5));
println!("Fetching apub data");
match fetch_all(&conn) {
Ok(_) => {}
Err(e) => warn!("Error during apub fetch: {}", e),
}
});
println!(
"Starting http server at {}:{}",
settings.bind, settings.port
);
// Create Http server with websocket support
Ok(
HttpServer::new(move || {
App::new()
.wrap(middleware::Logger::default())
@ -65,5 +83,6 @@ async fn main() -> io::Result<()> {
})
.bind((settings.bind, settings.port))?
.run()
.await
.await?,
)
}