grishka/lemmy
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Share list of communities over apub, some refactoring

Browse source
This commit is contained in:
Felix Ableitner 2020-04-03 07:02:43 +02:00
parent fbdb14bd9f
commit 5f2d58bd26
9 changed files with 250 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
server/Cargo.lock generated vendored
View file

@ -2,10 +2,10 @@
# It is not intended for manual editing.
[[package]]
name = "activitystreams"
version = "0.5.0-alpha.10"
version = "0.5.0-alpha.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"activitystreams-derive 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
"activitystreams-derive 0.5.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
"mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
@ -16,7 +16,7 @@ dependencies = [
[[package]]
name = "activitystreams-derive"
version = "0.5.0-alpha.4"
version = "0.5.0-alpha.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1401,7 +1401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
name = "lemmy_server"
version = "0.0.1"
dependencies = [
"activitystreams 0.5.0-alpha.10 (registry+https://github.com/rust-lang/crates.io-index)",
"activitystreams 0.5.0-alpha.15 (registry+https://github.com/rust-lang/crates.io-index)",
"actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"actix-files 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"actix-rt 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2855,8 +2855,8 @@ dependencies = [
]
[metadata]
"checksum activitystreams 0.5.0-alpha.10 (registry+https://github.com/rust-lang/crates.io-index)" = "04827f3390831f772d15ff3171336cc4fbac714f59233d24731beb7c865293a6"
"checksum activitystreams-derive 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2bc640808dceb2efac81e6bcb77a7f4e2e76af7fb60e88f966b48123b625d2f"
"checksum activitystreams 0.5.0-alpha.15 (registry+https://github.com/rust-lang/crates.io-index)" = "1ecff5b18578fac8cff5f139009cd0a4f605b6f5eddc91f2b8319bfef76cc2c3"
"checksum activitystreams-derive 0.5.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d7498811842309cc5b54c123bcf328e33eebe9841037067b1e9b93caa820085d"
"checksum actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a4af87564ff659dee8f9981540cac9418c45e910c8072fdedd643a262a38fcaf"
"checksum actix-codec 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "09e55f0a5c2ca15795035d90c46bd0e73a5123b72f68f12596d6ba5282051380"
"checksum actix-connect 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c95cc9569221e9802bf4c377f6c18b90ef10227d787611decf79fd47d2a8e76c"

2
server/Cargo.toml vendored
View file

@ -9,7 +9,7 @@ diesel = { version = "1.4.2", features = ["postgres","chrono", "r2d2", "64-colum
diesel_migrations = "1.4.0"
dotenv = "0.15.0"
bcrypt = "0.6.1"
activitystreams = "0.5.0-alpha.10"
activitystreams = "0.5.0-alpha.15"
chrono = { version = "0.4.7", features = ["serde"] }
failure = "0.1.5"
serde_json = { version = "1.0.45", features = ["preserve_order"]}

10
server/src/api/community.rs
View file

@ -36,11 +36,11 @@ pub struct CommunityResponse {
#[derive(Serialize, Deserialize, Debug)]
pub struct ListCommunities {
sort: String,
page: Option<i64>,
limit: Option<i64>,
auth: Option<String>,
local_only: Option<bool>,
pub sort: String,
pub page: Option<i64>,
pub limit: Option<i64>,
pub auth: Option<String>,
pub local_only: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug)]

165
server/src/apub/community.rs
View file

@ -1,15 +1,19 @@
use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType};
use crate::api::community::ListCommunities;
use crate::api::{Oper, Perform};
use crate::apub::puller::{fetch_remote_object, format_community_name};
use crate::apub::{
create_apub_response, get_apub_protocol_string, make_apub_endpoint, EndpointType, GroupExt,
};
use crate::convert_datetime;
use crate::db::community::Community;
use crate::db::community_view::CommunityFollowerView;
use crate::db::community_view::{CommunityFollowerView, CommunityView};
use crate::db::establish_unpooled_connection;
use crate::db::post_view::{PostQueryBuilder, PostView};
use crate::settings::Settings;
use activitystreams::actor::properties::ApActorProperties;
use activitystreams::collection::OrderedCollection;
use activitystreams::{
actor::{properties::ApActorProperties, Group},
collection::UnorderedCollection,
context,
ext::Extensible,
actor::Group, collection::UnorderedCollection, context, ext::Extensible,
object::properties::ObjectProperties,
};
use actix_web::body::Body;
@ -26,41 +30,126 @@ pub struct CommunityQuery {
community_name: String,
}
pub async fn get_apub_community(
pub async fn get_apub_community_list(
db: web::Data<Pool<ConnectionManager<PgConnection>>>,
) -> Result<HttpResponse<Body>, Error> {
// TODO: implement pagination
let query = ListCommunities {
sort: "Hot".to_string(),
page: None,
limit: None,
auth: None,
local_only: Some(true),
};
let communities = Oper::new(query)
.perform(&db.get().unwrap())
.unwrap()
.communities
.iter()
.map(|c| c.as_group())
.collect::<Result<Vec<GroupExt>, failure::Error>>()?;
let mut collection = UnorderedCollection::default();
let oprops: &mut ObjectProperties = collection.as_mut();
oprops.set_context_xsd_any_uri(context())?.set_id(format!(
"{}://{}/federation/communities",
get_apub_protocol_string(),
Settings::get().hostname
))?;
collection
.collection_props
.set_total_items(communities.len() as u64)?
.set_many_items_base_boxes(communities)?;
Ok(create_apub_response(&collection))
}
impl CommunityView {
fn as_group(&self) -> Result<GroupExt, Error> {
let base_url = make_apub_endpoint(EndpointType::Community, &self.name);
let mut group = Group::default();
let oprops: &mut ObjectProperties = group.as_mut();
oprops
.set_context_xsd_any_uri(context())?
.set_id(base_url.to_owned())?
.set_name_xsd_string(self.name.to_owned())?
.set_published(convert_datetime(self.published))?
.set_attributed_to_xsd_any_uri(make_apub_endpoint(
EndpointType::User,
&self.creator_id.to_string(),
))?;
if let Some(u) = self.updated.to_owned() {
oprops.set_updated(convert_datetime(u))?;
}
if let Some(d) = self.description.to_owned() {
oprops.set_summary_xsd_string(d)?;
}
let mut actor_props = ApActorProperties::default();
actor_props
.set_preferred_username(self.title.to_owned())?
.set_inbox(format!("{}/inbox", &base_url))?
.set_outbox(format!("{}/outbox", &base_url))?
.set_followers(format!("{}/followers", &base_url))?;
Ok(group.extend(actor_props))
}
pub fn from_group(group: &GroupExt, domain: &str) -> Result<CommunityView, Error> {
let followers_uri = &group.extension.get_followers().unwrap().to_string();
let outbox_uri = &group.extension.get_outbox().to_string();
let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
let followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
let oprops = &group.base.object_props;
let aprops = &group.extension;
Ok(CommunityView {
// TODO: we need to merge id and name into a single thing (stuff like @user@instance.com)
id: 1337, //community.object_props.get_id()
name: format_community_name(&oprops.get_name_xsd_string().unwrap().to_string(), domain),
title: aprops.get_preferred_username().unwrap().to_string(),
description: oprops.get_summary_xsd_string().map(|s| s.to_string()),
category_id: -1,
creator_id: -1, //community.object_props.get_attributed_to_xsd_any_uri()
removed: false,
published: oprops
.get_published()
.unwrap()
.as_ref()
.naive_local()
.to_owned(),
updated: oprops
.get_updated()
.map(|u| u.as_ref().to_owned().naive_local()),
deleted: false,
nsfw: false,
creator_name: "".to_string(),
creator_avatar: None,
category_name: "".to_string(),
number_of_subscribers: *followers
.collection_props
.get_total_items()
.unwrap()
.as_ref() as i64,
number_of_posts: *outbox.collection_props.get_total_items().unwrap().as_ref() as i64,
number_of_comments: -1,
hot_rank: -1,
user_id: None,
subscribed: None,
})
}
}
pub async fn get_apub_community_http(
info: Path<CommunityQuery>,
db: web::Data<Pool<ConnectionManager<PgConnection>>>,
) -> Result<HttpResponse<Body>, Error> {
let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?;
let base_url = make_apub_endpoint(EndpointType::Community, &community.name);
let mut group = Group::default();
let oprops: &mut ObjectProperties = group.as_mut();
oprops
.set_context_xsd_any_uri(context())?
.set_id(base_url.to_owned())?
.set_name_xsd_string(community.title.to_owned())?
.set_published(convert_datetime(community.published))?
.set_attributed_to_xsd_any_uri(make_apub_endpoint(
EndpointType::User,
&community.creator_id.to_string(),
))?;
if let Some(u) = community.updated.to_owned() {
oprops.set_updated(convert_datetime(u))?;
}
if let Some(d) = community.description {
oprops.set_summary_xsd_string(d)?;
}
let mut actor_props = ApActorProperties::default();
actor_props
.set_inbox(format!("{}/inbox", &base_url))?
.set_outbox(format!("{}/outbox", &base_url))?
.set_followers(format!("{}/followers", &base_url))?;
Ok(create_apub_response(&group.extend(actor_props)))
let community_view = CommunityView::read(&&db.get()?, community.id, None)?;
let c = community_view.as_group()?;
Ok(create_apub_response(&c))
}
pub async fn get_apub_community_followers(
@ -107,7 +196,7 @@ pub async fn get_apub_community_outbox(
.set_id(base_url)?;
collection
.collection_props
.set_many_items_object_boxs(
.set_many_items_base_boxes(
community_posts
.iter()
.map(|c| c.as_page().unwrap())

4
server/src/apub/mod.rs
View file

@ -4,10 +4,14 @@ pub mod puller;
pub mod user;
use crate::Settings;
use activitystreams::actor::{properties::ApActorProperties, Group};
use activitystreams::ext::Ext;
use actix_web::body::Body;
use actix_web::HttpResponse;
use url::Url;
type GroupExt = Ext<Group, ApActorProperties>;
fn create_apub_response<T>(json: &T) -> HttpResponse<Body>
where
T: serde::ser::Serialize,

51
server/src/apub/post.rs
View file

@ -1,6 +1,6 @@
use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType};
use crate::convert_datetime;
use crate::db::post_view::PostView;
use crate::{convert_datetime, naive_now};
use activitystreams::{object::properties::ObjectProperties, object::Page};
use actix_web::body::Body;
use actix_web::web::Path;
@ -59,4 +59,53 @@ impl PostView {
Ok(page)
}
pub fn from_page(page: &Page) -> Result<PostView, Error> {
let oprops = &page.object_props;
Ok(PostView {
id: -1,
name: oprops.get_name_xsd_string().unwrap().to_string(),
url: oprops.get_url_xsd_any_uri().map(|u| u.to_string()),
body: oprops.get_content_xsd_string().map(|c| c.to_string()),
creator_id: -1,
community_id: -1,
removed: false,
locked: false,
published: oprops
.get_published()
.unwrap()
.as_ref()
.naive_local()
.to_owned(),
updated: oprops
.get_updated()
.map(|u| u.as_ref().to_owned().naive_local()),
deleted: false,
nsfw: false,
stickied: false,
embed_title: None,
embed_description: None,
embed_html: None,
thumbnail_url: None,
banned: false,
banned_from_community: false,
creator_name: "".to_string(),
creator_avatar: None,
community_name: "".to_string(),
community_removed: false,
community_deleted: false,
community_nsfw: false,
number_of_comments: -1,
score: -1,
upvotes: -1,
downvotes: -1,
hot_rank: -1,
newest_activity_time: naive_now(),
user_id: None,
my_vote: None,
subscribed: None,
read: None,
saved: None,
})
}
}

175
server/src/apub/puller.rs
View file

@ -1,16 +1,13 @@
use crate::api::community::{GetCommunityResponse, ListCommunitiesResponse};
use crate::api::community::GetCommunityResponse;
use crate::api::post::GetPostsResponse;
use crate::apub::get_apub_protocol_string;
use crate::apub::{get_apub_protocol_string, GroupExt};
use crate::db::community_view::CommunityView;
use crate::db::post_view::PostView;
use crate::naive_now;
use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown};
use crate::settings::Settings;
use activitystreams::actor::{properties::ApActorProperties, Group};
use activitystreams::collection::{OrderedCollection, UnorderedCollection};
use activitystreams::ext::Ext;
use activitystreams::object::ObjectBox;
use activitystreams::object::Page;
use activitystreams::BaseBox;
use chttp::prelude::*;
use failure::Error;
use log::warn;
@ -28,39 +25,44 @@ fn fetch_node_info(domain: &str) -> Result<NodeInfo, Error> {
fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityView>, Error> {
let node_info = fetch_node_info(domain)?;
if node_info.software.name != "lemmy" {
return Err(format_err!(
if let Some(community_list_url) = node_info.metadata.community_list_url {
let collection = fetch_remote_object::<UnorderedCollection>(&community_list_url)?;
let object_boxes = collection
.collection_props
.get_many_items_base_boxes()
.unwrap();
let communities: Result<Vec<CommunityView>, Error> = object_boxes
.map(|c| -> Result<CommunityView, Error> {
let group = c.to_owned().to_concrete::<GroupExt>()?;
CommunityView::from_group(&group, domain)
})
.collect();
Ok(communities?)
} else {
Err(format_err!(
"{} is not a Lemmy instance, federation is not supported",
domain
));
))
}
}
// TODO: follow pagination (seems like page count is missing?)
// TODO: see if there is any standard for discovering remote actors, so we dont have to rely on lemmy apis
let communities_uri = format!(
"http://{}/api/v1/communities/list?sort=Hot&local_only=true",
domain
);
let communities1 = fetch_remote_object::<ListCommunitiesResponse>(&communities_uri)?;
let mut communities2 = communities1.communities;
for c in &mut communities2 {
c.name = format_community_name(&c.name, domain);
}
Ok(communities2)
/// Returns a tuple of (username, domain) from an identifier like "main@dev.lemmy.ml"
fn split_identifier(identifier: &str) -> (String, String) {
let x: Vec<&str> = identifier.split('@').collect();
(x[0].replace("!", ""), x[1].to_string())
}
fn get_remote_community_uri(identifier: &str) -> String {
let x: Vec<&str> = identifier.split('@').collect();
let name = x[0].replace("!", "");
let instance = x[1];
format!("http://{}/federation/c/{}", instance, name)
let (name, domain) = split_identifier(identifier);
format!("http://{}/federation/c/{}", domain, name)
}
fn fetch_remote_object<Response>(uri: &str) -> Result<Response, Error>
pub fn fetch_remote_object<Response>(uri: &str) -> Result<Response, Error>
where
Response: for<'de> Deserialize<'de>,
{
if Settings::get().federation.tls_enabled && !uri.starts_with("https") {
if Settings::get().federation.tls_enabled && !uri.starts_with("https://") {
return Err(format_err!("Activitypub uri is insecure: {}", uri));
}
// TODO: should cache responses here when we are in production
@ -71,131 +73,32 @@ where
}
pub fn get_remote_community_posts(identifier: &str) -> Result<GetPostsResponse, Error> {
let community =
fetch_remote_object::<Ext<Group, ApActorProperties>>(&get_remote_community_uri(identifier))?;
let community = fetch_remote_object::<GroupExt>(&get_remote_community_uri(identifier))?;
let outbox_uri = &community.extension.get_outbox().to_string();
let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
let items = outbox.collection_props.get_many_items_object_boxs();
let items = outbox.collection_props.get_many_items_base_boxes();
let posts: Vec<PostView> = items
let posts: Result<Vec<PostView>, Error> = items
.unwrap()
.map(|obox: &ObjectBox| {
let page: Page = obox.clone().to_concrete::<Page>().unwrap();
PostView {
id: -1,
name: page.object_props.get_name_xsd_string().unwrap().to_string(),
url: page
.object_props
.get_url_xsd_any_uri()
.map(|u| u.to_string()),
body: page
.object_props
.get_content_xsd_string()
.map(|c| c.to_string()),
creator_id: -1,
community_id: -1,
removed: false,
locked: false,
published: page
.object_props
.get_published()
.unwrap()
.as_ref()
.naive_local()
.to_owned(),
updated: page
.object_props
.get_updated()
.map(|u| u.as_ref().to_owned().naive_local()),
deleted: false,
nsfw: false,
stickied: false,
embed_title: None,
embed_description: None,
embed_html: None,
thumbnail_url: None,
banned: false,
banned_from_community: false,
creator_name: "".to_string(),
creator_avatar: None,
community_name: "".to_string(),
community_removed: false,
community_deleted: false,
community_nsfw: false,
number_of_comments: -1,
score: -1,
upvotes: -1,
downvotes: -1,
hot_rank: -1,
newest_activity_time: naive_now(),
user_id: None,
my_vote: None,
subscribed: None,
read: None,
saved: None,
}
.map(|obox: &BaseBox| {
let page = obox.clone().to_concrete::<Page>().unwrap();
PostView::from_page(&page)
})
.collect();
Ok(GetPostsResponse { posts })
Ok(GetPostsResponse { posts: posts? })
}
pub fn get_remote_community(identifier: &str) -> Result<GetCommunityResponse, failure::Error> {
let community =
fetch_remote_object::<Ext<Group, ApActorProperties>>(&get_remote_community_uri(identifier))?;
let followers_uri = &community.extension.get_followers().unwrap().to_string();
let outbox_uri = &community.extension.get_outbox().to_string();
let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
let followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
let group = fetch_remote_object::<GroupExt>(&get_remote_community_uri(identifier))?;
// TODO: this is only for testing until we can call that function from GetPosts
// (once string ids are supported)
//dbg!(get_remote_community_posts(identifier)?);
let (_, domain) = split_identifier(identifier);
Ok(GetCommunityResponse {
moderators: vec![],
admins: vec![],
community: CommunityView {
// TODO: we need to merge id and name into a single thing (stuff like @user@instance.com)
id: 1337, //community.object_props.get_id()
name: identifier.to_string(),
title: community
.as_ref()
.get_name_xsd_string()
.unwrap()
.to_string(),
description: community
.as_ref()
.get_summary_xsd_string()
.map(|s| s.to_string()),
category_id: -1,
creator_id: -1, //community.object_props.get_attributed_to_xsd_any_uri()
removed: false,
published: community
.as_ref()
.get_published()
.unwrap()
.as_ref()
.naive_local()
.to_owned(),
updated: community
.as_ref()
.get_updated()
.map(|u| u.as_ref().to_owned().naive_local()),
deleted: false,
nsfw: false,
creator_name: "".to_string(),
creator_avatar: None,
category_name: "".to_string(),
number_of_subscribers: *followers
.collection_props
.get_total_items()
.unwrap()
.as_ref() as i64, // TODO: need to use the same type
number_of_posts: *outbox.collection_props.get_total_items().unwrap().as_ref() as i64,
number_of_comments: -1,
hot_rank: -1,
user_id: None,
subscribed: None,
},
community: CommunityView::from_group(&group, &domain)?,
online: 0,
})
}

6
server/src/routes/federation.rs
View file

@ -12,9 +12,13 @@ pub fn config(cfg: &mut web::ServiceConfig) {
if Settings::get().federation.enabled {
println!("federation enabled, host is {}", Settings::get().hostname);
cfg
.route(
"/federation/communities",
web::get().to(apub::community::get_apub_community_list),
)
.route(
"/federation/c/{community_name}",
web::get().to(apub::community::get_apub_community),
web::get().to(apub::community::get_apub_community_http),
)
.route(
"/federation/c/{community_name}/followers",

13
server/src/routes/nodeinfo.rs
View file

@ -60,6 +60,13 @@ async fn node_info(
local_comments: site_view.number_of_comments,
open_registrations: site_view.open_registration,
},
metadata: NodeInfoMetadata {
community_list_url: Some(format!(
"{}://{}/federation/communities",
get_apub_protocol_string(),
Settings::get().hostname
)),
},
})
})
.await
@ -85,6 +92,7 @@ pub struct NodeInfo {
pub software: NodeInfoSoftware,
pub protocols: Vec<String>,
pub usage: NodeInfoUsage,
pub metadata: NodeInfoMetadata,
}
#[derive(Serialize, Deserialize, Debug)]
@ -106,3 +114,8 @@ pub struct NodeInfoUsage {
pub struct NodeInfoUsers {
pub total: i64,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct NodeInfoMetadata {
pub community_list_url: Option<String>,
}