From 51b410ce018f37a23bc0c4653cb8176f3fcd9972 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 7 Aug 2020 16:24:26 +0200 Subject: [PATCH 1/7] Disable rate limiting for pictrs --- ansible/templates/nginx.conf | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ansible/templates/nginx.conf b/ansible/templates/nginx.conf index 092f8552..886ce0b7 100644 --- a/ansible/templates/nginx.conf +++ b/ansible/templates/nginx.conf @@ -51,9 +51,6 @@ server { # Upload limit for pictrs client_max_body_size 20M; - # Rate limit - limit_req zone=lemmy_ratelimit burst=30 nodelay; - location / { proxy_pass http://0.0.0.0:8536; proxy_set_header X-Real-IP $remote_addr; @@ -67,6 +64,9 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; + + # Rate limit + limit_req zone=lemmy_ratelimit burst=30 nodelay; } # Redirect pictshare images to pictrs @@ -74,6 +74,14 @@ server { return 301 /pictrs/image/$1; } + # Separate location block to disable rate limiting for images + location /pictrs { + proxy_pass http://0.0.0.0:8536/pictrs; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location /iframely/ { proxy_pass http://0.0.0.0:8061/; proxy_set_header X-Real-IP $remote_addr; From 0acd0bba3a187a01630a6c83231a3f574832c6d0 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Mon, 10 Aug 2020 11:41:18 -0400 Subject: [PATCH 2/7] Version v0.7.48 --- ansible/VERSION | 2 +- docker/prod/docker-compose.yml | 2 +- docker/travis/docker_push.sh | 4 ++-- server/src/version.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/VERSION b/ansible/VERSION index cfdfa5f8..4c03c0cf 100644 --- a/ansible/VERSION +++ b/ansible/VERSION @@ -1 +1 @@ -v0.7.47 +v0.7.48 diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index d63b3d4b..eb84cd77 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -12,7 +12,7 @@ services: restart: always lemmy: - image: dessalines/lemmy:v0.7.47 + image: dessalines/lemmy:v0.7.48 ports: - "127.0.0.1:8536:8536" restart: always diff --git a/docker/travis/docker_push.sh b/docker/travis/docker_push.sh index aa87bb72..d5699414 100644 --- a/docker/travis/docker_push.sh +++ b/docker/travis/docker_push.sh @@ -1,5 +1,5 @@ #!/bin/sh echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin docker tag dessalines/lemmy:travis \ - dessalines/lemmy:v0.7.47 -docker push dessalines/lemmy:v0.7.47 + dessalines/lemmy:v0.7.48 +docker push dessalines/lemmy:v0.7.48 diff --git a/server/src/version.rs b/server/src/version.rs index e2bdf3b2..083162ad 100644 --- a/server/src/version.rs +++ b/server/src/version.rs @@ -1 +1 @@ -pub const VERSION: &str = "v0.7.47"; +pub const VERSION: &str = "v0.7.48"; From 95cd37827d5ef48f1fb46ceaedc4f1a484ee1e3b Mon Sep 17 00:00:00 2001 From: Dessalines Date: Mon, 10 Aug 2020 20:09:30 -0400 Subject: [PATCH 3/7] Updating sponsor list. --- ui/src/components/sponsors.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/src/components/sponsors.tsx b/ui/src/components/sponsors.tsx index 3317d1c7..509ba96c 100644 --- a/ui/src/components/sponsors.tsx +++ b/ui/src/components/sponsors.tsx @@ -19,6 +19,7 @@ interface SilverUser { } let general = [ + 'Brendan', 'mexicanhalloween', 'William Moore', 'Rachel Schmitz', From c1f48d6a0e8b2e845f8a23e8d70eb1c3a3b4237f Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 11 Aug 2020 14:12:05 +0200 Subject: [PATCH 4/7] In federation test setup, run yarn build by default --- docker/federation/run-federation-test.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/federation/run-federation-test.bash b/docker/federation/run-federation-test.bash index bc73fff6..77cc981f 100755 --- a/docker/federation/run-federation-test.bash +++ b/docker/federation/run-federation-test.bash @@ -6,7 +6,7 @@ pushd ../../server/ || exit cargo build & popd || exit -if [ "$1" = "-yarn" ]; then +if [ "$1" != "--no-yarn-build" ]; then pushd ../../ui/ || exit yarn yarn build From 3da47352befd37fe93c6d3e14874b5b90d8b8694 Mon Sep 17 00:00:00 2001 From: nutomic Date: Tue, 11 Aug 2020 14:31:05 +0000 Subject: [PATCH 5/7] Remove usage of Option::unwrap() in activitypub code (#80) Remove remaining usages of unwrap() from activitypub code Remove most usage of Option::unwrap() from activitypub code Co-authored-by: Felix Ableitner Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/80 --- server/lemmy_utils/src/lib.rs | 12 ++ server/src/api/community.rs | 4 +- server/src/apub/activities.rs | 11 +- server/src/apub/comment.rs | 49 ++++--- server/src/apub/community.rs | 101 +++++++++------ server/src/apub/extensions/signatures.rs | 19 +-- server/src/apub/fetcher.rs | 39 +++--- server/src/apub/inbox/activities/announce.rs | 8 +- server/src/apub/inbox/activities/create.rs | 13 +- server/src/apub/inbox/activities/delete.rs | 13 +- server/src/apub/inbox/activities/dislike.rs | 22 +++- server/src/apub/inbox/activities/like.rs | 10 +- server/src/apub/inbox/activities/remove.rs | 14 +- server/src/apub/inbox/activities/undo.rs | 69 +++++++--- server/src/apub/inbox/activities/update.rs | 13 +- server/src/apub/inbox/community_inbox.rs | 14 +- server/src/apub/inbox/shared_inbox.rs | 34 +++-- server/src/apub/inbox/user_inbox.rs | 72 ++++++++--- server/src/apub/mod.rs | 79 +++++++----- server/src/apub/post.rs | 128 ++++++++++++------- server/src/apub/private_message.rs | 29 +++-- server/src/apub/user.rs | 57 +++++---- 22 files changed, 528 insertions(+), 282 deletions(-) diff --git a/server/lemmy_utils/src/lib.rs b/server/lemmy_utils/src/lib.rs index 6d851b03..fc50e199 100644 --- a/server/lemmy_utils/src/lib.rs +++ b/server/lemmy_utils/src/lib.rs @@ -31,6 +31,18 @@ use regex::{Regex, RegexBuilder}; use std::io::{Error, ErrorKind}; use url::Url; +#[macro_export] +macro_rules! location_info { + () => { + format!( + "None value at {}:{}, column {}", + file!(), + line!(), + column!() + ) + }; +} + pub fn to_datetime_utc(ndt: NaiveDateTime) -> DateTime { DateTime::::from_utc(ndt, Utc) } diff --git a/server/src/api/community.rs b/server/src/api/community.rs index c7160848..fb153895 100644 --- a/server/src/api/community.rs +++ b/server/src/api/community.rs @@ -605,11 +605,11 @@ impl Perform for Oper { // Dont actually add to the community followers here, because you need // to wait for the accept user - .send_follow(&community.actor_id, &self.client, pool) + .send_follow(&community.actor_id()?, &self.client, pool) .await?; } else { user - .send_unfollow(&community.actor_id, &self.client, pool) + .send_unfollow(&community.actor_id()?, &self.client, pool) .await?; let unfollow = move |conn: &'_ _| CommunityFollower::unfollow(conn, &community_follower_form); if blocking(pool, unfollow).await?.is_err() { diff --git a/server/src/apub/activities.rs b/server/src/apub/activities.rs index b5d6ce46..6b6b17b8 100644 --- a/server/src/apub/activities.rs +++ b/server/src/apub/activities.rs @@ -21,7 +21,7 @@ use uuid::Uuid; pub async fn send_activity_to_community( creator: &User_, community: &Community, - to: Vec, + to: Vec, activity: AnyBase, client: &Client, pool: &DbPool, @@ -43,17 +43,18 @@ pub async fn send_activity( client: &Client, activity: &AnyBase, actor: &dyn ActorType, - to: Vec, + to: Vec, ) -> Result<(), LemmyError> { let activity = serde_json::to_string(&activity)?; debug!("Sending activitypub activity {} to {:?}", activity, to); - for t in to { - let to_url = Url::parse(&t)?; + for to_url in to { check_is_apub_id_valid(&to_url)?; let res = retry_custom(|| async { - let request = client.post(&t).header("Content-Type", "application/json"); + let request = client + .post(to_url.as_str()) + .header("Content-Type", "application/json"); match sign(request, actor, activity.clone()).await { Ok(signed) => Ok(signed.send().await), diff --git a/server/src/apub/comment.rs b/server/src/apub/comment.rs index fbec5905..d90d8227 100644 --- a/server/src/apub/comment.rs +++ b/server/src/apub/comment.rs @@ -41,6 +41,7 @@ use activitystreams::{ public, }; use actix_web::{body::Body, client::Client, web::Path, HttpResponse}; +use anyhow::Context; use itertools::Itertools; use lemmy_db::{ comment::{Comment, CommentForm}, @@ -49,7 +50,13 @@ use lemmy_db::{ user::User_, Crud, }; -use lemmy_utils::{convert_datetime, remove_slurs, scrape_text_for_mentions, MentionData}; +use lemmy_utils::{ + convert_datetime, + location_info, + remove_slurs, + scrape_text_for_mentions, + MentionData, +}; use log::debug; use serde::Deserialize; use serde_json::Error; @@ -136,21 +143,21 @@ impl FromApub for CommentForm { ) -> Result { let creator_actor_id = ¬e .attributed_to() - .unwrap() + .context(location_info!())? .as_single_xsd_any_uri() - .unwrap(); + .context(location_info!())?; let creator = get_or_fetch_and_upsert_user(creator_actor_id, client, pool).await?; let mut in_reply_tos = note .in_reply_to() .as_ref() - .unwrap() + .context(location_info!())? .as_many() - .unwrap() + .context(location_info!())? .iter() - .map(|i| i.as_xsd_any_uri().unwrap()); - let post_ap_id = in_reply_tos.next().unwrap(); + .map(|i| i.as_xsd_any_uri().context("")); + let post_ap_id = in_reply_tos.next().context(location_info!())??; // This post, or the parent comment might not yet exist on this server yet, fetch them. let post = get_or_fetch_and_insert_post(&post_ap_id, client, pool).await?; @@ -159,7 +166,7 @@ impl FromApub for CommentForm { // For deeply nested comments, FromApub automatically gets called recursively let parent_id: Option = match in_reply_tos.next() { Some(parent_comment_uri) => { - let parent_comment_ap_id = &parent_comment_uri; + let parent_comment_ap_id = &parent_comment_uri?; let parent_comment = get_or_fetch_and_insert_comment(&parent_comment_ap_id, client, pool).await?; @@ -169,9 +176,9 @@ impl FromApub for CommentForm { }; let content = note .content() - .unwrap() + .context(location_info!())? .as_single_xsd_string() - .unwrap() + .context(location_info!())? .to_string(); let content_slurs_removed = remove_slurs(&content); @@ -295,7 +302,7 @@ impl ApubObjectType for Comment { send_activity_to_community( &creator, &community, - vec![community.get_shared_inbox_url()], + vec![community.get_shared_inbox_url()?], delete.into_any_base()?, client, pool, @@ -337,7 +344,7 @@ impl ApubObjectType for Comment { send_activity_to_community( &creator, &community, - vec![community.get_shared_inbox_url()], + vec![community.get_shared_inbox_url()?], undo.into_any_base()?, client, pool, @@ -370,7 +377,7 @@ impl ApubObjectType for Comment { send_activity_to_community( &mod_, &community, - vec![community.get_shared_inbox_url()], + vec![community.get_shared_inbox_url()?], remove.into_any_base()?, client, pool, @@ -412,7 +419,7 @@ impl ApubObjectType for Comment { send_activity_to_community( &mod_, &community, - vec![community.get_shared_inbox_url()], + vec![community.get_shared_inbox_url()?], undo.into_any_base()?, client, pool, @@ -448,7 +455,7 @@ impl ApubLikeableType for Comment { send_activity_to_community( &creator, &community, - vec![community.get_shared_inbox_url()], + vec![community.get_shared_inbox_url()?], like.into_any_base()?, client, pool, @@ -481,7 +488,7 @@ impl ApubLikeableType for Comment { send_activity_to_community( &creator, &community, - vec![community.get_shared_inbox_url()], + vec![community.get_shared_inbox_url()?], dislike.into_any_base()?, client, pool, @@ -522,7 +529,7 @@ impl ApubLikeableType for Comment { send_activity_to_community( &creator, &community, - vec![community.get_shared_inbox_url()], + vec![community.get_shared_inbox_url()?], undo.into_any_base()?, client, pool, @@ -534,7 +541,7 @@ impl ApubLikeableType for Comment { struct MentionsAndAddresses { addressed_ccs: Vec, - inboxes: Vec, + inboxes: Vec, tags: Vec, } @@ -569,7 +576,7 @@ async fn collect_non_local_mentions_and_addresses( .filter(|m| !m.is_local()) .collect::>(); - let mut mention_inboxes = Vec::new(); + let mut mention_inboxes: Vec = Vec::new(); for mention in &mentions { // TODO should it be fetching it every time? if let Ok(actor_id) = fetch_webfinger_url(mention, client).await { @@ -577,7 +584,7 @@ async fn collect_non_local_mentions_and_addresses( addressed_ccs.push(actor_id.to_owned().to_string()); let mention_user = get_or_fetch_and_upsert_user(&actor_id, client, pool).await?; - let shared_inbox = mention_user.get_shared_inbox_url(); + let shared_inbox = mention_user.get_shared_inbox_url()?; mention_inboxes.push(shared_inbox); let mut mention_tag = Mention::new(); @@ -586,7 +593,7 @@ async fn collect_non_local_mentions_and_addresses( } } - let mut inboxes = vec![community.get_shared_inbox_url()]; + let mut inboxes = vec![community.get_shared_inbox_url()?]; inboxes.extend(mention_inboxes); inboxes = inboxes.into_iter().unique().collect(); diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index 8b522b44..fb8ef9bc 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -7,8 +7,7 @@ use crate::{ create_apub_tombstone_response, create_tombstone, extensions::group_extensions::GroupExtension, - fetcher::get_or_fetch_and_upsert_user, - get_shared_inbox, + fetcher::{get_or_fetch_and_upsert_actor, get_or_fetch_and_upsert_user}, insert_activity, ActorType, FromApub, @@ -40,6 +39,7 @@ use activitystreams::{ }; use activitystreams_ext::Ext2; use actix_web::{body::Body, client::Client, web, HttpResponse}; +use anyhow::Context; use itertools::Itertools; use lemmy_db::{ community::{Community, CommunityForm}, @@ -48,7 +48,7 @@ use lemmy_db::{ post::Post, user::User_, }; -use lemmy_utils::convert_datetime; +use lemmy_utils::{convert_datetime, get_apub_protocol_string, location_info}; use serde::Deserialize; use url::Url; @@ -99,7 +99,7 @@ impl ToApub for Community { .set_following(self.get_following_url().parse()?) .set_liked(self.get_liked_url().parse()?) .set_endpoints(Endpoints { - shared_inbox: Some(self.get_shared_inbox_url().parse()?), + shared_inbox: Some(self.get_shared_inbox_url()?), ..Default::default() }); @@ -113,7 +113,7 @@ impl ToApub for Community { Ok(Ext2::new( ap_actor, group_extension, - self.get_public_key_ext(), + self.get_public_key_ext()?, )) } @@ -128,11 +128,11 @@ impl ActorType for Community { self.actor_id.to_owned() } - fn public_key(&self) -> String { - self.public_key.to_owned().unwrap() + fn public_key(&self) -> Option { + self.public_key.to_owned() } - fn private_key(&self) -> String { - self.private_key.to_owned().unwrap() + fn private_key(&self) -> Option { + self.private_key.to_owned() } /// As a local community, accept the follow request from a remote user. @@ -142,10 +142,14 @@ impl ActorType for Community { client: &Client, pool: &DbPool, ) -> Result<(), LemmyError> { - let actor_uri = follow.actor()?.as_single_xsd_any_uri().unwrap().to_string(); + let actor_uri = follow + .actor()? + .as_single_xsd_any_uri() + .context(location_info!())?; + let actor = get_or_fetch_and_upsert_actor(actor_uri, client, pool).await?; let mut accept = Accept::new(self.actor_id.to_owned(), follow.into_any_base()?); - let to = format!("{}/inbox", actor_uri); + let to = actor.get_inbox_url()?; accept .set_context(context()) .set_id(generate_activity_id(AcceptType::Accept)?) @@ -279,7 +283,10 @@ impl ActorType for Community { } /// For a given community, returns the inboxes of all followers. - async fn get_follower_inboxes(&self, pool: &DbPool) -> Result, LemmyError> { + /// + /// TODO: this function is very badly implemented, we should just store shared_inbox_url in + /// CommunityFollowerView + async fn get_follower_inboxes(&self, pool: &DbPool) -> Result, LemmyError> { let id = self.id; let inboxes = blocking(pool, move |conn| { @@ -288,8 +295,22 @@ impl ActorType for Community { .await??; let inboxes = inboxes .into_iter() - .map(|c| get_shared_inbox(&Url::parse(&c.user_actor_id).unwrap())) - .filter(|s| !s.is_empty()) + .map(|u| -> Result { + let url = Url::parse(&u.user_actor_id)?; + let domain = url.domain().context(location_info!())?; + let port = if let Some(port) = url.port() { + format!(":{}", port) + } else { + "".to_string() + }; + Ok(Url::parse(&format!( + "{}://{}{}/inbox", + get_apub_protocol_string(), + domain, + port, + ))?) + }) + .filter_map(Result::ok) .unique() .collect(); @@ -298,7 +319,7 @@ impl ActorType for Community { async fn send_follow( &self, - _follow_actor_id: &str, + _follow_actor_id: &Url, _client: &Client, _pool: &DbPool, ) -> Result<(), LemmyError> { @@ -307,7 +328,7 @@ impl ActorType for Community { async fn send_unfollow( &self, - _follow_actor_id: &str, + _follow_actor_id: &Url, _client: &Client, _pool: &DbPool, ) -> Result<(), LemmyError> { @@ -330,44 +351,50 @@ impl FromApub for CommunityForm { pool: &DbPool, expected_domain: Option, ) -> Result { - let creator_and_moderator_uris = group.inner.attributed_to().unwrap(); + let creator_and_moderator_uris = group.inner.attributed_to().context(location_info!())?; let creator_uri = creator_and_moderator_uris .as_many() - .unwrap() + .context(location_info!())? .iter() .next() - .unwrap() + .context(location_info!())? .as_xsd_any_uri() - .unwrap(); + .context(location_info!())?; let creator = get_or_fetch_and_upsert_user(creator_uri, client, pool).await?; let name = group .inner .name() - .unwrap() + .context(location_info!())? .as_one() - .unwrap() + .context(location_info!())? .as_xsd_string() - .unwrap() + .context(location_info!())? + .to_string(); + let title = group + .inner + .preferred_username() + .context(location_info!())? .to_string(); - let title = group.inner.preferred_username().unwrap().to_string(); // TODO: should be parsed as html and tags like