From a14ebefd24a7f086db1d5a46eb5ad9b0b9663a31 Mon Sep 17 00:00:00 2001 From: jim-taylor-business Date: Mon, 8 Apr 2024 11:05:54 +0100 Subject: [PATCH 1/4] When env variable is set, any config file will be ignored and the default settings will be used (#4594) * do not panic when no config file found use defaults * formatting * implement env variable * ermove commented code * remove redundant comment * remove redundant space * simplify check logic * format * returns and messages * correct mistake --- crates/utils/src/settings/mod.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/utils/src/settings/mod.rs b/crates/utils/src/settings/mod.rs index f630d0217..d0f802e5b 100644 --- a/crates/utils/src/settings/mod.rs +++ b/crates/utils/src/settings/mod.rs @@ -13,8 +13,17 @@ use structs::{DatabaseConnection, PictrsConfig, PictrsImageMode, Settings}; static DEFAULT_CONFIG_FILE: &str = "config/config.hjson"; pub static SETTINGS: Lazy = Lazy::new(|| { - Settings::init().expect("Failed to load settings file, see documentation (https://join-lemmy.org/docs/en/administration/configuration.html)") + if env::var("LEMMY_INITIALIZE_WITH_DEFAULT_SETTINGS").is_ok() { + println!( + "LEMMY_INITIALIZE_WITH_DEFAULT_SETTINGS was set, any configuration file has been ignored." + ); + println!("Use with other environment variables to configure this instance further; e.g. LEMMY_DATABASE_URL."); + Settings::default() + } else { + Settings::init().expect("Failed to load settings file, see documentation (https://join-lemmy.org/docs/en/administration/configuration.html).") + } }); + static WEBFINGER_REGEX: Lazy = Lazy::new(|| { Regex::new(&format!( "^acct:([a-zA-Z0-9_]{{3,}})@{}$", @@ -30,9 +39,7 @@ impl Settings { /// `lemmy_db_schema/src/lib.rs::get_database_url_from_env()` /// Warning: Only call this once. pub(crate) fn init() -> Result { - // Read the config file let config = from_str::(&Self::read_config_file()?)?; - if config.hostname == "unset" { Err(anyhow!("Hostname variable is not set!").into()) } else { From 8e54a4a6ccc9491bb9defbfe82ef6353846098b8 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Mon, 8 Apr 2024 10:26:24 -0400 Subject: [PATCH 2/4] Fixing bug where comment replies wouldn't be sent to blocked instances. (#4595) * Fixing bug where comment replies wouldn't be sent to blocked instances. - Instance blocks should only affect communities, not comments. - Fixes #4590 * Revert "Fixing bug where comment replies wouldn't be sent to blocked instances." This reverts commit 1349aa351a195fdf0d179c27a8057257ca37e255. * Only block replies from the community's instance id. - Also refactor send_local_notifs slightly, since it has to fetch the community now. - Fixes #4590 --------- Co-authored-by: SleeplessOne1917 <28871516+SleeplessOne1917@users.noreply.github.com> --- crates/api_common/src/build_response.rs | 23 +++++++++++-------- crates/api_common/src/utils.rs | 4 ++-- crates/api_crud/src/comment/create.rs | 3 +-- crates/api_crud/src/comment/delete.rs | 18 +++------------ crates/api_crud/src/comment/remove.rs | 6 +---- crates/api_crud/src/comment/update.rs | 3 +-- .../activities/create_or_update/comment.rs | 4 +--- 7 files changed, 23 insertions(+), 38 deletions(-) diff --git a/crates/api_common/src/build_response.rs b/crates/api_common/src/build_response.rs index 19e0bb46b..990b96544 100644 --- a/crates/api_common/src/build_response.rs +++ b/crates/api_common/src/build_response.rs @@ -19,7 +19,6 @@ use lemmy_db_schema::{ comment_reply::{CommentReply, CommentReplyInsertForm}, person::Person, person_mention::{PersonMention, PersonMentionInsertForm}, - post::Post, }, traits::Crud, }; @@ -91,16 +90,19 @@ pub async fn build_post_response( #[tracing::instrument(skip_all)] pub async fn send_local_notifs( mentions: Vec, - comment: &Comment, + comment_id: CommentId, person: &Person, - post: &Post, do_send_email: bool, context: &LemmyContext, ) -> Result, LemmyError> { let mut recipient_ids = Vec::new(); let inbox_link = format!("{}/inbox", context.settings().get_protocol_and_hostname()); - let community_id = post.community_id; + // Read the comment view to get extra info + let comment_view = CommentView::read(&mut context.pool(), comment_id, None).await?; + let comment = comment_view.comment; + let post = comment_view.post; + let community = comment_view.community; // Send the local mentions for mention in mentions @@ -117,7 +119,7 @@ pub async fn send_local_notifs( let user_mention_form = PersonMentionInsertForm { recipient_id: mention_user_view.person.id, - comment_id: comment.id, + comment_id, read: None, }; @@ -152,8 +154,9 @@ pub async fn send_local_notifs( let check_blocks = check_person_instance_community_block( person.id, parent_creator_id, - person.instance_id, - community_id, + // Only block from the community's instance_id + community.instance_id, + community.id, &mut context.pool(), ) .await @@ -194,11 +197,13 @@ pub async fn send_local_notifs( } } } else { + // Use the post creator to check blocks let check_blocks = check_person_instance_community_block( person.id, post.creator_id, - person.instance_id, - community_id, + // Only block from the community's instance_id + community.instance_id, + community.id, &mut context.pool(), ) .await diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index ad416ffbb..dd0a8c0c6 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -295,12 +295,12 @@ async fn check_instance_block( pub async fn check_person_instance_community_block( my_id: PersonId, potential_blocker_id: PersonId, - instance_id: InstanceId, + community_instance_id: InstanceId, community_id: CommunityId, pool: &mut DbPool<'_>, ) -> Result<(), LemmyError> { check_person_block(my_id, potential_blocker_id, pool).await?; - check_instance_block(instance_id, potential_blocker_id, pool).await?; + check_instance_block(community_instance_id, potential_blocker_id, pool).await?; check_community_block(community_id, potential_blocker_id, pool).await?; Ok(()) } diff --git a/crates/api_crud/src/comment/create.rs b/crates/api_crud/src/comment/create.rs index acb386c60..f8092268f 100644 --- a/crates/api_crud/src/comment/create.rs +++ b/crates/api_crud/src/comment/create.rs @@ -138,9 +138,8 @@ pub async fn create_comment( let mentions = scrape_text_for_mentions(&content); let recipient_ids = send_local_notifs( mentions, - &updated_comment, + inserted_comment_id, &local_user_view.person, - &post, true, &context, ) diff --git a/crates/api_crud/src/comment/delete.rs b/crates/api_crud/src/comment/delete.rs index 2de2a7955..9db1ed034 100644 --- a/crates/api_crud/src/comment/delete.rs +++ b/crates/api_crud/src/comment/delete.rs @@ -8,10 +8,7 @@ use lemmy_api_common::{ utils::check_community_user_action, }; use lemmy_db_schema::{ - source::{ - comment::{Comment, CommentUpdateForm}, - post::Post, - }, + source::comment::{Comment, CommentUpdateForm}, traits::Crud, }; use lemmy_db_views::structs::{CommentView, LocalUserView}; @@ -56,17 +53,8 @@ pub async fn delete_comment( .await .with_lemmy_type(LemmyErrorType::CouldntUpdateComment)?; - let post_id = updated_comment.post_id; - let post = Post::read(&mut context.pool(), post_id).await?; - let recipient_ids = send_local_notifs( - vec![], - &updated_comment, - &local_user_view.person, - &post, - false, - &context, - ) - .await?; + let recipient_ids = + send_local_notifs(vec![], comment_id, &local_user_view.person, false, &context).await?; let updated_comment_id = updated_comment.id; ActivityChannel::submit_activity( diff --git a/crates/api_crud/src/comment/remove.rs b/crates/api_crud/src/comment/remove.rs index 5bb6f55b1..d735c4462 100644 --- a/crates/api_crud/src/comment/remove.rs +++ b/crates/api_crud/src/comment/remove.rs @@ -12,7 +12,6 @@ use lemmy_db_schema::{ comment::{Comment, CommentUpdateForm}, comment_report::CommentReport, moderator::{ModRemoveComment, ModRemoveCommentForm}, - post::Post, }, traits::{Crud, Reportable}, }; @@ -61,13 +60,10 @@ pub async fn remove_comment( }; ModRemoveComment::create(&mut context.pool(), &form).await?; - let post_id = updated_comment.post_id; - let post = Post::read(&mut context.pool(), post_id).await?; let recipient_ids = send_local_notifs( vec![], - &updated_comment, + comment_id, &local_user_view.person.clone(), - &post, false, &context, ) diff --git a/crates/api_crud/src/comment/update.rs b/crates/api_crud/src/comment/update.rs index e814ebd6b..b35333ec5 100644 --- a/crates/api_crud/src/comment/update.rs +++ b/crates/api_crud/src/comment/update.rs @@ -79,9 +79,8 @@ pub async fn update_comment( let mentions = scrape_text_for_mentions(&updated_comment_content); let recipient_ids = send_local_notifs( mentions, - &updated_comment, + comment_id, &local_user_view.person, - &orig_comment.post, false, &context, ) diff --git a/crates/apub/src/activities/create_or_update/comment.rs b/crates/apub/src/activities/create_or_update/comment.rs index c8d9a017e..ea6af2fcf 100644 --- a/crates/apub/src/activities/create_or_update/comment.rs +++ b/crates/apub/src/activities/create_or_update/comment.rs @@ -159,8 +159,6 @@ impl ActivityHandler for CreateOrUpdateNote { CommentAggregates::update_hot_rank(&mut context.pool(), comment.id).await?; let do_send_email = self.kind == CreateOrUpdateType::Create; - let post_id = comment.post_id; - let post = Post::read(&mut context.pool(), post_id).await?; let actor = self.actor.dereference(context).await?; // Note: @@ -169,7 +167,7 @@ impl ActivityHandler for CreateOrUpdateNote { // anyway. // TODO: for compatibility with other projects, it would be much better to read this from cc or tags let mentions = scrape_text_for_mentions(&comment.content); - send_local_notifs(mentions, &comment.0, &actor, &post, do_send_email, context).await?; + send_local_notifs(mentions, comment.id, &actor, do_send_email, context).await?; Ok(()) } } From 1d0a6ac08fd1e5e488bbd5bbf0105c8fe146da73 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Tue, 9 Apr 2024 16:10:20 +0200 Subject: [PATCH 3/4] Avoid breaking api change, reduce api cache duration (#4610) * Dont mark site.public_key as `serde(skip)` to avoid breaking change (fixes #4605) * Reduce cache duration for api --- crates/api_common/src/utils.rs | 4 ++-- crates/api_crud/src/site/read.rs | 4 ++-- crates/apub/src/lib.rs | 4 ++-- crates/db_schema/src/impls/local_site.rs | 4 ++-- crates/db_schema/src/source/site.rs | 2 +- crates/federate/src/util.rs | 9 ++++++--- crates/utils/src/lib.rs | 6 ++++-- 7 files changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index dd0a8c0c6..58e3f382b 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -42,7 +42,7 @@ use lemmy_utils::{ markdown::{markdown_check_for_blocked_urls, markdown_rewrite_image_links}, slurs::{build_slur_regex, remove_slurs}, }, - CACHE_DURATION_SHORT, + CACHE_DURATION_FEDERATION, }; use moka::future::Cache; use once_cell::sync::Lazy; @@ -524,7 +524,7 @@ pub async fn get_url_blocklist(context: &LemmyContext) -> LemmyResult static URL_BLOCKLIST: Lazy> = Lazy::new(|| { Cache::builder() .max_capacity(1) - .time_to_live(CACHE_DURATION_SHORT) + .time_to_live(CACHE_DURATION_FEDERATION) .build() }); diff --git a/crates/api_crud/src/site/read.rs b/crates/api_crud/src/site/read.rs index 0d3685a94..c4b27de1e 100644 --- a/crates/api_crud/src/site/read.rs +++ b/crates/api_crud/src/site/read.rs @@ -20,7 +20,7 @@ use lemmy_db_views_actor::structs::{ }; use lemmy_utils::{ error::{LemmyError, LemmyErrorExt, LemmyErrorType}, - CACHE_DURATION_SHORT, + CACHE_DURATION_API, VERSION, }; use moka::future::Cache; @@ -34,7 +34,7 @@ pub async fn get_site( static CACHE: Lazy> = Lazy::new(|| { Cache::builder() .max_capacity(1) - .time_to_live(CACHE_DURATION_SHORT) + .time_to_live(CACHE_DURATION_API) .build() }); diff --git a/crates/apub/src/lib.rs b/crates/apub/src/lib.rs index c09a3007c..5d10dd93d 100644 --- a/crates/apub/src/lib.rs +++ b/crates/apub/src/lib.rs @@ -11,7 +11,7 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::{LemmyError, LemmyErrorType, LemmyResult}, - CACHE_DURATION_SHORT, + CACHE_DURATION_FEDERATION, }; use moka::future::Cache; use once_cell::sync::Lazy; @@ -127,7 +127,7 @@ pub(crate) async fn local_site_data_cached( static CACHE: Lazy>> = Lazy::new(|| { Cache::builder() .max_capacity(1) - .time_to_live(CACHE_DURATION_SHORT) + .time_to_live(CACHE_DURATION_FEDERATION) .build() }); Ok( diff --git a/crates/db_schema/src/impls/local_site.rs b/crates/db_schema/src/impls/local_site.rs index 5a6cf2a53..90b25fed5 100644 --- a/crates/db_schema/src/impls/local_site.rs +++ b/crates/db_schema/src/impls/local_site.rs @@ -5,7 +5,7 @@ use crate::{ }; use diesel::{dsl::insert_into, result::Error}; use diesel_async::RunQueryDsl; -use lemmy_utils::{error::LemmyError, CACHE_DURATION_SHORT}; +use lemmy_utils::{error::LemmyError, CACHE_DURATION_API}; use moka::future::Cache; use once_cell::sync::Lazy; @@ -21,7 +21,7 @@ impl LocalSite { static CACHE: Lazy> = Lazy::new(|| { Cache::builder() .max_capacity(1) - .time_to_live(CACHE_DURATION_SHORT) + .time_to_live(CACHE_DURATION_API) .build() }); Ok( diff --git a/crates/db_schema/src/source/site.rs b/crates/db_schema/src/source/site.rs index 40ba14f96..83d23c779 100644 --- a/crates/db_schema/src/source/site.rs +++ b/crates/db_schema/src/source/site.rs @@ -36,7 +36,7 @@ pub struct Site { pub inbox_url: DbUrl, #[serde(skip)] pub private_key: Option, - #[serde(skip)] + // TODO: mark as `serde(skip)` in next major release as its not needed for api pub public_key: String, pub instance_id: InstanceId, /// If present, nsfw content is visible by default. Should be displayed by frontends/clients diff --git a/crates/federate/src/util.rs b/crates/federate/src/util.rs index 2809b9bb4..cedf3387f 100644 --- a/crates/federate/src/util.rs +++ b/crates/federate/src/util.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use diesel::prelude::*; use diesel_async::RunQueryDsl; -use lemmy_api_common::lemmy_utils::CACHE_DURATION_SHORT; +use lemmy_api_common::lemmy_utils::CACHE_DURATION_FEDERATION; use lemmy_apub::{ activity_lists::SharedInboxActivities, fetcher::{site_or_community_or_user::SiteOrCommunityOrUser, user_or_community::UserOrCommunity}, @@ -169,8 +169,11 @@ pub(crate) async fn get_activity_cached( /// return the most current activity id (with 1 second cache) pub(crate) async fn get_latest_activity_id(pool: &mut DbPool<'_>) -> Result { - static CACHE: Lazy> = - Lazy::new(|| Cache::builder().time_to_live(CACHE_DURATION_SHORT).build()); + static CACHE: Lazy> = Lazy::new(|| { + Cache::builder() + .time_to_live(CACHE_DURATION_FEDERATION) + .build() + }); CACHE .try_get_with((), async { use diesel::dsl::max; diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 46508eb50..95c1d0144 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -24,9 +24,11 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const REQWEST_TIMEOUT: Duration = Duration::from_secs(10); #[cfg(debug_assertions)] -pub const CACHE_DURATION_SHORT: Duration = Duration::from_millis(500); +pub const CACHE_DURATION_FEDERATION: Duration = Duration::from_millis(500); #[cfg(not(debug_assertions))] -pub const CACHE_DURATION_SHORT: Duration = Duration::from_secs(60); +pub const CACHE_DURATION_FEDERATION: Duration = Duration::from_secs(60); + +pub const CACHE_DURATION_API: Duration = Duration::from_secs(1); #[macro_export] macro_rules! location_info { From b4670988b55d1c0ce6bcb436fc3f6af82ec5c013 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Wed, 10 Apr 2024 01:33:01 +0200 Subject: [PATCH 4/4] Change exponential backoff algorithm for federation send (#4597) * Limit federation send retry interval to one hour * clippy * avoid overflow * change base for exp backoff * ignore first error * fix day duration --- crates/api_common/src/lib.rs | 38 +++++++++++++++++++++++++++++++++--- scripts/test.sh | 4 ++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/crates/api_common/src/lib.rs b/crates/api_common/src/lib.rs index b55dff32f..4bb7ada5b 100644 --- a/crates/api_common/src/lib.rs +++ b/crates/api_common/src/lib.rs @@ -27,7 +27,7 @@ pub extern crate lemmy_utils; pub use lemmy_utils::LemmyErrorType; use serde::{Deserialize, Serialize}; -use std::time::Duration; +use std::{cmp::min, time::Duration}; #[derive(Debug, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(ts_rs::TS))] @@ -43,7 +43,39 @@ impl Default for SuccessResponse { } } -/// how long to sleep based on how many retries have already happened +// TODO: use from_days once stabilized +// https://github.com/rust-lang/rust/issues/120301 +const DAY: Duration = Duration::from_secs(24 * 60 * 60); + +/// Calculate how long to sleep until next federation send based on how many +/// retries have already happened. Uses exponential backoff with maximum of one day. The first +/// error is ignored. pub fn federate_retry_sleep_duration(retry_count: i32) -> Duration { - Duration::from_secs_f64(2.0_f64.powf(f64::from(retry_count))) + debug_assert!(retry_count != 0); + if retry_count == 1 { + return Duration::from_secs(0); + } + let retry_count = retry_count - 1; + let pow = 1.25_f64.powf(retry_count.into()); + let pow = Duration::try_from_secs_f64(pow).unwrap_or(DAY); + min(DAY, pow) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + #[test] + fn test_federate_retry_sleep_duration() { + assert_eq!(Duration::from_secs(0), federate_retry_sleep_duration(1)); + assert_eq!( + Duration::new(1, 250000000), + federate_retry_sleep_duration(2) + ); + assert_eq!( + Duration::new(2, 441406250), + federate_retry_sleep_duration(5) + ); + assert_eq!(DAY, federate_retry_sleep_duration(100)); + } } diff --git a/scripts/test.sh b/scripts/test.sh index efe9b1513..3e0581fc7 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -6,7 +6,7 @@ CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" cd $CWD/../ PACKAGE="$1" -echo "$PACKAGE" +TEST="$2" source scripts/start_dev_db.sh @@ -17,7 +17,7 @@ export RUST_BACKTRACE=1 if [ -n "$PACKAGE" ]; then - cargo test -p $PACKAGE --all-features --no-fail-fast + cargo test -p $PACKAGE --all-features --no-fail-fast $TEST else cargo test --workspace --no-fail-fast fi