From 600ae662a57f1c2343eeba858141de30c699fd10 Mon Sep 17 00:00:00 2001 From: Andrew Yoon Date: Thu, 25 Feb 2021 14:43:39 -0500 Subject: [PATCH 01/16] Support plain `cargo test` and disable unused doctests for speed Since DB tests execute diesel migrations automatically, concurrent execution causes flaky failures from simultaneous migrations. This can be worked around using `cargo test --workspace -- --test-threads=1`, which is what the CI config does, but this is not intuitive for newcomer developers and unnecessarily slows down the test suite for the majority of tests which are safe to run concurrently. This fixes this issue by integrating with the small test crate `serial_test` and using it to explicitly mark DB tests to run sequentially while allowing all other tests to run in parallel. Additionally, this greatly improves the speed of `cargo test` by disabling doc-tests in all crates, since these are aren't currently used and cargo's doc-test pass, even when no doc-tests exist, has significant overhead. On my machine, this change significantly improves test suite times by about 85%, making it much more practical to develop with tools like `cargo watch` auto-running tests. --- .drone.yml | 2 -- Cargo.lock | 26 +++++++++++++++++++ Cargo.toml | 3 +++ crates/api/Cargo.toml | 1 + crates/apub/Cargo.toml | 1 + crates/db_queries/Cargo.toml | 4 +++ .../src/aggregates/comment_aggregates.rs | 2 ++ .../src/aggregates/community_aggregates.rs | 2 ++ .../src/aggregates/post_aggregates.rs | 2 ++ .../src/aggregates/site_aggregates.rs | 2 ++ .../src/aggregates/user_aggregates.rs | 2 ++ crates/db_queries/src/lib.rs | 3 +++ crates/db_queries/src/source/activity.rs | 2 ++ crates/db_queries/src/source/comment.rs | 2 ++ crates/db_queries/src/source/community.rs | 2 ++ crates/db_queries/src/source/moderator.rs | 2 ++ .../src/source/password_reset_request.rs | 2 ++ crates/db_queries/src/source/post.rs | 2 ++ .../db_queries/src/source/private_message.rs | 2 ++ crates/db_queries/src/source/user.rs | 2 ++ crates/db_queries/src/source/user_mention.rs | 2 ++ crates/db_schema/Cargo.toml | 3 +++ crates/db_views/Cargo.toml | 6 +++++ crates/db_views/src/comment_view.rs | 2 ++ crates/db_views/src/lib.rs | 3 +++ crates/db_views/src/post_view.rs | 2 ++ crates/db_views_actor/Cargo.toml | 3 +++ crates/db_views_moderator/Cargo.toml | 3 +++ crates/routes/Cargo.toml | 3 +++ crates/structs/Cargo.toml | 1 + crates/utils/Cargo.toml | 1 + crates/websocket/Cargo.toml | 1 + 32 files changed, 94 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 807d6c991..b6fa86db5 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,7 +29,6 @@ steps: environment: LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy RUST_BACKTRACE: 1 - RUST_TEST_THREADS: 1 commands: - sudo apt-get update - sudo apt-get -y install --no-install-recommends espeak postgresql-client @@ -107,7 +106,6 @@ steps: environment: LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy RUST_BACKTRACE: 1 - RUST_TEST_THREADS: 1 commands: - apt-get update - apt-get -y install --no-install-recommends espeak postgresql-client libssl-dev pkg-config libpq-dev diff --git a/Cargo.lock b/Cargo.lock index 2edcca8d7..cdf7f679c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "activitystreams" version = "0.7.0-alpha.10" @@ -1818,6 +1820,7 @@ dependencies = [ "regex", "serde 1.0.123", "serde_json", + "serial_test", "sha2", "strum", "strum_macros", @@ -1845,6 +1848,7 @@ dependencies = [ "lemmy_db_schema", "log", "serde 1.0.123", + "serial_test", "url", ] @@ -3080,6 +3084,28 @@ dependencies = [ "serde 1.0.123", ] +[[package]] +name = "serial_test" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0bccbcf40c8938196944a3da0e133e031a33f4d6b72db3bda3cc556e361905d" +dependencies = [ + "lazy_static", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha-1" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index edec85f60..ab7d90198 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,9 @@ name = "lemmy_server" version = "0.0.1" edition = "2018" +[lib] +doctest = false + [profile.dev] debug = 0 diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index e3226eb85..299121a6e 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [lib] name = "lemmy_api" path = "src/lib.rs" +doctest = false [dependencies] lemmy_apub = { path = "../apub" } diff --git a/crates/apub/Cargo.toml b/crates/apub/Cargo.toml index f2ad96838..9dd9bc82f 100644 --- a/crates/apub/Cargo.toml +++ b/crates/apub/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [lib] name = "lemmy_apub" path = "src/lib.rs" +doctest = false [dependencies] lemmy_utils = { path = "../utils" } diff --git a/crates/db_queries/Cargo.toml b/crates/db_queries/Cargo.toml index c396b39a3..0e489e228 100644 --- a/crates/db_queries/Cargo.toml +++ b/crates/db_queries/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [lib] name = "lemmy_db_queries" path = "src/lib.rs" +doctest = false [dependencies] lemmy_utils = { path = "../utils" } @@ -23,3 +24,6 @@ url = { version = "2.2.0", features = ["serde"] } lazy_static = "1.4.0" regex = "1.4.3" bcrypt = "0.9.0" + +[dev-dependencies] +serial_test = "0.5.1" \ No newline at end of file diff --git a/crates/db_queries/src/aggregates/comment_aggregates.rs b/crates/db_queries/src/aggregates/comment_aggregates.rs index a4db471b6..0826ae0d4 100644 --- a/crates/db_queries/src/aggregates/comment_aggregates.rs +++ b/crates/db_queries/src/aggregates/comment_aggregates.rs @@ -37,8 +37,10 @@ mod tests { post::{Post, PostForm}, user::{UserForm, User_}, }; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/aggregates/community_aggregates.rs b/crates/db_queries/src/aggregates/community_aggregates.rs index f5cd577e4..ee60da9b2 100644 --- a/crates/db_queries/src/aggregates/community_aggregates.rs +++ b/crates/db_queries/src/aggregates/community_aggregates.rs @@ -41,8 +41,10 @@ mod tests { post::{Post, PostForm}, user::{UserForm, User_}, }; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/aggregates/post_aggregates.rs b/crates/db_queries/src/aggregates/post_aggregates.rs index fa8b69255..404257a7b 100644 --- a/crates/db_queries/src/aggregates/post_aggregates.rs +++ b/crates/db_queries/src/aggregates/post_aggregates.rs @@ -41,8 +41,10 @@ mod tests { post::{Post, PostForm, PostLike, PostLikeForm}, user::{UserForm, User_}, }; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/aggregates/site_aggregates.rs b/crates/db_queries/src/aggregates/site_aggregates.rs index 46db2f0cd..81afed5c4 100644 --- a/crates/db_queries/src/aggregates/site_aggregates.rs +++ b/crates/db_queries/src/aggregates/site_aggregates.rs @@ -39,8 +39,10 @@ mod tests { site::{Site, SiteForm}, user::{UserForm, User_}, }; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/aggregates/user_aggregates.rs b/crates/db_queries/src/aggregates/user_aggregates.rs index fcda1d462..bdd16932a 100644 --- a/crates/db_queries/src/aggregates/user_aggregates.rs +++ b/crates/db_queries/src/aggregates/user_aggregates.rs @@ -37,8 +37,10 @@ mod tests { post::{Post, PostForm, PostLike, PostLikeForm}, user::{UserForm, User_}, }; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/lib.rs b/crates/db_queries/src/lib.rs index bf0de3366..5667b4262 100644 --- a/crates/db_queries/src/lib.rs +++ b/crates/db_queries/src/lib.rs @@ -9,6 +9,9 @@ extern crate lazy_static; #[macro_use] extern crate diesel_migrations; +#[cfg(test)] +extern crate serial_test; + use diesel::{result::Error, *}; use lemmy_db_schema::Url; use regex::Regex; diff --git a/crates/db_queries/src/source/activity.rs b/crates/db_queries/src/source/activity.rs index 59e1754aa..cf946d67d 100644 --- a/crates/db_queries/src/source/activity.rs +++ b/crates/db_queries/src/source/activity.rs @@ -133,8 +133,10 @@ mod tests { user::{UserForm, User_}, }; use serde_json::Value; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/source/comment.rs b/crates/db_queries/src/source/comment.rs index 709a8aa39..918ae4977 100644 --- a/crates/db_queries/src/source/comment.rs +++ b/crates/db_queries/src/source/comment.rs @@ -212,8 +212,10 @@ mod tests { post::*, user::{UserForm, User_}, }; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/source/community.rs b/crates/db_queries/src/source/community.rs index 08421bd5d..6a19d2211 100644 --- a/crates/db_queries/src/source/community.rs +++ b/crates/db_queries/src/source/community.rs @@ -340,8 +340,10 @@ mod tests { SortType, }; use lemmy_db_schema::source::{community::*, user::*}; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/source/moderator.rs b/crates/db_queries/src/source/moderator.rs index 9b6e58edf..469ada089 100644 --- a/crates/db_queries/src/source/moderator.rs +++ b/crates/db_queries/src/source/moderator.rs @@ -199,9 +199,11 @@ impl Crud for ModAdd { mod tests { use crate::{establish_unpooled_connection, Crud, ListingType, SortType}; use lemmy_db_schema::source::{comment::*, community::*, moderator::*, post::*, user::*}; + use serial_test::serial; // use Crud; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/source/password_reset_request.rs b/crates/db_queries/src/source/password_reset_request.rs index f58d2c014..600037d8a 100644 --- a/crates/db_queries/src/source/password_reset_request.rs +++ b/crates/db_queries/src/source/password_reset_request.rs @@ -80,8 +80,10 @@ mod tests { SortType, }; use lemmy_db_schema::source::{password_reset_request::PasswordResetRequest, user::*}; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/source/post.rs b/crates/db_queries/src/source/post.rs index 1c19e53d0..f6b1342ef 100644 --- a/crates/db_queries/src/source/post.rs +++ b/crates/db_queries/src/source/post.rs @@ -231,8 +231,10 @@ mod tests { community::{Community, CommunityForm}, user::*, }; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/source/private_message.rs b/crates/db_queries/src/source/private_message.rs index 8cace9384..f1908bc14 100644 --- a/crates/db_queries/src/source/private_message.rs +++ b/crates/db_queries/src/source/private_message.rs @@ -147,8 +147,10 @@ mod tests { SortType, }; use lemmy_db_schema::source::{private_message::*, user::*}; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/source/user.rs b/crates/db_queries/src/source/user.rs index 5f3fa6cbb..20b187b46 100644 --- a/crates/db_queries/src/source/user.rs +++ b/crates/db_queries/src/source/user.rs @@ -379,8 +379,10 @@ impl User for User_ { #[cfg(test)] mod tests { use crate::{establish_unpooled_connection, source::user::*, ListingType, SortType}; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_queries/src/source/user_mention.rs b/crates/db_queries/src/source/user_mention.rs index b0c97572f..5efc95bf6 100644 --- a/crates/db_queries/src/source/user_mention.rs +++ b/crates/db_queries/src/source/user_mention.rs @@ -81,8 +81,10 @@ mod tests { user::*, user_mention::*, }; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index a175f80e4..b637c0222 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -3,6 +3,9 @@ name = "lemmy_db_schema" version = "0.1.0" edition = "2018" +[lib] +doctest = false + [dependencies] diesel = { version = "1.4.5", features = ["postgres","chrono","r2d2","serde_json"] } chrono = { version = "0.4.19", features = ["serde"] } diff --git a/crates/db_views/Cargo.toml b/crates/db_views/Cargo.toml index 175e6eae6..e44f414a6 100644 --- a/crates/db_views/Cargo.toml +++ b/crates/db_views/Cargo.toml @@ -3,6 +3,9 @@ name = "lemmy_db_views" version = "0.1.0" edition = "2018" +[lib] +doctest = false + [dependencies] lemmy_db_queries = { path = "../db_queries" } lemmy_db_schema = { path = "../db_schema" } @@ -10,3 +13,6 @@ diesel = { version = "1.4.5", features = ["postgres","chrono","r2d2","serde_json serde = { version = "1.0.123", features = ["derive"] } log = "0.4.14" url = "2.2.0" + +[dev-dependencies] +serial_test = "0.5.1" \ No newline at end of file diff --git a/crates/db_views/src/comment_view.rs b/crates/db_views/src/comment_view.rs index 4f9a6a5f7..56b22a7f0 100644 --- a/crates/db_views/src/comment_view.rs +++ b/crates/db_views/src/comment_view.rs @@ -443,8 +443,10 @@ mod tests { SortType, }; use lemmy_db_schema::source::{comment::*, community::*, post::*, user::*}; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_views/src/lib.rs b/crates/db_views/src/lib.rs index b46ec5a46..a9369ed24 100644 --- a/crates/db_views/src/lib.rs +++ b/crates/db_views/src/lib.rs @@ -1,3 +1,6 @@ +#[cfg(test)] +extern crate serial_test; + pub mod comment_report_view; pub mod comment_view; pub mod post_report_view; diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index ed1ef4ce1..e7b488275 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -434,8 +434,10 @@ mod tests { SortType, }; use lemmy_db_schema::source::{community::*, post::*, user::*}; + use serial_test::serial; #[test] + #[serial] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/crates/db_views_actor/Cargo.toml b/crates/db_views_actor/Cargo.toml index 8e86d4c0a..e237941e1 100644 --- a/crates/db_views_actor/Cargo.toml +++ b/crates/db_views_actor/Cargo.toml @@ -3,6 +3,9 @@ name = "lemmy_db_views_actor" version = "0.1.0" edition = "2018" +[lib] +doctest = false + [dependencies] lemmy_db_queries = { path = "../db_queries" } lemmy_db_schema = { path = "../db_schema" } diff --git a/crates/db_views_moderator/Cargo.toml b/crates/db_views_moderator/Cargo.toml index aef6abdc3..9df01522d 100644 --- a/crates/db_views_moderator/Cargo.toml +++ b/crates/db_views_moderator/Cargo.toml @@ -3,6 +3,9 @@ name = "lemmy_db_views_moderator" version = "0.1.0" edition = "2018" +[lib] +doctest = false + [dependencies] lemmy_db_queries = { path = "../db_queries" } lemmy_db_schema = { path = "../db_schema" } diff --git a/crates/routes/Cargo.toml b/crates/routes/Cargo.toml index 0c5eac665..b71ffca70 100644 --- a/crates/routes/Cargo.toml +++ b/crates/routes/Cargo.toml @@ -3,6 +3,9 @@ name = "lemmy_routes" version = "0.1.0" edition = "2018" +[lib] +doctest = false + [dependencies] lemmy_utils = { path = "../utils" } lemmy_websocket = { path = "../websocket" } diff --git a/crates/structs/Cargo.toml b/crates/structs/Cargo.toml index f895a8c95..de4f38cbf 100644 --- a/crates/structs/Cargo.toml +++ b/crates/structs/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [lib] name = "lemmy_structs" path = "src/lib.rs" +doctest = false [dependencies] lemmy_db_queries = { path = "../db_queries" } diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index c10b7c2c7..3308669a4 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [lib] name = "lemmy_utils" path = "src/lib.rs" +doctest = false [dependencies] regex = "1.4.3" diff --git a/crates/websocket/Cargo.toml b/crates/websocket/Cargo.toml index 1cb71ea1d..3116d71e7 100644 --- a/crates/websocket/Cargo.toml +++ b/crates/websocket/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [lib] name = "lemmy_websocket" path = "src/lib.rs" +doctest = false [dependencies] lemmy_utils = { path = "../utils" } From 4cc341e4aa996cb7c3e5296168442a162b82ed6d Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 26 Feb 2021 14:03:49 +0100 Subject: [PATCH 02/16] Remove code for apub compatibility with Lemmy v0.8.9 and older --- .../apub/src/inbox/receive_for_community.rs | 43 +++++++------------ crates/apub/src/routes.rs | 4 +- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/crates/apub/src/inbox/receive_for_community.rs b/crates/apub/src/inbox/receive_for_community.rs index e3704d97a..6c087fc80 100644 --- a/crates/apub/src/inbox/receive_for_community.rs +++ b/crates/apub/src/inbox/receive_for_community.rs @@ -115,7 +115,10 @@ pub(in crate::inbox) async fn receive_like_for_community( verify_activity_domains_valid(&like, &expected_domain, false)?; is_addressed_to_public(&like)?; - let object_id = get_like_object_id(&like)?; + let object_id = like + .object() + .as_single_xsd_any_uri() + .context(location_info!())?; match fetch_post_or_comment_by_id(&object_id, context, request_counter).await? { PostOrComment::Post(post) => receive_like_post(like, post, context, request_counter).await, PostOrComment::Comment(comment) => { @@ -143,7 +146,10 @@ pub(in crate::inbox) async fn receive_dislike_for_community( verify_activity_domains_valid(&dislike, &expected_domain, false)?; is_addressed_to_public(&dislike)?; - let object_id = get_like_object_id(&dislike)?; + let object_id = dislike + .object() + .as_single_xsd_any_uri() + .context(location_info!())?; match fetch_post_or_comment_by_id(&object_id, context, request_counter).await? { PostOrComment::Post(post) => { receive_dislike_post(dislike, post, context, request_counter).await @@ -313,7 +319,10 @@ pub(in crate::inbox) async fn receive_undo_like_for_community( verify_activity_domains_valid(&like, &expected_domain, false)?; is_addressed_to_public(&like)?; - let object_id = get_like_object_id(&like)?; + let object_id = like + .object() + .as_single_xsd_any_uri() + .context(location_info!())?; match fetch_post_or_comment_by_id(&object_id, context, request_counter).await? { PostOrComment::Post(post) => { receive_undo_like_post(&like, post, context, request_counter).await @@ -336,7 +345,10 @@ pub(in crate::inbox) async fn receive_undo_dislike_for_community( verify_activity_domains_valid(&dislike, &expected_domain, false)?; is_addressed_to_public(&dislike)?; - let object_id = get_like_object_id(&dislike)?; + let object_id = dislike + .object() + .as_single_xsd_any_uri() + .context(location_info!())?; match fetch_post_or_comment_by_id(&object_id, context, request_counter).await? { PostOrComment::Post(post) => { receive_undo_dislike_post(&dislike, post, context, request_counter).await @@ -362,26 +374,3 @@ async fn fetch_post_or_comment_by_id( Err(NotFound.into()) } - -fn get_like_object_id(like_or_dislike: &Activity) -> Result -where - Activity: ActorAndObjectRefExt, -{ - // TODO: For backwards compatibility with older Lemmy versions where like.object contains a full - // post/comment. This can be removed after some time, using - // `activity.oject().as_single_xsd_any_uri()` instead. - let object = like_or_dislike.object(); - if let Some(xsd_uri) = object.as_single_xsd_any_uri() { - Ok(xsd_uri.to_owned()) - } else { - Ok( - object - .to_owned() - .one() - .context(location_info!())? - .id() - .context(location_info!())? - .to_owned(), - ) - } -} diff --git a/crates/apub/src/routes.rs b/crates/apub/src/routes.rs index 3717a894f..6fa82ea84 100644 --- a/crates/apub/src/routes.rs +++ b/crates/apub/src/routes.rs @@ -31,9 +31,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { .or(guard::Header("Accept", APUB_JSON_CONTENT_TYPE_LONG)); let header_guard_content_type = guard::Any(guard::Header("Content-Type", APUB_JSON_CONTENT_TYPE)) - .or(guard::Header("Content-Type", APUB_JSON_CONTENT_TYPE_LONG)) - // TODO: compatibility with previous lemmy versions, remove this later - .or(guard::Header("Content-Type", "application/json")); + .or(guard::Header("Content-Type", APUB_JSON_CONTENT_TYPE_LONG)); cfg .service( From c618b4efaa75ab2989da48317b32fc6725cb0e81 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 5 Feb 2021 14:43:18 +0100 Subject: [PATCH 03/16] Remove federation backward compatibility code (ref #1220) --- crates/apub/src/objects/mod.rs | 48 ++++++++++++++++----------------- crates/apub/src/objects/user.rs | 3 --- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/crates/apub/src/objects/mod.rs b/crates/apub/src/objects/mod.rs index 79b9ad405..a35ad7d13 100644 --- a/crates/apub/src/objects/mod.rs +++ b/crates/apub/src/objects/mod.rs @@ -15,7 +15,12 @@ use diesel::result::Error::NotFound; use lemmy_db_queries::{ApubObject, Crud, DbPool}; use lemmy_db_schema::source::community::Community; use lemmy_structs::blocking; -use lemmy_utils::{location_info, settings::Settings, utils::convert_datetime, LemmyError}; +use lemmy_utils::{ + location_info, + settings::Settings, + utils::{convert_datetime, markdown_to_html}, + LemmyError, +}; use lemmy_websocket::LemmyContext; use url::Url; @@ -114,11 +119,8 @@ where .set_media_type(mime_markdown()?); object.set_source(source.into_any_base()?); - // set `content` to markdown for compatibility with older Lemmy versions - // TODO: change this to HTML in a while - object.set_content(markdown_text); - object.set_media_type(mime_markdown()?); - //object.set_content(markdown_to_html(markdown_text)); + object.set_content(markdown_to_html(markdown_text)); + object.set_media_type(mime_html()?); Ok(()) } @@ -134,32 +136,28 @@ where .flatten() .map(|s| s.to_string()); if content.is_some() { - let source = object.source(); - // updated lemmy version, read markdown from `source.content` - if let Some(source) = source { - let source = Object::<()>::from_any_base(source.to_owned())?.context(location_info!())?; - check_is_markdown(source.media_type())?; - let source_content = source - .content() - .map(|s| s.as_single_xsd_string()) - .flatten() - .context(location_info!())? - .to_string(); - return Ok(Some(source_content)); - } - // older lemmy version, read markdown from `content` - // TODO: remove this after a while - else { - return Ok(content); - } + let source = object.source().context(location_info!())?; + let source = Object::<()>::from_any_base(source.to_owned())?.context(location_info!())?; + check_is_markdown(source.media_type())?; + let source_content = source + .content() + .map(|s| s.as_single_xsd_string()) + .flatten() + .context(location_info!())? + .to_string(); + return Ok(Some(source_content)); } Ok(None) } -pub(in crate::objects) fn mime_markdown() -> Result { +fn mime_markdown() -> Result { "text/markdown".parse() } +fn mime_html() -> Result { + "text/html".parse() +} + pub(in crate::objects) fn check_is_markdown(mime: Option<&Mime>) -> Result<(), LemmyError> { let mime = mime.context(location_info!())?; if !mime.eq(&mime_markdown()?) { diff --git a/crates/apub/src/objects/user.rs b/crates/apub/src/objects/user.rs index 8a911de27..7a85b20a9 100644 --- a/crates/apub/src/objects/user.rs +++ b/crates/apub/src/objects/user.rs @@ -62,9 +62,6 @@ impl ToApub for User_ { if let Some(bio) = &self.bio { set_content_and_source(&mut person, bio)?; - // Also set summary for compatibility with older Lemmy versions. - // TODO: remove this after a while. - person.set_summary(bio.to_owned()); } if let Some(i) = self.preferred_username.to_owned() { From a5e246309743e8f6004f2cbbc0771e223bf5a08e Mon Sep 17 00:00:00 2001 From: "Arjen P. de Vries" Date: Sat, 27 Feb 2021 20:26:30 +0100 Subject: [PATCH 04/16] Update README.md Federation is probably more complete than suggested here. In response to: https://github.com/LemmyNet/lemmy/issues/647#issuecomment-787068478 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 551dfa1d6..eebf4fbaf 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ The overall goal is to create an easily self-hostable, decentralized alternative Each Lemmy server can set its own moderation policy; appointing site-wide admins, and community moderators to keep out the trolls, and foster a healthy, non-toxic environment where all can feel comfortable contributing. -*Note: Federation is still in active development and the WebSocket, as well as, HTTP API are currently unstable* +*Note: The WebSocket and HTTP APIs are currently unstable* ### Why's it called Lemmy? From 3bdd78f3419df30713aeb7ed5c4cd0a40e4784e7 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 1 Mar 2021 14:08:41 +0100 Subject: [PATCH 05/16] Rename `lemmy_structs` to `lemmy_api_structs` --- Cargo.lock | 48 +++++++++---------- Cargo.toml | 4 +- crates/api/Cargo.toml | 3 +- crates/api/src/comment.rs | 2 +- crates/api/src/community.rs | 2 +- crates/api/src/lib.rs | 10 +++- crates/api/src/post.rs | 2 +- crates/api/src/routes.rs | 2 +- crates/api/src/site.rs | 2 +- crates/api/src/user.rs | 2 +- crates/api/src/websocket.rs | 2 +- crates/{structs => api_structs}/Cargo.toml | 5 +- .../{structs => api_structs}/src/comment.rs | 0 .../{structs => api_structs}/src/community.rs | 0 crates/{structs => api_structs}/src/lib.rs | 0 crates/{structs => api_structs}/src/post.rs | 0 crates/{structs => api_structs}/src/site.rs | 0 crates/{structs => api_structs}/src/user.rs | 0 .../{structs => api_structs}/src/websocket.rs | 0 crates/apub/Cargo.toml | 3 +- crates/apub/src/activities/receive/comment.rs | 2 +- .../src/activities/receive/comment_undo.rs | 2 +- .../apub/src/activities/receive/community.rs | 2 +- crates/apub/src/activities/receive/post.rs | 2 +- .../apub/src/activities/receive/post_undo.rs | 2 +- .../src/activities/receive/private_message.rs | 2 +- crates/apub/src/activities/send/comment.rs | 2 +- crates/apub/src/activities/send/community.rs | 2 +- crates/apub/src/activities/send/post.rs | 2 +- .../src/activities/send/private_message.rs | 2 +- crates/apub/src/activities/send/user.rs | 2 +- crates/apub/src/fetcher/community.rs | 2 +- crates/apub/src/fetcher/objects.rs | 2 +- crates/apub/src/fetcher/search.rs | 2 +- crates/apub/src/fetcher/user.rs | 2 +- crates/apub/src/http/comment.rs | 2 +- crates/apub/src/http/community.rs | 2 +- crates/apub/src/http/mod.rs | 2 +- crates/apub/src/http/post.rs | 2 +- crates/apub/src/http/user.rs | 2 +- crates/apub/src/inbox/community_inbox.rs | 2 +- crates/apub/src/inbox/mod.rs | 2 +- .../apub/src/inbox/receive_for_community.rs | 2 +- crates/apub/src/inbox/shared_inbox.rs | 2 +- crates/apub/src/inbox/user_inbox.rs | 2 +- crates/apub/src/lib.rs | 2 +- crates/apub/src/objects/comment.rs | 2 +- crates/apub/src/objects/community.rs | 2 +- crates/apub/src/objects/mod.rs | 2 +- crates/apub/src/objects/post.rs | 2 +- crates/apub/src/objects/private_message.rs | 2 +- crates/apub/src/objects/user.rs | 2 +- crates/routes/Cargo.toml | 2 +- crates/routes/src/feeds.rs | 2 +- crates/routes/src/nodeinfo.rs | 2 +- crates/routes/src/webfinger.rs | 2 +- crates/websocket/Cargo.toml | 3 +- crates/websocket/src/chat_server.rs | 2 +- crates/websocket/src/messages.rs | 2 +- src/main.rs | 2 +- 60 files changed, 86 insertions(+), 82 deletions(-) rename crates/{structs => api_structs}/Cargo.toml (87%) rename crates/{structs => api_structs}/src/comment.rs (100%) rename crates/{structs => api_structs}/src/community.rs (100%) rename crates/{structs => api_structs}/src/lib.rs (100%) rename crates/{structs => api_structs}/src/post.rs (100%) rename crates/{structs => api_structs}/src/site.rs (100%) rename crates/{structs => api_structs}/src/user.rs (100%) rename crates/{structs => api_structs}/src/websocket.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index cdf7f679c..13d67faa5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1734,13 +1734,13 @@ dependencies = [ "http-signature-normalization-actix", "itertools", "lazy_static", + "lemmy_api_structs", "lemmy_apub", "lemmy_db_queries", "lemmy_db_schema", "lemmy_db_views", "lemmy_db_views_actor", "lemmy_db_views_moderator", - "lemmy_structs", "lemmy_utils", "lemmy_websocket", "log", @@ -1758,6 +1758,25 @@ dependencies = [ "uuid", ] +[[package]] +name = "lemmy_api_structs" +version = "0.1.0" +dependencies = [ + "actix-web", + "chrono", + "diesel", + "lemmy_db_queries", + "lemmy_db_schema", + "lemmy_db_views", + "lemmy_db_views_actor", + "lemmy_db_views_moderator", + "lemmy_utils", + "log", + "serde 1.0.123", + "serde_json", + "url", +] + [[package]] name = "lemmy_apub" version = "0.1.0" @@ -1782,11 +1801,11 @@ dependencies = [ "http-signature-normalization-reqwest", "itertools", "lazy_static", + "lemmy_api_structs", "lemmy_db_queries", "lemmy_db_schema", "lemmy_db_views", "lemmy_db_views_actor", - "lemmy_structs", "lemmy_utils", "lemmy_websocket", "log", @@ -1884,11 +1903,11 @@ dependencies = [ "chrono", "diesel", "lazy_static", + "lemmy_api_structs", "lemmy_db_queries", "lemmy_db_schema", "lemmy_db_views", "lemmy_db_views_actor", - "lemmy_structs", "lemmy_utils", "lemmy_websocket", "log", @@ -1916,6 +1935,7 @@ dependencies = [ "env_logger", "http-signature-normalization-actix", "lemmy_api", + "lemmy_api_structs", "lemmy_apub", "lemmy_db_queries", "lemmy_db_schema", @@ -1923,7 +1943,6 @@ dependencies = [ "lemmy_db_views_actor", "lemmy_db_views_moderator", "lemmy_routes", - "lemmy_structs", "lemmy_utils", "lemmy_websocket", "log", @@ -1936,25 +1955,6 @@ dependencies = [ "url", ] -[[package]] -name = "lemmy_structs" -version = "0.1.0" -dependencies = [ - "actix-web", - "chrono", - "diesel", - "lemmy_db_queries", - "lemmy_db_schema", - "lemmy_db_views", - "lemmy_db_views_actor", - "lemmy_db_views_moderator", - "lemmy_utils", - "log", - "serde 1.0.123", - "serde_json", - "url", -] - [[package]] name = "lemmy_utils" version = "0.1.0" @@ -1998,9 +1998,9 @@ dependencies = [ "background-jobs", "chrono", "diesel", + "lemmy_api_structs", "lemmy_db_queries", "lemmy_db_schema", - "lemmy_structs", "lemmy_utils", "log", "rand 0.8.3", diff --git a/Cargo.toml b/Cargo.toml index ab7d90198..f76128cb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ members = [ "crates/db_views", "crates/db_views_actor", "crates/db_views_actor", - "crates/structs", + "crates/api_structs", "crates/websocket", "crates/routes" ] @@ -33,7 +33,7 @@ lemmy_db_queries = { path = "./crates/db_queries" } lemmy_db_views = { path = "./crates/db_views" } lemmy_db_views_moderator = { path = "./crates/db_views_moderator" } lemmy_db_views_actor = { path = "./crates/db_views_actor" } -lemmy_structs = { path = "./crates/structs" } +lemmy_api_structs = { path = "crates/api_structs" } lemmy_websocket = { path = "./crates/websocket" } lemmy_routes = { path = "./crates/routes" } diesel = "1.4.5" diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 299121a6e..5ab339584 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -1,7 +1,6 @@ [package] name = "lemmy_api" version = "0.1.0" -authors = ["Felix Ableitner "] edition = "2018" [lib] @@ -17,7 +16,7 @@ lemmy_db_schema = { path = "../db_schema" } lemmy_db_views = { path = "../db_views" } lemmy_db_views_moderator = { path = "../db_views_moderator" } lemmy_db_views_actor = { path = "../db_views_actor" } -lemmy_structs = { path = "../structs" } +lemmy_api_structs = { path = "../api_structs" } lemmy_websocket = { path = "../websocket" } diesel = "1.4.5" bcrypt = "0.9.0" diff --git a/crates/api/src/comment.rs b/crates/api/src/comment.rs index 02acc7f85..cb8b2d515 100644 --- a/crates/api/src/comment.rs +++ b/crates/api/src/comment.rs @@ -9,6 +9,7 @@ use crate::{ Perform, }; use actix_web::web::Data; +use lemmy_api_structs::{blocking, comment::*, send_local_notifs}; use lemmy_apub::{generate_apub_endpoint, ApubLikeableType, ApubObjectType, EndpointType}; use lemmy_db_queries::{ source::comment::Comment_, @@ -24,7 +25,6 @@ use lemmy_db_views::{ comment_report_view::{CommentReportQueryBuilder, CommentReportView}, comment_view::{CommentQueryBuilder, CommentView}, }; -use lemmy_structs::{blocking, comment::*, send_local_notifs}; use lemmy_utils::{ utils::{remove_slurs, scrape_text_for_mentions}, ApiError, diff --git a/crates/api/src/community.rs b/crates/api/src/community.rs index 128d8b303..4bfbaeb4a 100644 --- a/crates/api/src/community.rs +++ b/crates/api/src/community.rs @@ -9,6 +9,7 @@ use crate::{ }; use actix_web::web::Data; use anyhow::Context; +use lemmy_api_structs::{blocking, community::*}; use lemmy_apub::{ generate_apub_endpoint, generate_followers_url, @@ -43,7 +44,6 @@ use lemmy_db_views_actor::{ community_view::{CommunityQueryBuilder, CommunityView}, user_view::UserViewSafe, }; -use lemmy_structs::{blocking, community::*}; use lemmy_utils::{ apub::generate_actor_keypair, location_info, diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 5642c4b9c..871e8e9d8 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -1,4 +1,13 @@ use actix_web::{web, web::Data}; +use lemmy_api_structs::{ + blocking, + comment::*, + community::*, + post::*, + site::*, + user::*, + websocket::*, +}; use lemmy_db_queries::{ source::{ community::{CommunityModerator_, Community_}, @@ -18,7 +27,6 @@ use lemmy_db_views_actor::{ community_user_ban_view::CommunityUserBanView, community_view::CommunityView, }; -use lemmy_structs::{blocking, comment::*, community::*, post::*, site::*, user::*, websocket::*}; use lemmy_utils::{claims::Claims, settings::Settings, ApiError, ConnectionId, LemmyError}; use lemmy_websocket::{serialize_websocket_message, LemmyContext, UserOperation}; use serde::Deserialize; diff --git a/crates/api/src/post.rs b/crates/api/src/post.rs index 4ef07ae56..e905cd5cf 100644 --- a/crates/api/src/post.rs +++ b/crates/api/src/post.rs @@ -9,6 +9,7 @@ use crate::{ Perform, }; use actix_web::web::Data; +use lemmy_api_structs::{blocking, post::*}; use lemmy_apub::{generate_apub_endpoint, ApubLikeableType, ApubObjectType, EndpointType}; use lemmy_db_queries::{ source::post::Post_, @@ -36,7 +37,6 @@ use lemmy_db_views_actor::{ community_moderator_view::CommunityModeratorView, community_view::CommunityView, }; -use lemmy_structs::{blocking, post::*}; use lemmy_utils::{ request::fetch_iframely_and_pictrs_data, utils::{check_slurs, check_slurs_opt, is_valid_post_title}, diff --git a/crates/api/src/routes.rs b/crates/api/src/routes.rs index 7ca609f1a..a64e0bffd 100644 --- a/crates/api/src/routes.rs +++ b/crates/api/src/routes.rs @@ -1,6 +1,6 @@ use crate::Perform; use actix_web::{error::ErrorBadRequest, *}; -use lemmy_structs::{comment::*, community::*, post::*, site::*, user::*, websocket::*}; +use lemmy_api_structs::{comment::*, community::*, post::*, site::*, user::*, websocket::*}; use lemmy_utils::rate_limit::RateLimit; use lemmy_websocket::{routes::chat_route, LemmyContext}; use serde::Deserialize; diff --git a/crates/api/src/site.rs b/crates/api/src/site.rs index b545a72ea..716b67237 100644 --- a/crates/api/src/site.rs +++ b/crates/api/src/site.rs @@ -9,6 +9,7 @@ use crate::{ }; use actix_web::web::Data; use anyhow::Context; +use lemmy_api_structs::{blocking, site::*, user::Register}; use lemmy_apub::fetcher::search::search_by_apub_id; use lemmy_db_queries::{diesel_option_overwrite, source::site::Site_, Crud, SearchType, SortType}; use lemmy_db_schema::{ @@ -38,7 +39,6 @@ use lemmy_db_views_moderator::{ mod_remove_post_view::ModRemovePostView, mod_sticky_post_view::ModStickyPostView, }; -use lemmy_structs::{blocking, site::*, user::Register}; use lemmy_utils::{ location_info, settings::Settings, diff --git a/crates/api/src/user.rs b/crates/api/src/user.rs index c6877fe0d..d6395e6b7 100644 --- a/crates/api/src/user.rs +++ b/crates/api/src/user.rs @@ -12,6 +12,7 @@ use anyhow::Context; use bcrypt::verify; use captcha::{gen, Difficulty}; use chrono::Duration; +use lemmy_api_structs::{blocking, send_email_to_user, user::*}; use lemmy_apub::{ generate_apub_endpoint, generate_followers_url, @@ -65,7 +66,6 @@ use lemmy_db_views_actor::{ user_mention_view::{UserMentionQueryBuilder, UserMentionView}, user_view::UserViewSafe, }; -use lemmy_structs::{blocking, send_email_to_user, user::*}; use lemmy_utils::{ apub::generate_actor_keypair, claims::Claims, diff --git a/crates/api/src/websocket.rs b/crates/api/src/websocket.rs index 4342f15b8..58933712d 100644 --- a/crates/api/src/websocket.rs +++ b/crates/api/src/websocket.rs @@ -1,6 +1,6 @@ use crate::{get_user_from_jwt, Perform}; use actix_web::web::Data; -use lemmy_structs::websocket::*; +use lemmy_api_structs::websocket::*; use lemmy_utils::{ConnectionId, LemmyError}; use lemmy_websocket::{ messages::{JoinCommunityRoom, JoinModRoom, JoinPostRoom, JoinUserRoom}, diff --git a/crates/structs/Cargo.toml b/crates/api_structs/Cargo.toml similarity index 87% rename from crates/structs/Cargo.toml rename to crates/api_structs/Cargo.toml index de4f38cbf..fc91e759a 100644 --- a/crates/structs/Cargo.toml +++ b/crates/api_structs/Cargo.toml @@ -1,11 +1,10 @@ [package] -name = "lemmy_structs" +name = "lemmy_api_structs" version = "0.1.0" -authors = ["Felix Ableitner "] edition = "2018" [lib] -name = "lemmy_structs" +name = "lemmy_api_structs" path = "src/lib.rs" doctest = false diff --git a/crates/structs/src/comment.rs b/crates/api_structs/src/comment.rs similarity index 100% rename from crates/structs/src/comment.rs rename to crates/api_structs/src/comment.rs diff --git a/crates/structs/src/community.rs b/crates/api_structs/src/community.rs similarity index 100% rename from crates/structs/src/community.rs rename to crates/api_structs/src/community.rs diff --git a/crates/structs/src/lib.rs b/crates/api_structs/src/lib.rs similarity index 100% rename from crates/structs/src/lib.rs rename to crates/api_structs/src/lib.rs diff --git a/crates/structs/src/post.rs b/crates/api_structs/src/post.rs similarity index 100% rename from crates/structs/src/post.rs rename to crates/api_structs/src/post.rs diff --git a/crates/structs/src/site.rs b/crates/api_structs/src/site.rs similarity index 100% rename from crates/structs/src/site.rs rename to crates/api_structs/src/site.rs diff --git a/crates/structs/src/user.rs b/crates/api_structs/src/user.rs similarity index 100% rename from crates/structs/src/user.rs rename to crates/api_structs/src/user.rs diff --git a/crates/structs/src/websocket.rs b/crates/api_structs/src/websocket.rs similarity index 100% rename from crates/structs/src/websocket.rs rename to crates/api_structs/src/websocket.rs diff --git a/crates/apub/Cargo.toml b/crates/apub/Cargo.toml index 9dd9bc82f..d294cbd9d 100644 --- a/crates/apub/Cargo.toml +++ b/crates/apub/Cargo.toml @@ -1,7 +1,6 @@ [package] name = "lemmy_apub" version = "0.1.0" -authors = ["Felix Ableitner "] edition = "2018" [lib] @@ -15,7 +14,7 @@ lemmy_db_queries = { path = "../db_queries" } lemmy_db_schema = { path = "../db_schema" } lemmy_db_views = { path = "../db_views" } lemmy_db_views_actor = { path = "../db_views_actor" } -lemmy_structs = { path = "../structs" } +lemmy_api_structs = { path = "../api_structs" } lemmy_websocket = { path = "../websocket" } diesel = "1.4.5" activitystreams = "0.7.0-alpha.10" diff --git a/crates/apub/src/activities/receive/comment.rs b/crates/apub/src/activities/receive/comment.rs index 6136f63bb..591b6f33a 100644 --- a/crates/apub/src/activities/receive/comment.rs +++ b/crates/apub/src/activities/receive/comment.rs @@ -4,13 +4,13 @@ use activitystreams::{ base::ExtendsExt, }; use anyhow::Context; +use lemmy_api_structs::{blocking, comment::CommentResponse, send_local_notifs}; use lemmy_db_queries::{source::comment::Comment_, Crud, Likeable}; use lemmy_db_schema::source::{ comment::{Comment, CommentLike, CommentLikeForm}, post::Post, }; use lemmy_db_views::comment_view::CommentView; -use lemmy_structs::{blocking, comment::CommentResponse, send_local_notifs}; use lemmy_utils::{location_info, utils::scrape_text_for_mentions, LemmyError}; use lemmy_websocket::{messages::SendComment, LemmyContext, UserOperation}; diff --git a/crates/apub/src/activities/receive/comment_undo.rs b/crates/apub/src/activities/receive/comment_undo.rs index 5dc021ad3..702717155 100644 --- a/crates/apub/src/activities/receive/comment_undo.rs +++ b/crates/apub/src/activities/receive/comment_undo.rs @@ -1,9 +1,9 @@ use crate::activities::receive::get_actor_as_user; use activitystreams::activity::{Dislike, Like}; +use lemmy_api_structs::{blocking, comment::CommentResponse}; use lemmy_db_queries::{source::comment::Comment_, Likeable}; use lemmy_db_schema::source::comment::{Comment, CommentLike}; use lemmy_db_views::comment_view::CommentView; -use lemmy_structs::{blocking, comment::CommentResponse}; use lemmy_utils::LemmyError; use lemmy_websocket::{messages::SendComment, LemmyContext, UserOperation}; diff --git a/crates/apub/src/activities/receive/community.rs b/crates/apub/src/activities/receive/community.rs index 54a8cdb7a..854e75f26 100644 --- a/crates/apub/src/activities/receive/community.rs +++ b/crates/apub/src/activities/receive/community.rs @@ -4,10 +4,10 @@ use activitystreams::{ base::{AnyBase, ExtendsExt}, }; use anyhow::Context; +use lemmy_api_structs::{blocking, community::CommunityResponse}; use lemmy_db_queries::{source::community::Community_, ApubObject}; use lemmy_db_schema::source::community::Community; use lemmy_db_views_actor::community_view::CommunityView; -use lemmy_structs::{blocking, community::CommunityResponse}; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::{messages::SendCommunityRoomMessage, LemmyContext, UserOperation}; use url::Url; diff --git a/crates/apub/src/activities/receive/post.rs b/crates/apub/src/activities/receive/post.rs index 426358646..0fb6c8807 100644 --- a/crates/apub/src/activities/receive/post.rs +++ b/crates/apub/src/activities/receive/post.rs @@ -4,10 +4,10 @@ use activitystreams::{ prelude::*, }; use anyhow::Context; +use lemmy_api_structs::{blocking, post::PostResponse}; use lemmy_db_queries::{source::post::Post_, Likeable}; use lemmy_db_schema::source::post::{Post, PostLike, PostLikeForm}; use lemmy_db_views::post_view::PostView; -use lemmy_structs::{blocking, post::PostResponse}; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::{messages::SendPost, LemmyContext, UserOperation}; diff --git a/crates/apub/src/activities/receive/post_undo.rs b/crates/apub/src/activities/receive/post_undo.rs index 0b9d6f4a4..312010a9e 100644 --- a/crates/apub/src/activities/receive/post_undo.rs +++ b/crates/apub/src/activities/receive/post_undo.rs @@ -1,9 +1,9 @@ use crate::activities::receive::get_actor_as_user; use activitystreams::activity::{Dislike, Like}; +use lemmy_api_structs::{blocking, post::PostResponse}; use lemmy_db_queries::{source::post::Post_, Likeable}; use lemmy_db_schema::source::post::{Post, PostLike}; use lemmy_db_views::post_view::PostView; -use lemmy_structs::{blocking, post::PostResponse}; use lemmy_utils::LemmyError; use lemmy_websocket::{messages::SendPost, LemmyContext, UserOperation}; diff --git a/crates/apub/src/activities/receive/private_message.rs b/crates/apub/src/activities/receive/private_message.rs index 160b20ece..98c25a588 100644 --- a/crates/apub/src/activities/receive/private_message.rs +++ b/crates/apub/src/activities/receive/private_message.rs @@ -13,10 +13,10 @@ use activitystreams::{ public, }; use anyhow::{anyhow, Context}; +use lemmy_api_structs::{blocking, user::PrivateMessageResponse}; use lemmy_db_queries::source::private_message::PrivateMessage_; use lemmy_db_schema::source::private_message::PrivateMessage; use lemmy_db_views::private_message_view::PrivateMessageView; -use lemmy_structs::{blocking, user::PrivateMessageResponse}; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::{messages::SendUserRoomMessage, LemmyContext, UserOperation}; use url::Url; diff --git a/crates/apub/src/activities/send/comment.rs b/crates/apub/src/activities/send/comment.rs index f007cda42..f33de2ba7 100644 --- a/crates/apub/src/activities/send/comment.rs +++ b/crates/apub/src/activities/send/comment.rs @@ -26,9 +26,9 @@ use activitystreams::{ }; use anyhow::anyhow; use itertools::Itertools; +use lemmy_api_structs::{blocking, WebFingerResponse}; use lemmy_db_queries::{Crud, DbPool}; use lemmy_db_schema::source::{comment::Comment, community::Community, post::Post, user::User_}; -use lemmy_structs::{blocking, WebFingerResponse}; use lemmy_utils::{ request::{retry, RecvError}, settings::Settings, diff --git a/crates/apub/src/activities/send/community.rs b/crates/apub/src/activities/send/community.rs index a574c7b85..3e77248f8 100644 --- a/crates/apub/src/activities/send/community.rs +++ b/crates/apub/src/activities/send/community.rs @@ -23,10 +23,10 @@ use activitystreams::{ }; use anyhow::Context; use itertools::Itertools; +use lemmy_api_structs::blocking; use lemmy_db_queries::DbPool; use lemmy_db_schema::source::community::Community; use lemmy_db_views_actor::community_follower_view::CommunityFollowerView; -use lemmy_structs::blocking; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; use url::Url; diff --git a/crates/apub/src/activities/send/post.rs b/crates/apub/src/activities/send/post.rs index ce74d20a5..44f5b895f 100644 --- a/crates/apub/src/activities/send/post.rs +++ b/crates/apub/src/activities/send/post.rs @@ -21,9 +21,9 @@ use activitystreams::{ prelude::*, public, }; +use lemmy_api_structs::blocking; use lemmy_db_queries::Crud; use lemmy_db_schema::source::{community::Community, post::Post, user::User_}; -use lemmy_structs::blocking; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; diff --git a/crates/apub/src/activities/send/private_message.rs b/crates/apub/src/activities/send/private_message.rs index 31184a702..322eee5f7 100644 --- a/crates/apub/src/activities/send/private_message.rs +++ b/crates/apub/src/activities/send/private_message.rs @@ -16,9 +16,9 @@ use activitystreams::{ }, prelude::*, }; +use lemmy_api_structs::blocking; use lemmy_db_queries::Crud; use lemmy_db_schema::source::{private_message::PrivateMessage, user::User_}; -use lemmy_structs::blocking; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; diff --git a/crates/apub/src/activities/send/user.rs b/crates/apub/src/activities/send/user.rs index 1847ec5c5..1dc62e0bc 100644 --- a/crates/apub/src/activities/send/user.rs +++ b/crates/apub/src/activities/send/user.rs @@ -13,12 +13,12 @@ use activitystreams::{ base::{AnyBase, BaseExt, ExtendsExt}, object::ObjectExt, }; +use lemmy_api_structs::blocking; use lemmy_db_queries::{ApubObject, DbPool, Followable}; use lemmy_db_schema::source::{ community::{Community, CommunityFollower, CommunityFollowerForm}, user::User_, }; -use lemmy_structs::blocking; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use url::Url; diff --git a/crates/apub/src/fetcher/community.rs b/crates/apub/src/fetcher/community.rs index cb9ec8651..fb545ed6a 100644 --- a/crates/apub/src/fetcher/community.rs +++ b/crates/apub/src/fetcher/community.rs @@ -16,9 +16,9 @@ use activitystreams::{ }; use anyhow::Context; use diesel::result::Error::NotFound; +use lemmy_api_structs::blocking; use lemmy_db_queries::{source::community::Community_, ApubObject, Joinable}; use lemmy_db_schema::source::community::{Community, CommunityModerator, CommunityModeratorForm}; -use lemmy_structs::blocking; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; use log::debug; diff --git a/crates/apub/src/fetcher/objects.rs b/crates/apub/src/fetcher/objects.rs index df33bf740..6e0369bde 100644 --- a/crates/apub/src/fetcher/objects.rs +++ b/crates/apub/src/fetcher/objects.rs @@ -1,9 +1,9 @@ use crate::{fetcher::fetch::fetch_remote_object, objects::FromApub, NoteExt, PageExt}; use anyhow::anyhow; use diesel::result::Error::NotFound; +use lemmy_api_structs::blocking; use lemmy_db_queries::{ApubObject, Crud}; use lemmy_db_schema::source::{comment::Comment, post::Post}; -use lemmy_structs::blocking; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use log::debug; diff --git a/crates/apub/src/fetcher/search.rs b/crates/apub/src/fetcher/search.rs index a831ac404..6cbc42dc6 100644 --- a/crates/apub/src/fetcher/search.rs +++ b/crates/apub/src/fetcher/search.rs @@ -15,6 +15,7 @@ use crate::{ }; use activitystreams::base::BaseExt; use anyhow::{anyhow, Context}; +use lemmy_api_structs::{blocking, site::SearchResponse}; use lemmy_db_queries::{ source::{ comment::Comment_, @@ -34,7 +35,6 @@ use lemmy_db_schema::source::{ }; use lemmy_db_views::{comment_view::CommentView, post_view::PostView}; use lemmy_db_views_actor::{community_view::CommunityView, user_view::UserViewSafe}; -use lemmy_structs::{blocking, site::SearchResponse}; use lemmy_utils::{settings::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use log::debug; diff --git a/crates/apub/src/fetcher/user.rs b/crates/apub/src/fetcher/user.rs index 21cdfb348..e3ef41c74 100644 --- a/crates/apub/src/fetcher/user.rs +++ b/crates/apub/src/fetcher/user.rs @@ -5,9 +5,9 @@ use crate::{ }; use anyhow::anyhow; use diesel::result::Error::NotFound; +use lemmy_api_structs::blocking; use lemmy_db_queries::{source::user::User, ApubObject}; use lemmy_db_schema::source::user::User_; -use lemmy_structs::blocking; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use log::debug; diff --git a/crates/apub/src/http/comment.rs b/crates/apub/src/http/comment.rs index 44397db60..d4287224c 100644 --- a/crates/apub/src/http/comment.rs +++ b/crates/apub/src/http/comment.rs @@ -4,9 +4,9 @@ use crate::{ }; use actix_web::{body::Body, web, web::Path, HttpResponse}; use diesel::result::Error::NotFound; +use lemmy_api_structs::blocking; use lemmy_db_queries::Crud; use lemmy_db_schema::source::comment::Comment; -use lemmy_structs::blocking; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use serde::Deserialize; diff --git a/crates/apub/src/http/community.rs b/crates/apub/src/http/community.rs index 964ac2a15..2306286a7 100644 --- a/crates/apub/src/http/community.rs +++ b/crates/apub/src/http/community.rs @@ -9,10 +9,10 @@ use activitystreams::{ collection::{CollectionExt, OrderedCollection, UnorderedCollection}, }; use actix_web::{body::Body, web, HttpResponse}; +use lemmy_api_structs::blocking; use lemmy_db_queries::source::{activity::Activity_, community::Community_}; use lemmy_db_schema::source::{activity::Activity, community::Community}; use lemmy_db_views_actor::community_follower_view::CommunityFollowerView; -use lemmy_structs::blocking; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use serde::Deserialize; diff --git a/crates/apub/src/http/mod.rs b/crates/apub/src/http/mod.rs index f0ffbcb1e..ae8a3632a 100644 --- a/crates/apub/src/http/mod.rs +++ b/crates/apub/src/http/mod.rs @@ -1,9 +1,9 @@ use crate::APUB_JSON_CONTENT_TYPE; use actix_web::{body::Body, web, HttpResponse}; use http::StatusCode; +use lemmy_api_structs::blocking; use lemmy_db_queries::source::activity::Activity_; use lemmy_db_schema::source::activity::Activity; -use lemmy_structs::blocking; use lemmy_utils::{settings::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use serde::{Deserialize, Serialize}; diff --git a/crates/apub/src/http/post.rs b/crates/apub/src/http/post.rs index 66adae3ac..8bdded2a0 100644 --- a/crates/apub/src/http/post.rs +++ b/crates/apub/src/http/post.rs @@ -4,9 +4,9 @@ use crate::{ }; use actix_web::{body::Body, web, HttpResponse}; use diesel::result::Error::NotFound; +use lemmy_api_structs::blocking; use lemmy_db_queries::Crud; use lemmy_db_schema::source::post::Post; -use lemmy_structs::blocking; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use serde::Deserialize; diff --git a/crates/apub/src/http/user.rs b/crates/apub/src/http/user.rs index 7c7653e57..77c40d855 100644 --- a/crates/apub/src/http/user.rs +++ b/crates/apub/src/http/user.rs @@ -9,9 +9,9 @@ use activitystreams::{ collection::{CollectionExt, OrderedCollection}, }; use actix_web::{body::Body, web, HttpResponse}; +use lemmy_api_structs::blocking; use lemmy_db_queries::source::user::User; use lemmy_db_schema::source::user::User_; -use lemmy_structs::blocking; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use serde::Deserialize; diff --git a/crates/apub/src/inbox/community_inbox.rs b/crates/apub/src/inbox/community_inbox.rs index f9056a770..f003fb16f 100644 --- a/crates/apub/src/inbox/community_inbox.rs +++ b/crates/apub/src/inbox/community_inbox.rs @@ -26,13 +26,13 @@ use activitystreams::{ }; use actix_web::{web, HttpRequest, HttpResponse}; use anyhow::{anyhow, Context}; +use lemmy_api_structs::blocking; use lemmy_db_queries::{source::community::Community_, ApubObject, DbPool, Followable}; use lemmy_db_schema::source::{ community::{Community, CommunityFollower, CommunityFollowerForm}, user::User_, }; use lemmy_db_views_actor::community_user_ban_view::CommunityUserBanView; -use lemmy_structs::blocking; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; use log::info; diff --git a/crates/apub/src/inbox/mod.rs b/crates/apub/src/inbox/mod.rs index 765d5dff4..4f5b04633 100644 --- a/crates/apub/src/inbox/mod.rs +++ b/crates/apub/src/inbox/mod.rs @@ -12,13 +12,13 @@ use activitystreams::{ }; use actix_web::HttpRequest; use anyhow::{anyhow, Context}; +use lemmy_api_structs::blocking; use lemmy_db_queries::{ source::{activity::Activity_, community::Community_}, ApubObject, DbPool, }; use lemmy_db_schema::source::{activity::Activity, community::Community, user::User_}; -use lemmy_structs::blocking; use lemmy_utils::{location_info, settings::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use serde::Serialize; diff --git a/crates/apub/src/inbox/receive_for_community.rs b/crates/apub/src/inbox/receive_for_community.rs index 6c087fc80..9f8a6d6c1 100644 --- a/crates/apub/src/inbox/receive_for_community.rs +++ b/crates/apub/src/inbox/receive_for_community.rs @@ -43,9 +43,9 @@ use activitystreams::{ }; use anyhow::Context; use diesel::result::Error::NotFound; +use lemmy_api_structs::blocking; use lemmy_db_queries::Crud; use lemmy_db_schema::source::site::Site; -use lemmy_structs::blocking; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; use strum_macros::EnumString; diff --git a/crates/apub/src/inbox/shared_inbox.rs b/crates/apub/src/inbox/shared_inbox.rs index 93df6e728..8c197a85f 100644 --- a/crates/apub/src/inbox/shared_inbox.rs +++ b/crates/apub/src/inbox/shared_inbox.rs @@ -15,9 +15,9 @@ use crate::{ use activitystreams::{activity::ActorAndObject, prelude::*}; use actix_web::{web, HttpRequest, HttpResponse}; use anyhow::Context; +use lemmy_api_structs::blocking; use lemmy_db_queries::{ApubObject, DbPool}; use lemmy_db_schema::source::community::Community; -use lemmy_structs::blocking; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; use serde::{Deserialize, Serialize}; diff --git a/crates/apub/src/inbox/user_inbox.rs b/crates/apub/src/inbox/user_inbox.rs index 5657faf1e..467bee77c 100644 --- a/crates/apub/src/inbox/user_inbox.rs +++ b/crates/apub/src/inbox/user_inbox.rs @@ -48,13 +48,13 @@ use activitystreams::{ use actix_web::{web, HttpRequest, HttpResponse}; use anyhow::{anyhow, Context}; use diesel::NotFound; +use lemmy_api_structs::blocking; use lemmy_db_queries::{source::user::User, ApubObject, Followable}; use lemmy_db_schema::source::{ community::{Community, CommunityFollower}, private_message::PrivateMessage, user::User_, }; -use lemmy_structs::blocking; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; use log::debug; diff --git a/crates/apub/src/lib.rs b/crates/apub/src/lib.rs index 388d57e6a..c010a29bd 100644 --- a/crates/apub/src/lib.rs +++ b/crates/apub/src/lib.rs @@ -24,6 +24,7 @@ use activitystreams::{ use activitystreams_ext::{Ext1, Ext2}; use anyhow::{anyhow, Context}; use diesel::NotFound; +use lemmy_api_structs::blocking; use lemmy_db_queries::{source::activity::Activity_, ApubObject, DbPool}; use lemmy_db_schema::source::{ activity::Activity, @@ -33,7 +34,6 @@ use lemmy_db_schema::source::{ private_message::PrivateMessage, user::User_, }; -use lemmy_structs::blocking; use lemmy_utils::{location_info, settings::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use serde::Serialize; diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index 8452f3787..6c218190a 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -21,13 +21,13 @@ use activitystreams::{ public, }; use anyhow::{anyhow, Context}; +use lemmy_api_structs::blocking; use lemmy_db_queries::{Crud, DbPool}; use lemmy_db_schema::source::{ comment::{Comment, CommentForm}, post::Post, user::User_, }; -use lemmy_structs::blocking; use lemmy_utils::{ location_info, utils::{convert_datetime, remove_slurs}, diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index 827eae294..f516768ba 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -22,13 +22,13 @@ use activitystreams::{ }; use activitystreams_ext::Ext2; use anyhow::Context; +use lemmy_api_structs::blocking; use lemmy_db_queries::DbPool; use lemmy_db_schema::{ naive_now, source::community::{Community, CommunityForm}, }; use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView; -use lemmy_structs::blocking; use lemmy_utils::{ location_info, utils::{check_slurs, check_slurs_opt, convert_datetime}, diff --git a/crates/apub/src/objects/mod.rs b/crates/apub/src/objects/mod.rs index a35ad7d13..42e8bb628 100644 --- a/crates/apub/src/objects/mod.rs +++ b/crates/apub/src/objects/mod.rs @@ -12,9 +12,9 @@ use activitystreams::{ use anyhow::{anyhow, Context}; use chrono::NaiveDateTime; use diesel::result::Error::NotFound; +use lemmy_api_structs::blocking; use lemmy_db_queries::{ApubObject, Crud, DbPool}; use lemmy_db_schema::source::community::Community; -use lemmy_structs::blocking; use lemmy_utils::{ location_info, settings::Settings, diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 2cac67ea4..841b38069 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -22,13 +22,13 @@ use activitystreams::{ }; use activitystreams_ext::Ext1; use anyhow::Context; +use lemmy_api_structs::blocking; use lemmy_db_queries::{Crud, DbPool}; use lemmy_db_schema::source::{ community::Community, post::{Post, PostForm}, user::User_, }; -use lemmy_structs::blocking; use lemmy_utils::{ location_info, request::fetch_iframely_and_pictrs_data, diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index df91b03f0..0bb753e28 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -19,12 +19,12 @@ use activitystreams::{ prelude::*, }; use anyhow::Context; +use lemmy_api_structs::blocking; use lemmy_db_queries::{Crud, DbPool}; use lemmy_db_schema::source::{ private_message::{PrivateMessage, PrivateMessageForm}, user::User_, }; -use lemmy_structs::blocking; use lemmy_utils::{location_info, utils::convert_datetime, LemmyError}; use lemmy_websocket::LemmyContext; use url::Url; diff --git a/crates/apub/src/objects/user.rs b/crates/apub/src/objects/user.rs index 7a85b20a9..aad895674 100644 --- a/crates/apub/src/objects/user.rs +++ b/crates/apub/src/objects/user.rs @@ -18,12 +18,12 @@ use activitystreams::{ }; use activitystreams_ext::Ext1; use anyhow::Context; +use lemmy_api_structs::blocking; use lemmy_db_queries::{ApubObject, DbPool}; use lemmy_db_schema::{ naive_now, source::user::{UserForm, User_}, }; -use lemmy_structs::blocking; use lemmy_utils::{ location_info, settings::Settings, diff --git a/crates/routes/Cargo.toml b/crates/routes/Cargo.toml index b71ffca70..80b94aa7e 100644 --- a/crates/routes/Cargo.toml +++ b/crates/routes/Cargo.toml @@ -13,7 +13,7 @@ lemmy_db_queries = { path = "../db_queries" } lemmy_db_views = { path = "../db_views" } lemmy_db_views_actor = { path = "../db_views_actor" } lemmy_db_schema = { path = "../db_schema" } -lemmy_structs = { path = "../structs" } +lemmy_api_structs = { path = "../api_structs" } diesel = "1.4.5" actix = "0.10.0" actix-web = { version = "3.3.2", default-features = false, features = ["rustls"] } diff --git a/crates/routes/src/feeds.rs b/crates/routes/src/feeds.rs index 837e1489a..b8858f3f9 100644 --- a/crates/routes/src/feeds.rs +++ b/crates/routes/src/feeds.rs @@ -2,6 +2,7 @@ use actix_web::{error::ErrorBadRequest, *}; use anyhow::anyhow; use chrono::{DateTime, NaiveDateTime, Utc}; use diesel::PgConnection; +use lemmy_api_structs::blocking; use lemmy_db_queries::{ source::{community::Community_, user::User}, ListingType, @@ -14,7 +15,6 @@ use lemmy_db_views::{ site_view::SiteView, }; use lemmy_db_views_actor::user_mention_view::{UserMentionQueryBuilder, UserMentionView}; -use lemmy_structs::blocking; use lemmy_utils::{claims::Claims, settings::Settings, utils::markdown_to_html, LemmyError}; use lemmy_websocket::LemmyContext; use rss::{ diff --git a/crates/routes/src/nodeinfo.rs b/crates/routes/src/nodeinfo.rs index 46ef7ce4d..058ba8816 100644 --- a/crates/routes/src/nodeinfo.rs +++ b/crates/routes/src/nodeinfo.rs @@ -1,7 +1,7 @@ use actix_web::{body::Body, error::ErrorBadRequest, *}; use anyhow::anyhow; +use lemmy_api_structs::blocking; use lemmy_db_views::site_view::SiteView; -use lemmy_structs::blocking; use lemmy_utils::{settings::Settings, version, LemmyError}; use lemmy_websocket::LemmyContext; use serde::{Deserialize, Serialize}; diff --git a/crates/routes/src/webfinger.rs b/crates/routes/src/webfinger.rs index 4059e0f89..5c5387d58 100644 --- a/crates/routes/src/webfinger.rs +++ b/crates/routes/src/webfinger.rs @@ -1,8 +1,8 @@ use actix_web::{error::ErrorBadRequest, web::Query, *}; use anyhow::anyhow; +use lemmy_api_structs::{blocking, WebFingerLink, WebFingerResponse}; use lemmy_db_queries::source::{community::Community_, user::User}; use lemmy_db_schema::source::{community::Community, user::User_}; -use lemmy_structs::{blocking, WebFingerLink, WebFingerResponse}; use lemmy_utils::{ settings::Settings, LemmyError, diff --git a/crates/websocket/Cargo.toml b/crates/websocket/Cargo.toml index 3116d71e7..b957e9446 100644 --- a/crates/websocket/Cargo.toml +++ b/crates/websocket/Cargo.toml @@ -1,7 +1,6 @@ [package] name = "lemmy_websocket" version = "0.1.0" -authors = ["Felix Ableitner "] edition = "2018" [lib] @@ -11,7 +10,7 @@ doctest = false [dependencies] lemmy_utils = { path = "../utils" } -lemmy_structs = { path = "../structs" } +lemmy_api_structs = { path = "../api_structs" } lemmy_db_queries = { path = "../db_queries" } lemmy_db_schema = { path = "../db_schema" } reqwest = { version = "0.10.10", features = ["json"] } diff --git a/crates/websocket/src/chat_server.rs b/crates/websocket/src/chat_server.rs index fa1d90185..9978d1ec6 100644 --- a/crates/websocket/src/chat_server.rs +++ b/crates/websocket/src/chat_server.rs @@ -6,7 +6,7 @@ use diesel::{ r2d2::{ConnectionManager, Pool}, PgConnection, }; -use lemmy_structs::{comment::*, post::*}; +use lemmy_api_structs::{comment::*, post::*}; use lemmy_utils::{ location_info, rate_limit::RateLimit, diff --git a/crates/websocket/src/messages.rs b/crates/websocket/src/messages.rs index 4349b01b6..89f3f2b3e 100644 --- a/crates/websocket/src/messages.rs +++ b/crates/websocket/src/messages.rs @@ -1,6 +1,6 @@ use crate::UserOperation; use actix::{prelude::*, Recipient}; -use lemmy_structs::{comment::CommentResponse, post::PostResponse}; +use lemmy_api_structs::{comment::CommentResponse, post::PostResponse}; use lemmy_utils::{CommunityId, ConnectionId, IpAddr, PostId, UserId}; use serde::{Deserialize, Serialize}; diff --git a/src/main.rs b/src/main.rs index 4ebedb4f8..a18201c73 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,11 +8,11 @@ use diesel::{ PgConnection, }; use lemmy_api::match_websocket_operation; +use lemmy_api_structs::blocking; use lemmy_apub::activity_queue::create_activity_queue; use lemmy_db_queries::get_database_url_from_env; use lemmy_routes::{feeds, images, nodeinfo, webfinger}; use lemmy_server::{code_migrations::run_advanced_migrations, scheduled_tasks}; -use lemmy_structs::blocking; use lemmy_utils::{ rate_limit::{rate_limiter::RateLimiter, RateLimit}, settings::Settings, From 462c4a29544702885a34f335ed8fe8f59d29acf4 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Mon, 1 Mar 2021 12:24:11 -0500 Subject: [PATCH 06/16] Rewrite settings implementation. Fixes #1270 (#1433) * A first attempt at using deser-hjson. Fixes #1270 * Trying to fix tests, try 1 * Trying to fix tests, try 2 * A few fixes to deser_hjson - Removing unwrap_or_defaults, using impl functions. - Reorganized settings * Make clippy happy * hjson list strings must be quoted. * Adding support for env vars. * Moving to structs and defaults file. * Moving settings default and struct. --- Cargo.lock | 243 +++++++++------------ ansible/templates/config.hjson | 5 +- api_tests/prepare-drone-federation-test.sh | 45 ++-- config/defaults.hjson | 5 +- crates/api/src/lib.rs | 19 +- crates/api/src/site.rs | 4 +- crates/api/src/user.rs | 14 +- crates/api_structs/src/lib.rs | 4 +- crates/api_structs/src/site.rs | 4 +- crates/apub/src/activities/send/comment.rs | 2 +- crates/apub/src/activities/send/mod.rs | 2 +- crates/apub/src/activity_queue.rs | 6 +- crates/apub/src/fetcher/search.rs | 2 +- crates/apub/src/http/mod.rs | 2 +- crates/apub/src/inbox/mod.rs | 4 +- crates/apub/src/lib.rs | 19 +- crates/apub/src/objects/mod.rs | 4 +- crates/apub/src/objects/user.rs | 4 +- crates/apub/src/routes.rs | 6 +- crates/db_queries/src/source/user.rs | 2 +- crates/routes/src/feeds.rs | 9 +- crates/routes/src/images.rs | 14 +- crates/routes/src/nodeinfo.rs | 4 +- crates/routes/src/webfinger.rs | 4 +- crates/utils/Cargo.toml | 4 +- crates/utils/src/claims.rs | 6 +- crates/utils/src/email.rs | 6 +- crates/utils/src/lib.rs | 7 +- crates/utils/src/rate_limit/mod.rs | 4 +- crates/utils/src/request.rs | 6 +- crates/utils/src/settings.rs | 210 ------------------ crates/utils/src/settings/defaults.rs | 69 ++++++ crates/utils/src/settings/mod.rs | 172 +++++++++++++++ crates/utils/src/settings/structs.rs | 72 ++++++ crates/utils/src/utils.rs | 4 +- docker/dev/Dockerfile | 1 - docker/federation/docker-compose.yml | 75 +------ docker/federation/lemmy_alpha.hjson | 36 +++ docker/federation/lemmy_beta.hjson | 37 ++++ docker/federation/lemmy_delta.hjson | 37 ++++ docker/federation/lemmy_epsilon.hjson | 37 ++++ docker/federation/lemmy_gamma.hjson | 37 ++++ docker/lemmy.hjson | 4 + docker/prod/Dockerfile | 1 - docker/prod/Dockerfile.arm | 1 - src/code_migrations.rs | 2 +- src/main.rs | 9 +- 47 files changed, 737 insertions(+), 527 deletions(-) delete mode 100644 crates/utils/src/settings.rs create mode 100644 crates/utils/src/settings/defaults.rs create mode 100644 crates/utils/src/settings/mod.rs create mode 100644 crates/utils/src/settings/structs.rs create mode 100644 docker/federation/lemmy_alpha.hjson create mode 100644 docker/federation/lemmy_beta.hjson create mode 100644 docker/federation/lemmy_delta.hjson create mode 100644 docker/federation/lemmy_epsilon.hjson create mode 100644 docker/federation/lemmy_gamma.hjson diff --git a/Cargo.lock b/Cargo.lock index 13d67faa5..69b22f956 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,7 +10,7 @@ checksum = "fe7ceed015dfca322d3bcec3653909c77557e7e57df72e98cb8806e2c93cc919" dependencies = [ "chrono", "mime", - "serde 1.0.123", + "serde", "serde_json", "thiserror", "url", @@ -23,7 +23,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb8e19a0810cc25df3535061a08b7d8f8a734d309ea4411c57a9767e4a2ffa0e" dependencies = [ "activitystreams", - "serde 1.0.123", + "serde", "serde_json", ] @@ -130,7 +130,7 @@ dependencies = [ "pin-project 1.0.4", "rand 0.7.3", "regex", - "serde 1.0.123", + "serde", "serde_json", "serde_urlencoded", "sha-1 0.9.3", @@ -158,7 +158,7 @@ dependencies = [ "http", "log", "regex", - "serde 1.0.123", + "serde", ] [[package]] @@ -302,7 +302,7 @@ dependencies = [ "pin-project 1.0.4", "regex", "rustls", - "serde 1.0.123", + "serde", "serde_json", "serde_urlencoded", "socket2", @@ -391,12 +391,6 @@ version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "async-mutex" version = "1.4.0" @@ -466,7 +460,7 @@ dependencies = [ "percent-encoding", "rand 0.7.3", "rustls", - "serde 1.0.123", + "serde", "serde_json", "serde_urlencoded", ] @@ -496,7 +490,7 @@ dependencies = [ "log", "num_cpus", "rand 0.7.3", - "serde 1.0.123", + "serde", "serde_json", "thiserror", "tokio 0.2.25", @@ -515,7 +509,7 @@ dependencies = [ "async-trait", "chrono", "log", - "serde 1.0.123", + "serde", "serde_json", "thiserror", "tokio 0.2.25", @@ -735,8 +729,8 @@ checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ "libc", "num-integer", - "num-traits 0.2.14", - "serde 1.0.123", + "num-traits", + "serde", "time 0.1.44", "winapi 0.3.9", ] @@ -783,18 +777,6 @@ dependencies = [ "xdg", ] -[[package]] -name = "config" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" -dependencies = [ - "lazy_static", - "nom 5.1.2", - "serde 1.0.123", - "serde-hjson", -] - [[package]] name = "const_fn" version = "0.4.5" @@ -997,6 +979,15 @@ dependencies = [ "syn", ] +[[package]] +name = "deser-hjson" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d995b60ff81bc6af01a98f0bf5db70a7418a1ac8bd74ada633968f388139da5e" +dependencies = [ + "serde", +] + [[package]] name = "diesel" version = "1.4.5" @@ -1112,6 +1103,15 @@ dependencies = [ "termcolor", ] +[[package]] +name = "envy" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965" +dependencies = [ + "serde", +] + [[package]] name = "event-listener" version = "2.5.1" @@ -1592,7 +1592,7 @@ dependencies = [ "jpeg-decoder", "num-iter", "num-rational", - "num-traits 0.2.14", + "num-traits", "png", "scoped_threadpool", "tiff", @@ -1686,7 +1686,7 @@ dependencies = [ "base64 0.12.3", "pem", "ring", - "serde 1.0.123", + "serde", "serde_json", "simple_asn1", ] @@ -1747,7 +1747,7 @@ dependencies = [ "openssl", "rand 0.8.3", "reqwest", - "serde 1.0.123", + "serde", "serde_json", "sha2", "strum", @@ -1772,7 +1772,7 @@ dependencies = [ "lemmy_db_views_moderator", "lemmy_utils", "log", - "serde 1.0.123", + "serde", "serde_json", "url", ] @@ -1813,7 +1813,7 @@ dependencies = [ "percent-encoding", "rand 0.8.3", "reqwest", - "serde 1.0.123", + "serde", "serde_json", "sha2", "strum", @@ -1837,7 +1837,7 @@ dependencies = [ "lemmy_utils", "log", "regex", - "serde 1.0.123", + "serde", "serde_json", "serial_test", "sha2", @@ -1853,7 +1853,7 @@ dependencies = [ "chrono", "diesel", "log", - "serde 1.0.123", + "serde", "serde_json", "url", ] @@ -1866,7 +1866,7 @@ dependencies = [ "lemmy_db_queries", "lemmy_db_schema", "log", - "serde 1.0.123", + "serde", "serial_test", "url", ] @@ -1878,7 +1878,7 @@ dependencies = [ "diesel", "lemmy_db_queries", "lemmy_db_schema", - "serde 1.0.123", + "serde", ] [[package]] @@ -1888,7 +1888,7 @@ dependencies = [ "diesel", "lemmy_db_queries", "lemmy_db_schema", - "serde 1.0.123", + "serde", ] [[package]] @@ -1912,7 +1912,7 @@ dependencies = [ "lemmy_websocket", "log", "rss", - "serde 1.0.123", + "serde", "sha2", "strum", "url", @@ -1948,7 +1948,7 @@ dependencies = [ "log", "openssl", "reqwest", - "serde 1.0.123", + "serde", "serde_json", "strum", "tokio 0.3.7", @@ -1964,8 +1964,9 @@ dependencies = [ "anyhow", "chrono", "comrak", - "config", + "deser-hjson", "diesel", + "envy", "futures", "http", "itertools", @@ -1973,12 +1974,13 @@ dependencies = [ "lazy_static", "lettre", "log", + "merge", "openssl", "percent-encoding", "rand 0.8.3", "regex", "reqwest", - "serde 1.0.123", + "serde", "serde_json", "strum", "strum_macros", @@ -2005,7 +2007,7 @@ dependencies = [ "log", "rand 0.8.3", "reqwest", - "serde 1.0.123", + "serde", "serde_json", "strum", "strum_macros", @@ -2024,46 +2026,23 @@ dependencies = [ "idna", "mime", "native-tls", - "nom 6.1.0", + "nom", "once_cell", "quoted_printable", "r2d2", "rand 0.8.3", "regex", - "serde 1.0.123", + "serde", "serde_json", "uuid", ] -[[package]] -name = "lexical-core" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616" -dependencies = [ - "arrayvec", - "bitflags", - "cfg-if 0.1.10", - "ryu", - "static_assertions", -] - [[package]] name = "libc" version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff" -[[package]] -name = "linked-hash-map" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" -dependencies = [ - "serde 0.8.23", - "serde_test", -] - [[package]] name = "linked-hash-map" version = "0.5.4" @@ -2106,7 +2085,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" dependencies = [ - "linked-hash-map 0.5.4", + "linked-hash-map", ] [[package]] @@ -2148,6 +2127,28 @@ dependencies = [ "autocfg", ] +[[package]] +name = "merge" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10bbef93abb1da61525bbc45eeaff6473a41907d19f8f9aa5168d214e10693e9" +dependencies = [ + "merge_derive", + "num-traits", +] + +[[package]] +name = "merge_derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209d075476da2e63b4b29e72a2ef627b840589588e71400a25e3565c4f849d07" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "migrations_internals" version = "1.4.1" @@ -2275,17 +2276,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "nom" -version = "5.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" -dependencies = [ - "lexical-core", - "memchr", - "version_check", -] - [[package]] name = "nom" version = "6.1.0" @@ -2305,7 +2295,7 @@ checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ "autocfg", "num-integer", - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -2315,7 +2305,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg", - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -2326,7 +2316,7 @@ checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" dependencies = [ "autocfg", "num-integer", - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -2337,16 +2327,7 @@ checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ "autocfg", "num-integer", - "num-traits 0.2.14", -] - -[[package]] -name = "num-traits" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -dependencies = [ - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -2601,6 +2582,30 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -2842,7 +2847,7 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite 0.2.4", - "serde 1.0.123", + "serde", "serde_json", "serde_urlencoded", "tokio 0.2.25", @@ -3012,12 +3017,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -[[package]] -name = "serde" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" - [[package]] name = "serde" version = "1.0.123" @@ -3027,19 +3026,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-hjson" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8" -dependencies = [ - "lazy_static", - "linked-hash-map 0.3.0", - "num-traits 0.1.43", - "regex", - "serde 0.8.23", -] - [[package]] name = "serde_derive" version = "1.0.123" @@ -3060,16 +3046,7 @@ dependencies = [ "indexmap", "itoa", "ryu", - "serde 1.0.123", -] - -[[package]] -name = "serde_test" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5" -dependencies = [ - "serde 0.8.23", + "serde", ] [[package]] @@ -3081,7 +3058,7 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde 1.0.123", + "serde", ] [[package]] @@ -3173,7 +3150,7 @@ checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b" dependencies = [ "chrono", "num-bigint", - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -3214,12 +3191,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "stdweb" version = "0.4.20" @@ -3242,7 +3213,7 @@ checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ "proc-macro2", "quote", - "serde 1.0.123", + "serde", "serde_derive", "syn", ] @@ -3256,7 +3227,7 @@ dependencies = [ "base-x", "proc-macro2", "quote", - "serde 1.0.123", + "serde", "serde_derive", "serde_json", "sha1", @@ -3692,7 +3663,7 @@ dependencies = [ "idna", "matches", "percent-encoding", - "serde 1.0.123", + "serde", ] [[package]] @@ -3702,7 +3673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ "getrandom 0.2.2", - "serde 1.0.123", + "serde", ] [[package]] @@ -3746,7 +3717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55c0f7123de74f0dab9b7d00fd614e7b19349cd1e2f5252bbe9b1754b59433be" dependencies = [ "cfg-if 1.0.0", - "serde 1.0.123", + "serde", "serde_json", "wasm-bindgen-macro", ] diff --git a/ansible/templates/config.hjson b/ansible/templates/config.hjson index 29da81c34..55537ca50 100644 --- a/ansible/templates/config.hjson +++ b/ansible/templates/config.hjson @@ -29,8 +29,9 @@ # https://join.lemmy.ml/docs/en/federation/administration.html#instance-allowlist-and-blocklist # # comma separated list of instances with which federation is allowed - # allowed_instances: "" + # Only one of these blocks should be uncommented + # allowed_instances: ["instance1.tld","instance2.tld"] # comma separated list of instances which are blocked from federating - # blocked_instances: "" + # blocked_instances: [] } } diff --git a/api_tests/prepare-drone-federation-test.sh b/api_tests/prepare-drone-federation-test.sh index de9b7b844..dd357db83 100755 --- a/api_tests/prepare-drone-federation-test.sh +++ b/api_tests/prepare-drone-federation-test.sh @@ -1,13 +1,6 @@ #!/bin/bash set -e -export LEMMY_JWT_SECRET=changeme -export LEMMY_FEDERATION__ENABLED=true -export LEMMY_TLS_ENABLED=false -export LEMMY_SETUP__ADMIN_PASSWORD=lemmy -export LEMMY_RATE_LIMIT__POST=99999 -export LEMMY_RATE_LIMIT__REGISTER=99999 -export LEMMY_CAPTCHA__ENABLED=false export LEMMY_TEST_SEND_SYNC=1 export RUST_BACKTRACE=1 @@ -35,52 +28,40 @@ fi killall lemmy_server || true +echo "$PWD" + echo "start alpha" LEMMY_HOSTNAME=lemmy-alpha:8541 \ - LEMMY_PORT=8541 \ + LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_alpha.hjson \ LEMMY_DATABASE_URL="${LEMMY_DATABASE_URL}/lemmy_alpha" \ - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-beta,lemmy-gamma,lemmy-delta,lemmy-epsilon \ - LEMMY_SETUP__ADMIN_USERNAME=lemmy_alpha \ - LEMMY_SETUP__SITE_NAME=lemmy-alpha \ - target/lemmy_server >/dev/null 2>&1 & + LEMMY_HOSTNAME="lemmy-alpha:8541" \ + target/lemmy_server >/tmp/lemmy_alpha.out 2>&1 & echo "start beta" LEMMY_HOSTNAME=lemmy-beta:8551 \ - LEMMY_PORT=8551 \ + LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_beta.hjson \ LEMMY_DATABASE_URL="${LEMMY_DATABASE_URL}/lemmy_beta" \ - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-alpha,lemmy-gamma,lemmy-delta,lemmy-epsilon \ - LEMMY_SETUP__ADMIN_USERNAME=lemmy_beta \ - LEMMY_SETUP__SITE_NAME=lemmy-beta \ - target/lemmy_server >/dev/null 2>&1 & + target/lemmy_server >/tmp/lemmy_beta.out 2>&1 & echo "start gamma" LEMMY_HOSTNAME=lemmy-gamma:8561 \ - LEMMY_PORT=8561 \ + LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_gamma.hjson \ LEMMY_DATABASE_URL="${LEMMY_DATABASE_URL}/lemmy_gamma" \ - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-alpha,lemmy-beta,lemmy-delta,lemmy-epsilon \ - LEMMY_SETUP__ADMIN_USERNAME=lemmy_gamma \ - LEMMY_SETUP__SITE_NAME=lemmy-gamma \ - target/lemmy_server >/dev/null 2>&1 & + target/lemmy_server >/tmp/lemmy_gamma.out 2>&1 & echo "start delta" # An instance with only an allowlist for beta LEMMY_HOSTNAME=lemmy-delta:8571 \ - LEMMY_PORT=8571 \ + LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_delta.hjson \ LEMMY_DATABASE_URL="${LEMMY_DATABASE_URL}/lemmy_delta" \ - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-beta \ - LEMMY_SETUP__ADMIN_USERNAME=lemmy_delta \ - LEMMY_SETUP__SITE_NAME=lemmy-delta \ - target/lemmy_server >/dev/null 2>&1 & + target/lemmy_server >/tmp/lemmy_delta.out 2>&1 & echo "start epsilon" # An instance who has a blocklist, with lemmy-alpha blocked LEMMY_HOSTNAME=lemmy-epsilon:8581 \ - LEMMY_PORT=8581 \ + LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_epsilon.hjson \ LEMMY_DATABASE_URL="${LEMMY_DATABASE_URL}/lemmy_epsilon" \ - LEMMY_FEDERATION__BLOCKED_INSTANCES=lemmy-alpha \ - LEMMY_SETUP__ADMIN_USERNAME=lemmy_epsilon \ - LEMMY_SETUP__SITE_NAME=lemmy-epsilon \ - target/lemmy_server >/dev/null 2>&1 & + target/lemmy_server >/tmp/lemmy_epsilon.out 2>&1 & echo "wait for all instances to start" while [[ "$(curl -s -o /dev/null -w '%{http_code}' 'localhost:8541/api/v2/site')" != "200" ]]; do sleep 1; done diff --git a/config/defaults.hjson b/config/defaults.hjson index 6c6a68fc2..c3eaba6e5 100644 --- a/config/defaults.hjson +++ b/config/defaults.hjson @@ -66,9 +66,10 @@ # https://join.lemmy.ml/docs/en/federation/administration.html#instance-allowlist-and-blocklist # # comma separated list of instances with which federation is allowed - allowed_instances: "" + # Only one of these blocks should be uncommented + # allowed_instances: ["instance1.tld","instance2.tld"] # comma separated list of instances which are blocked from federating - blocked_instances: "" + # blocked_instances: [] } captcha: { enabled: true diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 871e8e9d8..6f4ea8f4d 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -27,7 +27,13 @@ use lemmy_db_views_actor::{ community_user_ban_view::CommunityUserBanView, community_view::CommunityView, }; -use lemmy_utils::{claims::Claims, settings::Settings, ApiError, ConnectionId, LemmyError}; +use lemmy_utils::{ + claims::Claims, + settings::structs::Settings, + ApiError, + ConnectionId, + LemmyError, +}; use lemmy_websocket::{serialize_websocket_message, LemmyContext, UserOperation}; use serde::Deserialize; use std::process::Command; @@ -192,7 +198,7 @@ pub(crate) fn check_optional_url(item: &Option>) -> Result<(), Le pub(crate) async fn build_federated_instances( pool: &DbPool, ) -> Result, LemmyError> { - if Settings::get().federation.enabled { + if Settings::get().federation().enabled { let distinct_communities = blocking(pool, move |conn| { Community::distinct_federated_communities(conn) }) @@ -206,8 +212,13 @@ pub(crate) async fn build_federated_instances( .map(|actor_id| Ok(Url::parse(actor_id)?.host_str().unwrap_or("").to_string())) .collect::, LemmyError>>()?; - linked.extend_from_slice(&allowed); - linked.retain(|a| !blocked.contains(a) && !a.eq("") && !a.eq(&Settings::get().hostname)); + if let Some(allowed) = allowed.as_ref() { + linked.extend_from_slice(allowed); + } + + if let Some(blocked) = blocked.as_ref() { + linked.retain(|a| !blocked.contains(a) && !a.eq(&Settings::get().hostname())); + } // Sort and remove dupes linked.sort_unstable(); diff --git a/crates/api/src/site.rs b/crates/api/src/site.rs index 716b67237..c4792f369 100644 --- a/crates/api/src/site.rs +++ b/crates/api/src/site.rs @@ -41,7 +41,7 @@ use lemmy_db_views_moderator::{ }; use lemmy_utils::{ location_info, - settings::Settings, + settings::structs::Settings, utils::{check_slurs, check_slurs_opt}, version, ApiError, @@ -245,7 +245,7 @@ impl Perform for GetSite { Ok(site_view) => Some(site_view), // If the site isn't created yet, check the setup Err(_) => { - if let Some(setup) = Settings::get().setup.as_ref() { + if let Some(setup) = Settings::get().setup().as_ref() { let register = Register { username: setup.admin_username.to_owned(), email: setup.admin_email.to_owned(), diff --git a/crates/api/src/user.rs b/crates/api/src/user.rs index d6395e6b7..6d37581e0 100644 --- a/crates/api/src/user.rs +++ b/crates/api/src/user.rs @@ -71,7 +71,7 @@ use lemmy_utils::{ claims::Claims, email::send_email, location_info, - settings::Settings, + settings::structs::Settings, utils::{ check_slurs, generate_random_string, @@ -121,7 +121,7 @@ impl Perform for Login { // Return the jwt Ok(LoginResponse { - jwt: Claims::jwt(user.id, Settings::get().hostname)?, + jwt: Claims::jwt(user.id, Settings::get().hostname())?, }) } } @@ -161,7 +161,7 @@ impl Perform for Register { .await??; // If its not the admin, check the captcha - if !no_admins && Settings::get().captcha.enabled { + if !no_admins && Settings::get().captcha().enabled { let check = context .chat_server() .send(CheckCaptcha { @@ -302,7 +302,7 @@ impl Perform for Register { // Return the jwt Ok(LoginResponse { - jwt: Claims::jwt(inserted_user.id, Settings::get().hostname)?, + jwt: Claims::jwt(inserted_user.id, Settings::get().hostname())?, }) } } @@ -316,7 +316,7 @@ impl Perform for GetCaptcha { context: &Data, _websocket_id: Option, ) -> Result { - let captcha_settings = Settings::get().captcha; + let captcha_settings = Settings::get().captcha(); if !captcha_settings.enabled { return Ok(GetCaptchaResponse { ok: None }); @@ -475,7 +475,7 @@ impl Perform for SaveUserSettings { // Return the jwt Ok(LoginResponse { - jwt: Claims::jwt(updated_user.id, Settings::get().hostname)?, + jwt: Claims::jwt(updated_user.id, Settings::get().hostname())?, }) } } @@ -1011,7 +1011,7 @@ impl Perform for PasswordChange { // Return the jwt Ok(LoginResponse { - jwt: Claims::jwt(updated_user.id, Settings::get().hostname)?, + jwt: Claims::jwt(updated_user.id, Settings::get().hostname())?, }) } } diff --git a/crates/api_structs/src/lib.rs b/crates/api_structs/src/lib.rs index 8b56fab6b..800fe6c8b 100644 --- a/crates/api_structs/src/lib.rs +++ b/crates/api_structs/src/lib.rs @@ -13,7 +13,7 @@ use lemmy_db_schema::source::{ user::User_, user_mention::{UserMention, UserMentionForm}, }; -use lemmy_utils::{email::send_email, settings::Settings, utils::MentionData, LemmyError}; +use lemmy_utils::{email::send_email, settings::structs::Settings, utils::MentionData, LemmyError}; use log::error; use serde::{Deserialize, Serialize}; use url::Url; @@ -152,7 +152,7 @@ pub fn send_email_to_user(user: User_, subject_text: &str, body_text: &str, comm let subject = &format!( "{} - {} {}", subject_text, - Settings::get().hostname, + Settings::get().hostname(), user.name, ); let html = &format!( diff --git a/crates/api_structs/src/site.rs b/crates/api_structs/src/site.rs index ef878ba52..edee17a85 100644 --- a/crates/api_structs/src/site.rs +++ b/crates/api_structs/src/site.rs @@ -126,6 +126,6 @@ pub struct SaveSiteConfig { #[derive(Serialize)] pub struct FederatedInstances { pub linked: Vec, - pub allowed: Vec, - pub blocked: Vec, + pub allowed: Option>, + pub blocked: Option>, } diff --git a/crates/apub/src/activities/send/comment.rs b/crates/apub/src/activities/send/comment.rs index f33de2ba7..ac7e884a9 100644 --- a/crates/apub/src/activities/send/comment.rs +++ b/crates/apub/src/activities/send/comment.rs @@ -31,7 +31,7 @@ use lemmy_db_queries::{Crud, DbPool}; use lemmy_db_schema::source::{comment::Comment, community::Community, post::Post, user::User_}; use lemmy_utils::{ request::{retry, RecvError}, - settings::Settings, + settings::structs::Settings, utils::{scrape_text_for_mentions, MentionData}, LemmyError, }; diff --git a/crates/apub/src/activities/send/mod.rs b/crates/apub/src/activities/send/mod.rs index 166855e20..2da0b48c2 100644 --- a/crates/apub/src/activities/send/mod.rs +++ b/crates/apub/src/activities/send/mod.rs @@ -1,4 +1,4 @@ -use lemmy_utils::settings::Settings; +use lemmy_utils::settings::structs::Settings; use url::{ParseError, Url}; use uuid::Uuid; diff --git a/crates/apub/src/activity_queue.rs b/crates/apub/src/activity_queue.rs index c0c4ac468..152fcaf9b 100644 --- a/crates/apub/src/activity_queue.rs +++ b/crates/apub/src/activity_queue.rs @@ -22,7 +22,7 @@ use background_jobs::{ use itertools::Itertools; use lemmy_db_queries::DbPool; use lemmy_db_schema::source::{community::Community, user::User_}; -use lemmy_utils::{location_info, settings::Settings, LemmyError}; +use lemmy_utils::{location_info, settings::structs::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use log::{debug, warn}; use reqwest::Client; @@ -88,7 +88,7 @@ where .await? .iter() .unique() - .filter(|inbox| inbox.host_str() != Some(&Settings::get().hostname)) + .filter(|inbox| inbox.host_str() != Some(&Settings::get().hostname())) .filter(|inbox| check_is_apub_id_valid(inbox).is_ok()) .map(|inbox| inbox.to_owned()) .collect(); @@ -215,7 +215,7 @@ where Kind: Serialize, >::Error: From + Send + Sync + 'static, { - if !Settings::get().federation.enabled || inboxes.is_empty() { + if !Settings::get().federation().enabled || inboxes.is_empty() { return Ok(()); } diff --git a/crates/apub/src/fetcher/search.rs b/crates/apub/src/fetcher/search.rs index 6cbc42dc6..acaccff28 100644 --- a/crates/apub/src/fetcher/search.rs +++ b/crates/apub/src/fetcher/search.rs @@ -35,7 +35,7 @@ use lemmy_db_schema::source::{ }; use lemmy_db_views::{comment_view::CommentView, post_view::PostView}; use lemmy_db_views_actor::{community_view::CommunityView, user_view::UserViewSafe}; -use lemmy_utils::{settings::Settings, LemmyError}; +use lemmy_utils::{settings::structs::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use log::debug; use url::Url; diff --git a/crates/apub/src/http/mod.rs b/crates/apub/src/http/mod.rs index ae8a3632a..f117b6ea2 100644 --- a/crates/apub/src/http/mod.rs +++ b/crates/apub/src/http/mod.rs @@ -4,7 +4,7 @@ use http::StatusCode; use lemmy_api_structs::blocking; use lemmy_db_queries::source::activity::Activity_; use lemmy_db_schema::source::activity::Activity; -use lemmy_utils::{settings::Settings, LemmyError}; +use lemmy_utils::{settings::structs::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use serde::{Deserialize, Serialize}; diff --git a/crates/apub/src/inbox/mod.rs b/crates/apub/src/inbox/mod.rs index 4f5b04633..2adefabe7 100644 --- a/crates/apub/src/inbox/mod.rs +++ b/crates/apub/src/inbox/mod.rs @@ -19,7 +19,7 @@ use lemmy_db_queries::{ DbPool, }; use lemmy_db_schema::source::{activity::Activity, community::Community, user::User_}; -use lemmy_utils::{location_info, settings::Settings, LemmyError}; +use lemmy_utils::{location_info, settings::structs::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use serde::Serialize; use std::fmt::Debug; @@ -167,7 +167,7 @@ where let id = activity.id_unchecked().context(location_info!())?; let activity_domain = id.domain().context(location_info!())?; - if activity_domain == Settings::get().hostname { + if activity_domain == Settings::get().hostname() { return Err( anyhow!( "Error: received activity which was sent by local instance: {:?}", diff --git a/crates/apub/src/lib.rs b/crates/apub/src/lib.rs index c010a29bd..5c0b267a6 100644 --- a/crates/apub/src/lib.rs +++ b/crates/apub/src/lib.rs @@ -34,7 +34,7 @@ use lemmy_db_schema::source::{ private_message::PrivateMessage, user::User_, }; -use lemmy_utils::{location_info, settings::Settings, LemmyError}; +use lemmy_utils::{location_info, settings::structs::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use serde::Serialize; use std::net::IpAddr; @@ -64,7 +64,7 @@ fn check_is_apub_id_valid(apub_id: &Url) -> Result<(), LemmyError> { let domain = apub_id.domain().context(location_info!())?.to_string(); let local_instance = settings.get_hostname_without_port()?; - if !settings.federation.enabled { + if !settings.federation().enabled { return if domain == local_instance { Ok(()) } else { @@ -88,22 +88,23 @@ fn check_is_apub_id_valid(apub_id: &Url) -> Result<(), LemmyError> { return Err(anyhow!("invalid apub id scheme {}: {}", apub_id.scheme(), apub_id).into()); } - let mut allowed_instances = Settings::get().get_allowed_instances(); + let allowed_instances = Settings::get().get_allowed_instances(); let blocked_instances = Settings::get().get_blocked_instances(); - if allowed_instances.is_empty() && blocked_instances.is_empty() { + + if allowed_instances.is_none() && blocked_instances.is_none() { Ok(()) - } else if !allowed_instances.is_empty() { + } else if let Some(mut allowed) = allowed_instances { // need to allow this explicitly because apub receive might contain objects from our local // instance. split is needed to remove the port in our federation test setup. - allowed_instances.push(local_instance); + allowed.push(local_instance); - if allowed_instances.contains(&domain) { + if allowed.contains(&domain) { Ok(()) } else { Err(anyhow!("{} not in federation allowlist", domain).into()) } - } else if !blocked_instances.is_empty() { - if blocked_instances.contains(&domain) { + } else if let Some(blocked) = blocked_instances { + if blocked.contains(&domain) { Err(anyhow!("{} is in federation blocklist", domain).into()) } else { Ok(()) diff --git a/crates/apub/src/objects/mod.rs b/crates/apub/src/objects/mod.rs index 42e8bb628..5667e1001 100644 --- a/crates/apub/src/objects/mod.rs +++ b/crates/apub/src/objects/mod.rs @@ -17,7 +17,7 @@ use lemmy_db_queries::{ApubObject, Crud, DbPool}; use lemmy_db_schema::source::community::Community; use lemmy_utils::{ location_info, - settings::Settings, + settings::structs::Settings, utils::{convert_datetime, markdown_to_html}, LemmyError, }; @@ -187,7 +187,7 @@ where let domain = object_id.domain().context(location_info!())?; // if its a local object, return it directly from the database - if Settings::get().hostname == domain { + if Settings::get().hostname() == domain { let object = blocking(context.pool(), move |conn| { To::read_from_apub_id(conn, &object_id.into()) }) diff --git a/crates/apub/src/objects/user.rs b/crates/apub/src/objects/user.rs index aad895674..c979e3ee4 100644 --- a/crates/apub/src/objects/user.rs +++ b/crates/apub/src/objects/user.rs @@ -26,7 +26,7 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ location_info, - settings::Settings, + settings::structs::Settings, utils::{check_slurs, check_slurs_opt, convert_datetime}, LemmyError, }; @@ -96,7 +96,7 @@ impl FromApub for User_ { ) -> Result { let user_id = person.id_unchecked().context(location_info!())?.to_owned(); let domain = user_id.domain().context(location_info!())?; - if domain == Settings::get().hostname { + if domain == Settings::get().hostname() { let user = blocking(context.pool(), move |conn| { User_::read_from_apub_id(conn, &user_id.into()) }) diff --git a/crates/apub/src/routes.rs b/crates/apub/src/routes.rs index 6fa82ea84..07dcc7f88 100644 --- a/crates/apub/src/routes.rs +++ b/crates/apub/src/routes.rs @@ -16,15 +16,15 @@ use crate::{ }; use actix_web::*; use http_signature_normalization_actix::digest::middleware::VerifyDigest; -use lemmy_utils::settings::Settings; +use lemmy_utils::settings::structs::Settings; use sha2::{Digest, Sha256}; static APUB_JSON_CONTENT_TYPE_LONG: &str = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""; pub fn config(cfg: &mut web::ServiceConfig) { - if Settings::get().federation.enabled { - println!("federation enabled, host is {}", Settings::get().hostname); + if Settings::get().federation().enabled { + println!("federation enabled, host is {}", Settings::get().hostname()); let digest_verifier = VerifyDigest::new(Sha256::new()); let header_guard_accept = guard::Any(guard::Header("Accept", APUB_JSON_CONTENT_TYPE)) diff --git a/crates/db_queries/src/source/user.rs b/crates/db_queries/src/source/user.rs index 20b187b46..56023f09f 100644 --- a/crates/db_queries/src/source/user.rs +++ b/crates/db_queries/src/source/user.rs @@ -7,7 +7,7 @@ use lemmy_db_schema::{ source::user::{UserForm, UserSafeSettings, User_}, Url, }; -use lemmy_utils::settings::Settings; +use lemmy_utils::settings::structs::Settings; mod safe_type { use crate::ToSafe; diff --git a/crates/routes/src/feeds.rs b/crates/routes/src/feeds.rs index b8858f3f9..40d80f25a 100644 --- a/crates/routes/src/feeds.rs +++ b/crates/routes/src/feeds.rs @@ -15,7 +15,12 @@ use lemmy_db_views::{ site_view::SiteView, }; use lemmy_db_views_actor::user_mention_view::{UserMentionQueryBuilder, UserMentionView}; -use lemmy_utils::{claims::Claims, settings::Settings, utils::markdown_to_html, LemmyError}; +use lemmy_utils::{ + claims::Claims, + settings::structs::Settings, + utils::markdown_to_html, + LemmyError, +}; use lemmy_websocket::LemmyContext; use rss::{ extension::dublincore::DublinCoreExtensionBuilder, @@ -163,7 +168,7 @@ fn get_feed_user( ) -> Result { let site_view = SiteView::read(&conn)?; let user = User_::find_by_username(&conn, &user_name)?; - let user_url = user.get_profile_url(&Settings::get().hostname); + let user_url = user.get_profile_url(&Settings::get().hostname()); let posts = PostQueryBuilder::create(&conn) .listing_type(&ListingType::All) diff --git a/crates/routes/src/images.rs b/crates/routes/src/images.rs index 4d7656e1b..fe23fe029 100644 --- a/crates/routes/src/images.rs +++ b/crates/routes/src/images.rs @@ -1,7 +1,7 @@ use actix::clock::Duration; use actix_web::{body::BodyStream, http::StatusCode, *}; use awc::Client; -use lemmy_utils::{claims::Claims, rate_limit::RateLimit, settings::Settings}; +use lemmy_utils::{claims::Claims, rate_limit::RateLimit, settings::structs::Settings}; use serde::{Deserialize, Serialize}; pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimit) { @@ -54,8 +54,10 @@ async fn upload( return Ok(HttpResponse::Unauthorized().finish()); }; - let mut client_req = - client.request_from(format!("{}/image", Settings::get().pictrs_url), req.head()); + let mut client_req = client.request_from( + format!("{}/image", Settings::get().pictrs_url()), + req.head(), + ); if let Some(addr) = req.head().peer_addr { client_req = client_req.header("X-Forwarded-For", addr.to_string()) @@ -78,14 +80,14 @@ async fn full_res( // If there are no query params, the URL is original let url = if params.format.is_none() && params.thumbnail.is_none() { - format!("{}/image/original/{}", Settings::get().pictrs_url, name,) + format!("{}/image/original/{}", Settings::get().pictrs_url(), name,) } else { // Use jpg as a default when none is given let format = params.format.unwrap_or_else(|| "jpg".to_string()); let mut url = format!( "{}/image/process.{}?src={}", - Settings::get().pictrs_url, + Settings::get().pictrs_url(), format, name, ); @@ -134,7 +136,7 @@ async fn delete( let url = format!( "{}/image/delete/{}/{}", - Settings::get().pictrs_url, + Settings::get().pictrs_url(), &token, &file ); diff --git a/crates/routes/src/nodeinfo.rs b/crates/routes/src/nodeinfo.rs index 058ba8816..1333279cc 100644 --- a/crates/routes/src/nodeinfo.rs +++ b/crates/routes/src/nodeinfo.rs @@ -2,7 +2,7 @@ use actix_web::{body::Body, error::ErrorBadRequest, *}; use anyhow::anyhow; use lemmy_api_structs::blocking; use lemmy_db_views::site_view::SiteView; -use lemmy_utils::{settings::Settings, version, LemmyError}; +use lemmy_utils::{settings::structs::Settings, version, LemmyError}; use lemmy_websocket::LemmyContext; use serde::{Deserialize, Serialize}; use url::Url; @@ -31,7 +31,7 @@ async fn node_info(context: web::Data) -> Result( &jwt, - &DecodingKey::from_secret(Settings::get().jwt_secret.as_ref()), + &DecodingKey::from_secret(Settings::get().jwt_secret().as_ref()), &v, ) } @@ -31,7 +31,7 @@ impl Claims { encode( &Header::default(), &my_claims, - &EncodingKey::from_secret(Settings::get().jwt_secret.as_ref()), + &EncodingKey::from_secret(Settings::get().jwt_secret().as_ref()), ) } } diff --git a/crates/utils/src/email.rs b/crates/utils/src/email.rs index 8e61500c0..ead534924 100644 --- a/crates/utils/src/email.rs +++ b/crates/utils/src/email.rs @@ -1,4 +1,4 @@ -use crate::settings::Settings; +use crate::settings::structs::Settings; use lettre::{ message::{header, Mailbox, MultiPart, SinglePart}, transport::smtp::{ @@ -19,8 +19,8 @@ pub fn send_email( to_username: &str, html: &str, ) -> Result<(), String> { - let email_config = Settings::get().email.ok_or("no_email_setup")?; - let domain = Settings::get().hostname; + let email_config = Settings::get().email().ok_or("no_email_setup")?; + let domain = Settings::get().hostname(); let (smtp_server, smtp_port) = { let email_and_port = email_config.smtp_server.split(':').collect::>(); diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index e64271568..58d2abb9d 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -9,12 +9,13 @@ pub mod email; pub mod rate_limit; pub mod request; pub mod settings; + #[cfg(test)] mod test; pub mod utils; pub mod version; -use crate::settings::Settings; +use crate::settings::structs::Settings; use http::StatusCode; use regex::Regex; use thiserror::Error; @@ -83,12 +84,12 @@ impl actix_web::error::ResponseError for LemmyError { lazy_static! { pub static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!( "^group:([a-z0-9_]{{3, 20}})@{}$", - Settings::get().hostname + Settings::get().hostname() )) .unwrap(); pub static ref WEBFINGER_USER_REGEX: Regex = Regex::new(&format!( "^acct:([a-z0-9_]{{3, 20}})@{}$", - Settings::get().hostname + Settings::get().hostname() )) .unwrap(); } diff --git a/crates/utils/src/rate_limit/mod.rs b/crates/utils/src/rate_limit/mod.rs index 5a18ffd54..d3e74ed5b 100644 --- a/crates/utils/src/rate_limit/mod.rs +++ b/crates/utils/src/rate_limit/mod.rs @@ -1,5 +1,5 @@ use crate::{ - settings::{RateLimitConfig, Settings}, + settings::structs::{RateLimitConfig, Settings}, utils::get_ip, LemmyError, }; @@ -70,7 +70,7 @@ impl RateLimited { { // Does not need to be blocking because the RwLock in settings never held across await points, // and the operation here locks only long enough to clone - let rate_limit: RateLimitConfig = Settings::get().rate_limit; + let rate_limit: RateLimitConfig = Settings::get().rate_limit(); // before { diff --git a/crates/utils/src/request.rs b/crates/utils/src/request.rs index c3207dddd..428d78974 100644 --- a/crates/utils/src/request.rs +++ b/crates/utils/src/request.rs @@ -1,4 +1,4 @@ -use crate::{settings::Settings, LemmyError}; +use crate::{settings::structs::Settings, LemmyError}; use anyhow::anyhow; use log::error; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; @@ -58,7 +58,7 @@ pub(crate) async fn fetch_iframely( client: &Client, url: &str, ) -> Result { - let fetch_url = format!("{}/oembed?url={}", Settings::get().iframely_url, url); + let fetch_url = format!("{}/oembed?url={}", Settings::get().iframely_url(), url); let response = retry(|| client.get(&fetch_url).send()).await?; @@ -89,7 +89,7 @@ pub(crate) async fn fetch_pictrs( let fetch_url = format!( "{}/image/download?url={}", - Settings::get().pictrs_url, + Settings::get().pictrs_url(), utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed ); diff --git a/crates/utils/src/settings.rs b/crates/utils/src/settings.rs deleted file mode 100644 index 8301dcee4..000000000 --- a/crates/utils/src/settings.rs +++ /dev/null @@ -1,210 +0,0 @@ -use crate::location_info; -use anyhow::Context; -use config::{Config, ConfigError, Environment, File}; -use serde::Deserialize; -use std::{env, fs, io::Error, net::IpAddr, sync::RwLock}; - -static CONFIG_FILE_DEFAULTS: &str = "config/defaults.hjson"; -static CONFIG_FILE: &str = "config/config.hjson"; - -#[derive(Debug, Deserialize, Clone)] -pub struct Settings { - pub setup: Option, - pub database: DatabaseConfig, - pub hostname: String, - pub bind: IpAddr, - pub port: u16, - pub tls_enabled: bool, - pub jwt_secret: String, - pub pictrs_url: String, - pub iframely_url: String, - pub rate_limit: RateLimitConfig, - pub email: Option, - pub federation: FederationConfig, - pub captcha: CaptchaConfig, -} - -#[derive(Debug, Deserialize, Clone)] -pub struct Setup { - pub admin_username: String, - pub admin_password: String, - pub admin_email: Option, - pub site_name: String, -} - -#[derive(Debug, Deserialize, Clone)] -pub struct RateLimitConfig { - pub message: i32, - pub message_per_second: i32, - pub post: i32, - pub post_per_second: i32, - pub register: i32, - pub register_per_second: i32, - pub image: i32, - pub image_per_second: i32, -} - -#[derive(Debug, Deserialize, Clone)] -pub struct EmailConfig { - pub smtp_server: String, - pub smtp_login: Option, - pub smtp_password: Option, - pub smtp_from_address: String, - pub use_tls: bool, -} - -#[derive(Debug, Deserialize, Clone)] -pub struct CaptchaConfig { - pub enabled: bool, - pub difficulty: String, // easy, medium, or hard -} - -#[derive(Debug, Deserialize, Clone)] -pub struct DatabaseConfig { - pub user: String, - pub password: String, - pub host: String, - pub port: i32, - pub database: String, - pub pool_size: u32, -} - -#[derive(Debug, Deserialize, Clone)] -pub struct FederationConfig { - pub enabled: bool, - pub allowed_instances: String, - pub blocked_instances: String, -} - -lazy_static! { - static ref SETTINGS: RwLock = RwLock::new(match Settings::init() { - Ok(c) => c, - Err(e) => panic!("{}", e), - }); -} - -impl Settings { - /// Reads config from the files and environment. - /// First, defaults are loaded from CONFIG_FILE_DEFAULTS, then these values can be overwritten - /// from CONFIG_FILE (optional). Finally, values from the environment (with prefix LEMMY) are - /// added to the config. - /// - /// Note: The env var `LEMMY_DATABASE_URL` is parsed in - /// `lemmy_db_queries/src/lib.rs::get_database_url_from_env()` - fn init() -> Result { - let mut s = Config::new(); - - s.merge(File::with_name(&Self::get_config_defaults_location()))?; - - s.merge(File::with_name(&Self::get_config_location()).required(false))?; - - // Add in settings from the environment (with a prefix of LEMMY) - // Eg.. `LEMMY_DEBUG=1 ./target/app` would set the `debug` key - // Note: we need to use double underscore here, because otherwise variables containing - // underscore cant be set from environmnet. - // https://github.com/mehcode/config-rs/issues/73 - s.merge(Environment::with_prefix("LEMMY").separator("__"))?; - - s.try_into() - } - - /// Returns the config as a struct. - pub fn get() -> Self { - SETTINGS.read().unwrap().to_owned() - } - - pub fn get_database_url(&self) -> String { - format!( - "postgres://{}:{}@{}:{}/{}", - self.database.user, - self.database.password, - self.database.host, - self.database.port, - self.database.database - ) - } - - pub fn get_config_defaults_location() -> String { - env::var("LEMMY_CONFIG_DEFAULTS_LOCATION").unwrap_or_else(|_| CONFIG_FILE_DEFAULTS.to_string()) - } - - pub fn get_config_location() -> String { - env::var("LEMMY_CONFIG_LOCATION").unwrap_or_else(|_| CONFIG_FILE.to_string()) - } - - pub fn read_config_file() -> Result { - fs::read_to_string(Self::get_config_location()) - } - - pub fn get_allowed_instances(&self) -> Vec { - let mut allowed_instances: Vec = self - .federation - .allowed_instances - .split(',') - .map(|d| d.trim().to_string()) - .collect(); - - // The defaults.hjson config always returns a [""] - allowed_instances.retain(|d| !d.eq("")); - - allowed_instances - } - - pub fn get_blocked_instances(&self) -> Vec { - let mut blocked_instances: Vec = self - .federation - .blocked_instances - .split(',') - .map(|d| d.trim().to_string()) - .collect(); - - // The defaults.hjson config always returns a [""] - blocked_instances.retain(|d| !d.eq("")); - - blocked_instances - } - - /// Returns either "http" or "https", depending on tls_enabled setting - pub fn get_protocol_string(&self) -> &'static str { - if self.tls_enabled { - "https" - } else { - "http" - } - } - - /// Returns something like `http://localhost` or `https://lemmy.ml`, - /// with the correct protocol and hostname. - pub fn get_protocol_and_hostname(&self) -> String { - format!("{}://{}", self.get_protocol_string(), self.hostname) - } - - /// When running the federation test setup in `api_tests/` or `docker/federation`, the `hostname` - /// variable will be like `lemmy-alpha:8541`. This method removes the port and returns - /// `lemmy-alpha` instead. It has no effect in production. - pub fn get_hostname_without_port(&self) -> Result { - Ok( - self - .hostname - .split(':') - .collect::>() - .first() - .context(location_info!())? - .to_string(), - ) - } - - pub fn save_config_file(data: &str) -> Result { - fs::write(CONFIG_FILE, data)?; - - // Reload the new settings - // From https://stackoverflow.com/questions/29654927/how-do-i-assign-a-string-to-a-mutable-static-variable/47181804#47181804 - let mut new_settings = SETTINGS.write().unwrap(); - *new_settings = match Settings::init() { - Ok(c) => c, - Err(e) => panic!("{}", e), - }; - - Self::read_config_file() - } -} diff --git a/crates/utils/src/settings/defaults.rs b/crates/utils/src/settings/defaults.rs new file mode 100644 index 000000000..56d24c7fb --- /dev/null +++ b/crates/utils/src/settings/defaults.rs @@ -0,0 +1,69 @@ +use crate::settings::{CaptchaConfig, DatabaseConfig, FederationConfig, RateLimitConfig, Settings}; +use std::net::{IpAddr, Ipv4Addr}; + +impl Default for Settings { + fn default() -> Self { + Self { + database: Some(DatabaseConfig::default()), + rate_limit: Some(RateLimitConfig::default()), + federation: Some(FederationConfig::default()), + captcha: Some(CaptchaConfig::default()), + email: None, + setup: None, + hostname: None, + bind: Some(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))), + port: Some(8536), + tls_enabled: Some(true), + jwt_secret: Some("changeme".into()), + pictrs_url: Some("http://pictrs:8080".into()), + iframely_url: Some("http://iframely".into()), + } + } +} + +impl Default for DatabaseConfig { + fn default() -> Self { + Self { + user: "lemmy".into(), + password: "password".into(), + host: "localhost".into(), + port: 5432, + database: "lemmy".into(), + pool_size: 5, + } + } +} + +impl Default for CaptchaConfig { + fn default() -> Self { + Self { + enabled: true, + difficulty: "medium".into(), + } + } +} + +impl Default for FederationConfig { + fn default() -> Self { + Self { + enabled: false, + allowed_instances: None, + blocked_instances: None, + } + } +} + +impl Default for RateLimitConfig { + fn default() -> Self { + Self { + message: 180, + message_per_second: 60, + post: 6, + post_per_second: 600, + register: 3, + register_per_second: 3600, + image: 6, + image_per_second: 3600, + } + } +} diff --git a/crates/utils/src/settings/mod.rs b/crates/utils/src/settings/mod.rs new file mode 100644 index 000000000..abd6f733c --- /dev/null +++ b/crates/utils/src/settings/mod.rs @@ -0,0 +1,172 @@ +use crate::{ + location_info, + settings::structs::{ + CaptchaConfig, + DatabaseConfig, + EmailConfig, + FederationConfig, + RateLimitConfig, + Settings, + SetupConfig, + }, + LemmyError, +}; +use anyhow::{anyhow, Context}; +use deser_hjson::from_str; +use merge::Merge; +use std::{env, fs, io::Error, net::IpAddr, sync::RwLock}; + +pub(crate) mod defaults; +pub mod structs; + +static CONFIG_FILE: &str = "config/config.hjson"; + +lazy_static! { + static ref SETTINGS: RwLock = RwLock::new(match Settings::init() { + Ok(c) => c, + Err(e) => panic!("{}", e), + }); +} + +impl Settings { + /// Reads config from the files and environment. + /// First, defaults are loaded from CONFIG_FILE_DEFAULTS, then these values can be overwritten + /// from CONFIG_FILE (optional). Finally, values from the environment (with prefix LEMMY) are + /// added to the config. + /// + /// Note: The env var `LEMMY_DATABASE_URL` is parsed in + /// `lemmy_db_queries/src/lib.rs::get_database_url_from_env()` + fn init() -> Result { + // Read the config file + let mut custom_config = from_str::(&Self::read_config_file()?)?; + + // Merge with env vars + custom_config.merge(envy::prefixed("LEMMY_").from_env::()?); + + // Merge with default + custom_config.merge(Settings::default()); + + if custom_config.hostname == Settings::default().hostname { + return Err(anyhow!("Hostname variable is not set!").into()); + } + + Ok(custom_config) + } + + /// Returns the config as a struct. + pub fn get() -> Self { + SETTINGS.read().unwrap().to_owned() + } + + pub fn get_database_url(&self) -> String { + let conf = self.database(); + format!( + "postgres://{}:{}@{}:{}/{}", + conf.user, conf.password, conf.host, conf.port, conf.database, + ) + } + + pub fn get_config_location() -> String { + env::var("LEMMY_CONFIG_LOCATION").unwrap_or_else(|_| CONFIG_FILE.to_string()) + } + + pub fn read_config_file() -> Result { + fs::read_to_string(Self::get_config_location()) + } + + pub fn get_allowed_instances(&self) -> Option> { + self.federation().allowed_instances + } + + pub fn get_blocked_instances(&self) -> Option> { + self.federation().blocked_instances + } + + /// Returns either "http" or "https", depending on tls_enabled setting + pub fn get_protocol_string(&self) -> &'static str { + if let Some(tls_enabled) = self.tls_enabled { + if tls_enabled { + "https" + } else { + "http" + } + } else { + "http" + } + } + + /// Returns something like `http://localhost` or `https://lemmy.ml`, + /// with the correct protocol and hostname. + pub fn get_protocol_and_hostname(&self) -> String { + format!("{}://{}", self.get_protocol_string(), self.hostname()) + } + + /// When running the federation test setup in `api_tests/` or `docker/federation`, the `hostname` + /// variable will be like `lemmy-alpha:8541`. This method removes the port and returns + /// `lemmy-alpha` instead. It has no effect in production. + pub fn get_hostname_without_port(&self) -> Result { + Ok( + self + .hostname() + .split(':') + .collect::>() + .first() + .context(location_info!())? + .to_string(), + ) + } + + pub fn save_config_file(data: &str) -> Result { + fs::write(CONFIG_FILE, data)?; + + // Reload the new settings + // From https://stackoverflow.com/questions/29654927/how-do-i-assign-a-string-to-a-mutable-static-variable/47181804#47181804 + let mut new_settings = SETTINGS.write().unwrap(); + *new_settings = match Settings::init() { + Ok(c) => c, + Err(e) => panic!("{}", e), + }; + + Self::read_config_file() + } + + pub fn database(&self) -> DatabaseConfig { + self.database.to_owned().unwrap_or_default() + } + pub fn hostname(&self) -> String { + self.hostname.to_owned().unwrap_or_default() + } + pub fn bind(&self) -> IpAddr { + self.bind.unwrap() + } + pub fn port(&self) -> u16 { + self.port.unwrap_or_default() + } + pub fn tls_enabled(&self) -> bool { + self.tls_enabled.unwrap_or_default() + } + pub fn jwt_secret(&self) -> String { + self.jwt_secret.to_owned().unwrap_or_default() + } + pub fn pictrs_url(&self) -> String { + self.pictrs_url.to_owned().unwrap_or_default() + } + pub fn iframely_url(&self) -> String { + self.iframely_url.to_owned().unwrap_or_default() + } + pub fn rate_limit(&self) -> RateLimitConfig { + self.rate_limit.to_owned().unwrap_or_default() + } + pub fn federation(&self) -> FederationConfig { + self.federation.to_owned().unwrap_or_default() + } + pub fn captcha(&self) -> CaptchaConfig { + self.captcha.to_owned().unwrap_or_default() + } + pub fn email(&self) -> Option { + self.email.to_owned() + } + pub fn setup(&self) -> Option { + self.setup.to_owned() + } +} diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs new file mode 100644 index 000000000..21d89b98a --- /dev/null +++ b/crates/utils/src/settings/structs.rs @@ -0,0 +1,72 @@ +use merge::Merge; +use serde::Deserialize; +use std::net::IpAddr; + +#[derive(Debug, Deserialize, Clone, Merge)] +pub struct Settings { + pub(crate) database: Option, + pub(crate) rate_limit: Option, + pub(crate) federation: Option, + pub(crate) hostname: Option, + pub(crate) bind: Option, + pub(crate) port: Option, + pub(crate) tls_enabled: Option, + pub(crate) jwt_secret: Option, + pub(crate) pictrs_url: Option, + pub(crate) iframely_url: Option, + pub(crate) captcha: Option, + pub(crate) email: Option, + pub(crate) setup: Option, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct CaptchaConfig { + pub enabled: bool, + pub difficulty: String, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct DatabaseConfig { + pub user: String, + pub password: String, + pub host: String, + pub port: i32, + pub database: String, + pub pool_size: u32, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct EmailConfig { + pub smtp_server: String, + pub smtp_login: Option, + pub smtp_password: Option, + pub smtp_from_address: String, + pub use_tls: bool, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct FederationConfig { + pub enabled: bool, + pub allowed_instances: Option>, + pub blocked_instances: Option>, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct RateLimitConfig { + pub message: i32, + pub message_per_second: i32, + pub post: i32, + pub post_per_second: i32, + pub register: i32, + pub register_per_second: i32, + pub image: i32, + pub image_per_second: i32, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct SetupConfig { + pub admin_username: String, + pub admin_password: String, + pub admin_email: Option, + pub site_name: String, +} diff --git a/crates/utils/src/utils.rs b/crates/utils/src/utils.rs index e0bbb88e7..98eada07e 100644 --- a/crates/utils/src/utils.rs +++ b/crates/utils/src/utils.rs @@ -1,4 +1,4 @@ -use crate::{settings::Settings, ApiError}; +use crate::{settings::structs::Settings, ApiError}; use actix_web::dev::ConnectionInfo; use chrono::{DateTime, FixedOffset, NaiveDateTime}; use itertools::Itertools; @@ -85,7 +85,7 @@ pub struct MentionData { impl MentionData { pub fn is_local(&self) -> bool { - Settings::get().hostname.eq(&self.domain) + Settings::get().hostname().eq(&self.domain) } pub fn full_name(&self) -> String { format!("@{}@{}", &self.name, &self.domain) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 3e3963d29..1d3775ff3 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -56,7 +56,6 @@ RUN addgroup -g 1000 lemmy RUN adduser -D -s /bin/sh -u 1000 -G lemmy lemmy # Copy resources -COPY --chown=lemmy:lemmy config/defaults.hjson /config/defaults.hjson COPY --chown=lemmy:lemmy --from=builder /app/lemmy_server /app/lemmy RUN chown lemmy:lemmy /app/lemmy diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index b2b5ef7ea..c40de9022 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -38,20 +38,9 @@ services: - lemmy-alpha lemmy-alpha: image: lemmy-federation:latest + volumes: + - ./lemmy_alpha.hjson:/config/config.hjson environment: - - LEMMY_HOSTNAME=lemmy-alpha:8541 - - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_alpha:5432/lemmy - - LEMMY_JWT_SECRET=changeme - - LEMMY_FEDERATION__ENABLED=true - - LEMMY_TLS_ENABLED=false - - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-beta,lemmy-gamma,lemmy-delta,lemmy-epsilon - - LEMMY_PORT=8541 - - LEMMY_SETUP__ADMIN_USERNAME=lemmy_alpha - - LEMMY_SETUP__ADMIN_PASSWORD=lemmy - - LEMMY_SETUP__SITE_NAME=lemmy-alpha - - LEMMY_RATE_LIMIT__POST=99999 - - LEMMY_RATE_LIMIT__REGISTER=99999 - - LEMMY_CAPTCHA__ENABLED=false - LEMMY_TEST_SEND_SYNC=1 - RUST_BACKTRACE=1 - RUST_LOG=debug @@ -78,20 +67,9 @@ services: - lemmy-beta lemmy-beta: image: lemmy-federation:latest + volumes: + - ./lemmy_beta.hjson:/config/config.hjson environment: - - LEMMY_HOSTNAME=lemmy-beta:8551 - - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_beta:5432/lemmy - - LEMMY_JWT_SECRET=changeme - - LEMMY_FEDERATION__ENABLED=true - - LEMMY_TLS_ENABLED=false - - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-alpha,lemmy-gamma,lemmy-delta,lemmy-epsilon - - LEMMY_PORT=8551 - - LEMMY_SETUP__ADMIN_USERNAME=lemmy_beta - - LEMMY_SETUP__ADMIN_PASSWORD=lemmy - - LEMMY_SETUP__SITE_NAME=lemmy-beta - - LEMMY_RATE_LIMIT__POST=99999 - - LEMMY_RATE_LIMIT__REGISTER=99999 - - LEMMY_CAPTCHA__ENABLED=false - LEMMY_TEST_SEND_SYNC=1 - RUST_BACKTRACE=1 - RUST_LOG=debug @@ -118,20 +96,9 @@ services: - lemmy-gamma lemmy-gamma: image: lemmy-federation:latest + volumes: + - ./lemmy_gamma.hjson:/config/config.hjson environment: - - LEMMY_HOSTNAME=lemmy-gamma:8561 - - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_gamma:5432/lemmy - - LEMMY_JWT_SECRET=changeme - - LEMMY_FEDERATION__ENABLED=true - - LEMMY_TLS_ENABLED=false - - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-alpha,lemmy-beta,lemmy-delta,lemmy-epsilon - - LEMMY_PORT=8561 - - LEMMY_SETUP__ADMIN_USERNAME=lemmy_gamma - - LEMMY_SETUP__ADMIN_PASSWORD=lemmy - - LEMMY_SETUP__SITE_NAME=lemmy-gamma - - LEMMY_RATE_LIMIT__POST=99999 - - LEMMY_RATE_LIMIT__REGISTER=99999 - - LEMMY_CAPTCHA__ENABLED=false - LEMMY_TEST_SEND_SYNC=1 - RUST_BACKTRACE=1 - RUST_LOG=debug @@ -159,20 +126,9 @@ services: - lemmy-delta lemmy-delta: image: lemmy-federation:latest + volumes: + - ./lemmy_delta.hjson:/config/config.hjson environment: - - LEMMY_HOSTNAME=lemmy-delta:8571 - - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_delta:5432/lemmy - - LEMMY_JWT_SECRET=changeme - - LEMMY_FEDERATION__ENABLED=true - - LEMMY_TLS_ENABLED=false - - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-beta - - LEMMY_PORT=8571 - - LEMMY_SETUP__ADMIN_USERNAME=lemmy_delta - - LEMMY_SETUP__ADMIN_PASSWORD=lemmy - - LEMMY_SETUP__SITE_NAME=lemmy-delta - - LEMMY_RATE_LIMIT__POST=99999 - - LEMMY_RATE_LIMIT__REGISTER=99999 - - LEMMY_CAPTCHA__ENABLED=false - LEMMY_TEST_SEND_SYNC=1 - RUST_BACKTRACE=1 - RUST_LOG=debug @@ -200,20 +156,9 @@ services: - lemmy-epsilon lemmy-epsilon: image: lemmy-federation:latest + volumes: + - ./lemmy_epsilon.hjson:/config/config.hjson environment: - - LEMMY_HOSTNAME=lemmy-epsilon:8581 - - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_epsilon:5432/lemmy - - LEMMY_JWT_SECRET=changeme - - LEMMY_FEDERATION__ENABLED=true - - LEMMY_TLS_ENABLED=false - - LEMMY_FEDERATION__BLOCKED_INSTANCES=lemmy-alpha - - LEMMY_PORT=8581 - - LEMMY_SETUP__ADMIN_USERNAME=lemmy_epsilon - - LEMMY_SETUP__ADMIN_PASSWORD=lemmy - - LEMMY_SETUP__SITE_NAME=lemmy-epsilon - - LEMMY_RATE_LIMIT__POST=99999 - - LEMMY_RATE_LIMIT__REGISTER=99999 - - LEMMY_CAPTCHA__ENABLED=false - LEMMY_TEST_SEND_SYNC=1 - RUST_BACKTRACE=1 - RUST_LOG=debug diff --git a/docker/federation/lemmy_alpha.hjson b/docker/federation/lemmy_alpha.hjson new file mode 100644 index 000000000..e806397a8 --- /dev/null +++ b/docker/federation/lemmy_alpha.hjson @@ -0,0 +1,36 @@ +{ + port: 8541 + tls_enabled: false + jwt_secret: changeme + setup: { + admin_username: lemmy_alpha + admin_password: lemmy + site_name: lemmy-alpha + } + database: { + database: lemmy + user: lemmy + password: password + host: postgres_alpha + port: 5432 + pool_size: 5 + } + federation: { + enabled: true + allowed_instances: ["lemmy-beta","lemmy-gamma","lemmy-delta","lemmy-epsilon"] + } + captcha: { + enabled: false + difficulty: medium + } + rate_limit: { + message: 180 + message_per_second: 60 + post: 99999 + post_per_second: 600 + register: 99999 + register_per_second: 3600 + image: 6 + image_per_second: 3600 + } +} diff --git a/docker/federation/lemmy_beta.hjson b/docker/federation/lemmy_beta.hjson new file mode 100644 index 000000000..e16ecd53b --- /dev/null +++ b/docker/federation/lemmy_beta.hjson @@ -0,0 +1,37 @@ +{ + hostname: lemmy-beta:8551 + port: 8551 + tls_enabled: false + jwt_secret: changeme + setup: { + admin_username: lemmy_beta + admin_password: lemmy + site_name: lemmy-beta + } + database: { + database: lemmy + user: lemmy + password: password + host: postgres_beta + port: 5432 + pool_size: 5 + } + federation: { + enabled: true + allowed_instances: ["lemmy-alpha","lemmy-gamma","lemmy-delta","lemmy-epsilon"] + } + captcha: { + enabled: false + difficulty: medium + } + rate_limit: { + message: 180 + message_per_second: 60 + post: 99999 + post_per_second: 600 + register: 99999 + register_per_second: 3600 + image: 6 + image_per_second: 3600 + } +} diff --git a/docker/federation/lemmy_delta.hjson b/docker/federation/lemmy_delta.hjson new file mode 100644 index 000000000..321775fb5 --- /dev/null +++ b/docker/federation/lemmy_delta.hjson @@ -0,0 +1,37 @@ +{ + hostname: lemmy-delta:8571 + port: 8571 + tls_enabled: false + jwt_secret: changeme + setup: { + admin_username: lemmy_delta + admin_password: lemmy + site_name: lemmy-delta + } + database: { + database: lemmy + user: lemmy + password: password + host: postgres_delta + port: 5432 + pool_size: 5 + } + federation: { + enabled: true + allowed_instances: ["lemmy-beta"] + } + captcha: { + enabled: false + difficulty: medium + } + rate_limit: { + message: 180 + message_per_second: 60 + post: 99999 + post_per_second: 600 + register: 99999 + register_per_second: 3600 + image: 6 + image_per_second: 3600 + } +} diff --git a/docker/federation/lemmy_epsilon.hjson b/docker/federation/lemmy_epsilon.hjson new file mode 100644 index 000000000..20fb2c353 --- /dev/null +++ b/docker/federation/lemmy_epsilon.hjson @@ -0,0 +1,37 @@ +{ + hostname: lemmy-epsilon:8581 + port: 8581 + tls_enabled: false + jwt_secret: changeme + setup: { + admin_username: lemmy_epsilon + admin_password: lemmy + site_name: lemmy-epsilon + } + database: { + database: lemmy + user: lemmy + password: password + host: postgres_epsilon + port: 5432 + pool_size: 5 + } + federation: { + enabled: true + blocked_instances: ["lemmy-alpha"] + } + captcha: { + enabled: false + difficulty: medium + } + rate_limit: { + message: 180 + message_per_second: 60 + post: 99999 + post_per_second: 600 + register: 99999 + register_per_second: 3600 + image: 6 + image_per_second: 3600 + } +} diff --git a/docker/federation/lemmy_gamma.hjson b/docker/federation/lemmy_gamma.hjson new file mode 100644 index 000000000..784f98eb7 --- /dev/null +++ b/docker/federation/lemmy_gamma.hjson @@ -0,0 +1,37 @@ +{ + hostname: lemmy-gamma:8561 + port: 8561 + tls_enabled: false + jwt_secret: changeme + setup: { + admin_username: lemmy_gamma + admin_password: lemmy + site_name: lemmy-gamma + } + database: { + database: lemmy + user: lemmy + password: password + host: postgres_gamma + port: 5432 + pool_size: 5 + } + federation: { + enabled: true + allowed_instances: ["lemmy-alpha","lemmy-beta","lemmy-delta","lemmy-epsilon"] + } + captcha: { + enabled: false + difficulty: medium + } + rate_limit: { + message: 180 + message_per_second: 60 + post: 99999 + post_per_second: 600 + register: 99999 + register_per_second: 3600 + image: 6 + image_per_second: 3600 + } +} diff --git a/docker/lemmy.hjson b/docker/lemmy.hjson index bea65d8f9..bceb5f6cd 100644 --- a/docker/lemmy.hjson +++ b/docker/lemmy.hjson @@ -29,6 +29,10 @@ password: "password" # host where postgres is running host: "postgres" + # port where postgres can be accessed + port: 5432 + # maximum number of active sql connections + pool_size: 5 } # # optional: email sending configuration # email: { diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile index 66e8217fe..2f2418e49 100644 --- a/docker/prod/Dockerfile +++ b/docker/prod/Dockerfile @@ -56,7 +56,6 @@ RUN addgroup -g 1000 lemmy RUN adduser -D -s /bin/sh -u 1000 -G lemmy lemmy # Copy resources -COPY --chown=lemmy:lemmy config/defaults.hjson /config/defaults.hjson COPY --chown=lemmy:lemmy --from=builder /app/lemmy_server /app/lemmy RUN chown lemmy:lemmy /app/lemmy diff --git a/docker/prod/Dockerfile.arm b/docker/prod/Dockerfile.arm index bcf590034..a6e698f0d 100644 --- a/docker/prod/Dockerfile.arm +++ b/docker/prod/Dockerfile.arm @@ -31,7 +31,6 @@ RUN addgroup --gid 1000 lemmy RUN adduser --no-create-home --shell /bin/sh --uid 1000 --gid 1000 lemmy # Copy resources -COPY --chown=lemmy:lemmy config/defaults.hjson /config/defaults.hjson COPY --chown=lemmy:lemmy --from=builder /app/lemmy_server /app/lemmy RUN chown lemmy:lemmy /app/lemmy diff --git a/src/code_migrations.rs b/src/code_migrations.rs index ea11e32d3..f3bc3df53 100644 --- a/src/code_migrations.rs +++ b/src/code_migrations.rs @@ -24,7 +24,7 @@ use lemmy_db_schema::{ user::{UserForm, User_}, }, }; -use lemmy_utils::{apub::generate_actor_keypair, settings::Settings, LemmyError}; +use lemmy_utils::{apub::generate_actor_keypair, settings::structs::Settings, LemmyError}; use log::info; pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), LemmyError> { diff --git a/src/main.rs b/src/main.rs index a18201c73..fa110b51e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,7 +15,7 @@ use lemmy_routes::{feeds, images, nodeinfo, webfinger}; use lemmy_server::{code_migrations::run_advanced_migrations, scheduled_tasks}; use lemmy_utils::{ rate_limit::{rate_limiter::RateLimiter, RateLimit}, - settings::Settings, + settings::structs::Settings, LemmyError, }; use lemmy_websocket::{chat_server::ChatServer, LemmyContext}; @@ -37,7 +37,7 @@ async fn main() -> Result<(), LemmyError> { }; let manager = ConnectionManager::::new(&db_url); let pool = Pool::builder() - .max_size(settings.database.pool_size) + .max_size(settings.database().pool_size) .build(manager) .unwrap_or_else(|_| panic!("Error connecting to {}", db_url)); @@ -61,7 +61,8 @@ async fn main() -> Result<(), LemmyError> { println!( "Starting http server at {}:{}", - settings.bind, settings.port + settings.bind(), + settings.port() ); let activity_queue = create_activity_queue(); @@ -94,7 +95,7 @@ async fn main() -> Result<(), LemmyError> { .configure(nodeinfo::config) .configure(webfinger::config) }) - .bind((settings.bind, settings.port))? + .bind((settings.bind(), settings.port()))? .run() .await?; From 66946117e1ab61c50a896a478a884aa3ad65a8ee Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 1 Mar 2021 18:46:56 +0100 Subject: [PATCH 07/16] Upgrade Rust version --- .drone.yml | 12 ++++++------ api_tests/run-federation-test.sh | 2 +- docker/dev/Dockerfile | 2 +- docker/dev/volume_mount.dockerfile | 2 +- docker/prod/Dockerfile | 2 +- docker/prod/Dockerfile.arm | 2 +- scripts/compilation_benchmark.sh | 2 +- scripts/test.sh | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.drone.yml b/.drone.yml index b6fa86db5..1a83c09a3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -9,7 +9,7 @@ platform: steps: - name: chown repo - image: ekidd/rust-musl-builder:1.47.0 + image: ekidd/rust-musl-builder:1.50.0 user: root commands: - chown 1000:1000 . -R @@ -20,12 +20,12 @@ steps: - /root/.cargo/bin/cargo fmt -- --check - name: cargo clippy - image: ekidd/rust-musl-builder:1.47.0 + image: ekidd/rust-musl-builder:1.50.0 commands: - cargo clippy --workspace --tests --all-targets --all-features -- -D warnings -D deprecated -D clippy::perf -D clippy::complexity -D clippy::dbg_macro - name: cargo test - image: ekidd/rust-musl-builder:1.47.0 + image: ekidd/rust-musl-builder:1.50.0 environment: LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy RUST_BACKTRACE: 1 @@ -35,7 +35,7 @@ steps: - cargo test --workspace --no-fail-fast - name: cargo build - image: ekidd/rust-musl-builder:1.47.0 + image: ekidd/rust-musl-builder:1.50.0 commands: - cargo build - mv target/x86_64-unknown-linux-musl/debug/lemmy_server target/lemmy_server @@ -102,7 +102,7 @@ platform: steps: - name: cargo test - image: rust:1.47-slim-buster + image: rust:1.50-slim-buster environment: LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy RUST_BACKTRACE: 1 @@ -114,7 +114,7 @@ steps: # Using Debian here because there seems to be no official Alpine-based Rust docker image for ARM. - name: cargo build - image: rust:1.47-slim-buster + image: rust:1.50-slim-buster commands: - apt-get update - apt-get -y install --no-install-recommends libssl-dev pkg-config libpq-dev diff --git a/api_tests/run-federation-test.sh b/api_tests/run-federation-test.sh index d624f9c25..ebde0c04d 100755 --- a/api_tests/run-federation-test.sh +++ b/api_tests/run-federation-test.sh @@ -4,7 +4,7 @@ set -e export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432 pushd .. -cargo +1.47.0 build +cargo build rm target/lemmy_server || true cp target/debug/lemmy_server target/lemmy_server ./api_tests/prepare-drone-federation-test.sh diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 1d3775ff3..954c85c84 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,4 +1,4 @@ -ARG RUST_BUILDER_IMAGE=ekidd/rust-musl-builder:1.47.0 +ARG RUST_BUILDER_IMAGE=ekidd/rust-musl-builder:1.50.0 # Cargo chef plan FROM $RUST_BUILDER_IMAGE as planner diff --git a/docker/dev/volume_mount.dockerfile b/docker/dev/volume_mount.dockerfile index 3bb0b81d5..62d77fa36 100644 --- a/docker/dev/volume_mount.dockerfile +++ b/docker/dev/volume_mount.dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:experimental -FROM rust:1.47-buster as rust +FROM rust:1.50-buster as rust ENV HOME=/home/root diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile index 2f2418e49..2ad601aec 100644 --- a/docker/prod/Dockerfile +++ b/docker/prod/Dockerfile @@ -1,4 +1,4 @@ -ARG RUST_BUILDER_IMAGE=ekidd/rust-musl-builder:1.47.0 +ARG RUST_BUILDER_IMAGE=ekidd/rust-musl-builder:1.50.0 # Cargo chef plan FROM $RUST_BUILDER_IMAGE as planner diff --git a/docker/prod/Dockerfile.arm b/docker/prod/Dockerfile.arm index a6e698f0d..61d1f86c9 100644 --- a/docker/prod/Dockerfile.arm +++ b/docker/prod/Dockerfile.arm @@ -1,4 +1,4 @@ -ARG RUST_BUILDER_IMAGE=rust:1.47-slim-buster +ARG RUST_BUILDER_IMAGE=rust:1.50-slim-buster # Build Lemmy FROM $RUST_BUILDER_IMAGE as builder diff --git a/scripts/compilation_benchmark.sh b/scripts/compilation_benchmark.sh index 760037980..6d454795d 100755 --- a/scripts/compilation_benchmark.sh +++ b/scripts/compilation_benchmark.sh @@ -12,7 +12,7 @@ for ((i=0; i < times; i++)) ; do cargo clean echo "cargo build" start=$(date +%s.%N) - RUSTC_WRAPPER='' cargo +1.47.0 build -q + RUSTC_WRAPPER='' cargo build -q end=$(date +%s.%N) echo "Finished iteration $i after $(bc <<< "scale=0; $end - $start") seconds" duration=$(bc <<< "$duration + $end - $start") diff --git a/scripts/test.sh b/scripts/test.sh index b47f09fae..217912a0a 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -10,4 +10,4 @@ export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy # Integration tests only work on stable due to a bug in config-rs # https://github.com/mehcode/config-rs/issues/158 RUST_BACKTRACE=1 RUST_TEST_THREADS=1 \ - cargo +1.47.0 test --workspace --no-fail-fast + cargo test --workspace --no-fail-fast From 7f56281c26c56e094ccb1db7a5f7a70ec8d77e68 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 1 Mar 2021 13:56:07 +0100 Subject: [PATCH 08/16] Forbid usage of unwrap --- .drone.yml | 1 + crates/apub/src/activity_queue.rs | 2 +- crates/apub/src/inbox/user_inbox.rs | 9 ++++++++- crates/db_queries/src/lib.rs | 5 +++-- crates/utils/src/lib.rs | 4 ++-- crates/utils/src/request.rs | 2 +- crates/utils/src/settings/mod.rs | 10 +++++----- crates/utils/src/utils.rs | 18 +++++++++--------- src/scheduled_tasks.rs | 12 ++++++++---- 9 files changed, 38 insertions(+), 25 deletions(-) diff --git a/.drone.yml b/.drone.yml index 1a83c09a3..8b9f15919 100644 --- a/.drone.yml +++ b/.drone.yml @@ -23,6 +23,7 @@ steps: image: ekidd/rust-musl-builder:1.50.0 commands: - cargo clippy --workspace --tests --all-targets --all-features -- -D warnings -D deprecated -D clippy::perf -D clippy::complexity -D clippy::dbg_macro + - cargo clippy --workspace -- -D clippy::unwrap_used - name: cargo test image: ekidd/rust-musl-builder:1.50.0 diff --git a/crates/apub/src/activity_queue.rs b/crates/apub/src/activity_queue.rs index 152fcaf9b..f607dafe0 100644 --- a/crates/apub/src/activity_queue.rs +++ b/crates/apub/src/activity_queue.rs @@ -223,7 +223,7 @@ where let hostname = Settings::get().get_hostname_without_port()?; let inboxes: Vec<&Url> = inboxes .iter() - .filter(|i| i.domain().unwrap() != hostname) + .filter(|i| i.domain().expect("valid inbox url") != hostname) .collect(); let activity = activity.into_any_base()?; diff --git a/crates/apub/src/inbox/user_inbox.rs b/crates/apub/src/inbox/user_inbox.rs index 467bee77c..1a906d627 100644 --- a/crates/apub/src/inbox/user_inbox.rs +++ b/crates/apub/src/inbox/user_inbox.rs @@ -143,7 +143,14 @@ pub(crate) async fn user_receive_message( let actor_url = actor.actor_id(); match kind { UserValidTypes::Accept => { - receive_accept(&context, any_base, actor, to_user.unwrap(), request_counter).await?; + receive_accept( + &context, + any_base, + actor, + to_user.expect("user provided"), + request_counter, + ) + .await?; } UserValidTypes::Announce => { receive_announce(&context, any_base, actor, request_counter).await? diff --git a/crates/db_queries/src/lib.rs b/crates/db_queries/src/lib.rs index 5667b4262..0f1c4ce62 100644 --- a/crates/db_queries/src/lib.rs +++ b/crates/db_queries/src/lib.rs @@ -231,13 +231,14 @@ pub fn establish_unpooled_connection() -> PgConnection { }; let conn = PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url)); - embedded_migrations::run(&conn).unwrap(); + embedded_migrations::run(&conn).expect("load migrations"); conn } lazy_static! { static ref EMAIL_REGEX: Regex = - Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap(); + Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$") + .expect("compile email regex"); } pub mod functions { diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 58d2abb9d..bd2b56844 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -86,10 +86,10 @@ lazy_static! { "^group:([a-z0-9_]{{3, 20}})@{}$", Settings::get().hostname() )) - .unwrap(); + .expect("compile webfinger regex"); pub static ref WEBFINGER_USER_REGEX: Regex = Regex::new(&format!( "^acct:([a-z0-9_]{{3, 20}})@{}$", Settings::get().hostname() )) - .unwrap(); + .expect("compile webfinger regex"); } diff --git a/crates/utils/src/request.rs b/crates/utils/src/request.rs index 428d78974..bc12f1394 100644 --- a/crates/utils/src/request.rs +++ b/crates/utils/src/request.rs @@ -43,7 +43,7 @@ where } } - response.unwrap() + response.expect("retry http request") } #[derive(Deserialize, Debug)] diff --git a/crates/utils/src/settings/mod.rs b/crates/utils/src/settings/mod.rs index abd6f733c..3911a18f9 100644 --- a/crates/utils/src/settings/mod.rs +++ b/crates/utils/src/settings/mod.rs @@ -55,7 +55,7 @@ impl Settings { /// Returns the config as a struct. pub fn get() -> Self { - SETTINGS.read().unwrap().to_owned() + SETTINGS.read().expect("read config").to_owned() } pub fn get_database_url(&self) -> String { @@ -116,18 +116,18 @@ impl Settings { ) } - pub fn save_config_file(data: &str) -> Result { + pub fn save_config_file(data: &str) -> Result { fs::write(CONFIG_FILE, data)?; // Reload the new settings // From https://stackoverflow.com/questions/29654927/how-do-i-assign-a-string-to-a-mutable-static-variable/47181804#47181804 - let mut new_settings = SETTINGS.write().unwrap(); + let mut new_settings = SETTINGS.write().expect("write config"); *new_settings = match Settings::init() { Ok(c) => c, Err(e) => panic!("{}", e), }; - Self::read_config_file() + Ok(Self::read_config_file()?) } pub fn database(&self) -> DatabaseConfig { @@ -137,7 +137,7 @@ impl Settings { self.hostname.to_owned().unwrap_or_default() } pub fn bind(&self) -> IpAddr { - self.bind.unwrap() + self.bind.expect("return bind address") } pub fn port(&self) -> u16 { self.port.unwrap_or_default() diff --git a/crates/utils/src/utils.rs b/crates/utils/src/utils.rs index 98eada07e..33ea833d5 100644 --- a/crates/utils/src/utils.rs +++ b/crates/utils/src/utils.rs @@ -6,15 +6,15 @@ use rand::{distributions::Alphanumeric, thread_rng, Rng}; use regex::{Regex, RegexBuilder}; lazy_static! { -static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap(); -static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?\b|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|mudslime?s?|kikes?|\bspi(c|k)s?\b|\bchinks?|gooks?|bitch(es|ing|y)?|whor(es?|ing)|\btr(a|@)nn?(y|ies?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap(); -static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap(); -// TODO keep this old one, it didn't work with port well tho -// static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap(); -static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").unwrap(); -static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap(); -static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap(); -static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").unwrap(); + static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").expect("compile regex"); + static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?\b|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|mudslime?s?|kikes?|\bspi(c|k)s?\b|\bchinks?|gooks?|bitch(es|ing|y)?|whor(es?|ing)|\btr(a|@)nn?(y|ies?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().expect("compile regex"); + static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").expect("compile regex"); + // TODO keep this old one, it didn't work with port well tho + // static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").expect("compile regex"); + static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").expect("compile regex"); + static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").expect("compile regex"); + static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").expect("compile regex"); + static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").expect("compile regex"); } pub fn naive_from_unix(time: i64) -> NaiveDateTime { diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index 53dfd6bf3..4751e9ea2 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -48,14 +48,14 @@ fn reindex_aggregates_tables(conn: &PgConnection) { fn reindex_table(conn: &PgConnection, table_name: &str) { info!("Reindexing table {} ...", table_name); let query = format!("reindex table concurrently {}", table_name); - sql_query(query).execute(conn).unwrap(); + sql_query(query).execute(conn).expect("reindex table"); info!("Done."); } /// Clear old activities (this table gets very large) fn clear_old_activities(conn: &PgConnection) { info!("Clearing old activities..."); - Activity::delete_olds(&conn).unwrap(); + Activity::delete_olds(&conn).expect("clear old activities"); info!("Done."); } @@ -75,10 +75,14 @@ fn active_counts(conn: &PgConnection) { "update site_aggregates set users_active_{} = (select * from site_aggregates_activity('{}'))", i.1, i.0 ); - sql_query(update_site_stmt).execute(conn).unwrap(); + sql_query(update_site_stmt) + .execute(conn) + .expect("update site stats"); let update_community_stmt = format!("update community_aggregates ca set users_active_{} = mv.count_ from community_aggregates_activity('{}') mv where ca.community_id = mv.community_id_", i.1, i.0); - sql_query(update_community_stmt).execute(conn).unwrap(); + sql_query(update_community_stmt) + .execute(conn) + .expect("update community stats"); } info!("Done."); From e78ba38e94cdc6f31b6922a71eae05cbd8a8e435 Mon Sep 17 00:00:00 2001 From: Andrew Yoon Date: Tue, 2 Mar 2021 07:41:48 -0500 Subject: [PATCH 09/16] Use URL type in most outstanding struct fields (#1468) * Use URL type in most outstanding struct fields This fixes all known remaining cases where url fields are stored as plain strings, with the exception of form fields where empty strings are used as sentinels (see `diesel_option_overwrite_to_url`). Tested for regressions in the federated docker setup attempting to exercise all changed fields, including through apub federation. Fixes #1385 * Add migration to fix blank-string post.url values to be null This also then fixes #602 * Address review feedback - Fixed some unwraps and err message formatting - Bumped the `url` library to 2.2.1 to fix a bug with serde error messages - Add unit tests for the two diesel option override functions - Fix migration teardown by adding a no-op * Rename lemmy_db_queries::Url to lemmy_db_queries::DbUrl * fix compile error * box PostOrComment variants --- Cargo.lock | 4 +- Cargo.toml | 2 +- crates/api/Cargo.toml | 2 +- crates/api/src/community.rs | 17 ++--- crates/api/src/lib.rs | 9 --- crates/api/src/post.rs | 17 +++-- crates/api/src/site.rs | 16 +++-- crates/api/src/user.rs | 10 +-- crates/api_structs/Cargo.toml | 2 +- crates/api_structs/src/post.rs | 5 +- crates/api_structs/src/site.rs | 5 +- crates/apub/Cargo.toml | 2 +- crates/apub/src/http/mod.rs | 6 +- crates/apub/src/inbox/mod.rs | 2 +- .../apub/src/inbox/receive_for_community.rs | 36 +++++------ crates/apub/src/lib.rs | 45 +++++++------- crates/apub/src/objects/community.rs | 8 +-- crates/apub/src/objects/mod.rs | 4 +- crates/apub/src/objects/post.rs | 32 +++++----- crates/apub/src/objects/user.rs | 12 ++-- crates/db_queries/Cargo.toml | 2 +- crates/db_queries/src/lib.rs | 50 ++++++++++++++- crates/db_queries/src/source/activity.rs | 29 +++++---- crates/db_queries/src/source/comment.rs | 8 +-- crates/db_queries/src/source/community.rs | 11 ++-- crates/db_queries/src/source/post.rs | 8 +-- .../db_queries/src/source/private_message.rs | 8 +-- crates/db_queries/src/source/user.rs | 4 +- crates/db_schema/Cargo.toml | 2 +- crates/db_schema/src/lib.rs | 29 ++++----- crates/db_schema/src/source/activity.rs | 6 +- crates/db_schema/src/source/comment.rs | 8 +-- crates/db_schema/src/source/community.rs | 32 +++++----- crates/db_schema/src/source/post.rs | 14 ++--- crates/db_schema/src/source/post_report.rs | 6 +- .../db_schema/src/source/private_message.rs | 6 +- crates/db_schema/src/source/site.rs | 10 +-- crates/db_schema/src/source/user.rs | 62 +++++++++---------- crates/db_views/Cargo.toml | 2 +- crates/routes/Cargo.toml | 2 +- crates/utils/Cargo.toml | 2 +- crates/utils/src/request.rs | 35 ++++++----- .../down.sql | 4 ++ .../up.sql | 1 + 44 files changed, 312 insertions(+), 265 deletions(-) create mode 100644 migrations/2021-02-28-162616_clean_empty_post_urls/down.sql create mode 100644 migrations/2021-02-28-162616_clean_empty_post_urls/up.sql diff --git a/Cargo.lock b/Cargo.lock index 69b22f956..959aec106 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3655,9 +3655,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" dependencies = [ "form_urlencoded", "idna", diff --git a/Cargo.toml b/Cargo.toml index f76128cb4..e7d92fdf1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ actix-web = { version = "3.3.2", default-features = false, features = ["rustls"] log = "0.4.14" env_logger = "0.8.2" strum = "0.20.0" -url = { version = "2.2.0", features = ["serde"] } +url = { version = "2.2.1", features = ["serde"] } openssl = "0.10.32" http-signature-normalization-actix = { version = "0.4.1", default-features = false, features = ["sha-2"] } tokio = "0.3.6" diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 5ab339584..ea3cd625c 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -32,7 +32,7 @@ rand = "0.8.3" strum = "0.20.0" strum_macros = "0.20.1" lazy_static = "1.4.0" -url = { version = "2.2.0", features = ["serde"] } +url = { version = "2.2.1", features = ["serde"] } openssl = "0.10.32" http = "0.2.3" http-signature-normalization-actix = { version = "0.4.1", default-features = false, features = ["sha-2"] } diff --git a/crates/api/src/community.rs b/crates/api/src/community.rs index 4bfbaeb4a..cee5d3710 100644 --- a/crates/api/src/community.rs +++ b/crates/api/src/community.rs @@ -1,6 +1,5 @@ use crate::{ check_community_ban, - check_optional_url, get_user_from_jwt, get_user_from_jwt_opt, is_admin, @@ -19,7 +18,7 @@ use lemmy_apub::{ EndpointType, }; use lemmy_db_queries::{ - diesel_option_overwrite, + diesel_option_overwrite_to_url, source::{ comment::Comment_, community::{CommunityModerator_, Community_}, @@ -155,11 +154,8 @@ impl Perform for CreateCommunity { } // Check to make sure the icon and banners are urls - let icon = diesel_option_overwrite(&data.icon); - let banner = diesel_option_overwrite(&data.banner); - - check_optional_url(&icon)?; - check_optional_url(&banner)?; + let icon = diesel_option_overwrite_to_url(&data.icon)?; + let banner = diesel_option_overwrite_to_url(&data.banner)?; // When you create a community, make sure the user becomes a moderator and a follower let keypair = generate_actor_keypair()?; @@ -260,11 +256,8 @@ impl Perform for EditCommunity { }) .await??; - let icon = diesel_option_overwrite(&data.icon); - let banner = diesel_option_overwrite(&data.banner); - - check_optional_url(&icon)?; - check_optional_url(&banner)?; + let icon = diesel_option_overwrite_to_url(&data.icon)?; + let banner = diesel_option_overwrite_to_url(&data.banner)?; let community_form = CommunityForm { name: read_community.name, diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 6f4ea8f4d..54d11c1e3 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -186,15 +186,6 @@ pub(crate) async fn collect_moderated_communities( } } -pub(crate) fn check_optional_url(item: &Option>) -> Result<(), LemmyError> { - if let Some(Some(item)) = &item { - if Url::parse(item).is_err() { - return Err(ApiError::err("invalid_url").into()); - } - } - Ok(()) -} - pub(crate) async fn build_federated_instances( pool: &DbPool, ) -> Result, LemmyError> { diff --git a/crates/api/src/post.rs b/crates/api/src/post.rs index e905cd5cf..202ea30b3 100644 --- a/crates/api/src/post.rs +++ b/crates/api/src/post.rs @@ -1,7 +1,6 @@ use crate::{ check_community_ban, check_downvotes_enabled, - check_optional_url, collect_moderated_communities, get_user_from_jwt, get_user_from_jwt_opt, @@ -72,15 +71,14 @@ impl Perform for CreatePost { check_community_ban(user.id, data.community_id, context.pool()).await?; - check_optional_url(&Some(data.url.to_owned()))?; - // Fetch Iframely and pictrs cached image + let data_url = data.url.as_ref(); let (iframely_title, iframely_description, iframely_html, pictrs_thumbnail) = - fetch_iframely_and_pictrs_data(context.client(), data.url.to_owned()).await; + fetch_iframely_and_pictrs_data(context.client(), data_url).await; let post_form = PostForm { name: data.name.trim().to_owned(), - url: data.url.to_owned(), + url: data_url.map(|u| u.to_owned().into()), body: data.body.to_owned(), community_id: data.community_id, creator_id: user.id, @@ -93,7 +91,7 @@ impl Perform for CreatePost { embed_title: iframely_title, embed_description: iframely_description, embed_html: iframely_html, - thumbnail_url: pictrs_thumbnail, + thumbnail_url: pictrs_thumbnail.map(|u| u.into()), ap_id: None, local: true, published: None, @@ -385,12 +383,13 @@ impl Perform for EditPost { } // Fetch Iframely and Pictrs cached image + let data_url = data.url.as_ref(); let (iframely_title, iframely_description, iframely_html, pictrs_thumbnail) = - fetch_iframely_and_pictrs_data(context.client(), data.url.to_owned()).await; + fetch_iframely_and_pictrs_data(context.client(), data_url).await; let post_form = PostForm { name: data.name.trim().to_owned(), - url: data.url.to_owned(), + url: data_url.map(|u| u.to_owned().into()), body: data.body.to_owned(), nsfw: data.nsfw, creator_id: orig_post.creator_id.to_owned(), @@ -403,7 +402,7 @@ impl Perform for EditPost { embed_title: iframely_title, embed_description: iframely_description, embed_html: iframely_html, - thumbnail_url: pictrs_thumbnail, + thumbnail_url: pictrs_thumbnail.map(|u| u.into()), ap_id: Some(orig_post.ap_id), local: orig_post.local, published: None, diff --git a/crates/api/src/site.rs b/crates/api/src/site.rs index c4792f369..af9d22c41 100644 --- a/crates/api/src/site.rs +++ b/crates/api/src/site.rs @@ -11,7 +11,13 @@ use actix_web::web::Data; use anyhow::Context; use lemmy_api_structs::{blocking, site::*, user::Register}; use lemmy_apub::fetcher::search::search_by_apub_id; -use lemmy_db_queries::{diesel_option_overwrite, source::site::Site_, Crud, SearchType, SortType}; +use lemmy_db_queries::{ + diesel_option_overwrite_to_url, + source::site::Site_, + Crud, + SearchType, + SortType, +}; use lemmy_db_schema::{ naive_now, source::{ @@ -157,8 +163,8 @@ impl Perform for CreateSite { let site_form = SiteForm { name: data.name.to_owned(), description: data.description.to_owned(), - icon: Some(data.icon.to_owned()), - banner: Some(data.banner.to_owned()), + icon: Some(data.icon.to_owned().map(|url| url.into())), + banner: Some(data.banner.to_owned().map(|url| url.into())), creator_id: user.id, enable_downvotes: data.enable_downvotes, open_registration: data.open_registration, @@ -196,8 +202,8 @@ impl Perform for EditSite { let found_site = blocking(context.pool(), move |conn| Site::read_simple(conn)).await??; - let icon = diesel_option_overwrite(&data.icon); - let banner = diesel_option_overwrite(&data.banner); + let icon = diesel_option_overwrite_to_url(&data.icon)?; + let banner = diesel_option_overwrite_to_url(&data.banner)?; let site_form = SiteForm { name: data.name.to_owned(), diff --git a/crates/api/src/user.rs b/crates/api/src/user.rs index 6d37581e0..903c00e72 100644 --- a/crates/api/src/user.rs +++ b/crates/api/src/user.rs @@ -1,6 +1,5 @@ use crate::{ captcha_espeak_wav_base64, - check_optional_url, collect_moderated_communities, get_user_from_jwt, get_user_from_jwt_opt, @@ -23,6 +22,7 @@ use lemmy_apub::{ }; use lemmy_db_queries::{ diesel_option_overwrite, + diesel_option_overwrite_to_url, source::{ comment::Comment_, community::Community_, @@ -366,17 +366,13 @@ impl Perform for SaveUserSettings { let data: &SaveUserSettings = &self; let user = get_user_from_jwt(&data.auth, context.pool()).await?; - let avatar = diesel_option_overwrite(&data.avatar); - let banner = diesel_option_overwrite(&data.banner); + let avatar = diesel_option_overwrite_to_url(&data.avatar)?; + let banner = diesel_option_overwrite_to_url(&data.banner)?; let email = diesel_option_overwrite(&data.email); let bio = diesel_option_overwrite(&data.bio); let preferred_username = diesel_option_overwrite(&data.preferred_username); let matrix_user_id = diesel_option_overwrite(&data.matrix_user_id); - // Check to make sure the avatar and banners are urls - check_optional_url(&avatar)?; - check_optional_url(&banner)?; - if let Some(Some(bio)) = &bio { if bio.chars().count() > 300 { return Err(ApiError::err("bio_length_overflow").into()); diff --git a/crates/api_structs/Cargo.toml b/crates/api_structs/Cargo.toml index fc91e759a..242383e6f 100644 --- a/crates/api_structs/Cargo.toml +++ b/crates/api_structs/Cargo.toml @@ -21,4 +21,4 @@ diesel = "1.4.5" actix-web = "3.3.2" chrono = { version = "0.4.19", features = ["serde"] } serde_json = { version = "1.0.61", features = ["preserve_order"] } -url = "2.2.0" +url = "2.2.1" diff --git a/crates/api_structs/src/post.rs b/crates/api_structs/src/post.rs index 4e2011e91..82be66170 100644 --- a/crates/api_structs/src/post.rs +++ b/crates/api_structs/src/post.rs @@ -8,11 +8,12 @@ use lemmy_db_views_actor::{ community_view::CommunityView, }; use serde::{Deserialize, Serialize}; +use url::Url; #[derive(Deserialize, Debug)] pub struct CreatePost { pub name: String, - pub url: Option, + pub url: Option, pub body: Option, pub nsfw: bool, pub community_id: i32, @@ -66,7 +67,7 @@ pub struct CreatePostLike { pub struct EditPost { pub post_id: i32, pub name: String, - pub url: Option, + pub url: Option, pub body: Option, pub nsfw: bool, pub auth: String, diff --git a/crates/api_structs/src/site.rs b/crates/api_structs/src/site.rs index edee17a85..9f69e63bc 100644 --- a/crates/api_structs/src/site.rs +++ b/crates/api_structs/src/site.rs @@ -13,6 +13,7 @@ use lemmy_db_views_moderator::{ mod_sticky_post_view::ModStickyPostView, }; use serde::{Deserialize, Serialize}; +use url::Url; #[derive(Deserialize, Debug)] pub struct Search { @@ -60,8 +61,8 @@ pub struct GetModlogResponse { pub struct CreateSite { pub name: String, pub description: Option, - pub icon: Option, - pub banner: Option, + pub icon: Option, + pub banner: Option, pub enable_downvotes: bool, pub open_registration: bool, pub enable_nsfw: bool, diff --git a/crates/apub/Cargo.toml b/crates/apub/Cargo.toml index d294cbd9d..b0538af6a 100644 --- a/crates/apub/Cargo.toml +++ b/crates/apub/Cargo.toml @@ -32,7 +32,7 @@ rand = "0.8.3" strum = "0.20.0" strum_macros = "0.20.1" lazy_static = "1.4.0" -url = { version = "2.2.0", features = ["serde"] } +url = { version = "2.2.1", features = ["serde"] } percent-encoding = "2.1.0" openssl = "0.10.32" http = "0.2.3" diff --git a/crates/apub/src/http/mod.rs b/crates/apub/src/http/mod.rs index f117b6ea2..6bf4bbde6 100644 --- a/crates/apub/src/http/mod.rs +++ b/crates/apub/src/http/mod.rs @@ -7,6 +7,7 @@ use lemmy_db_schema::source::activity::Activity; use lemmy_utils::{settings::structs::Settings, LemmyError}; use lemmy_websocket::LemmyContext; use serde::{Deserialize, Serialize}; +use url::Url; pub mod comment; pub mod community; @@ -46,12 +47,13 @@ pub async fn get_activity( context: web::Data, ) -> Result, LemmyError> { let settings = Settings::get(); - let activity_id = format!( + let activity_id = Url::parse(&format!( "{}/activities/{}/{}", settings.get_protocol_and_hostname(), info.type_, info.id - ); + ))? + .into(); let activity = blocking(context.pool(), move |conn| { Activity::read_from_apub_id(&conn, &activity_id) }) diff --git a/crates/apub/src/inbox/mod.rs b/crates/apub/src/inbox/mod.rs index 2adefabe7..21585aa6a 100644 --- a/crates/apub/src/inbox/mod.rs +++ b/crates/apub/src/inbox/mod.rs @@ -45,7 +45,7 @@ pub(crate) async fn is_activity_already_known( pool: &DbPool, activity_id: &Url, ) -> Result { - let activity_id = activity_id.to_string(); + let activity_id = activity_id.to_owned().into(); let existing = blocking(pool, move |conn| { Activity::read_from_apub_id(&conn, &activity_id) }) diff --git a/crates/apub/src/inbox/receive_for_community.rs b/crates/apub/src/inbox/receive_for_community.rs index 9f8a6d6c1..a3ffbf116 100644 --- a/crates/apub/src/inbox/receive_for_community.rs +++ b/crates/apub/src/inbox/receive_for_community.rs @@ -120,9 +120,9 @@ pub(in crate::inbox) async fn receive_like_for_community( .as_single_xsd_any_uri() .context(location_info!())?; match fetch_post_or_comment_by_id(&object_id, context, request_counter).await? { - PostOrComment::Post(post) => receive_like_post(like, post, context, request_counter).await, + PostOrComment::Post(post) => receive_like_post(like, *post, context, request_counter).await, PostOrComment::Comment(comment) => { - receive_like_comment(like, comment, context, request_counter).await + receive_like_comment(like, *comment, context, request_counter).await } } } @@ -152,10 +152,10 @@ pub(in crate::inbox) async fn receive_dislike_for_community( .context(location_info!())?; match fetch_post_or_comment_by_id(&object_id, context, request_counter).await? { PostOrComment::Post(post) => { - receive_dislike_post(dislike, post, context, request_counter).await + receive_dislike_post(dislike, *post, context, request_counter).await } PostOrComment::Comment(comment) => { - receive_dislike_comment(dislike, comment, context, request_counter).await + receive_dislike_comment(dislike, *comment, context, request_counter).await } } } @@ -177,8 +177,8 @@ pub(in crate::inbox) async fn receive_delete_for_community( .context(location_info!())?; match find_post_or_comment_by_id(context, object).await { - Ok(PostOrComment::Post(p)) => receive_delete_post(context, p).await, - Ok(PostOrComment::Comment(c)) => receive_delete_comment(context, c).await, + Ok(PostOrComment::Post(p)) => receive_delete_post(context, *p).await, + Ok(PostOrComment::Comment(c)) => receive_delete_comment(context, *c).await, // if we dont have the object, no need to do anything Err(_) => Ok(()), } @@ -215,8 +215,8 @@ pub(in crate::inbox) async fn receive_remove_for_community( remove.id(community_id.domain().context(location_info!())?)?; match find_post_or_comment_by_id(context, object).await { - Ok(PostOrComment::Post(p)) => receive_remove_post(context, remove, p).await, - Ok(PostOrComment::Comment(c)) => receive_remove_comment(context, remove, c).await, + Ok(PostOrComment::Post(p)) => receive_remove_post(context, remove, *p).await, + Ok(PostOrComment::Comment(c)) => receive_remove_comment(context, remove, *c).await, // if we dont have the object, no need to do anything Err(_) => Ok(()), } @@ -276,8 +276,8 @@ pub(in crate::inbox) async fn receive_undo_delete_for_community( .single_xsd_any_uri() .context(location_info!())?; match find_post_or_comment_by_id(context, object).await { - Ok(PostOrComment::Post(p)) => receive_undo_delete_post(context, p).await, - Ok(PostOrComment::Comment(c)) => receive_undo_delete_comment(context, c).await, + Ok(PostOrComment::Post(p)) => receive_undo_delete_post(context, *p).await, + Ok(PostOrComment::Comment(c)) => receive_undo_delete_comment(context, *c).await, // if we dont have the object, no need to do anything Err(_) => Ok(()), } @@ -300,8 +300,8 @@ pub(in crate::inbox) async fn receive_undo_remove_for_community( .single_xsd_any_uri() .context(location_info!())?; match find_post_or_comment_by_id(context, object).await { - Ok(PostOrComment::Post(p)) => receive_undo_remove_post(context, p).await, - Ok(PostOrComment::Comment(c)) => receive_undo_remove_comment(context, c).await, + Ok(PostOrComment::Post(p)) => receive_undo_remove_post(context, *p).await, + Ok(PostOrComment::Comment(c)) => receive_undo_remove_comment(context, *c).await, // if we dont have the object, no need to do anything Err(_) => Ok(()), } @@ -325,10 +325,10 @@ pub(in crate::inbox) async fn receive_undo_like_for_community( .context(location_info!())?; match fetch_post_or_comment_by_id(&object_id, context, request_counter).await? { PostOrComment::Post(post) => { - receive_undo_like_post(&like, post, context, request_counter).await + receive_undo_like_post(&like, *post, context, request_counter).await } PostOrComment::Comment(comment) => { - receive_undo_like_comment(&like, comment, context, request_counter).await + receive_undo_like_comment(&like, *comment, context, request_counter).await } } } @@ -351,10 +351,10 @@ pub(in crate::inbox) async fn receive_undo_dislike_for_community( .context(location_info!())?; match fetch_post_or_comment_by_id(&object_id, context, request_counter).await? { PostOrComment::Post(post) => { - receive_undo_dislike_post(&dislike, post, context, request_counter).await + receive_undo_dislike_post(&dislike, *post, context, request_counter).await } PostOrComment::Comment(comment) => { - receive_undo_dislike_comment(&dislike, comment, context, request_counter).await + receive_undo_dislike_comment(&dislike, *comment, context, request_counter).await } } } @@ -365,11 +365,11 @@ async fn fetch_post_or_comment_by_id( request_counter: &mut i32, ) -> Result { if let Ok(post) = get_or_fetch_and_insert_post(apub_id, context, request_counter).await { - return Ok(PostOrComment::Post(post)); + return Ok(PostOrComment::Post(Box::new(post))); } if let Ok(comment) = get_or_fetch_and_insert_comment(apub_id, context, request_counter).await { - return Ok(PostOrComment::Comment(comment)); + return Ok(PostOrComment::Comment(Box::new(comment))); } Err(NotFound.into()) diff --git a/crates/apub/src/lib.rs b/crates/apub/src/lib.rs index 5c0b267a6..850ef503e 100644 --- a/crates/apub/src/lib.rs +++ b/crates/apub/src/lib.rs @@ -26,13 +26,16 @@ use anyhow::{anyhow, Context}; use diesel::NotFound; use lemmy_api_structs::blocking; use lemmy_db_queries::{source::activity::Activity_, ApubObject, DbPool}; -use lemmy_db_schema::source::{ - activity::Activity, - comment::Comment, - community::Community, - post::Post, - private_message::PrivateMessage, - user::User_, +use lemmy_db_schema::{ + source::{ + activity::Activity, + comment::Comment, + community::Community, + post::Post, + private_message::PrivateMessage, + user::User_, + }, + DbUrl, }; use lemmy_utils::{location_info, settings::structs::Settings, LemmyError}; use lemmy_websocket::LemmyContext; @@ -216,7 +219,7 @@ pub enum EndpointType { pub fn generate_apub_endpoint( endpoint_type: EndpointType, name: &str, -) -> Result { +) -> Result { let point = match endpoint_type { EndpointType::Community => "c", EndpointType::User => "u", @@ -236,21 +239,15 @@ pub fn generate_apub_endpoint( ) } -pub fn generate_followers_url( - actor_id: &lemmy_db_schema::Url, -) -> Result { +pub fn generate_followers_url(actor_id: &DbUrl) -> Result { Ok(Url::parse(&format!("{}/followers", actor_id))?.into()) } -pub fn generate_inbox_url( - actor_id: &lemmy_db_schema::Url, -) -> Result { +pub fn generate_inbox_url(actor_id: &DbUrl) -> Result { Ok(Url::parse(&format!("{}/inbox", actor_id))?.into()) } -pub fn generate_shared_inbox_url( - actor_id: &lemmy_db_schema::Url, -) -> Result { +pub fn generate_shared_inbox_url(actor_id: &DbUrl) -> Result { let actor_id = actor_id.clone().into_inner(); let url = format!( "{}://{}{}/inbox", @@ -277,7 +274,7 @@ pub(crate) async fn insert_activity( where T: Serialize + std::fmt::Debug + Send + 'static, { - let ap_id = ap_id.to_string(); + let ap_id = ap_id.to_owned().into(); blocking(pool, move |conn| { Activity::insert(conn, ap_id, &activity, local, sensitive) }) @@ -286,8 +283,8 @@ where } pub(crate) enum PostOrComment { - Comment(Comment), - Post(Post), + Comment(Box), + Post(Box), } /// Tries to find a post or comment in the local database, without any network requests. @@ -303,7 +300,7 @@ pub(crate) async fn find_post_or_comment_by_id( }) .await?; if let Ok(p) = post { - return Ok(PostOrComment::Post(p)); + return Ok(PostOrComment::Post(Box::new(p))); } let ap_id = apub_id.clone(); @@ -312,7 +309,7 @@ pub(crate) async fn find_post_or_comment_by_id( }) .await?; if let Ok(c) = comment { - return Ok(PostOrComment::Comment(c)); + return Ok(PostOrComment::Comment(Box::new(c))); } Err(NotFound.into()) @@ -333,8 +330,8 @@ pub(crate) async fn find_object_by_id( let ap_id = apub_id.clone(); if let Ok(pc) = find_post_or_comment_by_id(context, ap_id.to_owned()).await { return Ok(match pc { - PostOrComment::Post(p) => Object::Post(p), - PostOrComment::Comment(c) => Object::Comment(c), + PostOrComment::Post(p) => Object::Post(*p), + PostOrComment::Comment(c) => Object::Comment(*c), }); } diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index f516768ba..200497b73 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -73,13 +73,13 @@ impl ToApub for Community { if let Some(icon_url) = &self.icon { let mut image = Image::new(); - image.set_url(Url::parse(icon_url)?); + image.set_url::(icon_url.to_owned().into()); group.set_icon(image.into_any_base()?); } if let Some(banner_url) = &self.banner { let mut image = Image::new(); - image.set_url(Url::parse(banner_url)?); + image.set_url::(banner_url.to_owned().into()); group.set_image(image.into_any_base()?); } @@ -173,7 +173,7 @@ impl FromApubToForm for CommunityForm { .url() .context(location_info!())? .as_single_xsd_any_uri() - .map(|u| u.to_string()), + .map(|u| u.to_owned().into()), ), None => None, }; @@ -185,7 +185,7 @@ impl FromApubToForm for CommunityForm { .url() .context(location_info!())? .as_single_xsd_any_uri() - .map(|u| u.to_string()), + .map(|u| u.to_owned().into()), ), None => None, }; diff --git a/crates/apub/src/objects/mod.rs b/crates/apub/src/objects/mod.rs index 5667e1001..6b59e5776 100644 --- a/crates/apub/src/objects/mod.rs +++ b/crates/apub/src/objects/mod.rs @@ -14,7 +14,7 @@ use chrono::NaiveDateTime; use diesel::result::Error::NotFound; use lemmy_api_structs::blocking; use lemmy_db_queries::{ApubObject, Crud, DbPool}; -use lemmy_db_schema::source::community::Community; +use lemmy_db_schema::{source::community::Community, DbUrl}; use lemmy_utils::{ location_info, settings::structs::Settings, @@ -96,7 +96,7 @@ where pub(in crate::objects) fn check_object_domain( apub: &T, expected_domain: Url, -) -> Result +) -> Result where T: Base + AsBase, { diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 841b38069..b066e6f8c 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -24,10 +24,13 @@ use activitystreams_ext::Ext1; use anyhow::Context; use lemmy_api_structs::blocking; use lemmy_db_queries::{Crud, DbPool}; -use lemmy_db_schema::source::{ - community::Community, - post::{Post, PostForm}, - user::User_, +use lemmy_db_schema::{ + self, + source::{ + community::Community, + post::{Post, PostForm}, + user::User_, + }, }; use lemmy_utils::{ location_info, @@ -70,16 +73,13 @@ impl ToApub for Post { set_content_and_source(&mut page, &body)?; } - // TODO: hacky code because we get self.url == Some("") - // https://github.com/LemmyNet/lemmy/issues/602 - let url = self.url.as_ref().filter(|u| !u.is_empty()); - if let Some(u) = url { - page.set_url(Url::parse(u)?); + if let Some(url) = &self.url { + page.set_url::(url.to_owned().into()); } if let Some(thumbnail_url) = &self.thumbnail_url { let mut image = Image::new(); - image.set_url(Url::parse(thumbnail_url)?); + image.set_url::(thumbnail_url.to_owned().into()); page.set_image(image.into_any_base()?); } @@ -146,7 +146,7 @@ impl FromApubToForm for PostForm { let community = get_to_community(page, context, request_counter).await?; - let thumbnail_url = match &page.inner.image() { + let thumbnail_url: Option = match &page.inner.image() { Some(any_image) => Image::from_any_base( any_image .to_owned() @@ -158,7 +158,7 @@ impl FromApubToForm for PostForm { .url() .context(location_info!())? .as_single_xsd_any_uri() - .map(|u| u.to_string()), + .map(|url| url.to_owned()), None => None, }; let url = page @@ -166,11 +166,11 @@ impl FromApubToForm for PostForm { .url() .map(|u| u.as_single_xsd_any_uri()) .flatten() - .map(|s| s.to_string()); + .map(|u| u.to_owned()); let (iframely_title, iframely_description, iframely_html, pictrs_thumbnail) = if let Some(url) = &url { - fetch_iframely_and_pictrs_data(context.client(), Some(url.to_owned())).await + fetch_iframely_and_pictrs_data(context.client(), Some(url)).await } else { (None, None, None, thumbnail_url) }; @@ -192,7 +192,7 @@ impl FromApubToForm for PostForm { let body_slurs_removed = body.map(|b| remove_slurs(&b)); Ok(PostForm { name, - url, + url: url.map(|u| u.into()), body: body_slurs_removed, creator_id: creator.id, community_id: community.id, @@ -214,7 +214,7 @@ impl FromApubToForm for PostForm { embed_title: iframely_title, embed_description: iframely_description, embed_html: iframely_html, - thumbnail_url: pictrs_thumbnail, + thumbnail_url: pictrs_thumbnail.map(|u| u.into()), ap_id: Some(check_object_domain(page, expected_domain)?), local: false, }) diff --git a/crates/apub/src/objects/user.rs b/crates/apub/src/objects/user.rs index c979e3ee4..83f75e49b 100644 --- a/crates/apub/src/objects/user.rs +++ b/crates/apub/src/objects/user.rs @@ -50,13 +50,13 @@ impl ToApub for User_ { if let Some(avatar_url) = &self.avatar { let mut image = Image::new(); - image.set_url(Url::parse(avatar_url)?); + image.set_url::(avatar_url.to_owned().into()); person.set_icon(image.into_any_base()?); } if let Some(banner_url) = &self.banner { let mut image = Image::new(); - image.set_url(Url::parse(banner_url)?); + image.set_url::(banner_url.to_owned().into()); person.set_image(image.into_any_base()?); } @@ -126,7 +126,7 @@ impl FromApubToForm for UserForm { .url() .context(location_info!())? .as_single_xsd_any_uri() - .map(|u| u.to_string()), + .map(|url| url.to_owned()), ), None => None, }; @@ -139,7 +139,7 @@ impl FromApubToForm for UserForm { .url() .context(location_info!())? .as_single_xsd_any_uri() - .map(|u| u.to_string()), + .map(|url| url.to_owned()), ), None => None, }; @@ -174,8 +174,8 @@ impl FromApubToForm for UserForm { admin: false, banned: None, email: None, - avatar, - banner, + avatar: avatar.map(|o| o.map(|i| i.into())), + banner: banner.map(|o| o.map(|i| i.into())), published: person.inner.published().map(|u| u.to_owned().naive_local()), updated: person.updated().map(|u| u.to_owned().naive_local()), show_nsfw: false, diff --git a/crates/db_queries/Cargo.toml b/crates/db_queries/Cargo.toml index 0e489e228..c950eea95 100644 --- a/crates/db_queries/Cargo.toml +++ b/crates/db_queries/Cargo.toml @@ -20,7 +20,7 @@ strum = "0.20.0" strum_macros = "0.20.1" log = "0.4.14" sha2 = "0.9.3" -url = { version = "2.2.0", features = ["serde"] } +url = { version = "2.2.1", features = ["serde"] } lazy_static = "1.4.0" regex = "1.4.3" bcrypt = "0.9.0" diff --git a/crates/db_queries/src/lib.rs b/crates/db_queries/src/lib.rs index 5667b4262..c36bc34af 100644 --- a/crates/db_queries/src/lib.rs +++ b/crates/db_queries/src/lib.rs @@ -13,10 +13,12 @@ extern crate diesel_migrations; extern crate serial_test; use diesel::{result::Error, *}; -use lemmy_db_schema::Url; +use lemmy_db_schema::DbUrl; +use lemmy_utils::ApiError; use regex::Regex; use serde::{Deserialize, Serialize}; use std::{env, env::VarError}; +use url::Url; pub mod aggregates; pub mod source; @@ -112,7 +114,7 @@ pub trait Reportable { } pub trait ApubObject { - fn read_from_apub_id(conn: &PgConnection, object_id: &Url) -> Result + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result where Self: Sized; fn upsert(conn: &PgConnection, user_form: &T) -> Result @@ -219,6 +221,20 @@ pub fn diesel_option_overwrite(opt: &Option) -> Option> { } } +pub fn diesel_option_overwrite_to_url( + opt: &Option, +) -> Result>, ApiError> { + match opt.as_ref().map(|s| s.as_str()) { + // An empty string is an erase + Some("") => Ok(Some(None)), + Some(str_url) => match Url::parse(str_url) { + Ok(url) => Ok(Some(Some(url.into()))), + Err(_) => Err(ApiError::err("invalid_url")), + }, + None => Ok(None), + } +} + embed_migrations!(); pub fn establish_unpooled_connection() -> PgConnection { @@ -250,7 +266,7 @@ pub mod functions { #[cfg(test)] mod tests { - use super::fuzzy_search; + use super::{fuzzy_search, *}; use crate::is_email_regex; #[test] @@ -264,4 +280,32 @@ mod tests { assert!(is_email_regex("gush@gmail.com")); assert!(!is_email_regex("nada_neutho")); } + + #[test] + fn test_diesel_option_overwrite() { + assert_eq!(diesel_option_overwrite(&None), None); + assert_eq!(diesel_option_overwrite(&Some("".to_string())), Some(None)); + assert_eq!( + diesel_option_overwrite(&Some("test".to_string())), + Some(Some("test".to_string())) + ); + } + + #[test] + fn test_diesel_option_overwrite_to_url() { + assert!(matches!(diesel_option_overwrite_to_url(&None), Ok(None))); + assert!(matches!( + diesel_option_overwrite_to_url(&Some("".to_string())), + Ok(Some(None)) + )); + assert!(matches!( + diesel_option_overwrite_to_url(&Some("invalid_url".to_string())), + Err(_) + )); + let example_url = "https://example.com"; + assert!(matches!( + diesel_option_overwrite_to_url(&Some(example_url.to_string())), + Ok(Some(Some(url))) if url == Url::parse(&example_url).unwrap().into() + )); + } } diff --git a/crates/db_queries/src/source/activity.rs b/crates/db_queries/src/source/activity.rs index cf946d67d..56b904e89 100644 --- a/crates/db_queries/src/source/activity.rs +++ b/crates/db_queries/src/source/activity.rs @@ -1,6 +1,6 @@ use crate::Crud; use diesel::{dsl::*, result::Error, sql_types::Text, *}; -use lemmy_db_schema::{source::activity::*, Url}; +use lemmy_db_schema::{source::activity::*, DbUrl}; use log::debug; use serde::Serialize; use serde_json::Value; @@ -41,7 +41,7 @@ impl Crud for Activity { pub trait Activity_ { fn insert( conn: &PgConnection, - ap_id: String, + ap_id: DbUrl, data: &T, local: bool, sensitive: bool, @@ -49,20 +49,20 @@ pub trait Activity_ { where T: Serialize + Debug; - fn read_from_apub_id(conn: &PgConnection, object_id: &str) -> Result; + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result; fn delete_olds(conn: &PgConnection) -> Result; /// Returns up to 20 activities of type `Announce/Create/Page` from the community fn read_community_outbox( conn: &PgConnection, - community_actor_id: &Url, + community_actor_id: &DbUrl, ) -> Result, Error>; } impl Activity_ for Activity { fn insert( conn: &PgConnection, - ap_id: String, + ap_id: DbUrl, data: &T, local: bool, sensitive: bool, @@ -88,7 +88,7 @@ impl Activity_ for Activity { } } - fn read_from_apub_id(conn: &PgConnection, object_id: &str) -> Result { + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result { use lemmy_db_schema::schema::activity::dsl::*; activity.filter(ap_id.eq(object_id)).first::(conn) } @@ -100,7 +100,7 @@ impl Activity_ for Activity { fn read_community_outbox( conn: &PgConnection, - community_actor_id: &Url, + community_actor_id: &DbUrl, ) -> Result, Error> { use lemmy_db_schema::schema::activity::dsl::*; let res: Vec = activity @@ -121,6 +121,7 @@ impl Activity_ for Activity { #[cfg(test)] mod tests { + use super::*; use crate::{ establish_unpooled_connection, source::activity::Activity_, @@ -134,6 +135,7 @@ mod tests { }; use serde_json::Value; use serial_test::serial; + use url::Url; #[test] #[serial] @@ -171,8 +173,11 @@ mod tests { let inserted_creator = User_::create(&conn, &creator_form).unwrap(); - let ap_id = - "https://enterprise.lemmy.ml/activities/delete/f1b5d57c-80f8-4e03-a615-688d552e946c"; + let ap_id: DbUrl = Url::parse( + "https://enterprise.lemmy.ml/activities/delete/f1b5d57c-80f8-4e03-a615-688d552e946c", + ) + .unwrap() + .into(); let test_json: Value = serde_json::from_str( r#"{ "@context": "https://www.w3.org/ns/activitystreams", @@ -188,7 +193,7 @@ mod tests { ) .unwrap(); let activity_form = ActivityForm { - ap_id: ap_id.to_string(), + ap_id: ap_id.clone(), data: test_json.to_owned(), local: true, sensitive: false, @@ -198,7 +203,7 @@ mod tests { let inserted_activity = Activity::create(&conn, &activity_form).unwrap(); let expected_activity = Activity { - ap_id: Some(ap_id.to_string()), + ap_id: Some(ap_id.clone()), id: inserted_activity.id, data: test_json, local: true, @@ -208,7 +213,7 @@ mod tests { }; let read_activity = Activity::read(&conn, inserted_activity.id).unwrap(); - let read_activity_by_apub_id = Activity::read_from_apub_id(&conn, ap_id).unwrap(); + let read_activity_by_apub_id = Activity::read_from_apub_id(&conn, &ap_id).unwrap(); User_::delete(&conn, inserted_creator.id).unwrap(); Activity::delete(&conn, inserted_activity.id).unwrap(); diff --git a/crates/db_queries/src/source/comment.rs b/crates/db_queries/src/source/comment.rs index 918ae4977..8dc9050cb 100644 --- a/crates/db_queries/src/source/comment.rs +++ b/crates/db_queries/src/source/comment.rs @@ -10,11 +10,11 @@ use lemmy_db_schema::{ CommentSaved, CommentSavedForm, }, - Url, + DbUrl, }; pub trait Comment_ { - fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: Url) -> Result; + fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: DbUrl) -> Result; fn permadelete_for_creator( conn: &PgConnection, for_creator_id: i32, @@ -43,7 +43,7 @@ pub trait Comment_ { } impl Comment_ for Comment { - fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: Url) -> Result { + fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: DbUrl) -> Result { use lemmy_db_schema::schema::comment::dsl::*; diesel::update(comment.find(comment_id)) @@ -145,7 +145,7 @@ impl Crud for Comment { } impl ApubObject for Comment { - fn read_from_apub_id(conn: &PgConnection, object_id: &Url) -> Result { + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result { use lemmy_db_schema::schema::comment::dsl::*; comment.filter(ap_id.eq(object_id)).first::(conn) } diff --git a/crates/db_queries/src/source/community.rs b/crates/db_queries/src/source/community.rs index 6a19d2211..03484816c 100644 --- a/crates/db_queries/src/source/community.rs +++ b/crates/db_queries/src/source/community.rs @@ -12,7 +12,7 @@ use lemmy_db_schema::{ CommunityUserBan, CommunityUserBanForm, }, - Url, + DbUrl, }; mod safe_type { @@ -90,7 +90,7 @@ impl Crud for Community { } impl ApubObject for Community { - fn read_from_apub_id(conn: &PgConnection, for_actor_id: &Url) -> Result { + fn read_from_apub_id(conn: &PgConnection, for_actor_id: &DbUrl) -> Result { use lemmy_db_schema::schema::community::dsl::*; community .filter(actor_id.eq(for_actor_id)) @@ -131,7 +131,10 @@ pub trait Community_ { new_creator_id: i32, ) -> Result; fn distinct_federated_communities(conn: &PgConnection) -> Result, Error>; - fn read_from_followers_url(conn: &PgConnection, followers_url: &Url) -> Result; + fn read_from_followers_url( + conn: &PgConnection, + followers_url: &DbUrl, + ) -> Result; } impl Community_ for Community { @@ -194,7 +197,7 @@ impl Community_ for Community { fn read_from_followers_url( conn: &PgConnection, - followers_url_: &Url, + followers_url_: &DbUrl, ) -> Result { use lemmy_db_schema::schema::community::dsl::*; community diff --git a/crates/db_queries/src/source/post.rs b/crates/db_queries/src/source/post.rs index f6b1342ef..f105dc738 100644 --- a/crates/db_queries/src/source/post.rs +++ b/crates/db_queries/src/source/post.rs @@ -12,7 +12,7 @@ use lemmy_db_schema::{ PostSaved, PostSavedForm, }, - Url, + DbUrl, }; impl Crud for Post { @@ -42,7 +42,7 @@ impl Crud for Post { pub trait Post_ { //fn read(conn: &PgConnection, post_id: i32) -> Result; fn list_for_community(conn: &PgConnection, the_community_id: i32) -> Result, Error>; - fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: Url) -> Result; + fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: DbUrl) -> Result; fn permadelete_for_creator(conn: &PgConnection, for_creator_id: i32) -> Result, Error>; fn update_deleted(conn: &PgConnection, post_id: i32, new_deleted: bool) -> Result; fn update_removed(conn: &PgConnection, post_id: i32, new_removed: bool) -> Result; @@ -68,7 +68,7 @@ impl Post_ for Post { .load::(conn) } - fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: Url) -> Result { + fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: DbUrl) -> Result { use lemmy_db_schema::schema::post::dsl::*; diesel::update(post.find(post_id)) @@ -147,7 +147,7 @@ impl Post_ for Post { } impl ApubObject for Post { - fn read_from_apub_id(conn: &PgConnection, object_id: &Url) -> Result { + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result { use lemmy_db_schema::schema::post::dsl::*; post.filter(ap_id.eq(object_id)).first::(conn) } diff --git a/crates/db_queries/src/source/private_message.rs b/crates/db_queries/src/source/private_message.rs index f1908bc14..c437252ef 100644 --- a/crates/db_queries/src/source/private_message.rs +++ b/crates/db_queries/src/source/private_message.rs @@ -1,6 +1,6 @@ use crate::{ApubObject, Crud}; use diesel::{dsl::*, result::Error, *}; -use lemmy_db_schema::{naive_now, source::private_message::*, Url}; +use lemmy_db_schema::{naive_now, source::private_message::*, DbUrl}; impl Crud for PrivateMessage { fn read(conn: &PgConnection, private_message_id: i32) -> Result { @@ -28,7 +28,7 @@ impl Crud for PrivateMessage { } impl ApubObject for PrivateMessage { - fn read_from_apub_id(conn: &PgConnection, object_id: &Url) -> Result + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result where Self: Sized, { @@ -53,7 +53,7 @@ pub trait PrivateMessage_ { fn update_ap_id( conn: &PgConnection, private_message_id: i32, - apub_id: Url, + apub_id: DbUrl, ) -> Result; fn update_content( conn: &PgConnection, @@ -80,7 +80,7 @@ impl PrivateMessage_ for PrivateMessage { fn update_ap_id( conn: &PgConnection, private_message_id: i32, - apub_id: Url, + apub_id: DbUrl, ) -> Result { use lemmy_db_schema::schema::private_message::dsl::*; diff --git a/crates/db_queries/src/source/user.rs b/crates/db_queries/src/source/user.rs index 56023f09f..d0e7411a5 100644 --- a/crates/db_queries/src/source/user.rs +++ b/crates/db_queries/src/source/user.rs @@ -5,7 +5,7 @@ use lemmy_db_schema::{ naive_now, schema::user_::dsl::*, source::user::{UserForm, UserSafeSettings, User_}, - Url, + DbUrl, }; use lemmy_utils::settings::structs::Settings; @@ -242,7 +242,7 @@ impl Crud for User_ { } impl ApubObject for User_ { - fn read_from_apub_id(conn: &PgConnection, object_id: &Url) -> Result { + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result { use lemmy_db_schema::schema::user_::dsl::*; user_ .filter(deleted.eq(false)) diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index b637c0222..1da8b68fb 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -12,4 +12,4 @@ chrono = { version = "0.4.19", features = ["serde"] } serde = { version = "1.0.123", features = ["derive"] } serde_json = { version = "1.0.61", features = ["preserve_order"] } log = "0.4.14" -url = { version = "2.2.0", features = ["serde"] } +url = { version = "2.2.1", features = ["serde"] } diff --git a/crates/db_schema/src/lib.rs b/crates/db_schema/src/lib.rs index b0733884e..f44567b90 100644 --- a/crates/db_schema/src/lib.rs +++ b/crates/db_schema/src/lib.rs @@ -8,21 +8,22 @@ use diesel::{ serialize::{Output, ToSql}, sql_types::Text, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::{ fmt::{Display, Formatter}, io::Write, }; +use url::Url; pub mod schema; pub mod source; #[repr(transparent)] -#[derive(Clone, PartialEq, Serialize, Debug, AsExpression, FromSqlRow)] +#[derive(Clone, PartialEq, Serialize, Deserialize, Debug, AsExpression, FromSqlRow)] #[sql_type = "Text"] -pub struct Url(url::Url); +pub struct DbUrl(Url); -impl ToSql for Url +impl ToSql for DbUrl where String: ToSql, { @@ -31,37 +32,37 @@ where } } -impl FromSql for Url +impl FromSql for DbUrl where String: FromSql, { fn from_sql(bytes: Option<&DB::RawValue>) -> diesel::deserialize::Result { let str = String::from_sql(bytes)?; - Ok(Url(url::Url::parse(&str)?)) + Ok(DbUrl(Url::parse(&str)?)) } } -impl Url { - pub fn into_inner(self) -> url::Url { +impl DbUrl { + pub fn into_inner(self) -> Url { self.0 } } -impl Display for Url { +impl Display for DbUrl { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { self.to_owned().into_inner().fmt(f) } } -impl From for url::Url { - fn from(url: Url) -> Self { +impl From for Url { + fn from(url: DbUrl) -> Self { url.0 } } -impl From for Url { - fn from(url: url::Url) -> Self { - Url(url) +impl From for DbUrl { + fn from(url: Url) -> Self { + DbUrl(url) } } diff --git a/crates/db_schema/src/source/activity.rs b/crates/db_schema/src/source/activity.rs index cf81ab8c8..7b7f4aba3 100644 --- a/crates/db_schema/src/source/activity.rs +++ b/crates/db_schema/src/source/activity.rs @@ -1,4 +1,4 @@ -use crate::schema::activity; +use crate::{schema::activity, DbUrl}; use serde_json::Value; use std::fmt::Debug; @@ -10,7 +10,7 @@ pub struct Activity { pub local: bool, pub published: chrono::NaiveDateTime, pub updated: Option, - pub ap_id: Option, + pub ap_id: Option, pub sensitive: Option, } @@ -20,6 +20,6 @@ pub struct ActivityForm { pub data: Value, pub local: bool, pub updated: Option, - pub ap_id: String, + pub ap_id: DbUrl, pub sensitive: bool, } diff --git a/crates/db_schema/src/source/comment.rs b/crates/db_schema/src/source/comment.rs index 72b9e740a..a00738a06 100644 --- a/crates/db_schema/src/source/comment.rs +++ b/crates/db_schema/src/source/comment.rs @@ -1,7 +1,7 @@ use crate::{ schema::{comment, comment_alias_1, comment_like, comment_saved}, source::post::Post, - Url, + DbUrl, }; use serde::Serialize; @@ -26,7 +26,7 @@ pub struct Comment { pub published: chrono::NaiveDateTime, pub updated: Option, pub deleted: bool, - pub ap_id: Url, + pub ap_id: DbUrl, pub local: bool, } @@ -44,7 +44,7 @@ pub struct CommentAlias1 { pub published: chrono::NaiveDateTime, pub updated: Option, pub deleted: bool, - pub ap_id: Url, + pub ap_id: DbUrl, pub local: bool, } @@ -60,7 +60,7 @@ pub struct CommentForm { pub published: Option, pub updated: Option, pub deleted: Option, - pub ap_id: Option, + pub ap_id: Option, pub local: bool, } diff --git a/crates/db_schema/src/source/community.rs b/crates/db_schema/src/source/community.rs index b8702ca97..b9fe12493 100644 --- a/crates/db_schema/src/source/community.rs +++ b/crates/db_schema/src/source/community.rs @@ -1,6 +1,6 @@ use crate::{ schema::{community, community_follower, community_moderator, community_user_ban}, - Url, + DbUrl, }; use serde::Serialize; @@ -17,16 +17,16 @@ pub struct Community { pub updated: Option, pub deleted: bool, pub nsfw: bool, - pub actor_id: Url, + pub actor_id: DbUrl, pub local: bool, pub private_key: Option, pub public_key: Option, pub last_refreshed_at: chrono::NaiveDateTime, - pub icon: Option, - pub banner: Option, - pub followers_url: Url, - pub inbox_url: Url, - pub shared_inbox_url: Option, + pub icon: Option, + pub banner: Option, + pub followers_url: DbUrl, + pub inbox_url: DbUrl, + pub shared_inbox_url: Option, } /// A safe representation of community, without the sensitive info @@ -43,10 +43,10 @@ pub struct CommunitySafe { pub updated: Option, pub deleted: bool, pub nsfw: bool, - pub actor_id: Url, + pub actor_id: DbUrl, pub local: bool, - pub icon: Option, - pub banner: Option, + pub icon: Option, + pub banner: Option, } #[derive(Insertable, AsChangeset, Debug)] @@ -61,16 +61,16 @@ pub struct CommunityForm { pub updated: Option, pub deleted: Option, pub nsfw: bool, - pub actor_id: Option, + pub actor_id: Option, pub local: bool, pub private_key: Option, pub public_key: Option, pub last_refreshed_at: Option, - pub icon: Option>, - pub banner: Option>, - pub followers_url: Option, - pub inbox_url: Option, - pub shared_inbox_url: Option>, + pub icon: Option>, + pub banner: Option>, + pub followers_url: Option, + pub inbox_url: Option, + pub shared_inbox_url: Option>, } #[derive(Identifiable, Queryable, Associations, PartialEq, Debug)] diff --git a/crates/db_schema/src/source/post.rs b/crates/db_schema/src/source/post.rs index 4ec6b56d0..c8cc51cd0 100644 --- a/crates/db_schema/src/source/post.rs +++ b/crates/db_schema/src/source/post.rs @@ -1,6 +1,6 @@ use crate::{ schema::{post, post_like, post_read, post_saved}, - Url, + DbUrl, }; use serde::Serialize; @@ -9,7 +9,7 @@ use serde::Serialize; pub struct Post { pub id: i32, pub name: String, - pub url: Option, + pub url: Option, pub body: Option, pub creator_id: i32, pub community_id: i32, @@ -23,8 +23,8 @@ pub struct Post { pub embed_title: Option, pub embed_description: Option, pub embed_html: Option, - pub thumbnail_url: Option, - pub ap_id: Url, + pub thumbnail_url: Option, + pub ap_id: DbUrl, pub local: bool, } @@ -32,7 +32,7 @@ pub struct Post { #[table_name = "post"] pub struct PostForm { pub name: String, - pub url: Option, + pub url: Option, pub body: Option, pub creator_id: i32, pub community_id: i32, @@ -46,8 +46,8 @@ pub struct PostForm { pub embed_title: Option, pub embed_description: Option, pub embed_html: Option, - pub thumbnail_url: Option, - pub ap_id: Option, + pub thumbnail_url: Option, + pub ap_id: Option, pub local: bool, } diff --git a/crates/db_schema/src/source/post_report.rs b/crates/db_schema/src/source/post_report.rs index b75fb954a..62ef31cd8 100644 --- a/crates/db_schema/src/source/post_report.rs +++ b/crates/db_schema/src/source/post_report.rs @@ -1,4 +1,4 @@ -use crate::{schema::post_report, source::post::Post}; +use crate::{schema::post_report, source::post::Post, DbUrl}; use serde::{Deserialize, Serialize}; #[derive( @@ -11,7 +11,7 @@ pub struct PostReport { pub creator_id: i32, pub post_id: i32, pub original_post_name: String, - pub original_post_url: Option, + pub original_post_url: Option, pub original_post_body: Option, pub reason: String, pub resolved: bool, @@ -26,7 +26,7 @@ pub struct PostReportForm { pub creator_id: i32, pub post_id: i32, pub original_post_name: String, - pub original_post_url: Option, + pub original_post_url: Option, pub original_post_body: Option, pub reason: String, } diff --git a/crates/db_schema/src/source/private_message.rs b/crates/db_schema/src/source/private_message.rs index 376728a1b..949c97709 100644 --- a/crates/db_schema/src/source/private_message.rs +++ b/crates/db_schema/src/source/private_message.rs @@ -1,4 +1,4 @@ -use crate::{schema::private_message, Url}; +use crate::{schema::private_message, DbUrl}; use serde::Serialize; #[derive(Clone, Queryable, Associations, Identifiable, PartialEq, Debug, Serialize)] @@ -12,7 +12,7 @@ pub struct PrivateMessage { pub read: bool, pub published: chrono::NaiveDateTime, pub updated: Option, - pub ap_id: Url, + pub ap_id: DbUrl, pub local: bool, } @@ -26,6 +26,6 @@ pub struct PrivateMessageForm { pub read: Option, pub published: Option, pub updated: Option, - pub ap_id: Option, + pub ap_id: Option, pub local: bool, } diff --git a/crates/db_schema/src/source/site.rs b/crates/db_schema/src/source/site.rs index 66319548e..998e9f9d1 100644 --- a/crates/db_schema/src/source/site.rs +++ b/crates/db_schema/src/source/site.rs @@ -1,4 +1,4 @@ -use crate::schema::site; +use crate::{schema::site, DbUrl}; use serde::Serialize; #[derive(Queryable, Identifiable, PartialEq, Debug, Clone, Serialize)] @@ -13,8 +13,8 @@ pub struct Site { pub enable_downvotes: bool, pub open_registration: bool, pub enable_nsfw: bool, - pub icon: Option, - pub banner: Option, + pub icon: Option, + pub banner: Option, } #[derive(Insertable, AsChangeset)] @@ -28,6 +28,6 @@ pub struct SiteForm { pub open_registration: bool, pub enable_nsfw: bool, // when you want to null out a column, you have to send Some(None)), since sending None means you just don't want to update that column. - pub icon: Option>, - pub banner: Option>, + pub icon: Option>, + pub banner: Option>, } diff --git a/crates/db_schema/src/source/user.rs b/crates/db_schema/src/source/user.rs index d72929fa8..f04b9a609 100644 --- a/crates/db_schema/src/source/user.rs +++ b/crates/db_schema/src/source/user.rs @@ -1,6 +1,6 @@ use crate::{ schema::{user_, user_alias_1, user_alias_2}, - Url, + DbUrl, }; use serde::Serialize; @@ -12,7 +12,7 @@ pub struct User_ { pub preferred_username: Option, pub password_encrypted: String, pub email: Option, - pub avatar: Option, + pub avatar: Option, pub admin: bool, pub banned: bool, pub published: chrono::NaiveDateTime, @@ -25,16 +25,16 @@ pub struct User_ { pub show_avatars: bool, pub send_notifications_to_email: bool, pub matrix_user_id: Option, - pub actor_id: Url, + pub actor_id: DbUrl, pub bio: Option, pub local: bool, pub private_key: Option, pub public_key: Option, pub last_refreshed_at: chrono::NaiveDateTime, - pub banner: Option, + pub banner: Option, pub deleted: bool, - pub inbox_url: Url, - pub shared_inbox_url: Option, + pub inbox_url: DbUrl, + pub shared_inbox_url: Option, } /// A safe representation of user, without the sensitive info @@ -44,19 +44,19 @@ pub struct UserSafe { pub id: i32, pub name: String, pub preferred_username: Option, - pub avatar: Option, + pub avatar: Option, pub admin: bool, pub banned: bool, pub published: chrono::NaiveDateTime, pub updated: Option, pub matrix_user_id: Option, - pub actor_id: Url, + pub actor_id: DbUrl, pub bio: Option, pub local: bool, - pub banner: Option, + pub banner: Option, pub deleted: bool, - pub inbox_url: Url, - pub shared_inbox_url: Option, + pub inbox_url: DbUrl, + pub shared_inbox_url: Option, } /// A safe user view with only settings @@ -67,7 +67,7 @@ pub struct UserSafeSettings { pub name: String, pub preferred_username: Option, pub email: Option, - pub avatar: Option, + pub avatar: Option, pub admin: bool, pub banned: bool, pub published: chrono::NaiveDateTime, @@ -80,11 +80,11 @@ pub struct UserSafeSettings { pub show_avatars: bool, pub send_notifications_to_email: bool, pub matrix_user_id: Option, - pub actor_id: Url, + pub actor_id: DbUrl, pub bio: Option, pub local: bool, pub last_refreshed_at: chrono::NaiveDateTime, - pub banner: Option, + pub banner: Option, pub deleted: bool, } @@ -96,7 +96,7 @@ pub struct UserAlias1 { pub preferred_username: Option, pub password_encrypted: String, pub email: Option, - pub avatar: Option, + pub avatar: Option, pub admin: bool, pub banned: bool, pub published: chrono::NaiveDateTime, @@ -109,13 +109,13 @@ pub struct UserAlias1 { pub show_avatars: bool, pub send_notifications_to_email: bool, pub matrix_user_id: Option, - pub actor_id: Url, + pub actor_id: DbUrl, pub bio: Option, pub local: bool, pub private_key: Option, pub public_key: Option, pub last_refreshed_at: chrono::NaiveDateTime, - pub banner: Option, + pub banner: Option, pub deleted: bool, } @@ -125,16 +125,16 @@ pub struct UserSafeAlias1 { pub id: i32, pub name: String, pub preferred_username: Option, - pub avatar: Option, + pub avatar: Option, pub admin: bool, pub banned: bool, pub published: chrono::NaiveDateTime, pub updated: Option, pub matrix_user_id: Option, - pub actor_id: Url, + pub actor_id: DbUrl, pub bio: Option, pub local: bool, - pub banner: Option, + pub banner: Option, pub deleted: bool, } @@ -146,7 +146,7 @@ pub struct UserAlias2 { pub preferred_username: Option, pub password_encrypted: String, pub email: Option, - pub avatar: Option, + pub avatar: Option, pub admin: bool, pub banned: bool, pub published: chrono::NaiveDateTime, @@ -159,13 +159,13 @@ pub struct UserAlias2 { pub show_avatars: bool, pub send_notifications_to_email: bool, pub matrix_user_id: Option, - pub actor_id: Url, + pub actor_id: DbUrl, pub bio: Option, pub local: bool, pub private_key: Option, pub public_key: Option, pub last_refreshed_at: chrono::NaiveDateTime, - pub banner: Option, + pub banner: Option, pub deleted: bool, } @@ -175,16 +175,16 @@ pub struct UserSafeAlias2 { pub id: i32, pub name: String, pub preferred_username: Option, - pub avatar: Option, + pub avatar: Option, pub admin: bool, pub banned: bool, pub published: chrono::NaiveDateTime, pub updated: Option, pub matrix_user_id: Option, - pub actor_id: Url, + pub actor_id: DbUrl, pub bio: Option, pub local: bool, - pub banner: Option, + pub banner: Option, pub deleted: bool, } @@ -197,7 +197,7 @@ pub struct UserForm { pub admin: bool, pub banned: Option, pub email: Option>, - pub avatar: Option>, + pub avatar: Option>, pub published: Option, pub updated: Option, pub show_nsfw: bool, @@ -208,13 +208,13 @@ pub struct UserForm { pub show_avatars: bool, pub send_notifications_to_email: bool, pub matrix_user_id: Option>, - pub actor_id: Option, + pub actor_id: Option, pub bio: Option>, pub local: bool, pub private_key: Option, pub public_key: Option, pub last_refreshed_at: Option, - pub banner: Option>, - pub inbox_url: Option, - pub shared_inbox_url: Option>, + pub banner: Option>, + pub inbox_url: Option, + pub shared_inbox_url: Option>, } diff --git a/crates/db_views/Cargo.toml b/crates/db_views/Cargo.toml index e44f414a6..42a942b40 100644 --- a/crates/db_views/Cargo.toml +++ b/crates/db_views/Cargo.toml @@ -12,7 +12,7 @@ lemmy_db_schema = { path = "../db_schema" } diesel = { version = "1.4.5", features = ["postgres","chrono","r2d2","serde_json"] } serde = { version = "1.0.123", features = ["derive"] } log = "0.4.14" -url = "2.2.0" +url = "2.2.1" [dev-dependencies] serial_test = "0.5.1" \ No newline at end of file diff --git a/crates/routes/Cargo.toml b/crates/routes/Cargo.toml index 80b94aa7e..b6b464c33 100644 --- a/crates/routes/Cargo.toml +++ b/crates/routes/Cargo.toml @@ -25,6 +25,6 @@ chrono = { version = "0.4.19", features = ["serde"] } rss = "1.10.0" serde = { version = "1.0.123", features = ["derive"] } awc = { version = "2.0.3", default-features = false } -url = { version = "2.2.0", features = ["serde"] } +url = { version = "2.2.1", features = ["serde"] } strum = "0.20.0" lazy_static = "1.4.0" diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index bb87cd301..f8c2012db 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -22,7 +22,7 @@ thiserror = "1.0.23" comrak = { version = "0.9.0", default-features = false } lazy_static = "1.4.0" openssl = "0.10.32" -url = { version = "2.2.0", features = ["serde"] } +url = { version = "2.2.1", features = ["serde"] } actix-web = { version = "3.3.2", default-features = false, features = ["rustls"] } actix-rt = { version = "1.1.1", default-features = false } anyhow = "1.0.38" diff --git a/crates/utils/src/request.rs b/crates/utils/src/request.rs index 428d78974..963bc9a21 100644 --- a/crates/utils/src/request.rs +++ b/crates/utils/src/request.rs @@ -6,6 +6,7 @@ use reqwest::Client; use serde::Deserialize; use std::future::Future; use thiserror::Error; +use url::Url; #[derive(Clone, Debug, Error)] #[error("Error sending request, {0}")] @@ -50,13 +51,13 @@ where pub(crate) struct IframelyResponse { title: Option, description: Option, - thumbnail_url: Option, + thumbnail_url: Option, html: Option, } pub(crate) async fn fetch_iframely( client: &Client, - url: &str, + url: &Url, ) -> Result { let fetch_url = format!("{}/oembed?url={}", Settings::get().iframely_url(), url); @@ -83,14 +84,14 @@ pub(crate) struct PictrsFile { pub(crate) async fn fetch_pictrs( client: &Client, - image_url: &str, + image_url: &Url, ) -> Result { is_image_content_type(client, image_url).await?; let fetch_url = format!( "{}/image/download?url={}", Settings::get().pictrs_url(), - utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed + utf8_percent_encode(image_url.as_str(), NON_ALPHANUMERIC) // TODO this might not be needed ); let response = retry(|| client.get(&fetch_url).send()).await?; @@ -109,13 +110,8 @@ pub(crate) async fn fetch_pictrs( pub async fn fetch_iframely_and_pictrs_data( client: &Client, - url: Option, -) -> ( - Option, - Option, - Option, - Option, -) { + url: Option<&Url>, +) -> (Option, Option, Option, Option) { match &url { Some(url) => { // Fetch iframely data @@ -149,11 +145,19 @@ pub async fn fetch_iframely_and_pictrs_data( // The full urls are necessary for federation let pictrs_thumbnail = if let Some(pictrs_hash) = pictrs_hash { - Some(format!( + let url = Url::parse(&format!( "{}/pictrs/image/{}", Settings::get().get_protocol_and_hostname(), pictrs_hash - )) + )); + match url { + Ok(parsed_url) => Some(parsed_url), + Err(e) => { + // This really shouldn't happen unless the settings or hash are malformed + error!("Unexpected error constructing pictrs thumbnail URL: {}", e); + None + } + } } else { None }; @@ -169,9 +173,8 @@ pub async fn fetch_iframely_and_pictrs_data( } } -async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> { - let response = retry(|| client.get(test).send()).await?; - +async fn is_image_content_type(client: &Client, test: &Url) -> Result<(), LemmyError> { + let response = retry(|| client.get(test.to_owned()).send()).await?; if response .headers() .get("Content-Type") diff --git a/migrations/2021-02-28-162616_clean_empty_post_urls/down.sql b/migrations/2021-02-28-162616_clean_empty_post_urls/down.sql new file mode 100644 index 000000000..7195601c9 --- /dev/null +++ b/migrations/2021-02-28-162616_clean_empty_post_urls/down.sql @@ -0,0 +1,4 @@ +-- This is a clean-up migration that cannot be undone, +-- but Diesel requires a non-empty script so run a no-op. +SELECT 1; + diff --git a/migrations/2021-02-28-162616_clean_empty_post_urls/up.sql b/migrations/2021-02-28-162616_clean_empty_post_urls/up.sql new file mode 100644 index 000000000..24e2d16b8 --- /dev/null +++ b/migrations/2021-02-28-162616_clean_empty_post_urls/up.sql @@ -0,0 +1 @@ +UPDATE post SET url = NULL where url = ''; From 985dbcaada44af795ff0e1b920993a248c5e1b4b Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 2 Mar 2021 13:57:06 +0100 Subject: [PATCH 10/16] Update test script --- scripts/test.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 217912a0a..251e6a74a 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -5,9 +5,5 @@ psql -U lemmy -d postgres -c "DROP DATABASE lemmy;" psql -U lemmy -d postgres -c "CREATE DATABASE lemmy;" export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy -# Commenting since this will overwrite schema.rs, which will break things now -# diesel migration run -# Integration tests only work on stable due to a bug in config-rs -# https://github.com/mehcode/config-rs/issues/158 -RUST_BACKTRACE=1 RUST_TEST_THREADS=1 \ +RUST_BACKTRACE=1 \ cargo test --workspace --no-fail-fast From 134fece36da646cc34aa83865fe462e8a39ae6bc Mon Sep 17 00:00:00 2001 From: Dessalines Date: Tue, 2 Mar 2021 10:36:10 -0500 Subject: [PATCH 11/16] Adding a password length check to other API actions. (#1474) * Adding a password length check to other API actions. - Fixes #1473 * Fixing comment. --- crates/api/src/lib.rs | 9 +++++++++ crates/api/src/user.rs | 10 ++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 54d11c1e3..d87375ca7 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -465,6 +465,15 @@ pub(crate) fn espeak_wav_base64(text: &str) -> Result { Ok(base64) } +/// Checks the password length +pub(crate) fn password_length_check(pass: &str) -> Result<(), LemmyError> { + if pass.len() > 60 { + Err(ApiError::err("invalid_password").into()) + } else { + Ok(()) + } +} + #[cfg(test)] mod tests { use crate::captcha_espeak_wav_base64; diff --git a/crates/api/src/user.rs b/crates/api/src/user.rs index 903c00e72..93ffdfff4 100644 --- a/crates/api/src/user.rs +++ b/crates/api/src/user.rs @@ -4,6 +4,7 @@ use crate::{ get_user_from_jwt, get_user_from_jwt_opt, is_admin, + password_length_check, Perform, }; use actix_web::web::Data; @@ -144,10 +145,7 @@ impl Perform for Register { } } - // Password length check - if data.password.len() > 60 { - return Err(ApiError::err("invalid_password").into()); - } + password_length_check(&data.password)?; // Make sure passwords match if data.password != data.password_verify { @@ -390,6 +388,8 @@ impl Perform for SaveUserSettings { Some(new_password) => { match &data.new_password_verify { Some(new_password_verify) => { + password_length_check(&new_password)?; + // Make sure passwords match if new_password != new_password_verify { return Err(ApiError::err("passwords_dont_match").into()); @@ -989,6 +989,8 @@ impl Perform for PasswordChange { }) .await??; + password_length_check(&data.password)?; + // Make sure passwords match if data.password != data.password_verify { return Err(ApiError::err("passwords_dont_match").into()); From 7189328f803cc3e90961fe61fc35f7e39725a12d Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 2 Mar 2021 17:12:45 +0100 Subject: [PATCH 12/16] Remove category from activitypub context --- crates/apub/src/extensions/context.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/apub/src/extensions/context.rs b/crates/apub/src/extensions/context.rs index fe52ab95d..b670e60d8 100644 --- a/crates/apub/src/extensions/context.rs +++ b/crates/apub/src/extensions/context.rs @@ -6,12 +6,11 @@ pub(crate) fn lemmy_context() -> Result, LemmyError> { let context_ext = AnyBase::from_arbitrary_json(json!( { "sc": "http://schema.org#", - "category": "sc:category", "sensitive": "as:sensitive", "stickied": "as:stickied", "comments_enabled": { - "kind": "sc:Boolean", - "id": "pt:commentsEnabled" + "kind": "sc:Boolean", + "id": "pt:commentsEnabled" } }))?; Ok(vec![AnyBase::from(context()), context_ext]) From 817b4ff08ec162a3600285904667a7ffc439608b Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 3 Mar 2021 23:40:00 -0500 Subject: [PATCH 13/16] Fixing wrong user_ and community icon and banner urls. - Fixes #1477 --- docker/dev/volume_mount.dockerfile | 2 ++ migrations/2021-03-04-040229_clean_icon_urls/down.sql | 3 +++ migrations/2021-03-04-040229_clean_icon_urls/up.sql | 5 +++++ 3 files changed, 10 insertions(+) create mode 100644 migrations/2021-03-04-040229_clean_icon_urls/down.sql create mode 100644 migrations/2021-03-04-040229_clean_icon_urls/up.sql diff --git a/docker/dev/volume_mount.dockerfile b/docker/dev/volume_mount.dockerfile index 62d77fa36..0cb036247 100644 --- a/docker/dev/volume_mount.dockerfile +++ b/docker/dev/volume_mount.dockerfile @@ -1,4 +1,6 @@ # syntax=docker/dockerfile:experimental + +# Warning: this will not pick up migrations unless there are code changes FROM rust:1.50-buster as rust ENV HOME=/home/root diff --git a/migrations/2021-03-04-040229_clean_icon_urls/down.sql b/migrations/2021-03-04-040229_clean_icon_urls/down.sql new file mode 100644 index 000000000..f866b155c --- /dev/null +++ b/migrations/2021-03-04-040229_clean_icon_urls/down.sql @@ -0,0 +1,3 @@ +-- This is a clean-up migration that cannot be undone, +-- but Diesel requires a non-empty script so run a no-op. +SELECT 1; diff --git a/migrations/2021-03-04-040229_clean_icon_urls/up.sql b/migrations/2021-03-04-040229_clean_icon_urls/up.sql new file mode 100644 index 000000000..8b22b33fc --- /dev/null +++ b/migrations/2021-03-04-040229_clean_icon_urls/up.sql @@ -0,0 +1,5 @@ +-- If these are not urls, it will crash the server +update user_ set avatar = NULL where avatar not like 'http%'; +update user_ set banner = NULL where banner not like 'http%'; +update community set icon = NULL where icon not like 'http%'; +update community set banner = NULL where banner not like 'http%'; From a1c7584875581a055dd8f5d1159577d4b9226e8b Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 3 Mar 2021 23:44:07 -0500 Subject: [PATCH 14/16] Remove extra category_id s . Fixes #1429 --- crates/api_structs/src/community.rs | 2 -- crates/routes/src/feeds.rs | 3 --- 2 files changed, 5 deletions(-) diff --git a/crates/api_structs/src/community.rs b/crates/api_structs/src/community.rs index fbbf2e403..ee349cb04 100644 --- a/crates/api_structs/src/community.rs +++ b/crates/api_structs/src/community.rs @@ -27,7 +27,6 @@ pub struct CreateCommunity { pub description: Option, pub icon: Option, pub banner: Option, - pub category_id: i32, pub nsfw: bool, pub auth: String, } @@ -88,7 +87,6 @@ pub struct EditCommunity { pub description: Option, pub icon: Option, pub banner: Option, - pub category_id: i32, pub nsfw: bool, pub auth: String, } diff --git a/crates/routes/src/feeds.rs b/crates/routes/src/feeds.rs index 40d80f25a..105f9662d 100644 --- a/crates/routes/src/feeds.rs +++ b/crates/routes/src/feeds.rs @@ -396,9 +396,6 @@ fn create_post_items(posts: Vec) -> Result, LemmyError> { p.community.name ); - // TODO: for category we should just put the name of the category, but then we would have - // to read each community from the db - // TODO add images let mut description = format!("submitted by {} to {}
{} points | {} comments", p.creator.actor_id, From 126c6a23bb79103eb71b51741c5d9b0b50ba173f Mon Sep 17 00:00:00 2001 From: Avery Pierce Date: Thu, 4 Mar 2021 08:58:03 -0600 Subject: [PATCH 15/16] bump memory limit of iframely --- docker/prod/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index 5cbdc8d29..e219bdef8 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -53,4 +53,4 @@ services: volumes: - ./iframely.config.local.js:/iframely/config.local.js:ro restart: always - mem_limit: 100m + mem_limit: 200m From ff2c71a74a79e8e0fa2a46c185998e1af74d8778 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Thu, 4 Mar 2021 22:41:08 -0500 Subject: [PATCH 16/16] Adding more mem limits --- ansible/templates/docker-compose.yml | 1 + docker/dev/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/ansible/templates/docker-compose.yml b/ansible/templates/docker-compose.yml index 435fb281d..0ce715c69 100644 --- a/ansible/templates/docker-compose.yml +++ b/ansible/templates/docker-compose.yml @@ -53,6 +53,7 @@ services: volumes: - ./iframely.config.local.js:/iframely/config.local.js:ro restart: always + mem_limit: 200m postfix: image: mwader/postfix-relay diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index da6eef533..a23f1e0ed 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -57,3 +57,4 @@ services: volumes: - ../iframely.config.local.js:/iframely/config.local.js:ro restart: always + mem_limit: 200m