From 80aef61aed29d25099835ee4769bb8e1e363eb47 Mon Sep 17 00:00:00 2001 From: nutomic Date: Fri, 10 Jul 2020 18:15:41 +0000 Subject: [PATCH] Split code into cargo workspaces (#67) More fixes - fixed docker builds - fixed mentions regex test - fixed DATABASE_URL stuff - change schema path in diesel.toml Address review comments - add jsonb column back into activity table - remove authors field from cargo.toml - adjust LEMMY_DATABASE_URL env var usage - rename all occurences of LEMMY_DATABASE_URL to DATABASE_URL Decouple utils and db Split code into cargo workspaces Co-authored-by: Felix Ableitner Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/67 --- docker/dev/Dockerfile | 9 +- docker/prod/Dockerfile | 10 +- docs/src/contributing_tests.md | 2 +- install.sh | 2 +- server/Cargo.lock | 49 ++- server/Cargo.toml | 17 +- server/db-init.sh | 3 +- server/diesel.toml | 2 +- server/lemmy_db/Cargo.toml | 15 + server/{src/db => lemmy_db/src}/activity.rs | 50 +-- server/{src/db => lemmy_db/src}/category.rs | 5 +- server/{src/db => lemmy_db/src}/comment.rs | 21 +- .../{src/db => lemmy_db/src}/comment_view.rs | 14 +- server/{src/db => lemmy_db/src}/community.rs | 8 +- .../db => lemmy_db/src}/community_view.rs | 2 +- server/{src/db/mod.rs => lemmy_db/src/lib.rs} | 38 +- server/{src/db => lemmy_db/src}/moderator.rs | 15 +- .../db => lemmy_db/src}/moderator_views.rs | 2 +- .../src}/password_reset_request.rs | 4 +- server/{src/db => lemmy_db/src}/post.rs | 22 +- server/{src/db => lemmy_db/src}/post_view.rs | 13 +- .../db => lemmy_db/src}/private_message.rs | 28 +- .../src}/private_message_view.rs | 2 +- server/{ => lemmy_db}/src/schema.rs | 0 server/{src/db => lemmy_db/src}/site.rs | 2 +- server/{src/db => lemmy_db/src}/site_view.rs | 0 server/{src/db => lemmy_db/src}/user.rs | 83 +---- .../{src/db => lemmy_db/src}/user_mention.rs | 15 +- .../db => lemmy_db/src}/user_mention_view.rs | 2 +- server/{src/db => lemmy_db/src}/user_view.rs | 2 +- server/lemmy_utils/Cargo.toml | 22 ++ server/lemmy_utils/src/lib.rs | 324 +++++++++++++++++ server/{ => lemmy_utils}/src/settings.rs | 33 +- server/src/api/claims.rs | 73 ++++ server/src/api/comment.rs | 52 +-- server/src/api/community.rs | 26 +- server/src/api/mod.rs | 9 +- server/src/api/post.rs | 54 +-- server/src/api/site.rs | 37 +- server/src/api/user.rs | 97 ++--- server/src/apub/activities.rs | 12 +- server/src/apub/comment.rs | 18 +- server/src/apub/community.rs | 74 ++-- server/src/apub/community_inbox.rs | 12 +- .../src/apub/extensions/group_extensions.rs | 6 +- server/src/apub/extensions/signatures.rs | 18 - server/src/apub/fetcher.rs | 39 +-- server/src/apub/mod.rs | 60 ++-- server/src/apub/post.rs | 16 +- server/src/apub/private_message.rs | 14 +- server/src/apub/shared_inbox.rs | 29 +- server/src/apub/user.rs | 24 +- server/src/apub/user_inbox.rs | 20 +- server/src/{db => }/code_migrations.rs | 33 +- server/src/lib.rs | 331 +++--------------- server/src/main.rs | 16 +- server/src/rate_limit/mod.rs | 5 +- server/src/routes/federation.rs | 22 +- server/src/routes/feeds.rs | 31 +- server/src/routes/index.rs | 2 +- server/src/routes/nodeinfo.rs | 12 +- server/src/routes/webfinger.rs | 24 +- ui/package.json | 2 +- 63 files changed, 1067 insertions(+), 917 deletions(-) create mode 100644 server/lemmy_db/Cargo.toml rename server/{src/db => lemmy_db/src}/activity.rs (84%) rename server/{src/db => lemmy_db/src}/category.rs (95%) rename server/{src/db => lemmy_db/src}/comment.rs (96%) rename server/{src/db => lemmy_db/src}/comment_view.rs (98%) rename server/{src/db => lemmy_db/src}/community.rs (98%) rename server/{src/db => lemmy_db/src}/community_view.rs (99%) rename server/{src/db/mod.rs => lemmy_db/src/lib.rs} (78%) rename server/{src/db => lemmy_db/src}/moderator.rs (99%) rename server/{src/db => lemmy_db/src}/moderator_views.rs (99%) rename server/{src/db => lemmy_db/src}/password_reset_request.rs (98%) rename server/{src/db => lemmy_db/src}/post.rs (96%) rename server/{src/db => lemmy_db/src}/post_view.rs (98%) rename server/{src/db => lemmy_db/src}/private_message.rs (92%) rename server/{src/db => lemmy_db/src}/private_message_view.rs (98%) rename server/{ => lemmy_db}/src/schema.rs (100%) rename server/{src/db => lemmy_db/src}/site.rs (97%) rename server/{src/db => lemmy_db/src}/site_view.rs (100%) rename server/{src/db => lemmy_db/src}/user.rs (73%) rename server/{src/db => lemmy_db/src}/user_mention.rs (96%) rename server/{src/db => lemmy_db/src}/user_mention_view.rs (98%) rename server/{src/db => lemmy_db/src}/user_view.rs (98%) create mode 100644 server/lemmy_utils/Cargo.toml create mode 100644 server/lemmy_utils/src/lib.rs rename server/{ => lemmy_utils}/src/settings.rs (82%) create mode 100644 server/src/api/claims.rs rename server/src/{db => }/code_migrations.rs (86%) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 82a03f3c9..4445e4feb 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -17,13 +17,20 @@ WORKDIR /app RUN sudo chown -R rust:rust . RUN USER=root cargo new server WORKDIR /app/server +RUN mkdir -p lemmy_db/src/ lemmy_utils/src/ COPY server/Cargo.toml server/Cargo.lock ./ +COPY server/lemmy_db/Cargo.toml ./lemmy_db/ +COPY server/lemmy_utils/Cargo.toml ./lemmy_utils/ RUN sudo chown -R rust:rust . RUN mkdir -p ./src/bin \ - && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs + && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \ + && cp ./src/bin/main.rs ./lemmy_db/src/main.rs \ + && cp ./src/bin/main.rs ./lemmy_utils/src/main.rs RUN cargo build RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/lemmy_server* COPY server/src ./src/ +COPY server/lemmy_db ./lemmy_db/ +COPY server/lemmy_utils ./lemmy_utils/ COPY server/migrations ./migrations/ # Build for debug diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile index 54485a37e..9000ca3a8 100644 --- a/docker/prod/Dockerfile +++ b/docker/prod/Dockerfile @@ -10,13 +10,19 @@ WORKDIR /app RUN sudo chown -R rust:rust . RUN USER=root cargo new server WORKDIR /app/server +RUN mkdir -p lemmy_db/src/ lemmy_utils/src/ COPY --chown=rust:rust server/Cargo.toml server/Cargo.lock ./ -#RUN sudo chown -R rust:rust . +COPY --chown=rust:rust server/lemmy_db/Cargo.toml ./lemmy_db/ +COPY --chown=rust:rust server/lemmy_utils/Cargo.toml ./lemmy_utils/ RUN mkdir -p ./src/bin \ - && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs + && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \ + && cp ./src/bin/main.rs ./lemmy_db/src/main.rs \ + && cp ./src/bin/main.rs ./lemmy_utils/src/main.rs RUN cargo build --release RUN rm -f ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/deps/lemmy_server* COPY --chown=rust:rust server/src ./src/ +COPY --chown=rust:rust server/lemmy_db ./lemmy_db/ +COPY --chown=rust:rust server/lemmy_utils ./lemmy_utils/ COPY --chown=rust:rust server/migrations ./migrations/ # build for release diff --git a/docs/src/contributing_tests.md b/docs/src/contributing_tests.md index 13e5d1222..ccce67250 100644 --- a/docs/src/contributing_tests.md +++ b/docs/src/contributing_tests.md @@ -9,7 +9,7 @@ following commands in the `server` subfolder: psql -U lemmy -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy diesel migration run -RUST_TEST_THREADS=1 cargo test +RUST_TEST_THREADS=1 cargo test --workspace ``` ### Federation diff --git a/install.sh b/install.sh index fb42b26d1..19b847b1c 100755 --- a/install.sh +++ b/install.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e # Set the database variable to the default first. diff --git a/server/Cargo.lock b/server/Cargo.lock index 6d6364d4e..624385935 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -1399,12 +1399,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "htmlescape" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" - [[package]] name = "http" version = "0.2.1" @@ -1572,6 +1566,21 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lemmy_db" +version = "0.1.0" +dependencies = [ + "bcrypt", + "chrono", + "diesel", + "log", + "serde 1.0.114", + "serde_json", + "sha2", + "strum", + "strum_macros", +] + [[package]] name = "lemmy_server" version = "0.0.1" @@ -1589,27 +1598,23 @@ dependencies = [ "base64 0.12.3", "bcrypt", "chrono", - "comrak", - "config", "diesel", "diesel_migrations", "dotenv", "env_logger", "failure", "futures", - "htmlescape", "http", "http-signature-normalization-actix", "itertools", "jsonwebtoken", "lazy_static", - "lettre", - "lettre_email", + "lemmy_db", + "lemmy_utils", "log", "openssl", "percent-encoding", "rand 0.7.3", - "regex", "rss", "serde 1.0.114", "serde_json", @@ -1621,6 +1626,26 @@ dependencies = [ "uuid 0.8.1", ] +[[package]] +name = "lemmy_utils" +version = "0.1.0" +dependencies = [ + "chrono", + "comrak", + "config", + "itertools", + "lazy_static", + "lettre", + "lettre_email", + "log", + "openssl", + "rand 0.7.3", + "regex", + "serde 1.0.114", + "serde_json", + "url", +] + [[package]] name = "lettre" version = "0.9.3" diff --git a/server/Cargo.toml b/server/Cargo.toml index 8daf72c4a..a5e5a583b 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,14 +1,21 @@ [package] name = "lemmy_server" version = "0.0.1" -authors = ["Dessalines "] edition = "2018" [profile.release] lto = true +[workspace] +members = [ + "lemmy_utils", + "lemmy_db" +] + [dependencies] -diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] } +lemmy_utils = { path = "./lemmy_utils" } +lemmy_db = { path = "./lemmy_db" } +diesel = "1.4.4" diesel_migrations = "1.4.0" dotenv = "0.15.0" activitystreams = "0.6.2" @@ -31,16 +38,10 @@ rand = "0.7.3" strum = "0.18.0" strum_macros = "0.18.0" jsonwebtoken = "7.0.1" -regex = "1.3.5" lazy_static = "1.3.0" -lettre = "0.9.3" -lettre_email = "0.9.4" rss = "1.9.0" -htmlescape = "0.3.1" url = { version = "2.1.1", features = ["serde"] } -config = {version = "0.10.1", default-features = false, features = ["hjson"] } percent-encoding = "2.1.0" -comrak = "0.7" openssl = "0.10" http = "0.2.1" http-signature-normalization-actix = { version = "0.4.0-alpha.0", default-features = false, features = ["sha-2"] } diff --git a/server/db-init.sh b/server/db-init.sh index a2ad77b59..ccecb7de7 100755 --- a/server/db-init.sh +++ b/server/db-init.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/bin/bash +set -e # Default configurations username=lemmy diff --git a/server/diesel.toml b/server/diesel.toml index 92267c829..1644558f1 100644 --- a/server/diesel.toml +++ b/server/diesel.toml @@ -2,4 +2,4 @@ # see diesel.rs/guides/configuring-diesel-cli [print_schema] -file = "src/schema.rs" +file = "lemmy_db/src/schema.rs" diff --git a/server/lemmy_db/Cargo.toml b/server/lemmy_db/Cargo.toml new file mode 100644 index 000000000..d94cf5fc6 --- /dev/null +++ b/server/lemmy_db/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "lemmy_db" +version = "0.1.0" +edition = "2018" + +[dependencies] +diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] } +chrono = { version = "0.4.7", features = ["serde"] } +serde = { version = "1.0.105", features = ["derive"] } +serde_json = { version = "1.0.52", features = ["preserve_order"]} +strum = "0.18.0" +strum_macros = "0.18.0" +log = "0.4.0" +sha2 = "0.9" +bcrypt = "0.8.0" \ No newline at end of file diff --git a/server/src/db/activity.rs b/server/lemmy_db/src/activity.rs similarity index 84% rename from server/src/db/activity.rs rename to server/lemmy_db/src/activity.rs index 8c2b0c742..83f85ca1e 100644 --- a/server/src/db/activity.rs +++ b/server/lemmy_db/src/activity.rs @@ -1,9 +1,12 @@ -use crate::{blocking, db::Crud, schema::activity, DbPool, LemmyError}; +use crate::{schema::activity, Crud}; use diesel::{dsl::*, result::Error, *}; use log::debug; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::fmt::Debug; +use std::{ + fmt::Debug, + io::{Error as IoError, ErrorKind}, +}; #[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize)] #[table_name = "activity"] @@ -55,46 +58,43 @@ impl Crud for Activity { } } -pub async fn insert_activity( - user_id: i32, - data: T, - local: bool, - pool: &DbPool, -) -> Result<(), LemmyError> -where - T: Serialize + Debug + Send + 'static, -{ - blocking(pool, move |conn| { - do_insert_activity(conn, user_id, &data, local) - }) - .await??; - Ok(()) -} - -fn do_insert_activity( +pub fn do_insert_activity( conn: &PgConnection, user_id: i32, data: &T, local: bool, -) -> Result<(), LemmyError> +) -> Result where T: Serialize + Debug, { + debug!("inserting activity for user {}, data {:?}", user_id, &data); let activity_form = ActivityForm { user_id, data: serde_json::to_value(&data)?, local, updated: None, }; - debug!("inserting activity for user {}, data {:?}", user_id, data); - Activity::create(&conn, &activity_form)?; - Ok(()) + let result = Activity::create(&conn, &activity_form); + match result { + Ok(s) => Ok(s), + Err(e) => Err(IoError::new( + ErrorKind::Other, + format!("Failed to insert activity into database: {}", e), + )), + } } #[cfg(test)] mod tests { - use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, Crud, ListingType, SortType}; + use crate::{ + activity::{Activity, ActivityForm}, + tests::establish_unpooled_connection, + user::{UserForm, User_}, + Crud, + ListingType, + SortType, + }; + use serde_json::Value; #[test] fn test_crud() { diff --git a/server/src/db/category.rs b/server/lemmy_db/src/category.rs similarity index 95% rename from server/src/db/category.rs rename to server/lemmy_db/src/category.rs index ff49bbbee..ec2efc7b7 100644 --- a/server/src/db/category.rs +++ b/server/lemmy_db/src/category.rs @@ -1,6 +1,6 @@ use crate::{ - db::Crud, schema::{category, category::dsl::*}, + Crud, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -52,8 +52,7 @@ impl Category { #[cfg(test)] mod tests { - use super::*; - use crate::db::establish_unpooled_connection; + use crate::{category::Category, tests::establish_unpooled_connection}; #[test] fn test_crud() { diff --git a/server/src/db/comment.rs b/server/lemmy_db/src/comment.rs similarity index 96% rename from server/src/db/comment.rs rename to server/lemmy_db/src/comment.rs index 7e76770f6..602070d51 100644 --- a/server/src/db/comment.rs +++ b/server/lemmy_db/src/comment.rs @@ -1,9 +1,5 @@ use super::{post::Post, *}; -use crate::{ - apub::{make_apub_endpoint, EndpointType}, - naive_now, - schema::{comment, comment_like, comment_saved}, -}; +use crate::schema::{comment, comment_like, comment_saved}; // WITH RECURSIVE MyTree AS ( // SELECT * FROM comment WHERE parent_id IS NULL @@ -77,12 +73,15 @@ impl Crud for Comment { } impl Comment { - pub fn update_ap_id(conn: &PgConnection, comment_id: i32) -> Result { + pub fn update_ap_id( + conn: &PgConnection, + comment_id: i32, + apub_id: String, + ) -> Result { use crate::schema::comment::dsl::*; - let apid = make_apub_endpoint(EndpointType::Comment, &comment_id.to_string()).to_string(); diesel::update(comment.find(comment_id)) - .set(ap_id.eq(apid)) + .set(ap_id.eq(apub_id)) .get_result::(conn) } @@ -204,10 +203,8 @@ impl Saveable for CommentSaved { #[cfg(test)] mod tests { - use super::{ - super::{community::*, post::*, user::*}, - *, - }; + use crate::{comment::*, community::*, post::*, tests::establish_unpooled_connection, user::*}; + #[test] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/server/src/db/comment_view.rs b/server/lemmy_db/src/comment_view.rs similarity index 98% rename from server/src/db/comment_view.rs rename to server/lemmy_db/src/comment_view.rs index 7c853a817..914e568c8 100644 --- a/server/src/db/comment_view.rs +++ b/server/lemmy_db/src/comment_view.rs @@ -1,5 +1,5 @@ // TODO, remove the cross join here, just join to user directly -use crate::db::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; +use crate::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -460,11 +460,17 @@ impl<'a> ReplyQueryBuilder<'a> { #[cfg(test)] mod tests { - use super::{ - super::{comment::*, community::*, post::*, user::*}, + use crate::{ + comment::*, + comment_view::*, + community::*, + post::*, + tests::establish_unpooled_connection, + user::*, + Crud, + Likeable, *, }; - use crate::db::{establish_unpooled_connection, Crud, Likeable}; #[test] fn test_crud() { diff --git a/server/src/db/community.rs b/server/lemmy_db/src/community.rs similarity index 98% rename from server/src/db/community.rs rename to server/lemmy_db/src/community.rs index 461ba473a..607520803 100644 --- a/server/src/db/community.rs +++ b/server/lemmy_db/src/community.rs @@ -1,6 +1,9 @@ use crate::{ - db::{Bannable, Crud, Followable, Joinable}, schema::{community, community_follower, community_moderator, community_user_ban}, + Bannable, + Crud, + Followable, + Joinable, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -232,8 +235,7 @@ impl Followable for CommunityFollower { #[cfg(test)] mod tests { - use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{community::*, tests::establish_unpooled_connection, user::*, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/community_view.rs b/server/lemmy_db/src/community_view.rs similarity index 99% rename from server/src/db/community_view.rs rename to server/lemmy_db/src/community_view.rs index 4ec839acf..b465ddabc 100644 --- a/server/src/db/community_view.rs +++ b/server/lemmy_db/src/community_view.rs @@ -1,5 +1,5 @@ use super::community_view::community_fast_view::BoxedQuery; -use crate::db::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; +use crate::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; use diesel::{pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/db/mod.rs b/server/lemmy_db/src/lib.rs similarity index 78% rename from server/src/db/mod.rs rename to server/lemmy_db/src/lib.rs index da69f8dcd..b34919cdf 100644 --- a/server/src/db/mod.rs +++ b/server/lemmy_db/src/lib.rs @@ -1,10 +1,22 @@ -use crate::settings::Settings; +#[macro_use] +pub extern crate diesel; +#[macro_use] +pub extern crate strum_macros; +pub extern crate bcrypt; +pub extern crate chrono; +pub extern crate log; +pub extern crate serde; +pub extern crate serde_json; +pub extern crate sha2; +pub extern crate strum; + +use chrono::NaiveDateTime; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; +use std::{env, env::VarError}; pub mod activity; pub mod category; -pub mod code_migrations; pub mod comment; pub mod comment_view; pub mod community; @@ -16,6 +28,7 @@ pub mod post; pub mod post_view; pub mod private_message; pub mod private_message_view; +pub mod schema; pub mod site; pub mod site_view; pub mod user; @@ -111,9 +124,8 @@ impl MaybeOptional for Option { } } -pub fn establish_unpooled_connection() -> PgConnection { - let db_url = Settings::get().get_database_url(); - PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url)) +pub fn get_database_url_from_env() -> Result { + env::var("LEMMY_DATABASE_URL") } #[derive(EnumString, ToString, Debug, Serialize, Deserialize)] @@ -155,9 +167,25 @@ pub fn limit_and_offset(page: Option, limit: Option) -> (i64, i64) { let offset = limit * (page - 1); (limit, offset) } + +pub fn naive_now() -> NaiveDateTime { + chrono::prelude::Utc::now().naive_utc() +} + #[cfg(test)] mod tests { use super::fuzzy_search; + use crate::get_database_url_from_env; + use diesel::{Connection, PgConnection}; + + pub fn establish_unpooled_connection() -> PgConnection { + let db_url = match get_database_url_from_env() { + Ok(url) => url, + Err(_) => panic!("Failed to read database URL from env var LEMMY_DATABASE_URL"), + }; + PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url)) + } + #[test] fn test_fuzzy_search() { let test = "This is a fuzzy search"; diff --git a/server/src/db/moderator.rs b/server/lemmy_db/src/moderator.rs similarity index 99% rename from server/src/db/moderator.rs rename to server/lemmy_db/src/moderator.rs index 44b04ec63..f5d33d967 100644 --- a/server/src/db/moderator.rs +++ b/server/lemmy_db/src/moderator.rs @@ -1,5 +1,4 @@ use crate::{ - db::Crud, schema::{ mod_add, mod_add_community, @@ -11,6 +10,7 @@ use crate::{ mod_remove_post, mod_sticky_post, }, + Crud, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -437,11 +437,16 @@ impl Crud for ModAdd { #[cfg(test)] mod tests { - use super::{ - super::{comment::*, community::*, post::*, user::*}, - *, + use crate::{ + comment::*, + community::*, + moderator::*, + post::*, + tests::establish_unpooled_connection, + user::*, + ListingType, + SortType, }; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; // use Crud; #[test] diff --git a/server/src/db/moderator_views.rs b/server/lemmy_db/src/moderator_views.rs similarity index 99% rename from server/src/db/moderator_views.rs rename to server/lemmy_db/src/moderator_views.rs index f5b109fe6..024907c39 100644 --- a/server/src/db/moderator_views.rs +++ b/server/lemmy_db/src/moderator_views.rs @@ -1,4 +1,4 @@ -use crate::db::limit_and_offset; +use crate::limit_and_offset; use diesel::{result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/db/password_reset_request.rs b/server/lemmy_db/src/password_reset_request.rs similarity index 98% rename from server/src/db/password_reset_request.rs rename to server/lemmy_db/src/password_reset_request.rs index 4a071f078..a2692add8 100644 --- a/server/src/db/password_reset_request.rs +++ b/server/lemmy_db/src/password_reset_request.rs @@ -1,6 +1,6 @@ use crate::{ - db::Crud, schema::{password_reset_request, password_reset_request::dsl::*}, + Crud, }; use diesel::{dsl::*, result::Error, *}; use sha2::{Digest, Sha256}; @@ -82,7 +82,7 @@ impl PasswordResetRequest { #[cfg(test)] mod tests { use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{tests::establish_unpooled_connection, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/post.rs b/server/lemmy_db/src/post.rs similarity index 96% rename from server/src/db/post.rs rename to server/lemmy_db/src/post.rs index 91c1dcbff..1525a675f 100644 --- a/server/src/db/post.rs +++ b/server/lemmy_db/src/post.rs @@ -1,8 +1,10 @@ use crate::{ - apub::{make_apub_endpoint, EndpointType}, - db::{Crud, Likeable, Readable, Saveable}, naive_now, schema::{post, post_like, post_read, post_saved}, + Crud, + Likeable, + Readable, + Saveable, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -75,12 +77,11 @@ impl Post { post.filter(ap_id.eq(object_id)).first::(conn) } - pub fn update_ap_id(conn: &PgConnection, post_id: i32) -> Result { + pub fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: String) -> Result { use crate::schema::post::dsl::*; - let apid = make_apub_endpoint(EndpointType::Post, &post_id.to_string()).to_string(); diesel::update(post.find(post_id)) - .set(ap_id.eq(apid)) + .set(ap_id.eq(apub_id)) .get_result::(conn) } @@ -241,11 +242,14 @@ impl Readable for PostRead { #[cfg(test)] mod tests { - use super::{ - super::{community::*, user::*}, - *, + use crate::{ + community::*, + post::*, + tests::establish_unpooled_connection, + user::*, + ListingType, + SortType, }; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/post_view.rs b/server/lemmy_db/src/post_view.rs similarity index 98% rename from server/src/db/post_view.rs rename to server/lemmy_db/src/post_view.rs index cda5cecf0..b55359ea3 100644 --- a/server/src/db/post_view.rs +++ b/server/lemmy_db/src/post_view.rs @@ -1,5 +1,5 @@ use super::post_view::post_fast_view::BoxedQuery; -use crate::db::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; +use crate::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -367,11 +367,16 @@ impl PostView { #[cfg(test)] mod tests { - use super::{ - super::{community::*, post::*, user::*}, + use crate::{ + community::*, + post::*, + post_view::*, + tests::establish_unpooled_connection, + user::*, + Crud, + Likeable, *, }; - use crate::db::{establish_unpooled_connection, Crud, Likeable}; #[test] fn test_crud() { diff --git a/server/src/db/private_message.rs b/server/lemmy_db/src/private_message.rs similarity index 92% rename from server/src/db/private_message.rs rename to server/lemmy_db/src/private_message.rs index 9d362bbf1..1c0b455f3 100644 --- a/server/src/db/private_message.rs +++ b/server/lemmy_db/src/private_message.rs @@ -1,8 +1,4 @@ -use crate::{ - apub::{make_apub_endpoint, EndpointType}, - db::Crud, - schema::private_message, -}; +use crate::{schema::private_message, Crud}; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -66,16 +62,15 @@ impl Crud for PrivateMessage { } impl PrivateMessage { - pub fn update_ap_id(conn: &PgConnection, private_message_id: i32) -> Result { + pub fn update_ap_id( + conn: &PgConnection, + private_message_id: i32, + apub_id: String, + ) -> Result { use crate::schema::private_message::dsl::*; - let apid = make_apub_endpoint( - EndpointType::PrivateMessage, - &private_message_id.to_string(), - ) - .to_string(); diesel::update(private_message.find(private_message_id)) - .set(ap_id.eq(apid)) + .set(ap_id.eq(apub_id)) .get_result::(conn) } @@ -89,8 +84,13 @@ impl PrivateMessage { #[cfg(test)] mod tests { - use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{ + private_message::*, + tests::establish_unpooled_connection, + user::*, + ListingType, + SortType, + }; #[test] fn test_crud() { diff --git a/server/src/db/private_message_view.rs b/server/lemmy_db/src/private_message_view.rs similarity index 98% rename from server/src/db/private_message_view.rs rename to server/lemmy_db/src/private_message_view.rs index 899a1084d..dfb11c444 100644 --- a/server/src/db/private_message_view.rs +++ b/server/lemmy_db/src/private_message_view.rs @@ -1,4 +1,4 @@ -use crate::db::{limit_and_offset, MaybeOptional}; +use crate::{limit_and_offset, MaybeOptional}; use diesel::{pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/schema.rs b/server/lemmy_db/src/schema.rs similarity index 100% rename from server/src/schema.rs rename to server/lemmy_db/src/schema.rs diff --git a/server/src/db/site.rs b/server/lemmy_db/src/site.rs similarity index 97% rename from server/src/db/site.rs rename to server/lemmy_db/src/site.rs index c752bfe79..066ae0b1a 100644 --- a/server/src/db/site.rs +++ b/server/lemmy_db/src/site.rs @@ -1,4 +1,4 @@ -use crate::{db::Crud, schema::site}; +use crate::{schema::site, Crud}; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/db/site_view.rs b/server/lemmy_db/src/site_view.rs similarity index 100% rename from server/src/db/site_view.rs rename to server/lemmy_db/src/site_view.rs diff --git a/server/src/db/user.rs b/server/lemmy_db/src/user.rs similarity index 73% rename from server/src/db/user.rs rename to server/lemmy_db/src/user.rs index 4ca0a0419..556fc1a75 100644 --- a/server/src/db/user.rs +++ b/server/lemmy_db/src/user.rs @@ -1,14 +1,10 @@ use crate::{ - db::Crud, - is_email_regex, naive_now, schema::{user_, user_::dsl::*}, - settings::Settings, + Crud, }; use bcrypt::{hash, DEFAULT_COST}; use diesel::{dsl::*, result::Error, *}; -use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation}; -use serde::{Deserialize, Serialize}; #[derive(Clone, Queryable, Identifiable, PartialEq, Debug)] #[table_name = "user_"] @@ -131,90 +127,23 @@ impl User_ { } } -#[derive(Debug, Serialize, Deserialize)] -pub struct Claims { - pub id: i32, - pub username: String, - pub iss: String, - pub show_nsfw: bool, - pub theme: String, - pub default_sort_type: i16, - pub default_listing_type: i16, - pub lang: String, - pub avatar: Option, - pub show_avatars: bool, -} - -impl Claims { - pub fn decode(jwt: &str) -> Result, jsonwebtoken::errors::Error> { - let v = Validation { - validate_exp: false, - ..Validation::default() - }; - decode::( - &jwt, - &DecodingKey::from_secret(Settings::get().jwt_secret.as_ref()), - &v, - ) - } -} - -type Jwt = String; impl User_ { - pub fn jwt(&self) -> Jwt { - let my_claims = Claims { - id: self.id, - username: self.name.to_owned(), - iss: Settings::get().hostname, - show_nsfw: self.show_nsfw, - theme: self.theme.to_owned(), - default_sort_type: self.default_sort_type, - default_listing_type: self.default_listing_type, - lang: self.lang.to_owned(), - avatar: self.avatar.to_owned(), - show_avatars: self.show_avatars.to_owned(), - }; - encode( - &Header::default(), - &my_claims, - &EncodingKey::from_secret(Settings::get().jwt_secret.as_ref()), - ) - .unwrap() - } - - pub fn find_by_username(conn: &PgConnection, username: &str) -> Result { + pub fn find_by_username(conn: &PgConnection, username: &str) -> Result { user_.filter(name.eq(username)).first::(conn) } - pub fn find_by_email(conn: &PgConnection, from_email: &str) -> Result { + pub fn find_by_email(conn: &PgConnection, from_email: &str) -> Result { user_.filter(email.eq(from_email)).first::(conn) } - pub fn find_by_email_or_username( - conn: &PgConnection, - username_or_email: &str, - ) -> Result { - if is_email_regex(username_or_email) { - User_::find_by_email(conn, username_or_email) - } else { - User_::find_by_username(conn, username_or_email) - } - } - - pub fn get_profile_url(&self) -> String { - format!("https://{}/u/{}", Settings::get().hostname, self.name) - } - - pub fn find_by_jwt(conn: &PgConnection, jwt: &str) -> Result { - let claims: Claims = Claims::decode(&jwt).expect("Invalid token").claims; - Self::read(&conn, claims.id) + pub fn get_profile_url(&self, hostname: &str) -> String { + format!("https://{}/u/{}", hostname, self.name) } } #[cfg(test)] mod tests { - use super::{User_, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{tests::establish_unpooled_connection, user::*, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/user_mention.rs b/server/lemmy_db/src/user_mention.rs similarity index 96% rename from server/src/db/user_mention.rs rename to server/lemmy_db/src/user_mention.rs index 1d54fa988..9f23f4410 100644 --- a/server/src/db/user_mention.rs +++ b/server/lemmy_db/src/user_mention.rs @@ -1,5 +1,5 @@ use super::comment::Comment; -use crate::{db::Crud, schema::user_mention}; +use crate::{schema::user_mention, Crud}; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -54,11 +54,16 @@ impl Crud for UserMention { #[cfg(test)] mod tests { - use super::{ - super::{comment::*, community::*, post::*, user::*}, - *, + use crate::{ + comment::*, + community::*, + post::*, + tests::establish_unpooled_connection, + user::*, + user_mention::*, + ListingType, + SortType, }; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/user_mention_view.rs b/server/lemmy_db/src/user_mention_view.rs similarity index 98% rename from server/src/db/user_mention_view.rs rename to server/lemmy_db/src/user_mention_view.rs index 59aefb200..8bfbf453d 100644 --- a/server/src/db/user_mention_view.rs +++ b/server/lemmy_db/src/user_mention_view.rs @@ -1,4 +1,4 @@ -use crate::db::{limit_and_offset, MaybeOptional, SortType}; +use crate::{limit_and_offset, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/db/user_view.rs b/server/lemmy_db/src/user_view.rs similarity index 98% rename from server/src/db/user_view.rs rename to server/lemmy_db/src/user_view.rs index 490521721..84feba38f 100644 --- a/server/src/db/user_view.rs +++ b/server/lemmy_db/src/user_view.rs @@ -1,5 +1,5 @@ use super::user_view::user_fast::BoxedQuery; -use crate::db::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; +use crate::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/lemmy_utils/Cargo.toml b/server/lemmy_utils/Cargo.toml new file mode 100644 index 000000000..fed22f585 --- /dev/null +++ b/server/lemmy_utils/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "lemmy_utils" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +regex = "1.3.5" +config = { version = "0.10.1", default-features = false, features = ["hjson"] } +chrono = { version = "0.4.7", features = ["serde"] } +lettre = "0.9.3" +lettre_email = "0.9.4" +log = "0.4.0" +itertools = "0.9.0" +rand = "0.7.3" +serde = { version = "1.0.105", features = ["derive"] } +serde_json = { version = "1.0.52", features = ["preserve_order"]} +comrak = "0.7" +lazy_static = "1.3.0" +openssl = "0.10" +url = { version = "2.1.1", features = ["serde"] } \ No newline at end of file diff --git a/server/lemmy_utils/src/lib.rs b/server/lemmy_utils/src/lib.rs new file mode 100644 index 000000000..f576ea003 --- /dev/null +++ b/server/lemmy_utils/src/lib.rs @@ -0,0 +1,324 @@ +#[macro_use] +pub extern crate lazy_static; +pub extern crate comrak; +pub extern crate lettre; +pub extern crate lettre_email; +pub extern crate openssl; +pub extern crate rand; +pub extern crate regex; +pub extern crate serde_json; +pub extern crate url; + +pub mod settings; + +use crate::settings::Settings; +use chrono::{DateTime, FixedOffset, Local, NaiveDateTime, Utc}; +use itertools::Itertools; +use lettre::{ + smtp::{ + authentication::{Credentials, Mechanism}, + extension::ClientId, + ConnectionReuseParameters, + }, + ClientSecurity, + SmtpClient, + Transport, +}; +use lettre_email::Email; +use openssl::{pkey::PKey, rsa::Rsa}; +use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use regex::{Regex, RegexBuilder}; +use std::io::{Error, ErrorKind}; +use url::Url; + +pub fn to_datetime_utc(ndt: NaiveDateTime) -> DateTime { + DateTime::::from_utc(ndt, Utc) +} + +pub fn naive_from_unix(time: i64) -> NaiveDateTime { + NaiveDateTime::from_timestamp(time, 0) +} + +pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime { + let now = Local::now(); + DateTime::::from_utc(datetime, *now.offset()) +} + +pub fn is_email_regex(test: &str) -> bool { + EMAIL_REGEX.is_match(test) +} + +pub fn remove_slurs(test: &str) -> String { + SLUR_REGEX.replace_all(test, "*removed*").to_string() +} + +pub fn slur_check(test: &str) -> Result<(), Vec<&str>> { + let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect(); + + // Unique + matches.sort_unstable(); + matches.dedup(); + + if matches.is_empty() { + Ok(()) + } else { + Err(matches) + } +} + +pub fn slurs_vec_to_str(slurs: Vec<&str>) -> String { + let start = "No slurs - "; + let combined = &slurs.join(", "); + [start, combined].concat() +} + +pub fn generate_random_string() -> String { + thread_rng().sample_iter(&Alphanumeric).take(30).collect() +} + +pub fn send_email( + subject: &str, + to_email: &str, + to_username: &str, + html: &str, +) -> Result<(), String> { + let email_config = Settings::get().email.ok_or("no_email_setup")?; + + let email = Email::builder() + .to((to_email, to_username)) + .from(email_config.smtp_from_address.to_owned()) + .subject(subject) + .html(html) + .build() + .unwrap(); + + let mailer = if email_config.use_tls { + SmtpClient::new_simple(&email_config.smtp_server).unwrap() + } else { + SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap() + } + .hello_name(ClientId::Domain(Settings::get().hostname)) + .smtp_utf8(true) + .authentication_mechanism(Mechanism::Plain) + .connection_reuse(ConnectionReuseParameters::ReuseUnlimited); + let mailer = if let (Some(login), Some(password)) = + (&email_config.smtp_login, &email_config.smtp_password) + { + mailer.credentials(Credentials::new(login.to_owned(), password.to_owned())) + } else { + mailer + }; + + let mut transport = mailer.transport(); + let result = transport.send(email.into()); + transport.close(); + + match result { + Ok(_) => Ok(()), + Err(e) => Err(e.to_string()), + } +} + +pub fn markdown_to_html(text: &str) -> String { + comrak::markdown_to_html(text, &comrak::ComrakOptions::default()) +} + +// TODO nothing is done with community / group webfingers yet, so just ignore those for now +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct MentionData { + pub name: String, + pub domain: String, +} + +impl MentionData { + pub fn is_local(&self) -> bool { + Settings::get().hostname.eq(&self.domain) + } + pub fn full_name(&self) -> String { + format!("@{}@{}", &self.name, &self.domain) + } +} + +pub fn scrape_text_for_mentions(text: &str) -> Vec { + let mut out: Vec = Vec::new(); + for caps in MENTIONS_REGEX.captures_iter(text) { + out.push(MentionData { + name: caps["name"].to_string(), + domain: caps["domain"].to_string(), + }); + } + out.into_iter().unique().collect() +} + +pub fn is_valid_username(name: &str) -> bool { + VALID_USERNAME_REGEX.is_match(name) +} + +pub fn is_valid_community_name(name: &str) -> bool { + VALID_COMMUNITY_NAME_REGEX.is_match(name) +} + +#[cfg(test)] +mod tests { + use crate::{ + is_email_regex, + is_valid_community_name, + is_valid_username, + remove_slurs, + scrape_text_for_mentions, + slur_check, + slurs_vec_to_str, + }; + + #[test] + fn test_mentions_regex() { + let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)"; + let mentions = scrape_text_for_mentions(text); + + assert_eq!(mentions[0].name, "tedu".to_string()); + assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string()); + assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string()); + } + + #[test] + fn test_email() { + assert!(is_email_regex("gush@gmail.com")); + assert!(!is_email_regex("nada_neutho")); + } + + #[test] + fn test_valid_register_username() { + assert!(is_valid_username("Hello_98")); + assert!(is_valid_username("ten")); + assert!(!is_valid_username("Hello-98")); + assert!(!is_valid_username("a")); + assert!(!is_valid_username("")); + } + + #[test] + fn test_valid_community_name() { + assert!(is_valid_community_name("example")); + assert!(is_valid_community_name("example_community")); + assert!(!is_valid_community_name("Example")); + assert!(!is_valid_community_name("Ex")); + assert!(!is_valid_community_name("")); + } + + #[test] + fn test_slur_filter() { + let test = + "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text."; + let slur_free = "No slurs here"; + assert_eq!( + remove_slurs(&test), + "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text." + .to_string() + ); + + let has_slurs_vec = vec![ + "Niggerz", + "coons", + "dindu", + "ladyboy", + "retardeds", + "tranny", + ]; + let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny"; + + assert_eq!(slur_check(test), Err(has_slurs_vec)); + assert_eq!(slur_check(slur_free), Ok(())); + if let Err(slur_vec) = slur_check(test) { + assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str); + } + } + + // These helped with testing + // #[test] + // fn test_send_email() { + // let result = send_email("not a subject", "test_email@gmail.com", "ur user", "

HI there

"); + // assert!(result.is_ok()); + // } +} + +lazy_static! { + static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap(); + static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap(); + static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap(); + // TODO keep this old one, it didn't work with port well tho + // static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap(); + static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").unwrap(); + static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap(); + static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap(); + pub static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!( + "^group:([a-z0-9_]{{3, 20}})@{}$", + Settings::get().hostname + )) + .unwrap(); + pub static ref WEBFINGER_USER_REGEX: Regex = Regex::new(&format!( + "^acct:([a-z0-9_]{{3, 20}})@{}$", + Settings::get().hostname + )) + .unwrap(); + pub static ref CACHE_CONTROL_REGEX: Regex = + Regex::new("^((text|image)/.+|application/javascript)$").unwrap(); +} + +pub struct Keypair { + pub private_key: String, + pub public_key: String, +} + +/// Generate the asymmetric keypair for ActivityPub HTTP signatures. +pub fn generate_actor_keypair() -> Result { + let rsa = Rsa::generate(2048)?; + let pkey = PKey::from_rsa(rsa)?; + let public_key = pkey.public_key_to_pem()?; + let private_key = pkey.private_key_to_pem_pkcs8()?; + let key_to_string = |key| match String::from_utf8(key) { + Ok(s) => Ok(s), + Err(e) => Err(Error::new( + ErrorKind::Other, + format!("Failed converting key to string: {}", e), + )), + }; + Ok(Keypair { + private_key: key_to_string(private_key)?, + public_key: key_to_string(public_key)?, + }) +} + +pub enum EndpointType { + Community, + User, + Post, + Comment, + PrivateMessage, +} + +pub fn get_apub_protocol_string() -> &'static str { + if Settings::get().federation.tls_enabled { + "https" + } else { + "http" + } +} + +/// Generates the ActivityPub ID for a given object type and ID. +pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url { + let point = match endpoint_type { + EndpointType::Community => "c", + EndpointType::User => "u", + EndpointType::Post => "post", + EndpointType::Comment => "comment", + EndpointType::PrivateMessage => "private_message", + }; + + Url::parse(&format!( + "{}://{}/{}/{}", + get_apub_protocol_string(), + Settings::get().hostname, + point, + name + )) + .unwrap() +} diff --git a/server/src/settings.rs b/server/lemmy_utils/src/settings.rs similarity index 82% rename from server/src/settings.rs rename to server/lemmy_utils/src/settings.rs index 12ffaceab..2ce33f58c 100644 --- a/server/src/settings.rs +++ b/server/lemmy_utils/src/settings.rs @@ -1,7 +1,6 @@ -use crate::LemmyError; use config::{Config, ConfigError, Environment, File}; use serde::Deserialize; -use std::{env, fs, net::IpAddr, sync::RwLock}; +use std::{fs, io::Error, net::IpAddr, sync::RwLock}; static CONFIG_FILE_DEFAULTS: &str = "config/defaults.hjson"; static CONFIG_FILE: &str = "config/config.hjson"; @@ -76,6 +75,9 @@ impl Settings { /// First, defaults are loaded from CONFIG_FILE_DEFAULTS, then these values can be overwritten /// from CONFIG_FILE (optional). Finally, values from the environment (with prefix LEMMY) are /// added to the config. + /// + /// Note: The env var `LEMMY_DATABASE_URL` is parsed in + /// `server/lemmy_db/src/lib.rs::get_database_url_from_env()` fn init() -> Result { let mut s = Config::new(); @@ -98,31 +100,26 @@ impl Settings { SETTINGS.read().unwrap().to_owned() } - /// Returns the postgres connection url. If LEMMY_DATABASE_URL is set, that is used, - /// otherwise the connection url is generated from the config. pub fn get_database_url(&self) -> String { - match env::var("LEMMY_DATABASE_URL") { - Ok(url) => url, - Err(_) => format!( - "postgres://{}:{}@{}:{}/{}", - self.database.user, - self.database.password, - self.database.host, - self.database.port, - self.database.database - ), - } + format!( + "postgres://{}:{}@{}:{}/{}", + self.database.user, + self.database.password, + self.database.host, + self.database.port, + self.database.database + ) } pub fn api_endpoint(&self) -> String { format!("{}/api/v1", self.hostname) } - pub fn read_config_file() -> Result { - Ok(fs::read_to_string(CONFIG_FILE)?) + pub fn read_config_file() -> Result { + fs::read_to_string(CONFIG_FILE) } - pub fn save_config_file(data: &str) -> Result { + pub fn save_config_file(data: &str) -> Result { fs::write(CONFIG_FILE, data)?; // Reload the new settings diff --git a/server/src/api/claims.rs b/server/src/api/claims.rs new file mode 100644 index 000000000..eec9d1a71 --- /dev/null +++ b/server/src/api/claims.rs @@ -0,0 +1,73 @@ +use diesel::{result::Error, PgConnection}; +use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation}; +use lemmy_db::{user::User_, Crud}; +use lemmy_utils::{is_email_regex, settings::Settings}; +use serde::{Deserialize, Serialize}; + +type Jwt = String; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Claims { + pub id: i32, + pub username: String, + pub iss: String, + pub show_nsfw: bool, + pub theme: String, + pub default_sort_type: i16, + pub default_listing_type: i16, + pub lang: String, + pub avatar: Option, + pub show_avatars: bool, +} + +impl Claims { + pub fn decode(jwt: &str) -> Result, jsonwebtoken::errors::Error> { + let v = Validation { + validate_exp: false, + ..Validation::default() + }; + decode::( + &jwt, + &DecodingKey::from_secret(Settings::get().jwt_secret.as_ref()), + &v, + ) + } + + pub fn jwt(user: User_, hostname: String) -> Jwt { + let my_claims = Claims { + id: user.id, + username: user.name.to_owned(), + iss: hostname, + show_nsfw: user.show_nsfw, + theme: user.theme.to_owned(), + default_sort_type: user.default_sort_type, + default_listing_type: user.default_listing_type, + lang: user.lang.to_owned(), + avatar: user.avatar.to_owned(), + show_avatars: user.show_avatars.to_owned(), + }; + encode( + &Header::default(), + &my_claims, + &EncodingKey::from_secret(Settings::get().jwt_secret.as_ref()), + ) + .unwrap() + } + + // TODO: move these into user? + pub fn find_by_email_or_username( + conn: &PgConnection, + username_or_email: &str, + ) -> Result { + if is_email_regex(username_or_email) { + User_::find_by_email(conn, username_or_email) + } else { + User_::find_by_username(conn, username_or_email) + } + } + + pub fn find_by_jwt(conn: &PgConnection, jwt: &str) -> Result { + let claims: Claims = Claims::decode(&jwt).expect("Invalid token").claims; + User_::read(&conn, claims.id) + } +} diff --git a/server/src/api/comment.rs b/server/src/api/comment.rs index c7406b370..2007542fa 100644 --- a/server/src/api/comment.rs +++ b/server/src/api/comment.rs @@ -1,28 +1,7 @@ use crate::{ - api::{APIError, Oper, Perform}, + api::{claims::Claims, APIError, Oper, Perform}, apub::{ApubLikeableType, ApubObjectType}, blocking, - db::{ - comment::*, - comment_view::*, - community_view::*, - moderator::*, - post::*, - site_view::*, - user::*, - user_mention::*, - user_view::*, - Crud, - Likeable, - ListingType, - Saveable, - SortType, - }, - naive_now, - remove_slurs, - scrape_text_for_mentions, - send_email, - settings::Settings, websocket::{ server::{JoinCommunityRoom, SendComment}, UserOperation, @@ -30,6 +9,31 @@ use crate::{ }, DbPool, LemmyError, +}; +use lemmy_db::{ + comment::*, + comment_view::*, + community_view::*, + moderator::*, + naive_now, + post::*, + site_view::*, + user::*, + user_mention::*, + user_view::*, + Crud, + Likeable, + ListingType, + Saveable, + SortType, +}; +use lemmy_utils::{ + make_apub_endpoint, + remove_slurs, + scrape_text_for_mentions, + send_email, + settings::Settings, + EndpointType, MentionData, }; use log::error; @@ -155,7 +159,9 @@ impl Perform for Oper { let inserted_comment_id = inserted_comment.id; let updated_comment: Comment = match blocking(pool, move |conn| { - Comment::update_ap_id(&conn, inserted_comment_id) + let apub_id = + make_apub_endpoint(EndpointType::Comment, &inserted_comment_id.to_string()).to_string(); + Comment::update_ap_id(&conn, inserted_comment_id, apub_id) }) .await? { diff --git a/server/src/api/community.rs b/server/src/api/community.rs index 02071c577..e703dcf41 100644 --- a/server/src/api/community.rs +++ b/server/src/api/community.rs @@ -1,26 +1,24 @@ use super::*; use crate::{ - api::{APIError, Oper, Perform}, - apub::{ - extensions::signatures::generate_actor_keypair, - make_apub_endpoint, - ActorType, - EndpointType, - }, + api::{claims::Claims, APIError, Oper, Perform}, + apub::ActorType, blocking, - db::{Bannable, Crud, Followable, Joinable, SortType}, - is_valid_community_name, - naive_from_unix, - naive_now, - slur_check, - slurs_vec_to_str, websocket::{ server::{JoinCommunityRoom, SendCommunityRoomMessage}, UserOperation, WebsocketInfo, }, DbPool, - LemmyError, +}; +use lemmy_db::{naive_now, Bannable, Crud, Followable, Joinable, SortType}; +use lemmy_utils::{ + generate_actor_keypair, + is_valid_community_name, + make_apub_endpoint, + naive_from_unix, + slur_check, + slurs_vec_to_str, + EndpointType, }; use serde::{Deserialize, Serialize}; use std::str::FromStr; diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 6df9909c5..bb65815ad 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -1,11 +1,8 @@ -use crate::{ - db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*}, - websocket::WebsocketInfo, - DbPool, - LemmyError, -}; +use crate::{websocket::WebsocketInfo, DbPool, LemmyError}; use actix_web::client::Client; +use lemmy_db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*}; +pub mod claims; pub mod comment; pub mod community; pub mod post; diff --git a/server/src/api/post.rs b/server/src/api/post.rs index 840f15305..c56a00dfe 100644 --- a/server/src/api/post.rs +++ b/server/src/api/post.rs @@ -1,27 +1,8 @@ use crate::{ - api::{APIError, Oper, Perform}, + api::{claims::Claims, APIError, Oper, Perform}, apub::{ApubLikeableType, ApubObjectType}, blocking, - db::{ - comment_view::*, - community_view::*, - moderator::*, - post::*, - post_view::*, - site::*, - site_view::*, - user::*, - user_view::*, - Crud, - Likeable, - ListingType, - Saveable, - SortType, - }, fetch_iframely_and_pictrs_data, - naive_now, - slur_check, - slurs_vec_to_str, websocket::{ server::{JoinCommunityRoom, JoinPostRoom, SendPost}, UserOperation, @@ -30,6 +11,24 @@ use crate::{ DbPool, LemmyError, }; +use lemmy_db::{ + comment_view::*, + community_view::*, + moderator::*, + naive_now, + post::*, + post_view::*, + site::*, + site_view::*, + user::*, + user_view::*, + Crud, + Likeable, + ListingType, + Saveable, + SortType, +}; +use lemmy_utils::{make_apub_endpoint, slur_check, slurs_vec_to_str, EndpointType}; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -191,11 +190,16 @@ impl Perform for Oper { }; let inserted_post_id = inserted_post.id; - let updated_post = - match blocking(pool, move |conn| Post::update_ap_id(conn, inserted_post_id)).await? { - Ok(post) => post, - Err(_e) => return Err(APIError::err("couldnt_create_post").into()), - }; + let updated_post = match blocking(pool, move |conn| { + let apub_id = + make_apub_endpoint(EndpointType::Post, &inserted_post_id.to_string()).to_string(); + Post::update_ap_id(conn, inserted_post_id, apub_id) + }) + .await? + { + Ok(post) => post, + Err(_e) => return Err(APIError::err("couldnt_create_post").into()), + }; updated_post.send_create(&user, &self.client, pool).await?; diff --git a/server/src/api/site.rs b/server/src/api/site.rs index f45561a82..241a80e31 100644 --- a/server/src/api/site.rs +++ b/server/src/api/site.rs @@ -1,31 +1,28 @@ use super::user::Register; use crate::{ - api::{APIError, Oper, Perform}, + api::{claims::Claims, APIError, Oper, Perform}, apub::fetcher::search_by_apub_id, blocking, - db::{ - category::*, - comment_view::*, - community_view::*, - moderator::*, - moderator_views::*, - post_view::*, - site::*, - site_view::*, - user::*, - user_view::*, - Crud, - SearchType, - SortType, - }, - naive_now, - settings::Settings, - slur_check, - slurs_vec_to_str, websocket::{server::SendAllMessage, UserOperation, WebsocketInfo}, DbPool, LemmyError, }; +use lemmy_db::{ + category::*, + comment_view::*, + community_view::*, + moderator::*, + moderator_views::*, + naive_now, + post_view::*, + site::*, + site_view::*, + user_view::*, + Crud, + SearchType, + SortType, +}; +use lemmy_utils::{settings::Settings, slur_check, slurs_vec_to_str}; use log::{debug, info}; use serde::{Deserialize, Serialize}; use std::str::FromStr; diff --git a/server/src/api/user.rs b/server/src/api/user.rs index 1284c6608..9f33843f6 100644 --- a/server/src/api/user.rs +++ b/server/src/api/user.rs @@ -1,44 +1,7 @@ use crate::{ - api::{APIError, Oper, Perform}, - apub::{ - extensions::signatures::generate_actor_keypair, - make_apub_endpoint, - ApubObjectType, - EndpointType, - }, + api::{claims::Claims, APIError, Oper, Perform}, + apub::ApubObjectType, blocking, - db::{ - comment::*, - comment_view::*, - community::*, - community_view::*, - moderator::*, - password_reset_request::*, - post::*, - post_view::*, - private_message::*, - private_message_view::*, - site::*, - site_view::*, - user::*, - user_mention::*, - user_mention_view::*, - user_view::*, - Crud, - Followable, - Joinable, - ListingType, - SortType, - }, - generate_random_string, - is_valid_username, - naive_from_unix, - naive_now, - remove_slurs, - send_email, - settings::Settings, - slur_check, - slurs_vec_to_str, websocket::{ server::{JoinUserRoom, SendAllMessage, SendUserRoomMessage}, UserOperation, @@ -48,6 +11,43 @@ use crate::{ LemmyError, }; use bcrypt::verify; +use lemmy_db::{ + comment::*, + comment_view::*, + community::*, + community_view::*, + moderator::*, + naive_now, + password_reset_request::*, + post::*, + post_view::*, + private_message::*, + private_message_view::*, + site::*, + site_view::*, + user::*, + user_mention::*, + user_mention_view::*, + user_view::*, + Crud, + Followable, + Joinable, + ListingType, + SortType, +}; +use lemmy_utils::{ + generate_actor_keypair, + generate_random_string, + is_valid_username, + make_apub_endpoint, + naive_from_unix, + remove_slurs, + send_email, + settings::Settings, + slur_check, + slurs_vec_to_str, + EndpointType, +}; use log::error; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -264,7 +264,7 @@ impl Perform for Oper { // Fetch that username / email let username_or_email = data.username_or_email.clone(); let user = match blocking(pool, move |conn| { - User_::find_by_email_or_username(conn, &username_or_email) + Claims::find_by_email_or_username(conn, &username_or_email) }) .await? { @@ -279,7 +279,9 @@ impl Perform for Oper { } // Return the jwt - Ok(LoginResponse { jwt: user.jwt() }) + Ok(LoginResponse { + jwt: Claims::jwt(user, Settings::get().hostname), + }) } } @@ -421,7 +423,7 @@ impl Perform for Oper { // Return the jwt Ok(LoginResponse { - jwt: inserted_user.jwt(), + jwt: Claims::jwt(inserted_user, Settings::get().hostname), }) } } @@ -532,7 +534,7 @@ impl Perform for Oper { // Return the jwt Ok(LoginResponse { - jwt: updated_user.jwt(), + jwt: Claims::jwt(updated_user, Settings::get().hostname), }) } } @@ -1155,7 +1157,7 @@ impl Perform for Oper { // Return the jwt Ok(LoginResponse { - jwt: updated_user.jwt(), + jwt: Claims::jwt(updated_user, Settings::get().hostname), }) } } @@ -1213,7 +1215,12 @@ impl Perform for Oper { let inserted_private_message_id = inserted_private_message.id; let updated_private_message = match blocking(pool, move |conn| { - PrivateMessage::update_ap_id(&conn, inserted_private_message_id) + let apub_id = make_apub_endpoint( + EndpointType::PrivateMessage, + &inserted_private_message_id.to_string(), + ) + .to_string(); + PrivateMessage::update_ap_id(&conn, inserted_private_message_id, apub_id) }) .await? { diff --git a/server/src/apub/activities.rs b/server/src/apub/activities.rs index e5dc70457..204a380d3 100644 --- a/server/src/apub/activities.rs +++ b/server/src/apub/activities.rs @@ -1,12 +1,18 @@ use crate::{ - apub::{extensions::signatures::sign, is_apub_id_valid, ActorType}, - db::{activity::insert_activity, community::Community, user::User_}, + apub::{ + community::do_announce, + extensions::signatures::sign, + insert_activity, + is_apub_id_valid, + ActorType, + }, request::retry_custom, DbPool, LemmyError, }; use activitystreams::{context, object::properties::ObjectProperties, public, Activity, Base}; use actix_web::client::Client; +use lemmy_db::{community::Community, user::User_}; use log::debug; use serde::Serialize; use std::fmt::Debug; @@ -43,7 +49,7 @@ where // if this is a local community, we need to do an announce from the community instead if community.local { - Community::do_announce(activity, &community, creator, client, pool).await?; + do_announce(activity, &community, creator, client, pool).await?; } else { send_activity(client, &activity, creator, to).await?; } diff --git a/server/src/apub/comment.rs b/server/src/apub/comment.rs index dbc15909e..af3581cbb 100644 --- a/server/src/apub/comment.rs +++ b/server/src/apub/comment.rs @@ -17,19 +17,9 @@ use crate::{ ToApub, }, blocking, - convert_datetime, - db::{ - comment::{Comment, CommentForm}, - community::Community, - post::Post, - user::User_, - Crud, - }, routes::DbPoolParam, - scrape_text_for_mentions, DbPool, LemmyError, - MentionData, }; use activitystreams::{ activity::{Create, Delete, Dislike, Like, Remove, Undo, Update}, @@ -40,6 +30,14 @@ use activitystreams::{ use activitystreams_new::object::Tombstone; use actix_web::{body::Body, client::Client, web::Path, HttpResponse}; use itertools::Itertools; +use lemmy_db::{ + comment::{Comment, CommentForm}, + community::Community, + post::Post, + user::User_, + Crud, +}; +use lemmy_utils::{convert_datetime, scrape_text_for_mentions, MentionData}; use log::debug; use serde::Deserialize; diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index bfc896af6..8b623e713 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -7,20 +7,13 @@ use crate::{ extensions::group_extensions::GroupExtension, fetcher::get_or_fetch_and_upsert_remote_user, get_shared_inbox, + insert_activity, ActorType, FromApub, GroupExt, ToApub, }, blocking, - convert_datetime, - db::{ - activity::insert_activity, - community::{Community, CommunityForm}, - community_view::{CommunityFollowerView, CommunityModeratorView}, - user::User_, - }, - naive_now, routes::DbPoolParam, DbPool, LemmyError, @@ -44,6 +37,13 @@ use activitystreams_new::{ }; use actix_web::{body::Body, client::Client, web, HttpResponse}; use itertools::Itertools; +use lemmy_db::{ + community::{Community, CommunityForm}, + community_view::{CommunityFollowerView, CommunityModeratorView}, + naive_now, + user::User_, +}; +use lemmy_utils::convert_datetime; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, str::FromStr}; @@ -462,39 +462,37 @@ pub async fn get_apub_community_followers( Ok(create_apub_response(&collection)) } -impl Community { - pub async fn do_announce( - activity: A, - community: &Community, - sender: &dyn ActorType, - client: &Client, - pool: &DbPool, - ) -> Result - where - A: Activity + Base + Serialize + Debug, - { - let mut announce = Announce::default(); - populate_object_props( - &mut announce.object_props, - vec![community.get_followers_url()], - &format!("{}/announce/{}", community.actor_id, uuid::Uuid::new_v4()), - )?; - announce - .announce_props - .set_actor_xsd_any_uri(community.actor_id.to_owned())? - .set_object_base_box(BaseBox::from_concrete(activity)?)?; +pub async fn do_announce( + activity: A, + community: &Community, + sender: &dyn ActorType, + client: &Client, + pool: &DbPool, +) -> Result +where + A: Activity + Base + Serialize + Debug, +{ + let mut announce = Announce::default(); + populate_object_props( + &mut announce.object_props, + vec![community.get_followers_url()], + &format!("{}/announce/{}", community.actor_id, uuid::Uuid::new_v4()), + )?; + announce + .announce_props + .set_actor_xsd_any_uri(community.actor_id.to_owned())? + .set_object_base_box(BaseBox::from_concrete(activity)?)?; - insert_activity(community.creator_id, announce.clone(), true, pool).await?; + insert_activity(community.creator_id, announce.clone(), true, pool).await?; - // dont send to the instance where the activity originally came from, because that would result - // in a database error (same data inserted twice) - let mut to = community.get_follower_inboxes(pool).await?; + // dont send to the instance where the activity originally came from, because that would result + // in a database error (same data inserted twice) + let mut to = community.get_follower_inboxes(pool).await?; - // this seems to be the "easiest" stable alternative for remove_item() - to.retain(|x| *x != sender.get_shared_inbox_url()); + // this seems to be the "easiest" stable alternative for remove_item() + to.retain(|x| *x != sender.get_shared_inbox_url()); - send_activity(client, &announce, community, to).await?; + send_activity(client, &announce, community, to).await?; - Ok(HttpResponse::Ok().finish()) - } + Ok(HttpResponse::Ok().finish()) } diff --git a/server/src/apub/community_inbox.rs b/server/src/apub/community_inbox.rs index 996e0c251..8ea644341 100644 --- a/server/src/apub/community_inbox.rs +++ b/server/src/apub/community_inbox.rs @@ -2,21 +2,21 @@ use crate::{ apub::{ extensions::signatures::verify, fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user}, + insert_activity, ActorType, }, blocking, - db::{ - activity::insert_activity, - community::{Community, CommunityFollower, CommunityFollowerForm}, - user::User_, - Followable, - }, routes::{ChatServerParam, DbPoolParam}, LemmyError, }; use activitystreams::activity::Undo; use activitystreams_new::activity::Follow; use actix_web::{client::Client, web, HttpRequest, HttpResponse}; +use lemmy_db::{ + community::{Community, CommunityFollower, CommunityFollowerForm}, + user::User_, + Followable, +}; use log::debug; use serde::Deserialize; use std::fmt::Debug; diff --git a/server/src/apub/extensions/group_extensions.rs b/server/src/apub/extensions/group_extensions.rs index 1c24eef57..2120f6f14 100644 --- a/server/src/apub/extensions/group_extensions.rs +++ b/server/src/apub/extensions/group_extensions.rs @@ -1,9 +1,7 @@ -use crate::{ - db::{category::Category, Crud}, - LemmyError, -}; +use crate::LemmyError; use activitystreams::{ext::Extension, Actor}; use diesel::PgConnection; +use lemmy_db::{category::Category, Crud}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, Deserialize, Serialize)] diff --git a/server/src/apub/extensions/signatures.rs b/server/src/apub/extensions/signatures.rs index af46bc5ee..1c930a958 100644 --- a/server/src/apub/extensions/signatures.rs +++ b/server/src/apub/extensions/signatures.rs @@ -9,7 +9,6 @@ use log::debug; use openssl::{ hash::MessageDigest, pkey::PKey, - rsa::Rsa, sign::{Signer, Verifier}, }; use serde::{Deserialize, Serialize}; @@ -19,23 +18,6 @@ lazy_static! { static ref HTTP_SIG_CONFIG: Config = Config::new(); } -pub struct Keypair { - pub private_key: String, - pub public_key: String, -} - -/// Generate the asymmetric keypair for ActivityPub HTTP signatures. -pub fn generate_actor_keypair() -> Result { - let rsa = Rsa::generate(2048)?; - let pkey = PKey::from_rsa(rsa)?; - let public_key = pkey.public_key_to_pem()?; - let private_key = pkey.private_key_to_pem_pkcs8()?; - Ok(Keypair { - private_key: String::from_utf8(private_key)?, - public_key: String::from_utf8(public_key)?, - }) -} - /// Signs request headers with the given keypair. pub async fn sign( request: ClientRequest, diff --git a/server/src/apub/fetcher.rs b/server/src/apub/fetcher.rs index d8a1e764f..0604129d4 100644 --- a/server/src/apub/fetcher.rs +++ b/server/src/apub/fetcher.rs @@ -1,29 +1,7 @@ use crate::{ api::site::SearchResponse, - apub::{ - get_apub_protocol_string, - is_apub_id_valid, - FromApub, - GroupExt, - PageExt, - PersonExt, - APUB_JSON_CONTENT_TYPE, - }, + apub::{is_apub_id_valid, FromApub, GroupExt, PageExt, PersonExt, APUB_JSON_CONTENT_TYPE}, blocking, - db::{ - comment::{Comment, CommentForm}, - comment_view::CommentView, - community::{Community, CommunityForm, CommunityModerator, CommunityModeratorForm}, - community_view::CommunityView, - post::{Post, PostForm}, - post_view::PostView, - user::{UserForm, User_}, - user_view::UserView, - Crud, - Joinable, - SearchType, - }, - naive_now, request::{retry, RecvError}, routes::nodeinfo::{NodeInfo, NodeInfoWellKnown}, DbPool, @@ -34,6 +12,21 @@ use activitystreams_new::{base::BaseExt, prelude::*, primitives::XsdAnyUri}; use actix_web::client::Client; use chrono::NaiveDateTime; use diesel::{result::Error::NotFound, PgConnection}; +use lemmy_db::{ + comment::{Comment, CommentForm}, + comment_view::CommentView, + community::{Community, CommunityForm, CommunityModerator, CommunityModeratorForm}, + community_view::CommunityView, + naive_now, + post::{Post, PostForm}, + post_view::PostView, + user::{UserForm, User_}, + user_view::UserView, + Crud, + Joinable, + SearchType, +}; +use lemmy_utils::get_apub_protocol_string; use log::debug; use serde::Deserialize; use std::{fmt::Debug, time::Duration}; diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs index 561dc49a1..eeac5fec3 100644 --- a/server/src/apub/mod.rs +++ b/server/src/apub/mod.rs @@ -16,14 +16,11 @@ use crate::{ page_extension::PageExtension, signatures::{PublicKey, PublicKeyExtension}, }, - convert_datetime, - db::user::User_, + blocking, request::{retry, RecvError}, routes::webfinger::WebFingerResponse, DbPool, LemmyError, - MentionData, - Settings, }; use activitystreams::object::Page; use activitystreams_ext::{Ext1, Ext2}; @@ -35,6 +32,9 @@ use activitystreams_new::{ }; use actix_web::{body::Body, client::Client, HttpResponse}; use chrono::NaiveDateTime; +use failure::_core::fmt::Debug; +use lemmy_db::{activity::do_insert_activity, user::User_}; +use lemmy_utils::{convert_datetime, get_apub_protocol_string, settings::Settings, MentionData}; use log::debug; use serde::Serialize; use url::Url; @@ -45,14 +45,6 @@ type PageExt = Ext1; pub static APUB_JSON_CONTENT_TYPE: &str = "application/activity+json"; -pub enum EndpointType { - Community, - User, - Post, - Comment, - PrivateMessage, -} - /// Convert the data to json and turn it into an HTTP Response with the correct ActivityPub /// headers. fn create_apub_response(data: &T) -> HttpResponse @@ -73,34 +65,6 @@ where .json(data) } -/// Generates the ActivityPub ID for a given object type and ID. -pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url { - let point = match endpoint_type { - EndpointType::Community => "c", - EndpointType::User => "u", - EndpointType::Post => "post", - EndpointType::Comment => "comment", - EndpointType::PrivateMessage => "private_message", - }; - - Url::parse(&format!( - "{}://{}/{}/{}", - get_apub_protocol_string(), - Settings::get().hostname, - point, - name - )) - .unwrap() -} - -pub fn get_apub_protocol_string() -> &'static str { - if Settings::get().federation.tls_enabled { - "https" - } else { - "http" - } -} - // Checks if the ID has a valid format, correct scheme, and is in the allowed instance list. fn is_apub_id_valid(apub_id: &Url) -> bool { debug!("Checking {}", apub_id); @@ -374,3 +338,19 @@ pub async fn fetch_webfinger_url( .to_owned() .ok_or_else(|| format_err!("No href found.").into()) } + +pub async fn insert_activity( + user_id: i32, + data: T, + local: bool, + pool: &DbPool, +) -> Result<(), LemmyError> +where + T: Serialize + Debug + Send + 'static, +{ + blocking(pool, move |conn| { + do_insert_activity(conn, user_id, &data, local) + }) + .await??; + Ok(()) +} diff --git a/server/src/apub/post.rs b/server/src/apub/post.rs index 255629e8d..ba0372c44 100644 --- a/server/src/apub/post.rs +++ b/server/src/apub/post.rs @@ -6,7 +6,6 @@ use crate::{ create_tombstone, extensions::page_extension::PageExtension, fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user}, - get_apub_protocol_string, ActorType, ApubLikeableType, ApubObjectType, @@ -15,17 +14,9 @@ use crate::{ ToApub, }, blocking, - convert_datetime, - db::{ - community::Community, - post::{Post, PostForm}, - user::User_, - Crud, - }, routes::DbPoolParam, DbPool, LemmyError, - Settings, }; use activitystreams::{ activity::{Create, Delete, Dislike, Like, Remove, Undo, Update}, @@ -36,6 +27,13 @@ use activitystreams::{ use activitystreams_ext::Ext1; use activitystreams_new::object::Tombstone; use actix_web::{body::Body, client::Client, web, HttpResponse}; +use lemmy_db::{ + community::Community, + post::{Post, PostForm}, + user::User_, + Crud, +}; +use lemmy_utils::{convert_datetime, get_apub_protocol_string, settings::Settings}; use serde::Deserialize; #[derive(Deserialize)] diff --git a/server/src/apub/private_message.rs b/server/src/apub/private_message.rs index 48bbd1f0f..567a61784 100644 --- a/server/src/apub/private_message.rs +++ b/server/src/apub/private_message.rs @@ -3,18 +3,12 @@ use crate::{ activities::send_activity, create_tombstone, fetcher::get_or_fetch_and_upsert_remote_user, + insert_activity, ApubObjectType, FromApub, ToApub, }, blocking, - convert_datetime, - db::{ - activity::insert_activity, - private_message::{PrivateMessage, PrivateMessageForm}, - user::User_, - Crud, - }, DbPool, LemmyError, }; @@ -25,6 +19,12 @@ use activitystreams::{ }; use activitystreams_new::object::Tombstone; use actix_web::client::Client; +use lemmy_db::{ + private_message::{PrivateMessage, PrivateMessageForm}, + user::User_, + Crud, +}; +use lemmy_utils::convert_datetime; #[async_trait::async_trait(?Send)] impl ToApub for PrivateMessage { diff --git a/server/src/apub/shared_inbox.rs b/server/src/apub/shared_inbox.rs index fa9794b25..75ce34155 100644 --- a/server/src/apub/shared_inbox.rs +++ b/server/src/apub/shared_inbox.rs @@ -5,6 +5,7 @@ use crate::{ post::PostResponse, }, apub::{ + community::do_announce, extensions::signatures::verify, fetcher::{ get_or_fetch_and_insert_remote_comment, @@ -12,25 +13,13 @@ use crate::{ get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user, }, + insert_activity, FromApub, GroupExt, PageExt, }, blocking, - db::{ - activity::insert_activity, - comment::{Comment, CommentForm, CommentLike, CommentLikeForm}, - comment_view::CommentView, - community::{Community, CommunityForm}, - community_view::CommunityView, - post::{Post, PostForm, PostLike, PostLikeForm}, - post_view::PostView, - Crud, - Likeable, - }, - naive_now, routes::{ChatServerParam, DbPoolParam}, - scrape_text_for_mentions, websocket::{ server::{SendComment, SendCommunityRoomMessage, SendPost}, UserOperation, @@ -46,6 +35,18 @@ use activitystreams::{ BaseBox, }; use actix_web::{client::Client, web, HttpRequest, HttpResponse}; +use lemmy_db::{ + comment::{Comment, CommentForm, CommentLike, CommentLikeForm}, + comment_view::CommentView, + community::{Community, CommunityForm}, + community_view::CommunityView, + naive_now, + post::{Post, PostForm, PostLike, PostLikeForm}, + post_view::PostView, + Crud, + Likeable, +}; +use lemmy_utils::scrape_text_for_mentions; use log::debug; use serde::{Deserialize, Serialize}; use std::fmt::Debug; @@ -234,7 +235,7 @@ where if community.local { let sending_user = get_or_fetch_and_upsert_remote_user(sender, client, pool).await?; - Community::do_announce(activity, &community, &sending_user, client, pool).await + do_announce(activity, &community, &sending_user, client, pool).await } else { Ok(HttpResponse::NotFound().finish()) } diff --git a/server/src/apub/user.rs b/server/src/apub/user.rs index a3194355a..323407c72 100644 --- a/server/src/apub/user.rs +++ b/server/src/apub/user.rs @@ -1,12 +1,15 @@ use crate::{ - apub::{activities::send_activity, create_apub_response, ActorType, FromApub, PersonExt, ToApub}, - blocking, - convert_datetime, - db::{ - activity::insert_activity, - user::{UserForm, User_}, + api::claims::Claims, + apub::{ + activities::send_activity, + create_apub_response, + insert_activity, + ActorType, + FromApub, + PersonExt, + ToApub, }, - naive_now, + blocking, routes::DbPoolParam, DbPool, LemmyError, @@ -22,6 +25,11 @@ use activitystreams_new::{ }; use actix_web::{body::Body, client::Client, web, HttpResponse}; use failure::_core::str::FromStr; +use lemmy_db::{ + naive_now, + user::{UserForm, User_}, +}; +use lemmy_utils::convert_datetime; use serde::Deserialize; #[derive(Deserialize)] @@ -240,7 +248,7 @@ pub async fn get_apub_user_http( ) -> Result, LemmyError> { let user_name = info.into_inner().user_name; let user = blocking(&db, move |conn| { - User_::find_by_email_or_username(conn, &user_name) + Claims::find_by_email_or_username(conn, &user_name) }) .await??; let u = user.to_apub(&db).await?; diff --git a/server/src/apub/user_inbox.rs b/server/src/apub/user_inbox.rs index 80280adeb..226235e6a 100644 --- a/server/src/apub/user_inbox.rs +++ b/server/src/apub/user_inbox.rs @@ -3,19 +3,10 @@ use crate::{ apub::{ extensions::signatures::verify, fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user}, + insert_activity, FromApub, }, blocking, - db::{ - activity::insert_activity, - community::{CommunityFollower, CommunityFollowerForm}, - private_message::{PrivateMessage, PrivateMessageForm}, - private_message_view::PrivateMessageView, - user::User_, - Crud, - Followable, - }, - naive_now, routes::{ChatServerParam, DbPoolParam}, websocket::{server::SendUserRoomMessage, UserOperation}, DbPool, @@ -26,6 +17,15 @@ use activitystreams::{ object::Note, }; use actix_web::{client::Client, web, HttpRequest, HttpResponse}; +use lemmy_db::{ + community::{CommunityFollower, CommunityFollowerForm}, + naive_now, + private_message::{PrivateMessage, PrivateMessageForm}, + private_message_view::PrivateMessageView, + user::User_, + Crud, + Followable, +}; use log::debug; use serde::Deserialize; use std::fmt::Debug; diff --git a/server/src/db/code_migrations.rs b/server/src/code_migrations.rs similarity index 86% rename from server/src/db/code_migrations.rs rename to server/src/code_migrations.rs index 1810fae29..b28e120a1 100644 --- a/server/src/db/code_migrations.rs +++ b/server/src/code_migrations.rs @@ -1,18 +1,16 @@ // This is for db migrations that require code -use super::{ +use crate::LemmyError; +use diesel::*; +use lemmy_db::{ comment::Comment, community::{Community, CommunityForm}, + naive_now, post::Post, private_message::PrivateMessage, user::{UserForm, User_}, + Crud, }; -use crate::{ - apub::{extensions::signatures::generate_actor_keypair, make_apub_endpoint, EndpointType}, - db::Crud, - naive_now, - LemmyError, -}; -use diesel::*; +use lemmy_utils::{generate_actor_keypair, make_apub_endpoint, EndpointType}; use log::info; pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), LemmyError> { @@ -26,7 +24,7 @@ pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), LemmyError> { } fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { - use crate::schema::user_::dsl::*; + use lemmy_db::schema::user_::dsl::*; info!("Running user_updates_2020_04_02"); @@ -77,7 +75,7 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { } fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { - use crate::schema::community::dsl::*; + use lemmy_db::schema::community::dsl::*; info!("Running community_updates_2020_04_02"); @@ -121,7 +119,7 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { } fn post_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> { - use crate::schema::post::dsl::*; + use lemmy_db::schema::post::dsl::*; info!("Running post_updates_2020_04_03"); @@ -134,7 +132,8 @@ fn post_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> { sql_query("alter table post disable trigger refresh_post").execute(conn)?; for cpost in &incorrect_posts { - Post::update_ap_id(&conn, cpost.id)?; + let apub_id = make_apub_endpoint(EndpointType::Post, &cpost.id.to_string()).to_string(); + Post::update_ap_id(&conn, cpost.id, apub_id)?; } info!("{} post rows updated.", incorrect_posts.len()); @@ -145,7 +144,7 @@ fn post_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> { } fn comment_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> { - use crate::schema::comment::dsl::*; + use lemmy_db::schema::comment::dsl::*; info!("Running comment_updates_2020_04_03"); @@ -158,7 +157,8 @@ fn comment_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> { sql_query("alter table comment disable trigger refresh_comment").execute(conn)?; for ccomment in &incorrect_comments { - Comment::update_ap_id(&conn, ccomment.id)?; + let apub_id = make_apub_endpoint(EndpointType::Comment, &ccomment.id.to_string()).to_string(); + Comment::update_ap_id(&conn, ccomment.id, apub_id)?; } sql_query("alter table comment enable trigger refresh_comment").execute(conn)?; @@ -169,7 +169,7 @@ fn comment_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> { } fn private_message_updates_2020_05_05(conn: &PgConnection) -> Result<(), LemmyError> { - use crate::schema::private_message::dsl::*; + use lemmy_db::schema::private_message::dsl::*; info!("Running private_message_updates_2020_05_05"); @@ -180,7 +180,8 @@ fn private_message_updates_2020_05_05(conn: &PgConnection) -> Result<(), LemmyEr .load::(conn)?; for cpm in &incorrect_pms { - PrivateMessage::update_ap_id(&conn, cpm.id)?; + let apub_id = make_apub_endpoint(EndpointType::PrivateMessage, &cpm.id.to_string()).to_string(); + PrivateMessage::update_ap_id(&conn, cpm.id, apub_id)?; } info!("{} private message rows updated.", incorrect_pms.len()); diff --git a/server/src/lib.rs b/server/src/lib.rs index 08c3fa98a..4795cf01e 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -5,76 +5,34 @@ pub extern crate strum_macros; pub extern crate lazy_static; #[macro_use] pub extern crate failure; -#[macro_use] -pub extern crate diesel; pub extern crate actix; pub extern crate actix_web; pub extern crate bcrypt; pub extern crate chrono; -pub extern crate comrak; +pub extern crate diesel; pub extern crate dotenv; pub extern crate jsonwebtoken; -pub extern crate lettre; -pub extern crate lettre_email; extern crate log; pub extern crate openssl; -pub extern crate rand; -pub extern crate regex; pub extern crate rss; pub extern crate serde; pub extern crate serde_json; pub extern crate sha2; pub extern crate strum; -pub async fn blocking(pool: &DbPool, f: F) -> Result -where - F: FnOnce(&diesel::PgConnection) -> T + Send + 'static, - T: Send + 'static, -{ - let pool = pool.clone(); - let res = actix_web::web::block(move || { - let conn = pool.get()?; - let res = (f)(&conn); - Ok(res) as Result<_, LemmyError> - }) - .await?; - - Ok(res) -} - pub mod api; pub mod apub; -pub mod db; +pub mod code_migrations; pub mod rate_limit; pub mod request; pub mod routes; -pub mod schema; -pub mod settings; pub mod version; pub mod websocket; -use crate::{ - request::{retry, RecvError}, - settings::Settings, -}; +use crate::request::{retry, RecvError}; use actix_web::{client::Client, dev::ConnectionInfo}; -use chrono::{DateTime, FixedOffset, Local, NaiveDateTime, Utc}; -use itertools::Itertools; -use lettre::{ - smtp::{ - authentication::{Credentials, Mechanism}, - extension::ClientId, - ConnectionReuseParameters, - }, - ClientSecurity, - SmtpClient, - Transport, -}; -use lettre_email::Email; use log::error; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; -use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use regex::{Regex, RegexBuilder}; use serde::Deserialize; pub type DbPool = diesel::r2d2::Pool>; @@ -89,14 +47,6 @@ pub struct LemmyError { inner: failure::Error, } -impl std::fmt::Display for LemmyError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - self.inner.fmt(f) - } -} - -impl actix_web::error::ResponseError for LemmyError {} - impl From for LemmyError where T: Into, @@ -106,113 +56,13 @@ where } } -pub fn to_datetime_utc(ndt: NaiveDateTime) -> DateTime { - DateTime::::from_utc(ndt, Utc) -} - -pub fn naive_now() -> NaiveDateTime { - chrono::prelude::Utc::now().naive_utc() -} - -pub fn naive_from_unix(time: i64) -> NaiveDateTime { - NaiveDateTime::from_timestamp(time, 0) -} - -pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime { - let now = Local::now(); - DateTime::::from_utc(datetime, *now.offset()) -} - -pub fn is_email_regex(test: &str) -> bool { - EMAIL_REGEX.is_match(test) -} - -pub async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> { - let response = retry(|| client.get(test).send()).await?; - - if response - .headers() - .get("Content-Type") - .ok_or_else(|| format_err!("No Content-Type header"))? - .to_str()? - .starts_with("image/") - { - Ok(()) - } else { - Err(format_err!("Not an image type.").into()) +impl std::fmt::Display for LemmyError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.inner.fmt(f) } } -pub fn remove_slurs(test: &str) -> String { - SLUR_REGEX.replace_all(test, "*removed*").to_string() -} - -pub fn slur_check(test: &str) -> Result<(), Vec<&str>> { - let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect(); - - // Unique - matches.sort_unstable(); - matches.dedup(); - - if matches.is_empty() { - Ok(()) - } else { - Err(matches) - } -} - -pub fn slurs_vec_to_str(slurs: Vec<&str>) -> String { - let start = "No slurs - "; - let combined = &slurs.join(", "); - [start, combined].concat() -} - -pub fn generate_random_string() -> String { - thread_rng().sample_iter(&Alphanumeric).take(30).collect() -} - -pub fn send_email( - subject: &str, - to_email: &str, - to_username: &str, - html: &str, -) -> Result<(), String> { - let email_config = Settings::get().email.ok_or("no_email_setup")?; - - let email = Email::builder() - .to((to_email, to_username)) - .from(email_config.smtp_from_address.to_owned()) - .subject(subject) - .html(html) - .build() - .unwrap(); - - let mailer = if email_config.use_tls { - SmtpClient::new_simple(&email_config.smtp_server).unwrap() - } else { - SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap() - } - .hello_name(ClientId::Domain(Settings::get().hostname)) - .smtp_utf8(true) - .authentication_mechanism(Mechanism::Plain) - .connection_reuse(ConnectionReuseParameters::ReuseUnlimited); - let mailer = if let (Some(login), Some(password)) = - (&email_config.smtp_login, &email_config.smtp_password) - { - mailer.credentials(Credentials::new(login.to_owned(), password.to_owned())) - } else { - mailer - }; - - let mut transport = mailer.transport(); - let result = transport.send(email.into()); - transport.close(); - - match result { - Ok(_) => Ok(()), - Err(e) => Err(e.to_string()), - } -} +impl actix_web::error::ResponseError for LemmyError {} #[derive(Deserialize, Debug)] pub struct IframelyResponse { @@ -319,8 +169,20 @@ async fn fetch_iframely_and_pictrs_data( } } -pub fn markdown_to_html(text: &str) -> String { - comrak::markdown_to_html(text, &comrak::ComrakOptions::default()) +pub async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> { + let response = retry(|| client.get(test).send()).await?; + + if response + .headers() + .get("Content-Type") + .ok_or_else(|| format_err!("No Content-Type header"))? + .to_str()? + .starts_with("image/") + { + Ok(()) + } else { + Err(format_err!("Not an image type.").into()) + } } pub fn get_ip(conn_info: &ConnectionInfo) -> String { @@ -333,127 +195,37 @@ pub fn get_ip(conn_info: &ConnectionInfo) -> String { .to_string() } -// TODO nothing is done with community / group webfingers yet, so just ignore those for now -#[derive(Clone, PartialEq, Eq, Hash)] -pub struct MentionData { - pub name: String, - pub domain: String, -} +pub async fn blocking(pool: &DbPool, f: F) -> Result +where + F: FnOnce(&diesel::PgConnection) -> T + Send + 'static, + T: Send + 'static, +{ + let pool = pool.clone(); + let res = actix_web::web::block(move || { + let conn = pool.get()?; + let res = (f)(&conn); + Ok(res) as Result<_, LemmyError> + }) + .await?; -impl MentionData { - pub fn is_local(&self) -> bool { - Settings::get().hostname.eq(&self.domain) - } - pub fn full_name(&self) -> String { - format!("@{}@{}", &self.name, &self.domain) - } -} - -pub fn scrape_text_for_mentions(text: &str) -> Vec { - let mut out: Vec = Vec::new(); - for caps in WEBFINGER_USER_REGEX.captures_iter(text) { - out.push(MentionData { - name: caps["name"].to_string(), - domain: caps["domain"].to_string(), - }); - } - out.into_iter().unique().collect() -} - -pub fn is_valid_username(name: &str) -> bool { - VALID_USERNAME_REGEX.is_match(name) -} - -pub fn is_valid_community_name(name: &str) -> bool { - VALID_COMMUNITY_NAME_REGEX.is_match(name) + Ok(res) } #[cfg(test)] mod tests { - use crate::{ - is_email_regex, - is_image_content_type, - is_valid_community_name, - is_valid_username, - remove_slurs, - scrape_text_for_mentions, - slur_check, - slurs_vec_to_str, - }; - - #[test] - fn test_mentions_regex() { - let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)"; - let mentions = scrape_text_for_mentions(text); - - assert_eq!(mentions[0].name, "tedu".to_string()); - assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string()); - assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string()); - } + use crate::is_image_content_type; #[test] fn test_image() { actix_rt::System::new("tset_image").block_on(async move { - let client = actix_web::client::Client::default(); - assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok()); - assert!(is_image_content_type(&client, - "https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20" - ) - .await.is_err() - ); - }); - } - - #[test] - fn test_email() { - assert!(is_email_regex("gush@gmail.com")); - assert!(!is_email_regex("nada_neutho")); - } - - #[test] - fn test_valid_register_username() { - assert!(is_valid_username("Hello_98")); - assert!(is_valid_username("ten")); - assert!(!is_valid_username("Hello-98")); - assert!(!is_valid_username("a")); - assert!(!is_valid_username("")); - } - - #[test] - fn test_valid_community_name() { - assert!(is_valid_community_name("example")); - assert!(is_valid_community_name("example_community")); - assert!(!is_valid_community_name("Example")); - assert!(!is_valid_community_name("Ex")); - assert!(!is_valid_community_name("")); - } - - #[test] - fn test_slur_filter() { - let test = - "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text."; - let slur_free = "No slurs here"; - assert_eq!( - remove_slurs(&test), - "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text." - .to_string() - ); - - let has_slurs_vec = vec![ - "Niggerz", - "coons", - "dindu", - "ladyboy", - "retardeds", - "tranny", - ]; - let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny"; - - assert_eq!(slur_check(test), Err(has_slurs_vec)); - assert_eq!(slur_check(slur_free), Ok(())); - if let Err(slur_vec) = slur_check(test) { - assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str); - } + let client = actix_web::client::Client::default(); + assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok()); + assert!(is_image_content_type(&client, + "https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20" + ) + .await.is_err() + ); + }); } // These helped with testing @@ -470,21 +242,4 @@ mod tests { // let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu"); // assert!(res_other.is_err()); // } - - // #[test] - // fn test_send_email() { - // let result = send_email("not a subject", "test_email@gmail.com", "ur user", "

HI there

"); - // assert!(result.is_ok()); - // } -} - -lazy_static! { - static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap(); - static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap(); - static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap(); - // TODO keep this old one, it didn't work with port well tho - // static ref WEBFINGER_USER_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap(); - static ref WEBFINGER_USER_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").unwrap(); - static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap(); - static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap(); } diff --git a/server/src/main.rs b/server/src/main.rs index 30be711fa..7689d7ad1 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -22,22 +22,20 @@ use diesel::{ r2d2::{ConnectionManager, Pool}, PgConnection, }; +use lemmy_db::get_database_url_from_env; use lemmy_server::{ blocking, - db::code_migrations::run_advanced_migrations, + code_migrations::run_advanced_migrations, rate_limit::{rate_limiter::RateLimiter, RateLimit}, routes::{api, federation, feeds, index, nodeinfo, webfinger}, - settings::Settings, websocket::server::*, LemmyError, }; -use regex::Regex; +use lemmy_utils::{settings::Settings, CACHE_CONTROL_REGEX}; use std::sync::Arc; use tokio::sync::Mutex; lazy_static! { - static ref CACHE_CONTROL_REGEX: Regex = - Regex::new("^((text|image)/.+|application/javascript)$").unwrap(); // static ref CACHE_CONTROL_VALUE: String = format!("public, max-age={}", 365 * 24 * 60 * 60); // Test out 1 hour here, this is breaking some things static ref CACHE_CONTROL_VALUE: String = format!("public, max-age={}", 60 * 60); @@ -51,11 +49,15 @@ async fn main() -> Result<(), LemmyError> { let settings = Settings::get(); // Set up the r2d2 connection pool - let manager = ConnectionManager::::new(&settings.get_database_url()); + let db_url = match get_database_url_from_env() { + Ok(url) => url, + Err(_) => settings.get_database_url(), + }; + let manager = ConnectionManager::::new(&db_url); let pool = Pool::builder() .max_size(settings.database.pool_size) .build(manager) - .unwrap_or_else(|_| panic!("Error connecting to {}", settings.get_database_url())); + .unwrap_or_else(|_| panic!("Error connecting to {}", db_url)); // Run the migrations from code blocking(&pool, move |conn| { diff --git a/server/src/rate_limit/mod.rs b/server/src/rate_limit/mod.rs index e49a527e8..513c923c6 100644 --- a/server/src/rate_limit/mod.rs +++ b/server/src/rate_limit/mod.rs @@ -1,7 +1,8 @@ -use super::{IPAddr, Settings}; -use crate::{get_ip, settings::RateLimitConfig, LemmyError}; +use super::IPAddr; +use crate::{get_ip, LemmyError}; use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; use futures::future::{ok, Ready}; +use lemmy_utils::settings::{RateLimitConfig, Settings}; use rate_limiter::{RateLimitType, RateLimiter}; use std::{ future::Future, diff --git a/server/src/routes/federation.rs b/server/src/routes/federation.rs index fe6e33657..ebab139a7 100644 --- a/server/src/routes/federation.rs +++ b/server/src/routes/federation.rs @@ -1,17 +1,15 @@ -use crate::{ - apub::{ - comment::get_apub_comment, - community::*, - community_inbox::community_inbox, - post::get_apub_post, - shared_inbox::shared_inbox, - user::*, - user_inbox::user_inbox, - APUB_JSON_CONTENT_TYPE, - }, - settings::Settings, +use crate::apub::{ + comment::get_apub_comment, + community::*, + community_inbox::community_inbox, + post::get_apub_post, + shared_inbox::shared_inbox, + user::*, + user_inbox::user_inbox, + APUB_JSON_CONTENT_TYPE, }; use actix_web::*; +use lemmy_utils::settings::Settings; pub fn config(cfg: &mut web::ServiceConfig) { if Settings::get().federation.enabled { diff --git a/server/src/routes/feeds.rs b/server/src/routes/feeds.rs index a1c2ba58f..1322feb44 100644 --- a/server/src/routes/feeds.rs +++ b/server/src/routes/feeds.rs @@ -1,26 +1,21 @@ -use crate::{ - blocking, - db::{ - comment_view::{ReplyQueryBuilder, ReplyView}, - community::Community, - post_view::{PostQueryBuilder, PostView}, - site_view::SiteView, - user::{Claims, User_}, - user_mention_view::{UserMentionQueryBuilder, UserMentionView}, - ListingType, - SortType, - }, - markdown_to_html, - routes::DbPoolParam, - settings::Settings, - LemmyError, -}; +use crate::{api::claims::Claims, blocking, routes::DbPoolParam, LemmyError}; use actix_web::{error::ErrorBadRequest, *}; use chrono::{DateTime, NaiveDateTime, Utc}; use diesel::{ r2d2::{ConnectionManager, Pool}, PgConnection, }; +use lemmy_db::{ + comment_view::{ReplyQueryBuilder, ReplyView}, + community::Community, + post_view::{PostQueryBuilder, PostView}, + site_view::SiteView, + user::User_, + user_mention_view::{UserMentionQueryBuilder, UserMentionView}, + ListingType, + SortType, +}; +use lemmy_utils::{markdown_to_html, settings::Settings}; use rss::{CategoryBuilder, ChannelBuilder, GuidBuilder, Item, ItemBuilder}; use serde::Deserialize; use std::str::FromStr; @@ -131,7 +126,7 @@ fn get_feed_user( ) -> Result { let site_view = SiteView::read(&conn)?; let user = User_::find_by_username(&conn, &user_name)?; - let user_url = user.get_profile_url(); + let user_url = user.get_profile_url(&Settings::get().hostname); let posts = PostQueryBuilder::create(&conn) .listing_type(ListingType::All) diff --git a/server/src/routes/index.rs b/server/src/routes/index.rs index 2f462aa5f..b579a1958 100644 --- a/server/src/routes/index.rs +++ b/server/src/routes/index.rs @@ -1,6 +1,6 @@ -use crate::settings::Settings; use actix_files::NamedFile; use actix_web::*; +use lemmy_utils::settings::Settings; pub fn config(cfg: &mut web::ServiceConfig) { cfg diff --git a/server/src/routes/nodeinfo.rs b/server/src/routes/nodeinfo.rs index ff728fe3e..5094c2f15 100644 --- a/server/src/routes/nodeinfo.rs +++ b/server/src/routes/nodeinfo.rs @@ -1,13 +1,7 @@ -use crate::{ - apub::get_apub_protocol_string, - blocking, - db::site_view::SiteView, - routes::DbPoolParam, - version, - LemmyError, - Settings, -}; +use crate::{blocking, routes::DbPoolParam, version, LemmyError}; use actix_web::{body::Body, error::ErrorBadRequest, *}; +use lemmy_db::site_view::SiteView; +use lemmy_utils::{get_apub_protocol_string, settings::Settings}; use serde::{Deserialize, Serialize}; use url::Url; diff --git a/server/src/routes/webfinger.rs b/server/src/routes/webfinger.rs index af021dd5f..e616de0e8 100644 --- a/server/src/routes/webfinger.rs +++ b/server/src/routes/webfinger.rs @@ -1,12 +1,7 @@ -use crate::{ - blocking, - db::{community::Community, user::User_}, - routes::DbPoolParam, - LemmyError, - Settings, -}; +use crate::{blocking, routes::DbPoolParam, LemmyError}; use actix_web::{error::ErrorBadRequest, web::Query, *}; -use regex::Regex; +use lemmy_db::{community::Community, user::User_}; +use lemmy_utils::{settings::Settings, WEBFINGER_COMMUNITY_REGEX, WEBFINGER_USER_REGEX}; use serde::{Deserialize, Serialize}; #[derive(Deserialize)] @@ -40,19 +35,6 @@ pub fn config(cfg: &mut web::ServiceConfig) { } } -lazy_static! { - static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!( - "^group:([a-z0-9_]{{3, 20}})@{}$", - Settings::get().hostname - )) - .unwrap(); - static ref WEBFINGER_USER_REGEX: Regex = Regex::new(&format!( - "^acct:([a-z0-9_]{{3, 20}})@{}$", - Settings::get().hostname - )) - .unwrap(); -} - /// Responds to webfinger requests of the following format. There isn't any real documentation for /// this, but it described in this blog post: /// https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social diff --git a/ui/package.json b/ui/package.json index 2819433ad..a08cec5c5 100644 --- a/ui/package.json +++ b/ui/package.json @@ -70,7 +70,7 @@ "engineStrict": true, "husky": { "hooks": { - "pre-commit": "cargo clippy --manifest-path ../server/Cargo.toml --all-targets --all-features -- -D warnings && lint-staged" + "pre-commit": "cargo clippy --manifest-path ../server/Cargo.toml --all-targets --workspace -- -D warnings && lint-staged" } }, "lint-staged": {