Split lemmy_utils into multiple files

server/lemmy_db/src/activity.rs

@@ -1,7 +1,7 @@
 use crate::{schema::activity, Crud};
 use diesel::{dsl::*, result::Error, *};
 use log::debug;
-use serde::{Serialize};
+use serde::Serialize;
 use serde_json::Value;
 use std::{
   fmt::Debug,

server/lemmy_db/src/category.rs

@@ -3,7 +3,7 @@ use crate::{
   Crud,
 };
 use diesel::{dsl::*, result::Error, *};
-use serde::{Serialize};
+use serde::Serialize;
 
 #[derive(Queryable, Identifiable, PartialEq, Debug, Serialize)]
 #[table_name = "category"]

server/lemmy_db/src/comment_view.rs

@@ -84,9 +84,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "comment_fast_view"]
 pub struct CommentView {
   pub id: i32,

server/lemmy_db/src/community_view.rs

@@ -123,9 +123,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "community_fast_view"]
 pub struct CommunityView {
   pub id: i32,

server/lemmy_db/src/moderator_views.rs

@@ -1,6 +1,6 @@
 use crate::limit_and_offset;
 use diesel::{result::Error, *};
-use serde::{Serialize};
+use serde::Serialize;
 
 table! {
   mod_remove_post_view (id) {
@@ -17,9 +17,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "mod_remove_post_view"]
 pub struct ModRemovePostView {
   pub id: i32,
@@ -77,9 +75,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "mod_lock_post_view"]
 pub struct ModLockPostView {
   pub id: i32,
@@ -136,9 +132,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "mod_sticky_post_view"]
 pub struct ModStickyPostView {
   pub id: i32,
@@ -200,9 +194,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "mod_remove_comment_view"]
 pub struct ModRemoveCommentView {
   pub id: i32,
@@ -264,9 +256,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "mod_remove_community_view"]
 pub struct ModRemoveCommunityView {
   pub id: i32,
@@ -320,9 +310,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "mod_ban_from_community_view"]
 pub struct ModBanFromCommunityView {
   pub id: i32,
@@ -381,9 +369,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "mod_ban_view"]
 pub struct ModBanView {
   pub id: i32,
@@ -435,9 +421,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "mod_add_community_view"]
 pub struct ModAddCommunityView {
   pub id: i32,
@@ -492,9 +476,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "mod_add_view"]
 pub struct ModAddView {
   pub id: i32,

server/lemmy_db/src/post_view.rs

@@ -1,7 +1,7 @@
 use super::post_view::post_fast_view::BoxedQuery;
 use crate::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType};
 use diesel::{dsl::*, pg::Pg, result::Error, *};
-use serde::{Serialize};
+use serde::Serialize;
 
 // The faked schema since diesel doesn't do views
 table! {
@@ -106,9 +106,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "post_fast_view"]
 pub struct PostView {
   pub id: i32,

server/lemmy_db/src/private_message_view.rs

@@ -1,6 +1,6 @@
 use crate::{limit_and_offset, MaybeOptional};
 use diesel::{pg::Pg, result::Error, *};
-use serde::{Serialize};
+use serde::Serialize;
 
 // The faked schema since diesel doesn't do views
 table! {
@@ -28,9 +28,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "private_message_view"]
 pub struct PrivateMessageView {
   pub id: i32,

server/lemmy_db/src/site_view.rs

@@ -1,5 +1,5 @@
 use diesel::{result::Error, *};
-use serde::{Serialize};
+use serde::Serialize;
 
 table! {
   site_view (id) {
@@ -24,9 +24,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "site_view"]
 pub struct SiteView {
   pub id: i32,

server/lemmy_db/src/user.rs

@@ -6,7 +6,7 @@ use crate::{
 };
 use bcrypt::{hash, DEFAULT_COST};
 use diesel::{dsl::*, result::Error, *};
-use serde::{Serialize};
+use serde::Serialize;
 
 #[derive(Clone, Queryable, Identifiable, PartialEq, Debug, Serialize)]
 #[table_name = "user_"]

server/lemmy_db/src/user_mention_view.rs

@@ -1,6 +1,6 @@
 use crate::{limit_and_offset, MaybeOptional, SortType};
 use diesel::{dsl::*, pg::Pg, result::Error, *};
-use serde::{Serialize};
+use serde::Serialize;
 
 // The faked schema since diesel doesn't do views
 table! {
@@ -83,9 +83,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "user_mention_fast_view"]
 pub struct UserMentionView {
   pub id: i32,

server/lemmy_db/src/user_view.rs

@@ -1,7 +1,7 @@
 use super::user_view::user_fast::BoxedQuery;
 use crate::{fuzzy_search, limit_and_offset, MaybeOptional, SortType};
 use diesel::{dsl::*, pg::Pg, result::Error, *};
-use serde::{Serialize};
+use serde::Serialize;
 
 table! {
   user_view (id) {
@@ -51,9 +51,7 @@ table! {
   }
 }
 
-#[derive(
+#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)]
-  Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone,
-)]
 #[table_name = "user_fast"]
 pub struct UserView {
   pub id: i32,

server/lemmy_rate_limit/src/lib.rs

@@ -8,8 +8,8 @@ extern crate tokio;
 use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform};
 use futures::future::{ok, Ready};
 use lemmy_utils::{
-  get_ip,
   settings::{RateLimitConfig, Settings},
+  utils::get_ip,
   LemmyError,
 };
 use rate_limiter::{RateLimitType, RateLimiter};

server/lemmy_utils/src/apub.rs

@@ -0,0 +1,64 @@
+use crate::settings::Settings;
+use openssl::{pkey::PKey, rsa::Rsa};
+use std::io::{Error, ErrorKind};
+use url::Url;
+
+pub struct Keypair {
+  pub private_key: String,
+  pub public_key: String,
+}
+
+/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
+pub fn generate_actor_keypair() -> Result<Keypair, Error> {
+  let rsa = Rsa::generate(2048)?;
+  let pkey = PKey::from_rsa(rsa)?;
+  let public_key = pkey.public_key_to_pem()?;
+  let private_key = pkey.private_key_to_pem_pkcs8()?;
+  let key_to_string = |key| match String::from_utf8(key) {
+    Ok(s) => Ok(s),
+    Err(e) => Err(Error::new(
+      ErrorKind::Other,
+      format!("Failed converting key to string: {}", e),
+    )),
+  };
+  Ok(Keypair {
+    private_key: key_to_string(private_key)?,
+    public_key: key_to_string(public_key)?,
+  })
+}
+
+pub enum EndpointType {
+  Community,
+  User,
+  Post,
+  Comment,
+  PrivateMessage,
+}
+
+pub fn get_apub_protocol_string() -> &'static str {
+  if Settings::get().federation.tls_enabled {
+    "https"
+  } else {
+    "http"
+  }
+}
+
+/// Generates the ActivityPub ID for a given object type and ID.
+pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url {
+  let point = match endpoint_type {
+    EndpointType::Community => "c",
+    EndpointType::User => "u",
+    EndpointType::Post => "post",
+    EndpointType::Comment => "comment",
+    EndpointType::PrivateMessage => "private_message",
+  };
+
+  Url::parse(&format!(
+    "{}://{}/{}/{}",
+    get_apub_protocol_string(),
+    Settings::get().hostname,
+    point,
+    name
+  ))
+  .unwrap()
+}

server/lemmy_utils/src/email.rs

@@ -0,0 +1,55 @@
+use crate::settings::Settings;
+use lettre::{
+  smtp::{
+    authentication::{Credentials, Mechanism},
+    extension::ClientId,
+    ConnectionReuseParameters,
+  },
+  ClientSecurity,
+  SmtpClient,
+  Transport,
+};
+use lettre_email::Email;
+
+pub fn send_email(
+  subject: &str,
+  to_email: &str,
+  to_username: &str,
+  html: &str,
+) -> Result<(), String> {
+  let email_config = Settings::get().email.ok_or("no_email_setup")?;
+
+  let email = Email::builder()
+    .to((to_email, to_username))
+    .from(email_config.smtp_from_address.to_owned())
+    .subject(subject)
+    .html(html)
+    .build()
+    .unwrap();
+
+  let mailer = if email_config.use_tls {
+    SmtpClient::new_simple(&email_config.smtp_server).unwrap()
+  } else {
+    SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap()
+  }
+  .hello_name(ClientId::Domain(Settings::get().hostname))
+  .smtp_utf8(true)
+  .authentication_mechanism(Mechanism::Plain)
+  .connection_reuse(ConnectionReuseParameters::ReuseUnlimited);
+  let mailer = if let (Some(login), Some(password)) =
+    (&email_config.smtp_login, &email_config.smtp_password)
+  {
+    mailer.credentials(Credentials::new(login.to_owned(), password.to_owned()))
+  } else {
+    mailer
+  };
+
+  let mut transport = mailer.transport();
+  let result = transport.send(email.into());
+  transport.close();
+
+  match result {
+    Ok(_) => Ok(()),
+    Err(e) => Err(e.to_string()),
+  }
+}

server/lemmy_utils/src/lib.rs

@@ -12,29 +12,16 @@ extern crate serde_json;
 extern crate thiserror;
 extern crate url;
 
+pub mod apub;
+pub mod email;
 pub mod settings;
+#[cfg(test)]
+mod test;
+pub mod utils;
 
 use crate::settings::Settings;
-use actix_web::dev::ConnectionInfo;
+use regex::Regex;
-use chrono::{DateTime, FixedOffset, Local, NaiveDateTime};
-use itertools::Itertools;
-use lettre::{
-  smtp::{
-    authentication::{Credentials, Mechanism},
-    extension::ClientId,
-    ConnectionReuseParameters,
-  },
-  ClientSecurity,
-  SmtpClient,
-  Transport,
-};
-use lettre_email::Email;
-use openssl::{pkey::PKey, rsa::Rsa};
-use rand::{distributions::Alphanumeric, thread_rng, Rng};
-use regex::{Regex, RegexBuilder};
-use std::io::{Error, ErrorKind};
 use thiserror::Error;
-use url::Url;
 
 pub type ConnectionId = usize;
 pub type PostId = i32;
@@ -90,236 +77,7 @@ impl std::fmt::Display for LemmyError {
 
 impl actix_web::error::ResponseError for LemmyError {}
 
-pub fn naive_from_unix(time: i64) -> NaiveDateTime {
-  NaiveDateTime::from_timestamp(time, 0)
-}
-
-pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime<FixedOffset> {
-  let now = Local::now();
-  DateTime::<FixedOffset>::from_utc(datetime, *now.offset())
-}
-
-pub fn remove_slurs(test: &str) -> String {
-  SLUR_REGEX.replace_all(test, "*removed*").to_string()
-}
-
-pub fn slur_check(test: &str) -> Result<(), Vec<&str>> {
-  let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect();
-
-  // Unique
-  matches.sort_unstable();
-  matches.dedup();
-
-  if matches.is_empty() {
-    Ok(())
-  } else {
-    Err(matches)
-  }
-}
-
-pub fn slurs_vec_to_str(slurs: Vec<&str>) -> String {
-  let start = "No slurs - ";
-  let combined = &slurs.join(", ");
-  [start, combined].concat()
-}
-
-pub fn generate_random_string() -> String {
-  thread_rng().sample_iter(&Alphanumeric).take(30).collect()
-}
-
-pub fn send_email(
-  subject: &str,
-  to_email: &str,
-  to_username: &str,
-  html: &str,
-) -> Result<(), String> {
-  let email_config = Settings::get().email.ok_or("no_email_setup")?;
-
-  let email = Email::builder()
-    .to((to_email, to_username))
-    .from(email_config.smtp_from_address.to_owned())
-    .subject(subject)
-    .html(html)
-    .build()
-    .unwrap();
-
-  let mailer = if email_config.use_tls {
-    SmtpClient::new_simple(&email_config.smtp_server).unwrap()
-  } else {
-    SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap()
-  }
-  .hello_name(ClientId::Domain(Settings::get().hostname))
-  .smtp_utf8(true)
-  .authentication_mechanism(Mechanism::Plain)
-  .connection_reuse(ConnectionReuseParameters::ReuseUnlimited);
-  let mailer = if let (Some(login), Some(password)) =
-    (&email_config.smtp_login, &email_config.smtp_password)
-  {
-    mailer.credentials(Credentials::new(login.to_owned(), password.to_owned()))
-  } else {
-    mailer
-  };
-
-  let mut transport = mailer.transport();
-  let result = transport.send(email.into());
-  transport.close();
-
-  match result {
-    Ok(_) => Ok(()),
-    Err(e) => Err(e.to_string()),
-  }
-}
-
-pub fn markdown_to_html(text: &str) -> String {
-  comrak::markdown_to_html(text, &comrak::ComrakOptions::default())
-}
-
-// TODO nothing is done with community / group webfingers yet, so just ignore those for now
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub struct MentionData {
-  pub name: String,
-  pub domain: String,
-}
-
-impl MentionData {
-  pub fn is_local(&self) -> bool {
-    Settings::get().hostname.eq(&self.domain)
-  }
-  pub fn full_name(&self) -> String {
-    format!("@{}@{}", &self.name, &self.domain)
-  }
-}
-
-pub fn scrape_text_for_mentions(text: &str) -> Vec<MentionData> {
-  let mut out: Vec<MentionData> = Vec::new();
-  for caps in MENTIONS_REGEX.captures_iter(text) {
-    out.push(MentionData {
-      name: caps["name"].to_string(),
-      domain: caps["domain"].to_string(),
-    });
-  }
-  out.into_iter().unique().collect()
-}
-
-pub fn is_valid_username(name: &str) -> bool {
-  VALID_USERNAME_REGEX.is_match(name)
-}
-
-// Can't do a regex here, reverse lookarounds not supported
-pub fn is_valid_preferred_username(preferred_username: &str) -> bool {
-  !preferred_username.starts_with('@')
-    && preferred_username.len() >= 3
-    && preferred_username.len() <= 20
-}
-
-pub fn is_valid_community_name(name: &str) -> bool {
-  VALID_COMMUNITY_NAME_REGEX.is_match(name)
-}
-
-pub fn is_valid_post_title(title: &str) -> bool {
-  VALID_POST_TITLE_REGEX.is_match(title)
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::{
-    is_valid_community_name,
-    is_valid_post_title,
-    is_valid_preferred_username,
-    is_valid_username,
-    remove_slurs,
-    scrape_text_for_mentions,
-    slur_check,
-    slurs_vec_to_str,
-  };
-
-  #[test]
-  fn test_mentions_regex() {
-    let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)";
-    let mentions = scrape_text_for_mentions(text);
-
-    assert_eq!(mentions[0].name, "tedu".to_string());
-    assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string());
-    assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string());
-  }
-
-  #[test]
-  fn test_valid_register_username() {
-    assert!(is_valid_username("Hello_98"));
-    assert!(is_valid_username("ten"));
-    assert!(!is_valid_username("Hello-98"));
-    assert!(!is_valid_username("a"));
-    assert!(!is_valid_username(""));
-  }
-
-  #[test]
-  fn test_valid_preferred_username() {
-    assert!(is_valid_preferred_username("hello @there"));
-    assert!(!is_valid_preferred_username("@hello there"));
-  }
-
-  #[test]
-  fn test_valid_community_name() {
-    assert!(is_valid_community_name("example"));
-    assert!(is_valid_community_name("example_community"));
-    assert!(!is_valid_community_name("Example"));
-    assert!(!is_valid_community_name("Ex"));
-    assert!(!is_valid_community_name(""));
-  }
-
-  #[test]
-  fn test_valid_post_title() {
-    assert!(is_valid_post_title("Post Title"));
-    assert!(is_valid_post_title("   POST TITLE 😃😃😃😃😃"));
-    assert!(!is_valid_post_title("\n \n \n \n    		")); // tabs/spaces/newlines
-  }
-
-  #[test]
-  fn test_slur_filter() {
-    let test =
-      "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text.";
-    let slur_free = "No slurs here";
-    assert_eq!(
-      remove_slurs(&test),
-      "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text."
-        .to_string()
-    );
-
-    let has_slurs_vec = vec![
-      "Niggerz",
-      "coons",
-      "dindu",
-      "ladyboy",
-      "retardeds",
-      "tranny",
-    ];
-    let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny";
-
-    assert_eq!(slur_check(test), Err(has_slurs_vec));
-    assert_eq!(slur_check(slur_free), Ok(()));
-    if let Err(slur_vec) = slur_check(test) {
-      assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str);
-    }
-  }
-
-  // These helped with testing
-  // #[test]
-  // fn test_send_email() {
-  //  let result =  send_email("not a subject", "test_email@gmail.com", "ur user", "<h1>HI there</h1>");
-  //   assert!(result.is_ok());
-  // }
-}
-
 lazy_static! {
-  static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap();
-  static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap();
-  static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap();
-  // TODO keep this old one, it didn't work with port well tho
-  // static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap();
-  static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._:-]+)").unwrap();
-  static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap();
-  static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap();
-  static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").unwrap();
   pub static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!(
     "^group:([a-z0-9_]{{3, 20}})@{}$",
     Settings::get().hostname
@@ -333,73 +91,3 @@ lazy_static! {
   pub static ref CACHE_CONTROL_REGEX: Regex =
     Regex::new("^((text|image)/.+|application/javascript)$").unwrap();
 }
-
-pub struct Keypair {
-  pub private_key: String,
-  pub public_key: String,
-}
-
-/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
-pub fn generate_actor_keypair() -> Result<Keypair, Error> {
-  let rsa = Rsa::generate(2048)?;
-  let pkey = PKey::from_rsa(rsa)?;
-  let public_key = pkey.public_key_to_pem()?;
-  let private_key = pkey.private_key_to_pem_pkcs8()?;
-  let key_to_string = |key| match String::from_utf8(key) {
-    Ok(s) => Ok(s),
-    Err(e) => Err(Error::new(
-      ErrorKind::Other,
-      format!("Failed converting key to string: {}", e),
-    )),
-  };
-  Ok(Keypair {
-    private_key: key_to_string(private_key)?,
-    public_key: key_to_string(public_key)?,
-  })
-}
-
-pub enum EndpointType {
-  Community,
-  User,
-  Post,
-  Comment,
-  PrivateMessage,
-}
-
-pub fn get_apub_protocol_string() -> &'static str {
-  if Settings::get().federation.tls_enabled {
-    "https"
-  } else {
-    "http"
-  }
-}
-
-/// Generates the ActivityPub ID for a given object type and ID.
-pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url {
-  let point = match endpoint_type {
-    EndpointType::Community => "c",
-    EndpointType::User => "u",
-    EndpointType::Post => "post",
-    EndpointType::Comment => "comment",
-    EndpointType::PrivateMessage => "private_message",
-  };
-
-  Url::parse(&format!(
-    "{}://{}/{}/{}",
-    get_apub_protocol_string(),
-    Settings::get().hostname,
-    point,
-    name
-  ))
-  .unwrap()
-}
-
-pub fn get_ip(conn_info: &ConnectionInfo) -> String {
-  conn_info
-    .realip_remote_addr()
-    .unwrap_or("127.0.0.1:12345")
-    .split(':')
-    .next()
-    .unwrap_or("127.0.0.1")
-    .to_string()
-}

server/lemmy_utils/src/test.rs

@@ -0,0 +1,86 @@
+use crate::utils::{
+  is_valid_community_name,
+  is_valid_post_title,
+  is_valid_preferred_username,
+  is_valid_username,
+  remove_slurs,
+  scrape_text_for_mentions,
+  slur_check,
+  slurs_vec_to_str,
+};
+
+#[test]
+fn test_mentions_regex() {
+  let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)";
+  let mentions = scrape_text_for_mentions(text);
+
+  assert_eq!(mentions[0].name, "tedu".to_string());
+  assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string());
+  assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string());
+}
+
+#[test]
+fn test_valid_register_username() {
+  assert!(is_valid_username("Hello_98"));
+  assert!(is_valid_username("ten"));
+  assert!(!is_valid_username("Hello-98"));
+  assert!(!is_valid_username("a"));
+  assert!(!is_valid_username(""));
+}
+
+#[test]
+fn test_valid_preferred_username() {
+  assert!(is_valid_preferred_username("hello @there"));
+  assert!(!is_valid_preferred_username("@hello there"));
+}
+
+#[test]
+fn test_valid_community_name() {
+  assert!(is_valid_community_name("example"));
+  assert!(is_valid_community_name("example_community"));
+  assert!(!is_valid_community_name("Example"));
+  assert!(!is_valid_community_name("Ex"));
+  assert!(!is_valid_community_name(""));
+}
+
+#[test]
+fn test_valid_post_title() {
+  assert!(is_valid_post_title("Post Title"));
+  assert!(is_valid_post_title("   POST TITLE 😃😃😃😃😃"));
+  assert!(!is_valid_post_title("\n \n \n \n    		")); // tabs/spaces/newlines
+}
+
+#[test]
+fn test_slur_filter() {
+  let test =
+      "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text.";
+  let slur_free = "No slurs here";
+  assert_eq!(
+      remove_slurs(&test),
+      "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text."
+        .to_string()
+    );
+
+  let has_slurs_vec = vec![
+    "Niggerz",
+    "coons",
+    "dindu",
+    "ladyboy",
+    "retardeds",
+    "tranny",
+  ];
+  let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny";
+
+  assert_eq!(slur_check(test), Err(has_slurs_vec));
+  assert_eq!(slur_check(slur_free), Ok(()));
+  if let Err(slur_vec) = slur_check(test) {
+    assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str);
+  }
+}
+
+// These helped with testing
+// #[test]
+// fn test_send_email() {
+//  let result =  send_email("not a subject", "test_email@gmail.com", "ur user", "<h1>HI there</h1>");
+//   assert!(result.is_ok());
+// }

server/lemmy_utils/src/utils.rs

@@ -0,0 +1,130 @@
+use crate::{settings::Settings, APIError};
+use actix_web::dev::ConnectionInfo;
+use chrono::{DateTime, FixedOffset, Local, NaiveDateTime};
+use itertools::Itertools;
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use regex::{Regex, RegexBuilder};
+
+lazy_static! {
+static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap();
+static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap();
+static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap();
+// TODO keep this old one, it didn't work with port well tho
+// static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap();
+static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._:-]+)").unwrap();
+static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap();
+static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap();
+static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").unwrap();
+}
+
+pub fn naive_from_unix(time: i64) -> NaiveDateTime {
+  NaiveDateTime::from_timestamp(time, 0)
+}
+
+pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime<FixedOffset> {
+  let now = Local::now();
+  DateTime::<FixedOffset>::from_utc(datetime, *now.offset())
+}
+
+pub fn remove_slurs(test: &str) -> String {
+  SLUR_REGEX.replace_all(test, "*removed*").to_string()
+}
+
+pub(crate) fn slur_check(test: &str) -> Result<(), Vec<&str>> {
+  let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect();
+
+  // Unique
+  matches.sort_unstable();
+  matches.dedup();
+
+  if matches.is_empty() {
+    Ok(())
+  } else {
+    Err(matches)
+  }
+}
+
+pub fn check_slurs(text: &str) -> Result<(), APIError> {
+  if let Err(slurs) = slur_check(text) {
+    Err(APIError::err(&slurs_vec_to_str(slurs)))
+  } else {
+    Ok(())
+  }
+}
+
+pub fn check_slurs_opt(text: &Option<String>) -> Result<(), APIError> {
+  match text {
+    Some(t) => check_slurs(t),
+    None => Ok(()),
+  }
+}
+
+pub(crate) fn slurs_vec_to_str(slurs: Vec<&str>) -> String {
+  let start = "No slurs - ";
+  let combined = &slurs.join(", ");
+  [start, combined].concat()
+}
+
+pub fn generate_random_string() -> String {
+  thread_rng().sample_iter(&Alphanumeric).take(30).collect()
+}
+
+pub fn markdown_to_html(text: &str) -> String {
+  comrak::markdown_to_html(text, &comrak::ComrakOptions::default())
+}
+
+// TODO nothing is done with community / group webfingers yet, so just ignore those for now
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct MentionData {
+  pub name: String,
+  pub domain: String,
+}
+
+impl MentionData {
+  pub fn is_local(&self) -> bool {
+    Settings::get().hostname.eq(&self.domain)
+  }
+  pub fn full_name(&self) -> String {
+    format!("@{}@{}", &self.name, &self.domain)
+  }
+}
+
+pub fn scrape_text_for_mentions(text: &str) -> Vec<MentionData> {
+  let mut out: Vec<MentionData> = Vec::new();
+  for caps in MENTIONS_REGEX.captures_iter(text) {
+    out.push(MentionData {
+      name: caps["name"].to_string(),
+      domain: caps["domain"].to_string(),
+    });
+  }
+  out.into_iter().unique().collect()
+}
+
+pub fn is_valid_username(name: &str) -> bool {
+  VALID_USERNAME_REGEX.is_match(name)
+}
+
+// Can't do a regex here, reverse lookarounds not supported
+pub fn is_valid_preferred_username(preferred_username: &str) -> bool {
+  !preferred_username.starts_with('@')
+    && preferred_username.len() >= 3
+    && preferred_username.len() <= 20
+}
+
+pub fn is_valid_community_name(name: &str) -> bool {
+  VALID_COMMUNITY_NAME_REGEX.is_match(name)
+}
+
+pub fn is_valid_post_title(title: &str) -> bool {
+  VALID_POST_TITLE_REGEX.is_match(title)
+}
+
+pub fn get_ip(conn_info: &ConnectionInfo) -> String {
+  conn_info
+    .realip_remote_addr()
+    .unwrap_or("127.0.0.1:12345")
+    .split(':')
+    .next()
+    .unwrap_or("127.0.0.1")
+    .to_string()
+}

server/src/api/comment.rs

@@ -33,16 +33,13 @@ use lemmy_db::{
   SortType,
 };
 use lemmy_utils::{
-  make_apub_endpoint,
+  apub::{make_apub_endpoint, EndpointType},
-  remove_slurs,
+  email::send_email,
-  scrape_text_for_mentions,
-  send_email,
   settings::Settings,
+  utils::{remove_slurs, scrape_text_for_mentions, MentionData},
   APIError,
   ConnectionId,
-  EndpointType,
   LemmyError,
-  MentionData,
 };
 use log::error;
 use std::str::FromStr;

server/src/api/community.rs

@@ -1,13 +1,5 @@
 use crate::{
-  api::{
+  api::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, is_mod_or_admin, Perform},
-    check_slurs,
-    check_slurs_opt,
-    get_user_from_jwt,
-    get_user_from_jwt_opt,
-    is_admin,
-    is_mod_or_admin,
-    Perform,
-  },
   apub::ActorType,
   blocking,
   websocket::{
@@ -37,14 +29,11 @@ use lemmy_db::{
   SortType,
 };
 use lemmy_utils::{
-  generate_actor_keypair,
+  apub::{generate_actor_keypair, make_apub_endpoint, EndpointType},
-  is_valid_community_name,
   location_info,
-  make_apub_endpoint,
+  utils::{check_slurs, check_slurs_opt, is_valid_community_name, naive_from_unix},
-  naive_from_unix,
   APIError,
   ConnectionId,
-  EndpointType,
   LemmyError,
 };
 use std::str::FromStr;

server/src/api/mod.rs

@@ -7,7 +7,7 @@ use lemmy_db::{
   user::User_,
   Crud,
 };
-use lemmy_utils::{slur_check, slurs_vec_to_str, APIError, ConnectionId, LemmyError};
+use lemmy_utils::{APIError, ConnectionId, LemmyError};
 
 pub mod claims;
 pub mod comment;
@@ -83,19 +83,6 @@ pub(in crate::api) async fn get_user_from_jwt_opt(
   }
 }
 
-pub(in crate) fn check_slurs(text: &str) -> Result<(), APIError> {
-  if let Err(slurs) = slur_check(text) {
-    Err(APIError::err(&slurs_vec_to_str(slurs)))
-  } else {
-    Ok(())
-  }
-}
-pub(in crate) fn check_slurs_opt(text: &Option<String>) -> Result<(), APIError> {
-  match text {
-    Some(t) => check_slurs(t),
-    None => Ok(()),
-  }
-}
 pub(in crate::api) async fn check_community_ban(
   user_id: i32,
   community_id: i32,

server/src/api/post.rs

@@ -1,13 +1,5 @@
 use crate::{
-  api::{
+  api::{check_community_ban, get_user_from_jwt, get_user_from_jwt_opt, is_mod_or_admin, Perform},
-    check_community_ban,
-    check_slurs,
-    check_slurs_opt,
-    get_user_from_jwt,
-    get_user_from_jwt_opt,
-    is_mod_or_admin,
-    Perform,
-  },
   apub::{ApubLikeableType, ApubObjectType},
   blocking,
   fetch_iframely_and_pictrs_data,
@@ -34,11 +26,10 @@ use lemmy_db::{
   SortType,
 };
 use lemmy_utils::{
-  is_valid_post_title,
+  apub::{make_apub_endpoint, EndpointType},
-  make_apub_endpoint,
+  utils::{check_slurs, check_slurs_opt, is_valid_post_title},
   APIError,
   ConnectionId,
-  EndpointType,
   LemmyError,
 };
 use std::str::FromStr;

server/src/api/site.rs

@@ -1,12 +1,5 @@
 use crate::{
-  api::{
+  api::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform},
-    check_slurs,
-    check_slurs_opt,
-    get_user_from_jwt,
-    get_user_from_jwt_opt,
-    is_admin,
-    Perform,
-  },
   apub::fetcher::search_by_apub_id,
   blocking,
   version,
@@ -35,7 +28,14 @@ use lemmy_db::{
   SearchType,
   SortType,
 };
-use lemmy_utils::{location_info, settings::Settings, APIError, ConnectionId, LemmyError};
+use lemmy_utils::{
+  location_info,
+  settings::Settings,
+  utils::{check_slurs, check_slurs_opt},
+  APIError,
+  ConnectionId,
+  LemmyError,
+};
 use log::{debug, info};
 use std::str::FromStr;
 

server/src/api/user.rs

@@ -1,5 +1,5 @@
 use crate::{
-  api::{check_slurs, claims::Claims, get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform},
+  api::{claims::Claims, get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform},
   apub::ApubObjectType,
   blocking,
   captcha_espeak_wav_base64,
@@ -41,19 +41,20 @@ use lemmy_db::{
   SortType,
 };
 use lemmy_utils::{
-  generate_actor_keypair,
+  apub::{generate_actor_keypair, make_apub_endpoint, EndpointType},
-  generate_random_string,
+  email::send_email,
-  is_valid_preferred_username,
-  is_valid_username,
   location_info,
-  make_apub_endpoint,
-  naive_from_unix,
-  remove_slurs,
-  send_email,
   settings::Settings,
+  utils::{
+    check_slurs,
+    generate_random_string,
+    is_valid_preferred_username,
+    is_valid_username,
+    naive_from_unix,
+    remove_slurs,
+  },
   APIError,
   ConnectionId,
-  EndpointType,
   LemmyError,
 };
 use log::error;

server/src/apub/activities.rs

@@ -7,7 +7,7 @@ use activitystreams::{
   object::AsObject,
 };
 use lemmy_db::{community::Community, user::User_};
-use lemmy_utils::{get_apub_protocol_string, settings::Settings, LemmyError};
+use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
 use serde::{export::fmt::Debug, Serialize};
 use url::{ParseError, Url};
 use uuid::Uuid;

server/src/apub/comment.rs

@@ -49,12 +49,9 @@ use lemmy_db::{
   Crud,
 };
 use lemmy_utils::{
-  convert_datetime,
   location_info,
-  remove_slurs,
+  utils::{convert_datetime, remove_slurs, scrape_text_for_mentions, MentionData},
-  scrape_text_for_mentions,
   LemmyError,
-  MentionData,
 };
 use log::debug;
 use serde::Deserialize;

server/src/apub/community.rs

@@ -1,5 +1,4 @@
 use crate::{
-  api::{check_slurs, check_slurs_opt},
   apub::{
     activities::generate_activity_id,
     activity_queue::send_activity,
@@ -47,7 +46,12 @@ use lemmy_db::{
   post::Post,
   user::User_,
 };
-use lemmy_utils::{convert_datetime, get_apub_protocol_string, location_info, LemmyError};
+use lemmy_utils::{
+  apub::get_apub_protocol_string,
+  location_info,
+  utils::{check_slurs, check_slurs_opt, convert_datetime},
+  LemmyError,
+};
 use serde::Deserialize;
 use url::Url;
 

server/src/apub/fetcher.rs

@@ -31,7 +31,7 @@ use lemmy_db::{
   Joinable,
   SearchType,
 };
-use lemmy_utils::{get_apub_protocol_string, location_info, LemmyError};
+use lemmy_utils::{apub::get_apub_protocol_string, location_info, LemmyError};
 use log::debug;
 use reqwest::Client;
 use serde::Deserialize;

server/src/apub/inbox/activities/create.rs

@@ -27,7 +27,7 @@ use lemmy_db::{
   post::{Post, PostForm},
   post_view::PostView,
 };
-use lemmy_utils::{location_info, scrape_text_for_mentions, LemmyError};
+use lemmy_utils::{location_info, utils::scrape_text_for_mentions, LemmyError};
 
 pub async fn receive_create(
   activity: AnyBase,

server/src/apub/inbox/activities/update.rs

@@ -29,7 +29,7 @@ use lemmy_db::{
   post_view::PostView,
   Crud,
 };
-use lemmy_utils::{location_info, scrape_text_for_mentions, LemmyError};
+use lemmy_utils::{location_info, utils::scrape_text_for_mentions, LemmyError};
 
 pub async fn receive_update(
   activity: AnyBase,

server/src/apub/mod.rs

@@ -35,12 +35,11 @@ use anyhow::{anyhow, Context};
 use chrono::NaiveDateTime;
 use lemmy_db::{activity::do_insert_activity, user::User_};
 use lemmy_utils::{
-  convert_datetime,
+  apub::get_apub_protocol_string,
-  get_apub_protocol_string,
   location_info,
   settings::Settings,
+  utils::{convert_datetime, MentionData},
   LemmyError,
-  MentionData,
 };
 use log::debug;
 use reqwest::Client;

server/src/apub/post.rs

@@ -1,5 +1,4 @@
 use crate::{
-  api::check_slurs,
   apub::{
     activities::{generate_activity_id, send_activity_to_community},
     check_actor_domain,
@@ -43,7 +42,11 @@ use lemmy_db::{
   user::User_,
   Crud,
 };
-use lemmy_utils::{convert_datetime, location_info, remove_slurs, LemmyError};
+use lemmy_utils::{
+  location_info,
+  utils::{check_slurs, convert_datetime, remove_slurs},
+  LemmyError,
+};
 use serde::Deserialize;
 use url::Url;
 

server/src/apub/private_message.rs

@@ -33,7 +33,7 @@ use lemmy_db::{
   user::User_,
   Crud,
 };
-use lemmy_utils::{convert_datetime, location_info, LemmyError};
+use lemmy_utils::{location_info, utils::convert_datetime, LemmyError};
 use url::Url;
 
 #[async_trait::async_trait(?Send)]

server/src/apub/user.rs

@@ -1,5 +1,4 @@
 use crate::{
-  api::{check_slurs, check_slurs_opt},
   apub::{
     activities::generate_activity_id,
     activity_queue::send_activity,
@@ -33,7 +32,11 @@ use lemmy_db::{
   naive_now,
   user::{UserForm, User_},
 };
-use lemmy_utils::{convert_datetime, location_info, LemmyError};
+use lemmy_utils::{
+  location_info,
+  utils::{check_slurs, check_slurs_opt, convert_datetime},
+  LemmyError,
+};
 use serde::Deserialize;
 use url::Url;
 

server/src/code_migrations.rs

@@ -13,11 +13,8 @@ use lemmy_db::{
   Crud,
 };
 use lemmy_utils::{
-  generate_actor_keypair,
+  apub::{generate_actor_keypair, get_apub_protocol_string, make_apub_endpoint, EndpointType},
-  get_apub_protocol_string,
-  make_apub_endpoint,
   settings::Settings,
-  EndpointType,
   LemmyError,
 };
 use log::info;

server/src/lib.rs

@@ -36,7 +36,7 @@ use crate::{
 use actix::Addr;
 use anyhow::anyhow;
 use background_jobs::QueueHandle;
-use lemmy_utils::{get_apub_protocol_string, settings::Settings, LemmyError};
+use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
 use log::error;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use reqwest::Client;

server/src/routes/feeds.rs

@@ -13,7 +13,7 @@ use lemmy_db::{
   ListingType,
   SortType,
 };
-use lemmy_utils::{markdown_to_html, settings::Settings, LemmyError};
+use lemmy_utils::{settings::Settings, utils::markdown_to_html, LemmyError};
 use rss::{CategoryBuilder, ChannelBuilder, GuidBuilder, Item, ItemBuilder};
 use serde::Deserialize;
 use std::str::FromStr;

server/src/routes/nodeinfo.rs

@@ -2,7 +2,7 @@ use crate::{blocking, version, LemmyContext};
 use actix_web::{body::Body, error::ErrorBadRequest, *};
 use anyhow::anyhow;
 use lemmy_db::site_view::SiteView;
-use lemmy_utils::{get_apub_protocol_string, settings::Settings, LemmyError};
+use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
 use serde::{Deserialize, Serialize};
 use url::Url;
 

server/src/routes/websocket.rs

@@ -8,7 +8,7 @@ use crate::{
 use actix::prelude::*;
 use actix_web::*;
 use actix_web_actors::ws;
-use lemmy_utils::get_ip;
+use lemmy_utils::utils::get_ip;
 use log::{debug, error, info};
 use std::time::{Duration, Instant};
 

server/test.sh

@@ -2,4 +2,4 @@
 export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
 diesel migration run
 export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
-RUST_TEST_THREADS=1 cargo test --workspace
+RUST_TEST_THREADS=1 cargo test --workspace --no-fail-fast