Finishing up fixing imports.

Cargo.lock

@@ -2410,6 +2410,7 @@ dependencies = [
  "tracing",
  "tracing-opentelemetry 0.17.4",
  "url",
+ "uuid",
  "webpage",
 ]
 

crates/api/src/community/ban.rs

@@ -18,7 +18,7 @@ use lemmy_db_schema::{
   },
   traits::{Bannable, Crud, Followable},
 };
-use lemmy_db_views_actor::structs::PersonView;
+use lemmy_db_views_actor::structs::PersonViewSafe;
 use lemmy_utils::{
   error::LemmyError,
   utils::{time::naive_from_unix, validation::is_valid_body_field},

crates/api/src/lib.rs

@@ -2,7 +2,7 @@ use actix_web::web::Data;
 use captcha::Captcha;
 use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex};
 use lemmy_db_schema::source::local_site::LocalSite;
-use lemmy_utils::{error::LemmyError, utils::check_slurs, ConnectionId};
+use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs, ConnectionId};
 
 mod comment;
 mod comment_report;

crates/api/src/local_user/ban_person.rs

@@ -13,7 +13,7 @@ use lemmy_db_schema::{
   },
   traits::Crud,
 };
-use lemmy_db_views_actor::structs::PersonView;
+use lemmy_db_views_actor::structs::PersonViewSafe;
 use lemmy_utils::{
   error::LemmyError,
   utils::{time::naive_from_unix, validation::is_valid_body_field},

crates/api/src/local_user/save_settings.rs

@@ -8,23 +8,17 @@ use lemmy_api_common::{
 use lemmy_db_schema::{
   source::{
     actor_language::LocalUserLanguage,
-    local_site::LocalSite,
     local_user::{LocalUser, LocalUserUpdateForm},
     person::{Person, PersonUpdateForm},
   },
   traits::Crud,
   utils::{diesel_option_overwrite, diesel_option_overwrite_to_url},
 };
+use lemmy_db_views::structs::SiteView;
 use lemmy_utils::{
   claims::Claims,
   error::LemmyError,
-  utils::validation::{
-    build_totp_2fa,
-    generate_totp_2fa_secret,
-    is_valid_bio_field,
-    is_valid_display_name,
-    is_valid_matrix_id,
-  },
+  utils::validation::{is_valid_bio_field, is_valid_display_name, is_valid_matrix_id},
   ConnectionId,
 };
 
@@ -41,7 +35,7 @@ impl Perform for SaveUserSettings {
     let data: &SaveUserSettings = self;
     let local_user_view =
       get_local_user_view_from_jwt(&data.auth, context.pool(), context.secret()).await?;
-    let local_site = LocalSite::read(context.pool()).await?;
+    let site_view = SiteView::read_local(context.pool()).await?;
 
     let avatar = diesel_option_overwrite_to_url(&data.avatar)?;
     let banner = diesel_option_overwrite_to_url(&data.banner)?;
@@ -63,7 +57,7 @@ impl Perform for SaveUserSettings {
 
     // When the site requires email, make sure email is not Some(None). IE, an overwrite to a None value
     if let Some(email) = &email {
-      if email.is_none() && local_site.require_email_verification {
+      if email.is_none() && site_view.local_site.require_email_verification {
         return Err(LemmyError::from_message("email_required"));
       }
     }

crates/api_common/Cargo.toml

@@ -47,6 +47,7 @@ tracing-opentelemetry = { workspace = true }
 actix-ws = { workspace = true }
 futures = { workspace = true }
 background-jobs = "0.13.0"
+uuid = { workspace = true }
 
 [dev-dependencies]
 actix-rt = { workspace = true }

crates/api_common/src/utils.rs

@@ -38,7 +38,7 @@ use lemmy_utils::{
   location_info,
   rate_limit::RateLimitConfig,
   settings::structs::Settings,
-  utils::{build_slur_regex, generate_random_string},
+  utils::slurs::build_slur_regex,
 };
 use regex::Regex;
 use reqwest_middleware::ClientWithMiddleware;
@@ -373,7 +373,7 @@ pub async fn send_password_reset_email(
   settings: &Settings,
 ) -> Result<(), LemmyError> {
   // Generate a random token
-  let token = generate_random_string();
+  let token = uuid::Uuid::new_v4().to_string();
 
   // Insert the row
   let token2 = token.clone();
@@ -399,7 +399,7 @@ pub async fn send_verification_email(
   let form = EmailVerificationForm {
     local_user_id: user.local_user.id,
     email: new_email.to_string(),
-    verification_token: generate_random_string(),
+    verification_token: uuid::Uuid::new_v4().to_string(),
   };
   let verify_link = format!(
     "{}/verify_email/{}",

crates/api_common/src/websocket/send.rs

@@ -22,7 +22,7 @@ use lemmy_db_schema::{
 };
 use lemmy_db_views::structs::{CommentView, LocalUserView, PostView, PrivateMessageView};
 use lemmy_db_views_actor::structs::CommunityView;
-use lemmy_utils::{error::LemmyError, utils::MentionData, ConnectionId};
+use lemmy_utils::{error::LemmyError, utils::mention::MentionData, ConnectionId};
 
 #[tracing::instrument(skip_all)]
 pub async fn send_post_ws_message<OP: ToString + Send + OperationType + 'static>(

crates/api_crud/src/community/remove.rs

@@ -13,7 +13,7 @@ use lemmy_db_schema::{
   },
   traits::Crud,
 };
-use lemmy_utils::{error::LemmyError, utils::naive_from_unix, ConnectionId};
+use lemmy_utils::{error::LemmyError, utils::time::naive_from_unix, ConnectionId};
 
 #[async_trait::async_trait(?Send)]
 impl PerformCrud for RemoveCommunity {

crates/api_crud/src/user/create.rs

@@ -30,7 +30,10 @@ use lemmy_db_views::structs::{LocalUserView, SiteView};
 use lemmy_utils::{
   claims::Claims,
   error::LemmyError,
-  utils::{check_slurs, check_slurs_opt, is_valid_actor_name},
+  utils::{
+    slurs::{check_slurs, check_slurs_opt},
+    validation::is_valid_actor_name,
+  },
   ConnectionId,
 };
 

crates/apub/src/activities/block/block_user.rs

@@ -40,7 +40,7 @@ use lemmy_db_schema::{
   },
   traits::{Bannable, Crud, Followable},
 };
-use lemmy_utils::{error::LemmyError, utils::convert_datetime};
+use lemmy_utils::{error::LemmyError, utils::time::convert_datetime};
 use url::Url;
 
 impl BlockUser {

crates/apub/src/activities/block/mod.rs

@@ -21,7 +21,7 @@ use lemmy_db_schema::{
   utils::DbPool,
 };
 use lemmy_db_views::structs::SiteView;
-use lemmy_utils::{error::LemmyError, utils::naive_from_unix};
+use lemmy_utils::{error::LemmyError, utils::time::naive_from_unix};
 use serde::Deserialize;
 use url::Url;
 

crates/apub/src/activities/create_or_update/mod.rs

@@ -6,7 +6,7 @@ use lemmy_db_schema::{
   source::{comment::Comment, post::Post},
   traits::Crud,
 };
-use lemmy_utils::{error::LemmyError, utils::scrape_text_for_mentions};
+use lemmy_utils::{error::LemmyError, utils::mention::scrape_text_for_mentions};
 
 pub mod comment;
 pub mod post;

crates/apub/src/mentions.rs

@@ -13,7 +13,7 @@ use lemmy_db_schema::{
 };
 use lemmy_utils::{
   error::LemmyError,
-  utils::{scrape_text_for_mentions, MentionData},
+  utils::mention::{scrape_text_for_mentions, MentionData},
 };
 use serde::{Deserialize, Serialize};
 use serde_json::Value;

crates/apub/src/objects/comment.rs

@@ -33,7 +33,7 @@ use lemmy_db_schema::{
 };
 use lemmy_utils::{
   error::LemmyError,
-  utils::{convert_datetime, markdown_to_html, remove_slurs},
+  utils::{markdown::markdown_to_html, slurs::remove_slurs, time::convert_datetime},
 };
 use std::ops::Deref;
 use url::Url;

crates/apub/src/objects/community.rs

@@ -32,7 +32,7 @@ use lemmy_db_schema::{
 use lemmy_db_views_actor::structs::CommunityFollowerView;
 use lemmy_utils::{
   error::LemmyError,
-  utils::{convert_datetime, markdown_to_html},
+  utils::{markdown::markdown_to_html, time::convert_datetime},
 };
 use std::ops::Deref;
 use tracing::debug;

crates/apub/src/objects/instance.rs

@@ -31,7 +31,11 @@ use lemmy_db_schema::{
 };
 use lemmy_utils::{
   error::LemmyError,
-  utils::{check_slurs, check_slurs_opt, convert_datetime, markdown_to_html},
+  utils::{
+    markdown::markdown_to_html,
+    slurs::{check_slurs, check_slurs_opt},
+    time::convert_datetime,
+  },
 };
 use std::ops::Deref;
 use tracing::debug;

crates/apub/src/objects/mod.rs

@@ -88,9 +88,6 @@ pub(crate) mod tests {
 
   // TODO: would be nice if we didnt have to use a full context for tests.
   pub(crate) async fn init_context() -> LemmyContext {
-    async fn x() -> Result<String, LemmyError> {
-      Ok(String::new())
-    }
     // call this to run migrations
     let pool = build_db_pool_for_tests().await;
 

crates/apub/src/objects/person.rs

@@ -29,7 +29,11 @@ use lemmy_db_schema::{
 };
 use lemmy_utils::{
   error::LemmyError,
-  utils::{check_slurs, check_slurs_opt, convert_datetime, markdown_to_html},
+  utils::{
+    markdown::markdown_to_html,
+    slurs::{check_slurs, check_slurs_opt},
+    time::convert_datetime,
+  },
 };
 use std::ops::Deref;
 use url::Url;

crates/apub/src/objects/post.rs

@@ -42,7 +42,11 @@ use lemmy_db_schema::{
 };
 use lemmy_utils::{
   error::LemmyError,
-  utils::{check_slurs_opt, convert_datetime, markdown_to_html, remove_slurs},
+  utils::{
+    markdown::markdown_to_html,
+    slurs::{check_slurs_opt, remove_slurs},
+    time::convert_datetime,
+  },
 };
 use std::ops::Deref;
 use url::Url;

crates/apub/src/objects/private_message.rs

@@ -25,7 +25,7 @@ use lemmy_db_schema::{
 };
 use lemmy_utils::{
   error::LemmyError,
-  utils::{convert_datetime, markdown_to_html},
+  utils::{markdown::markdown_to_html, time::convert_datetime},
 };
 use std::ops::Deref;
 use url::Url;

crates/apub/src/protocol/objects/group.rs

@@ -28,7 +28,7 @@ use lemmy_db_schema::{
 };
 use lemmy_utils::{
   error::LemmyError,
-  utils::{check_slurs, check_slurs_opt},
+  utils::slurs::{check_slurs, check_slurs_opt},
 };
 use serde::{Deserialize, Serialize};
 use serde_with::skip_serializing_none;

crates/routes/src/feeds.rs

@@ -20,7 +20,7 @@ use lemmy_db_views_actor::{
   person_mention_view::PersonMentionQuery,
   structs::{CommentReplyView, PersonMentionView},
 };
-use lemmy_utils::{claims::Claims, error::LemmyError, utils::markdown_to_html};
+use lemmy_utils::{claims::Claims, error::LemmyError, utils::markdown::markdown_to_html};
 use once_cell::sync::Lazy;
 use rss::{
   extension::dublincore::DublinCoreExtensionBuilder,
@@ -276,7 +276,7 @@ async fn get_feed_front(
   channel_builder
     .namespaces(RSS_NAMESPACE.clone())
     .title(&format!("{} - Subscribed", site_view.site.name))
-    .link(protocol_and_hostname)
+    .link(protocol_and_hostname.to_string())
     .items(items);
 
   if let Some(site_desc) = site_view.site.description {
@@ -400,12 +400,15 @@ fn build_item(
   let dt = DateTime::<Utc>::from_utc(*published, Utc);
   i.pub_date(dt.to_rfc2822());
   i.comments(url.to_owned());
-  let guid = GuidBuilder::default().permalink(true).value(url).build();
-  i.guid(guid);
+  let guid = GuidBuilder::default()
+    .permalink(true)
+    .value(url.to_string())
+    .build();
+  i.guid(Some(guid));
   i.link(url.to_owned());
   // TODO add images
   let html = markdown_to_html(content);
-  i.description(html);
+  i.description(Some(html));
   Ok(i.build())
 }
 
@@ -420,7 +423,7 @@ fn create_post_items(
     let mut i = ItemBuilder::default();
     let mut dc_extension = DublinCoreExtensionBuilder::default();
 
-    i.title(p.post.name);
+    i.title(Some(p.post.name));
 
     dc_extension.creators(vec![p.creator.actor_id.to_string()]);
 
@@ -434,7 +437,7 @@ fn create_post_items(
       .permalink(true)
       .value(&post_url)
       .build();
-    i.guid(guid);
+    i.guid(Some(guid));
 
     let community_url = format!("{}/c/{}", protocol_and_hostname, p.community.name);
 
@@ -459,7 +462,7 @@ fn create_post_items(
       description.push_str(&html);
     }
 
-    i.description(description);
+    i.description(Some(description));
 
     i.dublin_core_ext(dc_extension.build());
     items.push(i.build());

crates/utils/src/error.rs

@@ -9,6 +9,8 @@ struct ApiError {
   error: String,
 }
 
+pub type LemmyResult<T> = Result<T, LemmyError>;
+
 pub struct LemmyError {
   pub message: Option<String>,
   pub inner: anyhow::Error,

crates/utils/src/lib.rs

@@ -11,8 +11,6 @@ pub mod settings;
 pub mod claims;
 pub mod error;
 pub mod request;
-#[cfg(test)]
-mod test;
 pub mod utils;
 pub mod version;
 

crates/utils/src/rate_limit/mod.rs

@@ -1,5 +1,5 @@
-use crate::{error::LemmyError, utils::get_ip, IpAddr};
-use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform};
+use crate::{error::LemmyError, IpAddr};
+use actix_web::dev::{ConnectionInfo, Service, ServiceRequest, ServiceResponse, Transform};
 use futures::future::{ok, Ready};
 use rate_limiter::{RateLimitStorage, RateLimitType};
 use serde::{Deserialize, Serialize};
@@ -220,3 +220,15 @@ where
     })
   }
 }
+
+fn get_ip(conn_info: &ConnectionInfo) -> IpAddr {
+  IpAddr(
+    conn_info
+      .realip_remote_addr()
+      .unwrap_or("127.0.0.1:12345")
+      .split(':')
+      .next()
+      .unwrap_or("127.0.0.1")
+      .to_string(),
+  )
+}

crates/utils/src/test.rs

@@ -1,98 +0,0 @@
-use crate::utils::{
-  is_valid_actor_name,
-  is_valid_display_name,
-  is_valid_matrix_id,
-  is_valid_post_title,
-  remove_slurs,
-  scrape_text_for_mentions,
-  slur_check,
-  slurs_vec_to_str,
-};
-use regex::RegexBuilder;
-
-#[test]
-fn test_mentions_regex() {
-  let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)";
-  let mentions = scrape_text_for_mentions(text);
-
-  assert_eq!(mentions[0].name, "tedu".to_string());
-  assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string());
-  assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string());
-}
-
-#[test]
-fn test_valid_actor_name() {
-  let actor_name_max_length = 20;
-  assert!(is_valid_actor_name("Hello_98", actor_name_max_length));
-  assert!(is_valid_actor_name("ten", actor_name_max_length));
-  assert!(!is_valid_actor_name("Hello-98", actor_name_max_length));
-  assert!(!is_valid_actor_name("a", actor_name_max_length));
-  assert!(!is_valid_actor_name("", actor_name_max_length));
-}
-
-#[test]
-fn test_valid_display_name() {
-  let actor_name_max_length = 20;
-  assert!(is_valid_display_name("hello @there", actor_name_max_length));
-  assert!(!is_valid_display_name(
-    "@hello there",
-    actor_name_max_length
-  ));
-
-  // Make sure zero-space with an @ doesn't work
-  assert!(!is_valid_display_name(
-    &format!("{}@my name is", '\u{200b}'),
-    actor_name_max_length
-  ));
-}
-
-#[test]
-fn test_valid_post_title() {
-  assert!(is_valid_post_title("Post Title"));
-  assert!(is_valid_post_title("   POST TITLE 😃😃😃😃😃"));
-  assert!(!is_valid_post_title("\n \n \n \n    		")); // tabs/spaces/newlines
-}
-
-#[test]
-fn test_valid_matrix_id() {
-  assert!(is_valid_matrix_id("@dess:matrix.org"));
-  assert!(!is_valid_matrix_id("dess:matrix.org"));
-  assert!(!is_valid_matrix_id(" @dess:matrix.org"));
-  assert!(!is_valid_matrix_id("@dess:matrix.org t"));
-}
-
-#[test]
-fn test_slur_filter() {
-  let slur_regex = Some(RegexBuilder::new(r"(fag(g|got|tard)?\b|cock\s?sucker(s|ing)?|ni((g{2,}|q)+|[gq]{2,})[e3r]+(s|z)?|mudslime?s?|kikes?|\bspi(c|k)s?\b|\bchinks?|gooks?|bitch(es|ing|y)?|whor(es?|ing)|\btr(a|@)nn?(y|ies?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap());
-  let test =
-      "faggot test kike tranny cocksucker retardeds. Capitalized Niggerz. This is a bunch of other safe text.";
-  let slur_free = "No slurs here";
-  assert_eq!(
-      remove_slurs(test, &slur_regex),
-      "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text."
-        .to_string()
-    );
-
-  let has_slurs_vec = vec![
-    "Niggerz",
-    "cocksucker",
-    "faggot",
-    "kike",
-    "retardeds",
-    "tranny",
-  ];
-  let has_slurs_err_str = "No slurs - Niggerz, cocksucker, faggot, kike, retardeds, tranny";
-
-  assert_eq!(slur_check(test, &slur_regex), Err(has_slurs_vec));
-  assert_eq!(slur_check(slur_free, &slur_regex), Ok(()));
-  if let Err(slur_vec) = slur_check(test, &slur_regex) {
-    assert_eq!(&slurs_vec_to_str(&slur_vec), has_slurs_err_str);
-  }
-}
-
-// These helped with testing
-// #[test]
-// fn test_send_email() {
-//  let result =  send_email("not a subject", "test_email@gmail.com", "ur user", "<h1>HI there</h1>");
-//   assert!(result.is_ok());
-// }

crates/utils/src/utils.rs

@@ -1,224 +0,0 @@
-use crate::{error::LemmyError, location_info, IpAddr};
-use actix_web::dev::ConnectionInfo;
-use anyhow::Context;
-use chrono::{DateTime, FixedOffset, NaiveDateTime};
-use itertools::Itertools;
-use once_cell::sync::Lazy;
-use rand::{distributions::Alphanumeric, thread_rng, Rng};
-use regex::{Regex, RegexBuilder};
-use url::Url;
-
-static MENTIONS_REGEX: Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._:-]+)").expect("compile regex")
-});
-static VALID_ACTOR_NAME_REGEX: Lazy<Regex> =
-  Lazy::new(|| Regex::new(r"^[a-zA-Z0-9_]{3,}$").expect("compile regex"));
-static VALID_POST_TITLE_REGEX: Lazy<Regex> =
-  Lazy::new(|| Regex::new(r".*\S{3,}.*").expect("compile regex"));
-static VALID_MATRIX_ID_REGEX: Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r"^@[A-Za-z0-9._=-]+:[A-Za-z0-9.-]+\.[A-Za-z]{2,}$").expect("compile regex")
-});
-// taken from https://en.wikipedia.org/wiki/UTM_parameters
-static CLEAN_URL_PARAMS_REGEX: Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r"^utm_source|utm_medium|utm_campaign|utm_term|utm_content|gclid|gclsrc|dclid|fbclid$")
-    .expect("compile regex")
-});
-
-pub fn naive_from_unix(time: i64) -> NaiveDateTime {
-  NaiveDateTime::from_timestamp_opt(time, 0).expect("convert datetime")
-}
-
-pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime<FixedOffset> {
-  DateTime::<FixedOffset>::from_utc(
-    datetime,
-    FixedOffset::east_opt(0).expect("create fixed offset"),
-  )
-}
-
-pub fn remove_slurs(test: &str, slur_regex: &Option<Regex>) -> String {
-  if let Some(slur_regex) = slur_regex {
-    slur_regex.replace_all(test, "*removed*").to_string()
-  } else {
-    test.to_string()
-  }
-}
-
-pub(crate) fn slur_check<'a>(
-  test: &'a str,
-  slur_regex: &'a Option<Regex>,
-) -> Result<(), Vec<&'a str>> {
-  if let Some(slur_regex) = slur_regex {
-    let mut matches: Vec<&str> = slur_regex.find_iter(test).map(|mat| mat.as_str()).collect();
-
-    // Unique
-    matches.sort_unstable();
-    matches.dedup();
-
-    if matches.is_empty() {
-      Ok(())
-    } else {
-      Err(matches)
-    }
-  } else {
-    Ok(())
-  }
-}
-
-pub fn build_slur_regex(regex_str: Option<&str>) -> Option<Regex> {
-  regex_str.map(|slurs| {
-    RegexBuilder::new(slurs)
-      .case_insensitive(true)
-      .build()
-      .expect("compile regex")
-  })
-}
-
-pub fn check_slurs(text: &str, slur_regex: &Option<Regex>) -> Result<(), LemmyError> {
-  if let Err(slurs) = slur_check(text, slur_regex) {
-    Err(LemmyError::from_error_message(
-      anyhow::anyhow!("{}", slurs_vec_to_str(&slurs)),
-      "slurs",
-    ))
-  } else {
-    Ok(())
-  }
-}
-
-pub fn check_slurs_opt(
-  text: &Option<String>,
-  slur_regex: &Option<Regex>,
-) -> Result<(), LemmyError> {
-  match text {
-    Some(t) => check_slurs(t, slur_regex),
-    None => Ok(()),
-  }
-}
-
-pub(crate) fn slurs_vec_to_str(slurs: &[&str]) -> String {
-  let start = "No slurs - ";
-  let combined = &slurs.join(", ");
-  [start, combined].concat()
-}
-
-pub fn generate_random_string() -> String {
-  thread_rng()
-    .sample_iter(&Alphanumeric)
-    .map(char::from)
-    .take(30)
-    .collect()
-}
-
-pub fn markdown_to_html(text: &str) -> String {
-  comrak::markdown_to_html(text, &comrak::ComrakOptions::default())
-}
-
-// TODO nothing is done with community / group webfingers yet, so just ignore those for now
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub struct MentionData {
-  pub name: String,
-  pub domain: String,
-}
-
-impl MentionData {
-  pub fn is_local(&self, hostname: &str) -> bool {
-    hostname.eq(&self.domain)
-  }
-  pub fn full_name(&self) -> String {
-    format!("@{}@{}", &self.name, &self.domain)
-  }
-}
-
-pub fn scrape_text_for_mentions(text: &str) -> Vec<MentionData> {
-  let mut out: Vec<MentionData> = Vec::new();
-  for caps in MENTIONS_REGEX.captures_iter(text) {
-    if let Some(name) = caps.name("name").map(|c| c.as_str().to_string()) {
-      if let Some(domain) = caps.name("domain").map(|c| c.as_str().to_string()) {
-        out.push(MentionData { name, domain });
-      }
-    }
-  }
-  out.into_iter().unique().collect()
-}
-
-fn has_newline(name: &str) -> bool {
-  name.contains('\n')
-}
-
-pub fn is_valid_actor_name(name: &str, actor_name_max_length: usize) -> bool {
-  name.chars().count() <= actor_name_max_length
-    && VALID_ACTOR_NAME_REGEX.is_match(name)
-    && !has_newline(name)
-}
-
-// Can't do a regex here, reverse lookarounds not supported
-pub fn is_valid_display_name(name: &str, actor_name_max_length: usize) -> bool {
-  !name.starts_with('@')
-    && !name.starts_with('\u{200b}')
-    && name.chars().count() >= 3
-    && name.chars().count() <= actor_name_max_length
-    && !has_newline(name)
-}
-
-pub fn is_valid_matrix_id(matrix_id: &str) -> bool {
-  VALID_MATRIX_ID_REGEX.is_match(matrix_id) && !has_newline(matrix_id)
-}
-
-pub fn is_valid_post_title(title: &str) -> bool {
-  VALID_POST_TITLE_REGEX.is_match(title) && !has_newline(title)
-}
-
-pub fn get_ip(conn_info: &ConnectionInfo) -> IpAddr {
-  IpAddr(
-    conn_info
-      .realip_remote_addr()
-      .unwrap_or("127.0.0.1:12345")
-      .split(':')
-      .next()
-      .unwrap_or("127.0.0.1")
-      .to_string(),
-  )
-}
-
-pub fn clean_url_params(url: &Url) -> Url {
-  let mut url_out = url.clone();
-  if url.query().is_some() {
-    let new_query = url
-      .query_pairs()
-      .filter(|q| !CLEAN_URL_PARAMS_REGEX.is_match(&q.0))
-      .map(|q| format!("{}={}", q.0, q.1))
-      .join("&");
-    url_out.set_query(Some(&new_query));
-  }
-  url_out
-}
-
-pub fn generate_domain_url(actor_id: &Url) -> Result<String, LemmyError> {
-  Ok(actor_id.host_str().context(location_info!())?.to_string())
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::utils::{clean_url_params, is_valid_post_title};
-  use url::Url;
-
-  #[test]
-  fn test_clean_url_params() {
-    let url = Url::parse("https://example.com/path/123?utm_content=buffercf3b2&utm_medium=social&username=randomuser&id=123").unwrap();
-    let cleaned = clean_url_params(&url);
-    let expected = Url::parse("https://example.com/path/123?username=randomuser&id=123").unwrap();
-    assert_eq!(expected.to_string(), cleaned.to_string());
-
-    let url = Url::parse("https://example.com/path/123").unwrap();
-    let cleaned = clean_url_params(&url);
-    assert_eq!(url.to_string(), cleaned.to_string());
-  }
-
-  #[test]
-  fn regex_checks() {
-    assert!(!is_valid_post_title("hi"));
-    assert!(is_valid_post_title("him"));
-    assert!(!is_valid_post_title("n\n\n\n\nanother"));
-    assert!(!is_valid_post_title("hello there!\n this is a test."));
-    assert!(is_valid_post_title("hello there! this is a test."));
-  }
-}

crates/utils/src/utils/markdown.rs

@@ -0,0 +1,3 @@
+pub fn markdown_to_html(text: &str) -> String {
+  comrak::markdown_to_html(text, &comrak::ComrakOptions::default())
+}

crates/utils/src/utils/mod.rs

@@ -0,0 +1,5 @@
+pub mod markdown;
+pub mod mention;
+pub mod slurs;
+pub mod time;
+pub mod validation;

crates/utils/src/utils/slurs.rs

@@ -0,0 +1,109 @@
+use crate::error::LemmyError;
+use regex::{Regex, RegexBuilder};
+
+pub fn remove_slurs(test: &str, slur_regex: &Option<Regex>) -> String {
+  if let Some(slur_regex) = slur_regex {
+    slur_regex.replace_all(test, "*removed*").to_string()
+  } else {
+    test.to_string()
+  }
+}
+
+pub(crate) fn slur_check<'a>(
+  test: &'a str,
+  slur_regex: &'a Option<Regex>,
+) -> Result<(), Vec<&'a str>> {
+  if let Some(slur_regex) = slur_regex {
+    let mut matches: Vec<&str> = slur_regex.find_iter(test).map(|mat| mat.as_str()).collect();
+
+    // Unique
+    matches.sort_unstable();
+    matches.dedup();
+
+    if matches.is_empty() {
+      Ok(())
+    } else {
+      Err(matches)
+    }
+  } else {
+    Ok(())
+  }
+}
+
+pub fn build_slur_regex(regex_str: Option<&str>) -> Option<Regex> {
+  regex_str.map(|slurs| {
+    RegexBuilder::new(slurs)
+      .case_insensitive(true)
+      .build()
+      .expect("compile regex")
+  })
+}
+
+pub fn check_slurs(text: &str, slur_regex: &Option<Regex>) -> Result<(), LemmyError> {
+  if let Err(slurs) = slur_check(text, slur_regex) {
+    Err(LemmyError::from_error_message(
+      anyhow::anyhow!("{}", slurs_vec_to_str(&slurs)),
+      "slurs",
+    ))
+  } else {
+    Ok(())
+  }
+}
+
+pub fn check_slurs_opt(
+  text: &Option<String>,
+  slur_regex: &Option<Regex>,
+) -> Result<(), LemmyError> {
+  match text {
+    Some(t) => check_slurs(t, slur_regex),
+    None => Ok(()),
+  }
+}
+
+pub(crate) fn slurs_vec_to_str(slurs: &[&str]) -> String {
+  let start = "No slurs - ";
+  let combined = &slurs.join(", ");
+  [start, combined].concat()
+}
+
+#[cfg(test)]
+mod test {
+  use crate::utils::slurs::{remove_slurs, slur_check, slurs_vec_to_str};
+  use regex::RegexBuilder;
+
+  #[test]
+  fn test_slur_filter() {
+    let slur_regex = Some(RegexBuilder::new(r"(fag(g|got|tard)?\b|cock\s?sucker(s|ing)?|ni((g{2,}|q)+|[gq]{2,})[e3r]+(s|z)?|mudslime?s?|kikes?|\bspi(c|k)s?\b|\bchinks?|gooks?|bitch(es|ing|y)?|whor(es?|ing)|\btr(a|@)nn?(y|ies?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap());
+    let test =
+      "faggot test kike tranny cocksucker retardeds. Capitalized Niggerz. This is a bunch of other safe text.";
+    let slur_free = "No slurs here";
+    assert_eq!(
+      remove_slurs(test, &slur_regex),
+      "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text."
+        .to_string()
+    );
+
+    let has_slurs_vec = vec![
+      "Niggerz",
+      "cocksucker",
+      "faggot",
+      "kike",
+      "retardeds",
+      "tranny",
+    ];
+    let has_slurs_err_str = "No slurs - Niggerz, cocksucker, faggot, kike, retardeds, tranny";
+
+    assert_eq!(slur_check(test, &slur_regex), Err(has_slurs_vec));
+    assert_eq!(slur_check(slur_free, &slur_regex), Ok(()));
+    if let Err(slur_vec) = slur_check(test, &slur_regex) {
+      assert_eq!(&slurs_vec_to_str(&slur_vec), has_slurs_err_str);
+    }
+  }
+
+  // These helped with testing
+  // #[test]
+  // fn test_send_email() {
+  //  let result =  send_email("not a subject", "test_email@gmail.com", "ur user", "<h1>HI there</h1>");
+  //   assert!(result.is_ok());
+  // }
+}

crates/utils/src/utils/time.rs

@@ -0,0 +1,12 @@
+use chrono::{DateTime, FixedOffset, NaiveDateTime};
+
+pub fn naive_from_unix(time: i64) -> NaiveDateTime {
+  NaiveDateTime::from_timestamp_opt(time, 0).expect("convert datetime")
+}
+
+pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime<FixedOffset> {
+  DateTime::<FixedOffset>::from_utc(
+    datetime,
+    FixedOffset::east_opt(0).expect("create fixed offset"),
+  )
+}

crates/utils/src/utils/validation.rs

@@ -2,7 +2,6 @@ use crate::error::{LemmyError, LemmyResult};
 use itertools::Itertools;
 use once_cell::sync::Lazy;
 use regex::Regex;
-use totp_rs::{Secret, TOTP};
 use url::Url;
 
 static VALID_ACTOR_NAME_REGEX: Lazy<Regex> =
@@ -103,58 +102,10 @@ pub fn clean_url_params(url: &Url) -> Url {
   url_out
 }
 
-pub fn check_totp_2fa_valid(
-  totp_secret: &Option<String>,
-  totp_token: &Option<String>,
-  site_name: &str,
-  username: &str,
-) -> LemmyResult<()> {
-  // Check only if they have a totp_secret in the DB
-  if let Some(totp_secret) = totp_secret {
-    // Throw an error if their token is missing
-    let token = totp_token
-      .as_deref()
-      .ok_or_else(|| LemmyError::from_message("missing_totp_token"))?;
-
-    let totp = build_totp_2fa(site_name, username, totp_secret)?;
-
-    let check_passed = totp.check_current(token)?;
-    if !check_passed {
-      return Err(LemmyError::from_message("incorrect_totp token"));
-    }
-  }
-
-  Ok(())
-}
-
-pub fn generate_totp_2fa_secret() -> String {
-  Secret::generate_secret().to_string()
-}
-
-pub fn build_totp_2fa(site_name: &str, username: &str, secret: &str) -> Result<TOTP, LemmyError> {
-  let sec = Secret::Raw(secret.as_bytes().to_vec());
-  let sec_bytes = sec
-    .to_bytes()
-    .map_err(|_| LemmyError::from_message("Couldnt parse totp secret"))?;
-
-  TOTP::new(
-    totp_rs::Algorithm::SHA256,
-    6,
-    1,
-    30,
-    sec_bytes,
-    Some(site_name.to_string()),
-    username.to_string(),
-  )
-  .map_err(|e| LemmyError::from_error_message(e, "Couldnt generate TOTP"))
-}
-
 #[cfg(test)]
 mod tests {
-  use super::build_totp_2fa;
   use crate::utils::validation::{
     clean_url_params,
-    generate_totp_2fa_secret,
     is_valid_actor_name,
     is_valid_display_name,
     is_valid_matrix_id,
@@ -219,11 +170,4 @@ mod tests {
     assert!(is_valid_matrix_id(" @dess:matrix.org").is_err());
     assert!(is_valid_matrix_id("@dess:matrix.org t").is_err());
   }
-
-  #[test]
-  fn test_build_totp() {
-    let generated_secret = generate_totp_2fa_secret();
-    let totp = build_totp_2fa("lemmy", "my_name", &generated_secret);
-    assert!(totp.is_ok());
-  }
 }