Merge remote-tracking branch 'upstream/main' into db-traits-refactor

.gitignore

@@ -26,3 +26,7 @@ pictrs/
 
 # The generated typescript bindings
 bindings
+
+# Database cluster and sockets for testing
+dev_pgdata/
+*.PGSQL.*

Cargo.lock

@@ -399,6 +399,9 @@ name = "anyhow"
 version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
+dependencies = [
+ "backtrace",
+]
 
 [[package]]
 name = "argparse"
@@ -2590,6 +2593,7 @@ dependencies = [
  "tokio",
  "tracing",
  "uuid",
+ "wav",
 ]
 
 [[package]]
@@ -4445,6 +4449,12 @@ dependencies = [
  "bytemuck",
 ]
 
+[[package]]
+name = "riff"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1"
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -6256,6 +6266,15 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "wav"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a65e199c799848b4f997072aa4d673c034f80f40191f97fe2f0a23f410be1609"
+dependencies = [
+ "riff",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.60"

Cargo.toml

@@ -21,14 +21,9 @@ repository.workspace = true
 doctest = false
 
 [profile.release]
-strip = "symbols"
 debug = 0
 lto = "thin"
 
-[profile.dev]
-strip = "symbols"
-debug = 0
-
 [features]
 embed-pictrs = ["pict-rs"]
 console = ["console-subscriber", "opentelemetry", "opentelemetry-otlp", "tracing-opentelemetry", "reqwest-tracing/opentelemetry_0_16"]
@@ -85,7 +80,7 @@ base64 = "0.13.1"
 uuid = { version = "1.3.4", features = ["serde", "v4"] }
 async-trait = "0.1.68"
 captcha = "0.0.9"
-anyhow = "1.0.71"
+anyhow = { version = "1.0.71", features = ["backtrace"] } # backtrace is on by default on nightly, but not stable rust
 diesel_ltree = "0.3.0"
 typed-builder = "0.10.0"
 serial_test = "0.9.0"

crates/api/Cargo.toml

@@ -30,6 +30,7 @@ captcha = { workspace = true }
 anyhow = { workspace = true }
 tracing = { workspace = true }
 chrono = { workspace = true }
+wav = "1.0.0"
 
 [dev-dependencies]
 serial_test = { workspace = true }

crates/api/src/lib.rs

@@ -3,6 +3,7 @@ use captcha::Captcha;
 use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex};
 use lemmy_db_schema::source::local_site::LocalSite;
 use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs};
+use std::io::Cursor;
 
 mod comment;
 mod comment_report;
@@ -22,18 +23,42 @@ pub trait Perform {
 }
 
 /// Converts the captcha to a base64 encoded wav audio file
-pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String {
+pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> Result<String, LemmyError> {
   let letters = captcha.as_wav();
 
-  let mut concat_letters: Vec<u8> = Vec::new();
+  // Decode each wav file, concatenate the samples
-
+  let mut concat_samples: Vec<i16> = Vec::new();
+  let mut any_header: Option<wav::Header> = None;
   for letter in letters {
-    let bytes = letter.unwrap_or_default();
+    let mut cursor = Cursor::new(letter.unwrap_or_default());
-    concat_letters.extend(bytes);
+    let (header, samples) = wav::read(&mut cursor)?;
+    any_header = Some(header);
+    if let Some(samples16) = samples.as_sixteen() {
+      concat_samples.extend(samples16);
+    } else {
+      return Err(LemmyError::from_message("couldnt_create_audio_captcha"));
+    }
   }
 
-  // Convert to base64
+  // Encode the concatenated result as a wav file
-  base64::encode(concat_letters)
+  let mut output_buffer = Cursor::new(vec![]);
+  let header = match any_header {
+    Some(header) => header,
+    None => return Err(LemmyError::from_message("couldnt_create_audio_captcha")),
+  };
+  let wav_write_result = wav::write(
+    header,
+    &wav::BitDepth::Sixteen(concat_samples),
+    &mut output_buffer,
+  );
+  if let Err(e) = wav_write_result {
+    return Err(LemmyError::from_error_message(
+      e,
+      "couldnt_create_audio_captcha",
+    ));
+  }
+
+  Ok(base64::encode(output_buffer.into_inner()))
 }
 
 /// Check size of report and remove whitespace

crates/api/src/local_user/get_captcha.rs

@@ -33,7 +33,7 @@ impl Perform for GetCaptcha {
 
     let png = captcha.as_base64().expect("failed to generate captcha");
 
-    let wav = captcha_as_wav_base64(&captcha);
+    let wav = captcha_as_wav_base64(&captcha)?;
 
     let captcha_form: CaptchaAnswerForm = CaptchaAnswerForm { answer };
     // Stores the captcha item in the db

crates/api_common/src/request.rs

@@ -193,6 +193,7 @@ pub async fn fetch_site_data(
   client: &ClientWithMiddleware,
   settings: &Settings,
   url: Option<&Url>,
+  include_image: bool,
 ) -> (Option<SiteMetadata>, Option<DbUrl>) {
   match &url {
     Some(url) => {
@@ -200,6 +201,9 @@ pub async fn fetch_site_data(
       // Ignore errors, since it may be an image, or not have the data.
       // Warning, this may ignore SSL errors
       let metadata_option = fetch_site_metadata(client, url).await.ok();
+      if !include_image {
+        return (metadata_option, None);
+      }
 
       let missing_pictrs_file =
         |r: PictrsResponse| r.files.first().expect("missing pictrs file").file.clone();

crates/api_common/src/utils.rs

@@ -432,6 +432,13 @@ pub fn local_site_opt_to_slur_regex(local_site: &Option<LocalSite>) -> Option<Re
     .unwrap_or(None)
 }
 
+pub fn local_site_opt_to_sensitive(local_site: &Option<LocalSite>) -> bool {
+  local_site
+    .as_ref()
+    .map(|site| site.enable_nsfw)
+    .unwrap_or(false)
+}
+
 pub fn send_application_approved_email(
   user: &LocalUserView,
   settings: &Settings,

crates/api_crud/src/post/create.rs

@@ -84,7 +84,7 @@ impl PerformCrud for CreatePost {
 
     // Fetch post links and pictrs cached image
     let (metadata_res, thumbnail_url) =
-      fetch_site_data(context.client(), context.settings(), data_url).await;
+      fetch_site_data(context.client(), context.settings(), data_url, true).await;
     let (embed_title, embed_description, embed_video_url) = metadata_res
       .map(|u| (u.title, u.description, u.embed_video_url))
       .unwrap_or_default();

crates/api_crud/src/post/update.rs

@@ -69,7 +69,7 @@ impl PerformCrud for EditPost {
     // Fetch post links and Pictrs cached image
     let data_url = data.url.as_ref();
     let (metadata_res, thumbnail_url) =
-      fetch_site_data(context.client(), context.settings(), data_url).await;
+      fetch_site_data(context.client(), context.settings(), data_url, true).await;
     let (embed_title, embed_description, embed_video_url) = metadata_res
       .map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url)))
       .unwrap_or_default();

crates/apub/src/objects/post.rs

@@ -25,7 +25,7 @@ use html2md::parse_html;
 use lemmy_api_common::{
   context::LemmyContext,
   request::fetch_site_data,
-  utils::{is_mod_or_admin, local_site_opt_to_slur_regex},
+  utils::{is_mod_or_admin, local_site_opt_to_sensitive, local_site_opt_to_slur_regex},
 };
 use lemmy_db_schema::{
   self,
@@ -197,18 +197,33 @@ impl Object for ApubPost {
       } else {
         None
       };
+
+      let local_site = LocalSite::read(&mut *context.conn().await?).await.ok();
+      let allow_sensitive = local_site_opt_to_sensitive(&local_site);
+      let page_is_sensitive = page.sensitive.unwrap_or(false);
+      let include_image = allow_sensitive || !page_is_sensitive;
+
       // Only fetch metadata if the post has a url and was not seen previously. We dont want to
       // waste resources by fetching metadata for the same post multiple times.
-      let (metadata_res, thumbnail_url) = match &url {
+      // Additionally, only fetch image if content is not sensitive or is allowed on local site.
+      let (metadata_res, thumbnail) = match &url {
         Some(url) if old_post.is_err() => {
-          fetch_site_data(context.client(), context.settings(), Some(url)).await
+          fetch_site_data(
+            context.client(),
+            context.settings(),
+            Some(url),
+            include_image,
+          )
+          .await
         }
-        _ => (None, page.image.map(|i| i.url.into())),
+        _ => (None, None),
       };
+      // If no image was included with metadata, use post image instead when available.
+      let thumbnail_url = thumbnail.or_else(|| page.image.map(|i| i.url.into()));
+
       let (embed_title, embed_description, embed_video_url) = metadata_res
         .map(|u| (u.title, u.description, u.embed_video_url))
         .unwrap_or_default();
-      let local_site = LocalSite::read(&mut *context.conn().await?).await.ok();
       let slur_regex = &local_site_opt_to_slur_regex(&local_site);
 
       let body_slurs_removed =

crates/utils/src/error.rs

@@ -90,7 +90,11 @@ impl Display for LemmyError {
     if let Some(message) = &self.message {
       write!(f, "{message}: ")?;
     }
-    writeln!(f, "{}", self.inner)?;
+    // print anyhow including trace
+    // https://docs.rs/anyhow/latest/anyhow/struct.Error.html#display-representations
+    // this will print the anyhow trace (only if it exists)
+    // and if RUST_BACKTRACE=1, also a full backtrace
+    writeln!(f, "{:?}", self.inner)?;
     fmt::Display::fmt(&self.context, f)
   }
 }

scripts/start_dev_db.sh

@@ -0,0 +1,24 @@
+# This script is meant to be run with `source` so it can set environment variables.
+
+export PGDATA="$PWD/dev_pgdata"
+export PGHOST=$PWD
+export LEMMY_DATABASE_URL="postgresql://lemmy:password@/lemmy?host=$PWD"
+
+# If cluster exists, stop the server and delete the cluster
+if [ -d $PGDATA ]
+then
+  # Prevent `stop` from failing if server already stopped
+  pg_ctl restart > /dev/null
+  pg_ctl stop
+  rm -rf $PGDATA
+fi
+
+# Create cluster
+initdb --username=postgres --auth=trust --no-instructions
+
+# Start server that only listens to socket in current directory
+pg_ctl start --options="-c listen_addresses= -c unix_socket_directories=$PWD" > /dev/null
+
+# Setup database
+psql -c "CREATE USER lemmy WITH PASSWORD 'password' SUPERUSER;" -U postgres
+psql -c "CREATE DATABASE lemmy WITH OWNER lemmy;" -U postgres

scripts/test.sh

@@ -1,13 +1,15 @@
 #!/usr/bin/env bash
 set -e
 
+CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+
+cd $CWD/../
+
 PACKAGE="$1"
 echo "$PACKAGE"
 
-psql -U lemmy -d postgres -c "DROP DATABASE lemmy;"
+source scripts/start_dev_db.sh
-psql -U lemmy -d postgres -c "CREATE DATABASE lemmy;"
 
-export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
 # tests are executed in working directory crates/api (or similar),
 # so to load the config we need to traverse to the repo root
 export LEMMY_CONFIG_LOCATION=../../config/config.hjson
@@ -21,3 +23,6 @@ else
 fi
 
 # Add this to do printlns: -- --nocapture
+
+pg_ctl stop
+rm -rf $PGDATA