Merge remote-tracking branch 'upstream/main' into db-traits-refactor

This commit is contained in:
dull b 2023-06-30 19:17:13 +00:00
commit 6846210074
14 changed files with 128 additions and 25 deletions

4
.gitignore vendored
View file

@ -26,3 +26,7 @@ pictrs/
# The generated typescript bindings # The generated typescript bindings
bindings bindings
# Database cluster and sockets for testing
dev_pgdata/
*.PGSQL.*

19
Cargo.lock generated
View file

@ -399,6 +399,9 @@ name = "anyhow"
version = "1.0.71" version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
dependencies = [
"backtrace",
]
[[package]] [[package]]
name = "argparse" name = "argparse"
@ -2590,6 +2593,7 @@ dependencies = [
"tokio", "tokio",
"tracing", "tracing",
"uuid", "uuid",
"wav",
] ]
[[package]] [[package]]
@ -4445,6 +4449,12 @@ dependencies = [
"bytemuck", "bytemuck",
] ]
[[package]]
name = "riff"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1"
[[package]] [[package]]
name = "ring" name = "ring"
version = "0.16.20" version = "0.16.20"
@ -6256,6 +6266,15 @@ dependencies = [
"web-sys", "web-sys",
] ]
[[package]]
name = "wav"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a65e199c799848b4f997072aa4d673c034f80f40191f97fe2f0a23f410be1609"
dependencies = [
"riff",
]
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.60" version = "0.3.60"

View file

@ -21,14 +21,9 @@ repository.workspace = true
doctest = false doctest = false
[profile.release] [profile.release]
strip = "symbols"
debug = 0 debug = 0
lto = "thin" lto = "thin"
[profile.dev]
strip = "symbols"
debug = 0
[features] [features]
embed-pictrs = ["pict-rs"] embed-pictrs = ["pict-rs"]
console = ["console-subscriber", "opentelemetry", "opentelemetry-otlp", "tracing-opentelemetry", "reqwest-tracing/opentelemetry_0_16"] console = ["console-subscriber", "opentelemetry", "opentelemetry-otlp", "tracing-opentelemetry", "reqwest-tracing/opentelemetry_0_16"]
@ -85,7 +80,7 @@ base64 = "0.13.1"
uuid = { version = "1.3.4", features = ["serde", "v4"] } uuid = { version = "1.3.4", features = ["serde", "v4"] }
async-trait = "0.1.68" async-trait = "0.1.68"
captcha = "0.0.9" captcha = "0.0.9"
anyhow = "1.0.71" anyhow = { version = "1.0.71", features = ["backtrace"] } # backtrace is on by default on nightly, but not stable rust
diesel_ltree = "0.3.0" diesel_ltree = "0.3.0"
typed-builder = "0.10.0" typed-builder = "0.10.0"
serial_test = "0.9.0" serial_test = "0.9.0"

View file

@ -30,6 +30,7 @@ captcha = { workspace = true }
anyhow = { workspace = true } anyhow = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
chrono = { workspace = true } chrono = { workspace = true }
wav = "1.0.0"
[dev-dependencies] [dev-dependencies]
serial_test = { workspace = true } serial_test = { workspace = true }

View file

@ -3,6 +3,7 @@ use captcha::Captcha;
use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex}; use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex};
use lemmy_db_schema::source::local_site::LocalSite; use lemmy_db_schema::source::local_site::LocalSite;
use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs}; use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs};
use std::io::Cursor;
mod comment; mod comment;
mod comment_report; mod comment_report;
@ -22,18 +23,42 @@ pub trait Perform {
} }
/// Converts the captcha to a base64 encoded wav audio file /// Converts the captcha to a base64 encoded wav audio file
pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String { pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> Result<String, LemmyError> {
let letters = captcha.as_wav(); let letters = captcha.as_wav();
let mut concat_letters: Vec<u8> = Vec::new(); // Decode each wav file, concatenate the samples
let mut concat_samples: Vec<i16> = Vec::new();
let mut any_header: Option<wav::Header> = None;
for letter in letters { for letter in letters {
let bytes = letter.unwrap_or_default(); let mut cursor = Cursor::new(letter.unwrap_or_default());
concat_letters.extend(bytes); let (header, samples) = wav::read(&mut cursor)?;
any_header = Some(header);
if let Some(samples16) = samples.as_sixteen() {
concat_samples.extend(samples16);
} else {
return Err(LemmyError::from_message("couldnt_create_audio_captcha"));
}
} }
// Convert to base64 // Encode the concatenated result as a wav file
base64::encode(concat_letters) let mut output_buffer = Cursor::new(vec![]);
let header = match any_header {
Some(header) => header,
None => return Err(LemmyError::from_message("couldnt_create_audio_captcha")),
};
let wav_write_result = wav::write(
header,
&wav::BitDepth::Sixteen(concat_samples),
&mut output_buffer,
);
if let Err(e) = wav_write_result {
return Err(LemmyError::from_error_message(
e,
"couldnt_create_audio_captcha",
));
}
Ok(base64::encode(output_buffer.into_inner()))
} }
/// Check size of report and remove whitespace /// Check size of report and remove whitespace

View file

@ -33,7 +33,7 @@ impl Perform for GetCaptcha {
let png = captcha.as_base64().expect("failed to generate captcha"); let png = captcha.as_base64().expect("failed to generate captcha");
let wav = captcha_as_wav_base64(&captcha); let wav = captcha_as_wav_base64(&captcha)?;
let captcha_form: CaptchaAnswerForm = CaptchaAnswerForm { answer }; let captcha_form: CaptchaAnswerForm = CaptchaAnswerForm { answer };
// Stores the captcha item in the db // Stores the captcha item in the db

View file

@ -193,6 +193,7 @@ pub async fn fetch_site_data(
client: &ClientWithMiddleware, client: &ClientWithMiddleware,
settings: &Settings, settings: &Settings,
url: Option<&Url>, url: Option<&Url>,
include_image: bool,
) -> (Option<SiteMetadata>, Option<DbUrl>) { ) -> (Option<SiteMetadata>, Option<DbUrl>) {
match &url { match &url {
Some(url) => { Some(url) => {
@ -200,6 +201,9 @@ pub async fn fetch_site_data(
// Ignore errors, since it may be an image, or not have the data. // Ignore errors, since it may be an image, or not have the data.
// Warning, this may ignore SSL errors // Warning, this may ignore SSL errors
let metadata_option = fetch_site_metadata(client, url).await.ok(); let metadata_option = fetch_site_metadata(client, url).await.ok();
if !include_image {
return (metadata_option, None);
}
let missing_pictrs_file = let missing_pictrs_file =
|r: PictrsResponse| r.files.first().expect("missing pictrs file").file.clone(); |r: PictrsResponse| r.files.first().expect("missing pictrs file").file.clone();

View file

@ -432,6 +432,13 @@ pub fn local_site_opt_to_slur_regex(local_site: &Option<LocalSite>) -> Option<Re
.unwrap_or(None) .unwrap_or(None)
} }
pub fn local_site_opt_to_sensitive(local_site: &Option<LocalSite>) -> bool {
local_site
.as_ref()
.map(|site| site.enable_nsfw)
.unwrap_or(false)
}
pub fn send_application_approved_email( pub fn send_application_approved_email(
user: &LocalUserView, user: &LocalUserView,
settings: &Settings, settings: &Settings,

View file

@ -84,7 +84,7 @@ impl PerformCrud for CreatePost {
// Fetch post links and pictrs cached image // Fetch post links and pictrs cached image
let (metadata_res, thumbnail_url) = let (metadata_res, thumbnail_url) =
fetch_site_data(context.client(), context.settings(), data_url).await; fetch_site_data(context.client(), context.settings(), data_url, true).await;
let (embed_title, embed_description, embed_video_url) = metadata_res let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (u.title, u.description, u.embed_video_url)) .map(|u| (u.title, u.description, u.embed_video_url))
.unwrap_or_default(); .unwrap_or_default();

View file

@ -69,7 +69,7 @@ impl PerformCrud for EditPost {
// Fetch post links and Pictrs cached image // Fetch post links and Pictrs cached image
let data_url = data.url.as_ref(); let data_url = data.url.as_ref();
let (metadata_res, thumbnail_url) = let (metadata_res, thumbnail_url) =
fetch_site_data(context.client(), context.settings(), data_url).await; fetch_site_data(context.client(), context.settings(), data_url, true).await;
let (embed_title, embed_description, embed_video_url) = metadata_res let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url))) .map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url)))
.unwrap_or_default(); .unwrap_or_default();

View file

@ -25,7 +25,7 @@ use html2md::parse_html;
use lemmy_api_common::{ use lemmy_api_common::{
context::LemmyContext, context::LemmyContext,
request::fetch_site_data, request::fetch_site_data,
utils::{is_mod_or_admin, local_site_opt_to_slur_regex}, utils::{is_mod_or_admin, local_site_opt_to_sensitive, local_site_opt_to_slur_regex},
}; };
use lemmy_db_schema::{ use lemmy_db_schema::{
self, self,
@ -197,18 +197,33 @@ impl Object for ApubPost {
} else { } else {
None None
}; };
let local_site = LocalSite::read(&mut *context.conn().await?).await.ok();
let allow_sensitive = local_site_opt_to_sensitive(&local_site);
let page_is_sensitive = page.sensitive.unwrap_or(false);
let include_image = allow_sensitive || !page_is_sensitive;
// Only fetch metadata if the post has a url and was not seen previously. We dont want to // Only fetch metadata if the post has a url and was not seen previously. We dont want to
// waste resources by fetching metadata for the same post multiple times. // waste resources by fetching metadata for the same post multiple times.
let (metadata_res, thumbnail_url) = match &url { // Additionally, only fetch image if content is not sensitive or is allowed on local site.
let (metadata_res, thumbnail) = match &url {
Some(url) if old_post.is_err() => { Some(url) if old_post.is_err() => {
fetch_site_data(context.client(), context.settings(), Some(url)).await fetch_site_data(
context.client(),
context.settings(),
Some(url),
include_image,
)
.await
} }
_ => (None, page.image.map(|i| i.url.into())), _ => (None, None),
}; };
// If no image was included with metadata, use post image instead when available.
let thumbnail_url = thumbnail.or_else(|| page.image.map(|i| i.url.into()));
let (embed_title, embed_description, embed_video_url) = metadata_res let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (u.title, u.description, u.embed_video_url)) .map(|u| (u.title, u.description, u.embed_video_url))
.unwrap_or_default(); .unwrap_or_default();
let local_site = LocalSite::read(&mut *context.conn().await?).await.ok();
let slur_regex = &local_site_opt_to_slur_regex(&local_site); let slur_regex = &local_site_opt_to_slur_regex(&local_site);
let body_slurs_removed = let body_slurs_removed =

View file

@ -90,7 +90,11 @@ impl Display for LemmyError {
if let Some(message) = &self.message { if let Some(message) = &self.message {
write!(f, "{message}: ")?; write!(f, "{message}: ")?;
} }
writeln!(f, "{}", self.inner)?; // print anyhow including trace
// https://docs.rs/anyhow/latest/anyhow/struct.Error.html#display-representations
// this will print the anyhow trace (only if it exists)
// and if RUST_BACKTRACE=1, also a full backtrace
writeln!(f, "{:?}", self.inner)?;
fmt::Display::fmt(&self.context, f) fmt::Display::fmt(&self.context, f)
} }
} }

24
scripts/start_dev_db.sh Normal file
View file

@ -0,0 +1,24 @@
# This script is meant to be run with `source` so it can set environment variables.
export PGDATA="$PWD/dev_pgdata"
export PGHOST=$PWD
export LEMMY_DATABASE_URL="postgresql://lemmy:password@/lemmy?host=$PWD"
# If cluster exists, stop the server and delete the cluster
if [ -d $PGDATA ]
then
# Prevent `stop` from failing if server already stopped
pg_ctl restart > /dev/null
pg_ctl stop
rm -rf $PGDATA
fi
# Create cluster
initdb --username=postgres --auth=trust --no-instructions
# Start server that only listens to socket in current directory
pg_ctl start --options="-c listen_addresses= -c unix_socket_directories=$PWD" > /dev/null
# Setup database
psql -c "CREATE USER lemmy WITH PASSWORD 'password' SUPERUSER;" -U postgres
psql -c "CREATE DATABASE lemmy WITH OWNER lemmy;" -U postgres

View file

@ -1,13 +1,15 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -e set -e
CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
cd $CWD/../
PACKAGE="$1" PACKAGE="$1"
echo "$PACKAGE" echo "$PACKAGE"
psql -U lemmy -d postgres -c "DROP DATABASE lemmy;" source scripts/start_dev_db.sh
psql -U lemmy -d postgres -c "CREATE DATABASE lemmy;"
export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
# tests are executed in working directory crates/api (or similar), # tests are executed in working directory crates/api (or similar),
# so to load the config we need to traverse to the repo root # so to load the config we need to traverse to the repo root
export LEMMY_CONFIG_LOCATION=../../config/config.hjson export LEMMY_CONFIG_LOCATION=../../config/config.hjson
@ -21,3 +23,6 @@ else
fi fi
# Add this to do printlns: -- --nocapture # Add this to do printlns: -- --nocapture
pg_ctl stop
rm -rf $PGDATA