Merge branch 'main' into migration-runner

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace.package]
-version = "0.19.4-beta.7"
+version = "0.19.4-rc.1"
 edition = "2021"
 description = "A link aggregator for the fediverse"
 license = "AGPL-3.0"
@@ -88,17 +88,17 @@ unused_self = "deny"
 unwrap_used = "deny"
 
 [workspace.dependencies]
-lemmy_api = { version = "=0.19.4-beta.7", path = "./crates/api" }
+lemmy_api = { version = "=0.19.4-rc.1", path = "./crates/api" }
-lemmy_api_crud = { version = "=0.19.4-beta.7", path = "./crates/api_crud" }
+lemmy_api_crud = { version = "=0.19.4-rc.1", path = "./crates/api_crud" }
-lemmy_apub = { version = "=0.19.4-beta.7", path = "./crates/apub" }
+lemmy_apub = { version = "=0.19.4-rc.1", path = "./crates/apub" }
-lemmy_utils = { version = "=0.19.4-beta.7", path = "./crates/utils", default-features = false }
+lemmy_utils = { version = "=0.19.4-rc.1", path = "./crates/utils", default-features = false }
-lemmy_db_schema = { version = "=0.19.4-beta.7", path = "./crates/db_schema" }
+lemmy_db_schema = { version = "=0.19.4-rc.1", path = "./crates/db_schema" }
-lemmy_api_common = { version = "=0.19.4-beta.7", path = "./crates/api_common" }
+lemmy_api_common = { version = "=0.19.4-rc.1", path = "./crates/api_common" }
-lemmy_routes = { version = "=0.19.4-beta.7", path = "./crates/routes" }
+lemmy_routes = { version = "=0.19.4-rc.1", path = "./crates/routes" }
-lemmy_db_views = { version = "=0.19.4-beta.7", path = "./crates/db_views" }
+lemmy_db_views = { version = "=0.19.4-rc.1", path = "./crates/db_views" }
-lemmy_db_views_actor = { version = "=0.19.4-beta.7", path = "./crates/db_views_actor" }
+lemmy_db_views_actor = { version = "=0.19.4-rc.1", path = "./crates/db_views_actor" }
-lemmy_db_views_moderator = { version = "=0.19.4-beta.7", path = "./crates/db_views_moderator" }
+lemmy_db_views_moderator = { version = "=0.19.4-rc.1", path = "./crates/db_views_moderator" }
-lemmy_federate = { version = "=0.19.4-beta.7", path = "./crates/federate" }
+lemmy_federate = { version = "=0.19.4-rc.1", path = "./crates/federate" }
 activitypub_federation = { version = "0.5.6", default-features = false, features = [
   "actix-web",
 ] }
@@ -165,7 +165,7 @@ urlencoding = "2.1.3"
 enum-map = "2.7"
 moka = { version = "0.12.7", features = ["future"] }
 i-love-jesus = { version = "0.1.0" }
-clap = { version = "4.5.4", features = ["derive"] }
+clap = { version = "4.5.4", features = ["derive", "env"] }
 pretty_assertions = "1.4.0"
 
 [dependencies]

api_tests/package.json

@@ -6,11 +6,11 @@
   "repository": "https://github.com/LemmyNet/lemmy",
   "author": "Dessalines",
   "license": "AGPL-3.0",
-  "packageManager": "pnpm@9.0.6",
+  "packageManager": "pnpm@9.1.1+sha256.9551e803dcb7a1839fdf5416153a844060c7bce013218ce823410532504ac10b",
   "scripts": {
     "lint": "tsc --noEmit && eslint --report-unused-disable-directives --ext .js,.ts,.tsx src && prettier --check 'src/**/*.ts'",
     "fix": "prettier --write src && eslint --fix src",
-    "api-test": "jest -i follow.spec.ts && jest -i post.spec.ts && jest -i comment.spec.ts && jest -i private_message.spec.ts && jest -i user.spec.ts && jest -i community.spec.ts && jest -i image.spec.ts",
+    "api-test": "jest -i follow.spec.ts && jest -i image.spec.ts && jest -i user.spec.ts && jest -i private_message.spec.ts && jest -i community.spec.ts &&  jest -i post.spec.ts && jest -i comment.spec.ts ",
     "api-test-follow": "jest -i follow.spec.ts",
     "api-test-comment": "jest -i comment.spec.ts",
     "api-test-post": "jest -i post.spec.ts",

api_tests/prepare-drone-federation-test.sh

@@ -15,7 +15,7 @@ export LEMMY_TEST_FAST_FEDERATION=1 # by default, the persistent federation queu
 
 # pictrs setup
 if [ ! -f "api_tests/pict-rs" ]; then
-  curl "https://git.asonix.dog/asonix/pict-rs/releases/download/v0.5.0-beta.2/pict-rs-linux-amd64" -o api_tests/pict-rs
+  curl "https://git.asonix.dog/asonix/pict-rs/releases/download/v0.5.13/pict-rs-linux-amd64" -o api_tests/pict-rs
   chmod +x api_tests/pict-rs
 fi
 ./api_tests/pict-rs \

api_tests/src/comment.spec.ts

@@ -45,7 +45,6 @@ let postOnAlphaRes: PostResponse;
 
 beforeAll(async () => {
   await setupLogins();
-  await unfollows();
   await Promise.all([followBeta(alpha), followBeta(gamma)]);
   betaCommunity = (await resolveBetaCommunity(alpha)).community;
   if (betaCommunity) {

api_tests/src/community.spec.ts

@@ -380,8 +380,8 @@ test("User blocks instance, communities are hidden", async () => {
 test("Community follower count is federated", async () => {
   // Follow the beta community from alpha
   let community = await createCommunity(beta);
-  let community_id = community.community_view.community.actor_id;
+  let communityActorId = community.community_view.community.actor_id;
-  let resolved = await resolveCommunity(alpha, community_id);
+  let resolved = await resolveCommunity(alpha, communityActorId);
   if (!resolved.community) {
     throw "Missing beta community";
   }
@@ -389,7 +389,7 @@ test("Community follower count is federated", async () => {
   await followCommunity(alpha, true, resolved.community.community.id);
   let followed = (
     await waitUntil(
-      () => resolveCommunity(alpha, community_id),
+      () => resolveCommunity(alpha, communityActorId),
       c => c.community?.subscribed === "Subscribed",
     )
   ).community;
@@ -398,7 +398,7 @@ test("Community follower count is federated", async () => {
   expect(followed?.counts.subscribers).toBe(1);
 
   // Follow the community from gamma
-  resolved = await resolveCommunity(gamma, community_id);
+  resolved = await resolveCommunity(gamma, communityActorId);
   if (!resolved.community) {
     throw "Missing beta community";
   }
@@ -406,7 +406,7 @@ test("Community follower count is federated", async () => {
   await followCommunity(gamma, true, resolved.community.community.id);
   followed = (
     await waitUntil(
-      () => resolveCommunity(gamma, community_id),
+      () => resolveCommunity(gamma, communityActorId),
       c => c.community?.subscribed === "Subscribed",
     )
   ).community;
@@ -415,7 +415,7 @@ test("Community follower count is federated", async () => {
   expect(followed?.counts?.subscribers).toBe(2);
 
   // Follow the community from delta
-  resolved = await resolveCommunity(delta, community_id);
+  resolved = await resolveCommunity(delta, communityActorId);
   if (!resolved.community) {
     throw "Missing beta community";
   }
@@ -423,7 +423,7 @@ test("Community follower count is federated", async () => {
   await followCommunity(delta, true, resolved.community.community.id);
   followed = (
     await waitUntil(
-      () => resolveCommunity(delta, community_id),
+      () => resolveCommunity(delta, communityActorId),
       c => c.community?.subscribed === "Subscribed",
     )
   ).community;

api_tests/src/image.spec.ts

@@ -29,14 +29,16 @@ import {
   unfollows,
   getPost,
   waitUntil,
-  randomString,
   createPostWithThumbnail,
+  sampleImage,
 } from "./shared";
 const downloadFileSync = require("download-file-sync");
 
 beforeAll(setupLogins);
 
-afterAll(unfollows);
+afterAll(async () => {
+  await Promise.all([unfollows(), deleteAllImages(alpha)]);
+});
 
 test("Upload image and delete it", async () => {
   // Before running this test, you need to delete all previous images in the DB
@@ -159,7 +161,6 @@ test("Purge post, linked image removed", async () => {
   expect(post.post_view.post.url).toBe(upload.url);
 
   // purge post
-
   const purgeForm: PurgePost = {
     post_id: post.post_view.post.id,
   };
@@ -183,13 +184,13 @@ test("Images in remote post are proxied if setting enabled", async () => {
     gamma,
     community.community_view.community.id,
     upload.url,
-    "![](http://example.com/image2.png)",
+    `![](${sampleImage})`,
   );
   expect(post.post_view.post).toBeDefined();
 
   // remote image gets proxied after upload
   expect(
-    post.post_view.post.url?.startsWith(
+    post.post_view.post.thumbnail_url?.startsWith(
       "http://lemmy-gamma:8561/api/v3/image_proxy?url",
     ),
   ).toBeTruthy();
@@ -202,14 +203,20 @@ test("Images in remote post are proxied if setting enabled", async () => {
   let epsilonPost = await resolvePost(epsilon, post.post_view.post);
   expect(epsilonPost.post).toBeDefined();
 
-  // remote image gets proxied after federation
+  // Fetch the post again, the metadata should be backgrounded now
+  // Wait for the metadata to get fetched, since this is backgrounded now
+  let epsilonPost2 = await waitUntil(
+    () => getPost(epsilon, epsilonPost.post!.post.id),
+    p => p.post_view.post.thumbnail_url != undefined,
+  );
+
   expect(
-    epsilonPost.post!.post.url?.startsWith(
+    epsilonPost2.post_view.post.thumbnail_url?.startsWith(
       "http://lemmy-epsilon:8581/api/v3/image_proxy?url",
     ),
   ).toBeTruthy();
   expect(
-    epsilonPost.post!.post.body?.startsWith(
+    epsilonPost2.post_view.post.body?.startsWith(
       "![](http://lemmy-epsilon:8581/api/v3/image_proxy?url",
     ),
   ).toBeTruthy();
@@ -232,7 +239,7 @@ test("No image proxying if setting is disabled", async () => {
     alpha,
     community.community_view.community.id,
     upload.url,
-    "![](http://example.com/image2.png)",
+    `![](${sampleImage})`,
   );
   expect(post.post_view.post).toBeDefined();
 
@@ -240,7 +247,7 @@ test("No image proxying if setting is disabled", async () => {
   expect(
     post.post_view.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"),
   ).toBeTruthy();
-  expect(post.post_view.post.body).toBe("![](http://example.com/image2.png)");
+  expect(post.post_view.post.body).toBe(`![](${sampleImage})`);
 
   let betaPost = await waitForPost(
     beta,
@@ -253,8 +260,7 @@ test("No image proxying if setting is disabled", async () => {
   expect(
     betaPost.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"),
   ).toBeTruthy();
-  expect(betaPost.post.body).toBe("![](http://example.com/image2.png)");
+  expect(betaPost.post.body).toBe(`![](${sampleImage})`);
-
   // Make sure the alt text got federated
   expect(post.post_view.post.alt_text).toBe(betaPost.post.alt_text);
 });

api_tests/src/post.spec.ts

@@ -48,7 +48,6 @@ beforeAll(async () => {
   await setupLogins();
   betaCommunity = (await resolveBetaCommunity(alpha)).community;
   expect(betaCommunity).toBeDefined();
-  await unfollows();
 });
 
 afterAll(unfollows);
@@ -83,10 +82,7 @@ async function assertPostFederation(postOne: PostView, postTwo: PostView) {
 
 test("Create a post", async () => {
   // Setup some allowlists and blocklists
-  let editSiteForm: EditSite = {
+  const editSiteForm: EditSite = {};
-    allowed_instances: ["lemmy-beta"],
-  };
-  await delta.editSite(editSiteForm);
 
   editSiteForm.allowed_instances = [];
   editSiteForm.blocked_instances = ["lemmy-alpha"];
@@ -749,7 +745,7 @@ test("Block post that contains banned URL", async () => {
 
   await epsilon.editSite(editSiteForm);
 
-  await delay(500);
+  await delay();
 
   if (!betaCommunity) {
     throw "Missing beta community";

api_tests/src/shared.ts

@@ -81,6 +81,8 @@ import { ListingType } from "lemmy-js-client/dist/types/ListingType";
 
 export const fetchFunction = fetch;
 export const imageFetchLimit = 50;
+export const sampleImage =
+  "https://i.pinimg.com/originals/df/5f/5b/df5f5b1b174a2b4b6026cc6c8f9395c1.jpg";
 
 export let alphaUrl = "http://127.0.0.1:8541";
 export let betaUrl = "http://127.0.0.1:8551";
@@ -180,6 +182,10 @@ export async function setupLogins() {
   ];
   await gamma.editSite(editSiteForm);
 
+  // Setup delta allowed instance
+  editSiteForm.allowed_instances = ["lemmy-beta"];
+  await delta.editSite(editSiteForm);
+
   // Create the main alpha/beta communities
   // Ignore thrown errors of duplicates
   try {
@@ -693,8 +699,8 @@ export async function saveUserSettingsBio(
 export async function saveUserSettingsFederated(
   api: LemmyHttp,
 ): Promise<SuccessResponse> {
-  let avatar = "https://image.flaticon.com/icons/png/512/35/35896.png";
+  let avatar = sampleImage;
-  let banner = "https://image.flaticon.com/icons/png/512/36/35896.png";
+  let banner = sampleImage;
   let bio = "a changed bio";
   let form: SaveUserSettings = {
     show_nsfw: false,
@@ -760,6 +766,7 @@ export async function unfollowRemotes(
   await Promise.all(
     remoteFollowed.map(cu => followCommunity(api, false, cu.community.id)),
   );
+
   let siteRes = await getSite(api);
   return siteRes;
 }
@@ -889,14 +896,17 @@ export async function deleteAllImages(api: LemmyHttp) {
     limit: imageFetchLimit,
   });
   imagesRes.images;
-
+  Promise.all(
-  for (const image of imagesRes.images) {
+    imagesRes.images
+      .map(image => {
         const form: DeleteImage = {
           token: image.local_image.pictrs_delete_token,
           filename: image.local_image.pictrs_alias,
         };
-    await api.deleteImage(form);
+        return form;
-  }
+      })
+      .map(form => api.deleteImage(form)),
+  );
 }
 
 export async function unfollows() {
@@ -907,6 +917,24 @@ export async function unfollows() {
     unfollowRemotes(delta),
     unfollowRemotes(epsilon),
   ]);
+  await Promise.all([
+    purgeAllPosts(alpha),
+    purgeAllPosts(beta),
+    purgeAllPosts(gamma),
+    purgeAllPosts(delta),
+    purgeAllPosts(epsilon),
+  ]);
+}
+
+export async function purgeAllPosts(api: LemmyHttp) {
+  // The best way to get all federated items, is to find the posts
+  let res = await api.getPosts({ type_: "All", limit: 50 });
+  await Promise.all(
+    Array.from(new Set(res.posts.map(p => p.post.id)))
+      .map(post_id => api.purgePost({ post_id }))
+      // Ignore errors
+      .map(p => p.catch(e => e)),
+  );
 }
 
 export function getCommentParentId(comment: Comment): number | undefined {

crates/api/src/lib.rs

@@ -44,6 +44,7 @@ pub mod site;
 pub mod sitemap;
 
 /// Converts the captcha to a base64 encoded wav audio file
+#[allow(deprecated)]
 pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> LemmyResult<String> {
   let letters = captcha.as_wav();
 

crates/api_common/src/request.rs

@@ -3,9 +3,10 @@ use crate::{
   lemmy_db_schema::traits::Crud,
   post::{LinkMetadata, OpenGraphData},
   send_activity::{ActivityChannel, SendActivityData},
-  utils::{local_site_opt_to_sensitive, proxy_image_link, proxy_image_link_opt_apub},
+  utils::{local_site_opt_to_sensitive, proxy_image_link},
 };
 use activitypub_federation::config::Data;
+use chrono::{DateTime, Utc};
 use encoding_rs::{Encoding, UTF_8};
 use lemmy_db_schema::{
   newtypes::DbUrl,
@@ -18,14 +19,13 @@ use lemmy_db_schema::{
 use lemmy_utils::{
   error::{LemmyError, LemmyErrorType, LemmyResult},
   settings::structs::{PictrsImageMode, Settings},
-  spawn_try_task,
   REQWEST_TIMEOUT,
   VERSION,
 };
 use mime::Mime;
 use reqwest::{header::CONTENT_TYPE, Client, ClientBuilder};
 use reqwest_middleware::ClientWithMiddleware;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use tracing::info;
 use url::Url;
 use urlencoding::encode;
@@ -65,22 +65,20 @@ pub async fn fetch_link_metadata(url: &Url, context: &LemmyContext) -> LemmyResu
   })
 }
 
-/// Generate post thumbnail in background task, because some sites can be very slow to respond.
+/// Generates and saves a post thumbnail and metadata.
 ///
 /// Takes a callback to generate a send activity task, so that post can be federated with metadata.
 ///
 /// TODO: `federated_thumbnail` param can be removed once we federate full metadata and can
 ///       write it to db directly, without calling this function.
 ///       https://github.com/LemmyNet/lemmy/issues/4598
-pub fn generate_post_link_metadata(
+pub async fn generate_post_link_metadata(
   post: Post,
   custom_thumbnail: Option<Url>,
-  federated_thumbnail: Option<Url>,
   send_activity: impl FnOnce(Post) -> Option<SendActivityData> + Send + 'static,
   local_site: Option<LocalSite>,
   context: Data<LemmyContext>,
-) {
+) -> LemmyResult<()> {
-  spawn_try_task(async move {
   let metadata = match &post.url {
     Some(url) => fetch_link_metadata(url, &context).await.unwrap_or_default(),
     _ => Default::default(),
@@ -95,38 +93,28 @@ pub fn generate_post_link_metadata(
   let allow_sensitive = local_site_opt_to_sensitive(&local_site);
   let allow_generate_thumbnail = allow_sensitive || !post.nsfw;
 
-    // Use custom thumbnail if available and its not an image post
+  let image_url = if is_image_post {
-    let thumbnail_url = if !is_image_post && custom_thumbnail.is_some() {
+    post.url
-      custom_thumbnail
+  } else {
-    }
+    metadata.opengraph_data.image.clone()
-    // Use federated thumbnail if available
-    else if federated_thumbnail.is_some() {
-      federated_thumbnail
-    }
-    // Generate local thumbnail if allowed
-    else if allow_generate_thumbnail {
-      match post
-        .url
-        .filter(|_| is_image_post)
-        .or(metadata.opengraph_data.image)
-      {
-        Some(url) => generate_pictrs_thumbnail(&url, &context).await.ok(),
-        None => None,
-      }
-    }
-    // Otherwise use opengraph preview image directly
-    else {
-      metadata.opengraph_data.image.map(Into::into)
   };
 
-    // Proxy the image fetch if necessary
+  let thumbnail_url = if let (false, Some(url)) = (is_image_post, custom_thumbnail) {
-    let proxied_thumbnail_url = proxy_image_link_opt_apub(thumbnail_url, &context).await?;
+    proxy_image_link(url, &context).await.ok()
+  } else if let (true, Some(url)) = (allow_generate_thumbnail, image_url) {
+    generate_pictrs_thumbnail(&url, &context)
+      .await
+      .ok()
+      .map(Into::into)
+  } else {
+    metadata.opengraph_data.image.clone()
+  };
 
   let form = PostUpdateForm {
     embed_title: Some(metadata.opengraph_data.title),
     embed_description: Some(metadata.opengraph_data.description),
     embed_video_url: Some(metadata.opengraph_data.embed_video_url),
-      thumbnail_url: Some(proxied_thumbnail_url),
+    thumbnail_url: Some(thumbnail_url),
     url_content_type: Some(metadata.content_type),
     ..Default::default()
   };
@@ -135,25 +123,12 @@ pub fn generate_post_link_metadata(
     ActivityChannel::submit_activity(send_activity, &context).await?;
   }
   Ok(())
-  });
 }
 
 /// Extract site metadata from HTML Opengraph attributes.
 fn extract_opengraph_data(html_bytes: &[u8], url: &Url) -> LemmyResult<OpenGraphData> {
   let html = String::from_utf8_lossy(html_bytes);
 
-  // Make sure the first line is doctype html
-  let first_line = html
-    .trim_start()
-    .lines()
-    .next()
-    .ok_or(LemmyErrorType::NoLinesInHtml)?
-    .to_lowercase();
-
-  if !first_line.starts_with("<!doctype html") {
-    Err(LemmyErrorType::SiteMetadataPageIsNotDoctypeHtml)?
-  }
-
   let mut page = HTML::from_string(html.to_string(), None)?;
 
   // If the web page specifies that it isn't actually UTF-8, re-decode the received bytes with the
@@ -201,19 +176,40 @@ fn extract_opengraph_data(html_bytes: &[u8], url: &Url) -> LemmyResult<OpenGraph
   })
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize, Serialize, Debug)]
-struct PictrsResponse {
+pub struct PictrsResponse {
-  files: Vec<PictrsFile>,
+  pub files: Option<Vec<PictrsFile>>,
-  msg: String,
+  pub msg: String,
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize, Serialize, Debug)]
-struct PictrsFile {
+pub struct PictrsFile {
-  file: String,
+  pub file: String,
-  delete_token: String,
+  pub delete_token: String,
+  pub details: PictrsFileDetails,
 }
 
-#[derive(Deserialize, Debug)]
+impl PictrsFile {
+  pub fn thumbnail_url(&self, protocol_and_hostname: &str) -> Result<Url, url::ParseError> {
+    Url::parse(&format!(
+      "{protocol_and_hostname}/pictrs/image/{}",
+      self.file
+    ))
+  }
+}
+
+/// Stores extra details about a Pictrs image.
+#[derive(Deserialize, Serialize, Debug)]
+pub struct PictrsFileDetails {
+  /// In pixels
+  pub width: u16,
+  /// In pixels
+  pub height: u16,
+  pub content_type: String,
+  pub created_at: DateTime<Utc>,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
 struct PictrsPurgeResponse {
   msg: String,
 }
@@ -295,33 +291,34 @@ async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> L
     encode(image_url.as_str())
   );
 
-  let response = context
+  let res = context
     .client()
     .get(&fetch_url)
     .timeout(REQWEST_TIMEOUT)
     .send()
+    .await?
+    .json::<PictrsResponse>()
     .await?;
 
-  let response: PictrsResponse = response.json().await?;
+  let files = res.files.unwrap_or_default();
+
+  let image = files
+    .first()
+    .ok_or(LemmyErrorType::PictrsResponseError(res.msg))?;
 
-  if response.msg == "ok" {
-    let thumbnail_url = Url::parse(&format!(
-      "{}/pictrs/image/{}",
-      context.settings().get_protocol_and_hostname(),
-      response.files.first().expect("missing pictrs file").file
-    ))?;
-    for uploaded_image in response.files {
   let form = LocalImageForm {
+    // This is none because its an internal request.
+    // IE, a local user shouldn't get to delete the thumbnails for their link posts
     local_user_id: None,
-        pictrs_alias: uploaded_image.file.to_string(),
+    pictrs_alias: image.file.clone(),
-        pictrs_delete_token: uploaded_image.delete_token.to_string(),
+    pictrs_delete_token: image.delete_token.clone(),
   };
+  let protocol_and_hostname = context.settings().get_protocol_and_hostname();
+  let thumbnail_url = image.thumbnail_url(&protocol_and_hostname)?;
+
   LocalImage::create(&mut context.pool(), &form).await?;
-    }
+
   Ok(thumbnail_url)
-  } else {
-    Err(LemmyErrorType::PictrsResponseError(response.msg))?
-  }
 }
 
 // TODO: get rid of this by reading content type from db

crates/api_common/src/utils.rs

@@ -965,13 +965,10 @@ async fn proxy_image_link_internal(
   if link.domain() == Some(&context.settings().hostname) {
     Ok(link.into())
   } else if image_mode == PictrsImageMode::ProxyAllImages {
-    let proxied = format!(
+    let proxied = build_proxied_image_url(&link, &context.settings().get_protocol_and_hostname())?;
-      "{}/api/v3/image_proxy?url={}",
+
-      context.settings().get_protocol_and_hostname(),
-      encode(link.as_str())
-    );
     RemoteImage::create(&mut context.pool(), vec![link]).await?;
-    Ok(Url::parse(&proxied)?.into())
+    Ok(proxied.into())
   } else {
     Ok(link.into())
   }
@@ -1025,6 +1022,17 @@ pub async fn proxy_image_link_opt_apub(
   }
 }
 
+fn build_proxied_image_url(
+  link: &Url,
+  protocol_and_hostname: &str,
+) -> Result<Url, url::ParseError> {
+  Url::parse(&format!(
+    "{}/api/v3/image_proxy?url={}",
+    protocol_and_hostname,
+    encode(link.as_str())
+  ))
+}
+
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
 #[allow(clippy::indexing_slicing)]

crates/api_crud/src/comment/remove.rs

@@ -37,6 +37,12 @@ pub async fn remove_comment(
   )
   .await?;
 
+  // Don't allow removing or restoring comment which was deleted by user, as it would reveal
+  // the comment text in mod log.
+  if orig_comment.comment.deleted {
+    return Err(LemmyErrorType::CouldntUpdateComment.into());
+  }
+
   // Do the remove
   let removed = data.removed;
   let updated_comment = Comment::update(

crates/api_crud/src/post/create.rs

@@ -14,7 +14,6 @@ use lemmy_api_common::{
     local_site_to_slur_regex,
     mark_post_as_read,
     process_markdown_opt,
-    proxy_image_link_opt_apub,
     EndpointType,
   },
 };
@@ -75,7 +74,6 @@ pub async fn create_post(
   is_url_blocked(&url, &url_blocklist)?;
   check_url_scheme(&url)?;
   check_url_scheme(&custom_thumbnail)?;
-  let url = proxy_image_link_opt_apub(url, &context).await?;
 
   check_community_user_action(
     &local_user_view.person,
@@ -125,7 +123,7 @@ pub async fn create_post(
 
   let post_form = PostInsertForm::builder()
     .name(data.name.trim().to_string())
-    .url(url)
+    .url(url.map(Into::into))
     .body(body)
     .alt_text(data.alt_text.clone())
     .community_id(data.community_id)
@@ -159,11 +157,11 @@ pub async fn create_post(
   generate_post_link_metadata(
     updated_post.clone(),
     custom_thumbnail,
-    None,
     |post| Some(SendActivityData::CreatePost(post)),
     Some(local_site),
     context.reset_request_count(),
-  );
+  )
+  .await?;
 
   // They like their own post by default
   let person_id = local_user_view.person.id;

crates/api_crud/src/post/update.rs

@@ -11,7 +11,6 @@ use lemmy_api_common::{
     get_url_blocklist,
     local_site_to_slur_regex,
     process_markdown_opt,
-    proxy_image_link_opt_apub,
   },
 };
 use lemmy_db_schema::{
@@ -86,11 +85,6 @@ pub async fn update_post(
     Err(LemmyErrorType::NoPostEditAllowed)?
   }
 
-  let url = match url {
-    Some(url) => Some(proxy_image_link_opt_apub(Some(url), &context).await?),
-    _ => Default::default(),
-  };
-
   let language_id = data.language_id;
   CommunityLanguage::is_allowed_community_language(
     &mut context.pool(),
@@ -101,7 +95,7 @@ pub async fn update_post(
 
   let post_form = PostUpdateForm {
     name: data.name.clone(),
-    url,
+    url: Some(url.map(Into::into)),
     body: diesel_option_overwrite(body),
     alt_text: diesel_option_overwrite(data.alt_text.clone()),
     nsfw: data.nsfw,
@@ -118,11 +112,11 @@ pub async fn update_post(
   generate_post_link_metadata(
     updated_post.clone(),
     custom_thumbnail,
-    None,
     |post| Some(SendActivityData::UpdatePost(post)),
     Some(local_site),
     context.reset_request_count(),
-  );
+  )
+  .await?;
 
   build_post_response(
     context.deref(),

crates/apub/src/objects/post.rs

@@ -45,6 +45,7 @@ use lemmy_db_schema::{
 use lemmy_db_views_actor::structs::CommunityModeratorView;
 use lemmy_utils::{
   error::{LemmyError, LemmyErrorType, LemmyResult},
+  spawn_try_task,
   utils::{markdown::markdown_to_html, slurs::check_slurs_opt, validation::check_url_scheme},
 };
 use std::ops::Deref;
@@ -255,15 +256,13 @@ impl Object for ApubPost {
 
     let timestamp = page.updated.or(page.published).unwrap_or_else(naive_now);
     let post = Post::insert_apub(&mut context.pool(), timestamp, &form).await?;
+    let post_ = post.clone();
+    let context_ = context.reset_request_count();
 
-    generate_post_link_metadata(
+    // Generates a post thumbnail in background task, because some sites can be very slow to respond.
-      post.clone(),
+    spawn_try_task(async move {
-      None,
+      generate_post_link_metadata(post_, None, |_| None, local_site, context_).await
-      page.image.map(|i| i.url),
+    });
-      |_| None,
-      local_site,
-      context.reset_request_count(),
-    );
 
     Ok(post.into())
   }

crates/db_schema/src/impls/comment_reply.rs

@@ -87,117 +87,3 @@ impl CommentReply {
       .optional()
   }
 }
-
-#[cfg(test)]
-#[allow(clippy::unwrap_used)]
-#[allow(clippy::indexing_slicing)]
-mod tests {
-
-  use crate::{
-    source::{
-      comment::{Comment, CommentInsertForm},
-      comment_reply::{CommentReply, CommentReplyInsertForm, CommentReplyUpdateForm},
-      community::{Community, CommunityInsertForm},
-      instance::Instance,
-      person::{Person, PersonInsertForm},
-      post::{Post, PostInsertForm},
-    },
-    traits::Crud,
-    utils::build_db_pool_for_tests,
-  };
-  use pretty_assertions::assert_eq;
-  use serial_test::serial;
-
-  #[tokio::test]
-  #[serial]
-  async fn test_crud() {
-    let pool = &build_db_pool_for_tests().await;
-    let pool = &mut pool.into();
-
-    let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string())
-      .await
-      .unwrap();
-
-    let new_person = PersonInsertForm::builder()
-      .name("terrylake".into())
-      .public_key("pubkey".to_string())
-      .instance_id(inserted_instance.id)
-      .build();
-
-    let inserted_person = Person::create(pool, &new_person).await.unwrap();
-
-    let recipient_form = PersonInsertForm::builder()
-      .name("terrylakes recipient".into())
-      .public_key("pubkey".to_string())
-      .instance_id(inserted_instance.id)
-      .build();
-
-    let inserted_recipient = Person::create(pool, &recipient_form).await.unwrap();
-
-    let new_community = CommunityInsertForm::builder()
-      .name("test community lake".to_string())
-      .title("nada".to_owned())
-      .public_key("pubkey".to_string())
-      .instance_id(inserted_instance.id)
-      .build();
-
-    let inserted_community = Community::create(pool, &new_community).await.unwrap();
-
-    let new_post = PostInsertForm::builder()
-      .name("A test post".into())
-      .creator_id(inserted_person.id)
-      .community_id(inserted_community.id)
-      .build();
-
-    let inserted_post = Post::create(pool, &new_post).await.unwrap();
-
-    let comment_form = CommentInsertForm::builder()
-      .content("A test comment".into())
-      .creator_id(inserted_person.id)
-      .post_id(inserted_post.id)
-      .build();
-
-    let inserted_comment = Comment::create(pool, &comment_form, None).await.unwrap();
-
-    let comment_reply_form = CommentReplyInsertForm {
-      recipient_id: inserted_recipient.id,
-      comment_id: inserted_comment.id,
-      read: None,
-    };
-
-    let inserted_reply = CommentReply::create(pool, &comment_reply_form)
-      .await
-      .unwrap();
-
-    let expected_reply = CommentReply {
-      id: inserted_reply.id,
-      recipient_id: inserted_reply.recipient_id,
-      comment_id: inserted_reply.comment_id,
-      read: false,
-      published: inserted_reply.published,
-    };
-
-    let read_reply = CommentReply::read(pool, inserted_reply.id)
-      .await
-      .unwrap()
-      .unwrap();
-
-    let comment_reply_update_form = CommentReplyUpdateForm { read: Some(false) };
-    let updated_reply = CommentReply::update(pool, inserted_reply.id, &comment_reply_update_form)
-      .await
-      .unwrap();
-
-    Comment::delete(pool, inserted_comment.id).await.unwrap();
-    Post::delete(pool, inserted_post.id).await.unwrap();
-    Community::delete(pool, inserted_community.id)
-      .await
-      .unwrap();
-    Person::delete(pool, inserted_person.id).await.unwrap();
-    Person::delete(pool, inserted_recipient.id).await.unwrap();
-    Instance::delete(pool, inserted_instance.id).await.unwrap();
-
-    assert_eq!(expected_reply, read_reply);
-    assert_eq!(expected_reply, inserted_reply);
-    assert_eq!(expected_reply, updated_reply);
-  }
-}

crates/db_schema/src/impls/person_mention.rs

@@ -74,117 +74,3 @@ impl PersonMention {
       .optional()
   }
 }
-
-#[cfg(test)]
-#[allow(clippy::unwrap_used)]
-#[allow(clippy::indexing_slicing)]
-mod tests {
-
-  use crate::{
-    source::{
-      comment::{Comment, CommentInsertForm},
-      community::{Community, CommunityInsertForm},
-      instance::Instance,
-      person::{Person, PersonInsertForm},
-      person_mention::{PersonMention, PersonMentionInsertForm, PersonMentionUpdateForm},
-      post::{Post, PostInsertForm},
-    },
-    traits::Crud,
-    utils::build_db_pool_for_tests,
-  };
-  use pretty_assertions::assert_eq;
-  use serial_test::serial;
-
-  #[tokio::test]
-  #[serial]
-  async fn test_crud() {
-    let pool = &build_db_pool_for_tests().await;
-    let pool = &mut pool.into();
-
-    let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string())
-      .await
-      .unwrap();
-
-    let new_person = PersonInsertForm::builder()
-      .name("terrylake".into())
-      .public_key("pubkey".to_string())
-      .instance_id(inserted_instance.id)
-      .build();
-
-    let inserted_person = Person::create(pool, &new_person).await.unwrap();
-
-    let recipient_form = PersonInsertForm::builder()
-      .name("terrylakes recipient".into())
-      .public_key("pubkey".to_string())
-      .instance_id(inserted_instance.id)
-      .build();
-
-    let inserted_recipient = Person::create(pool, &recipient_form).await.unwrap();
-
-    let new_community = CommunityInsertForm::builder()
-      .name("test community lake".to_string())
-      .title("nada".to_owned())
-      .public_key("pubkey".to_string())
-      .instance_id(inserted_instance.id)
-      .build();
-
-    let inserted_community = Community::create(pool, &new_community).await.unwrap();
-
-    let new_post = PostInsertForm::builder()
-      .name("A test post".into())
-      .creator_id(inserted_person.id)
-      .community_id(inserted_community.id)
-      .build();
-
-    let inserted_post = Post::create(pool, &new_post).await.unwrap();
-
-    let comment_form = CommentInsertForm::builder()
-      .content("A test comment".into())
-      .creator_id(inserted_person.id)
-      .post_id(inserted_post.id)
-      .build();
-
-    let inserted_comment = Comment::create(pool, &comment_form, None).await.unwrap();
-
-    let person_mention_form = PersonMentionInsertForm {
-      recipient_id: inserted_recipient.id,
-      comment_id: inserted_comment.id,
-      read: None,
-    };
-
-    let inserted_mention = PersonMention::create(pool, &person_mention_form)
-      .await
-      .unwrap();
-
-    let expected_mention = PersonMention {
-      id: inserted_mention.id,
-      recipient_id: inserted_mention.recipient_id,
-      comment_id: inserted_mention.comment_id,
-      read: false,
-      published: inserted_mention.published,
-    };
-
-    let read_mention = PersonMention::read(pool, inserted_mention.id)
-      .await
-      .unwrap()
-      .unwrap();
-
-    let person_mention_update_form = PersonMentionUpdateForm { read: Some(false) };
-    let updated_mention =
-      PersonMention::update(pool, inserted_mention.id, &person_mention_update_form)
-        .await
-        .unwrap();
-    Comment::delete(pool, inserted_comment.id).await.unwrap();
-    Post::delete(pool, inserted_post.id).await.unwrap();
-    Community::delete(pool, inserted_community.id)
-      .await
-      .unwrap();
-    Person::delete(pool, inserted_person.id).await.unwrap();
-    Person::delete(pool, inserted_recipient.id).await.unwrap();
-    Instance::delete(pool, inserted_instance.id).await.unwrap();
-
-    assert_eq!(expected_mention, read_mention);
-    assert_eq!(expected_mention, inserted_mention);
-    assert_eq!(expected_mention, updated_mention);
-  }
-}

crates/db_schema/src/source/images.rs

@@ -22,7 +22,7 @@ use typed_builder::TypedBuilder;
   diesel(belongs_to(crate::source::local_user::LocalUser))
 )]
 #[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))]
-#[cfg_attr(feature = "full", diesel(primary_key(local_user_id)))]
+#[cfg_attr(feature = "full", diesel(primary_key(pictrs_alias)))]
 pub struct LocalImage {
   pub local_user_id: Option<LocalUserId>,
   pub pictrs_alias: String,
@@ -41,9 +41,11 @@ pub struct LocalImageForm {
 
 /// Stores all images which are hosted on remote domains. When attempting to proxy an image, it
 /// is checked against this table to avoid Lemmy being used as a general purpose proxy.
-#[derive(PartialEq, Eq, Debug, Clone)]
+#[skip_serializing_none]
-#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
+#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
+#[cfg_attr(feature = "full", derive(Queryable, Selectable, Identifiable))]
 #[cfg_attr(feature = "full", diesel(table_name = remote_image))]
+#[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))]
 pub struct RemoteImage {
   pub id: i32,
   pub link: DbUrl,

crates/db_views/src/comment_view.rs

@@ -437,7 +437,6 @@ impl<'a> CommentQuery<'a> {
 }
 
 #[cfg(test)]
-#[allow(clippy::unwrap_used)]
 #[allow(clippy::indexing_slicing)]
 mod tests {
 
@@ -498,32 +497,28 @@ mod tests {
     inserted_community: Community,
   }
 
-  async fn init_data(pool: &mut DbPool<'_>) -> Data {
+  async fn init_data(pool: &mut DbPool<'_>) -> LemmyResult<Data> {
-    let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string())
+    let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?;
-      .await
-      .unwrap();
 
     let timmy_person_form = PersonInsertForm::builder()
       .name("timmy".into())
       .public_key("pubkey".to_string())
       .instance_id(inserted_instance.id)
       .build();
-    let inserted_timmy_person = Person::create(pool, &timmy_person_form).await.unwrap();
+    let inserted_timmy_person = Person::create(pool, &timmy_person_form).await?;
     let timmy_local_user_form = LocalUserInsertForm::builder()
       .person_id(inserted_timmy_person.id)
       .admin(Some(true))
       .password_encrypted(String::new())
       .build();
-    let inserted_timmy_local_user = LocalUser::create(pool, &timmy_local_user_form, vec![])
+    let inserted_timmy_local_user = LocalUser::create(pool, &timmy_local_user_form, vec![]).await?;
-      .await
-      .unwrap();
 
     let sara_person_form = PersonInsertForm::builder()
       .name("sara".into())
       .public_key("pubkey".to_string())
       .instance_id(inserted_instance.id)
       .build();
-    let inserted_sara_person = Person::create(pool, &sara_person_form).await.unwrap();
+    let inserted_sara_person = Person::create(pool, &sara_person_form).await?;
 
     let new_community = CommunityInsertForm::builder()
       .name("test community 5".to_string())
@@ -532,7 +527,7 @@ mod tests {
       .instance_id(inserted_instance.id)
       .build();
 
-    let inserted_community = Community::create(pool, &new_community).await.unwrap();
+    let inserted_community = Community::create(pool, &new_community).await?;
 
     let new_post = PostInsertForm::builder()
       .name("A test post 2".into())
@@ -540,8 +535,8 @@ mod tests {
       .community_id(inserted_community.id)
       .build();
 
-    let inserted_post = Post::create(pool, &new_post).await.unwrap();
+    let inserted_post = Post::create(pool, &new_post).await?;
-    let english_id = Language::read_id_from_code(pool, Some("en")).await.unwrap();
+    let english_id = Language::read_id_from_code(pool, Some("en")).await?;
 
     // Create a comment tree with this hierarchy
     //       0
@@ -558,7 +553,7 @@ mod tests {
       .language_id(english_id)
       .build();
 
-    let inserted_comment_0 = Comment::create(pool, &comment_form_0, None).await.unwrap();
+    let inserted_comment_0 = Comment::create(pool, &comment_form_0, None).await?;
 
     let comment_form_1 = CommentInsertForm::builder()
       .content("Comment 1, A test blocked comment".into())
@@ -567,11 +562,10 @@ mod tests {
       .language_id(english_id)
       .build();
 
-    let inserted_comment_1 = Comment::create(pool, &comment_form_1, Some(&inserted_comment_0.path))
+    let inserted_comment_1 =
-      .await
+      Comment::create(pool, &comment_form_1, Some(&inserted_comment_0.path)).await?;
-      .unwrap();
 
-    let finnish_id = Language::read_id_from_code(pool, Some("fi")).await.unwrap();
+    let finnish_id = Language::read_id_from_code(pool, Some("fi")).await?;
     let comment_form_2 = CommentInsertForm::builder()
       .content("Comment 2".into())
       .creator_id(inserted_timmy_person.id)
@@ -579,9 +573,8 @@ mod tests {
       .language_id(finnish_id)
       .build();
 
-    let inserted_comment_2 = Comment::create(pool, &comment_form_2, Some(&inserted_comment_0.path))
+    let inserted_comment_2 =
-      .await
+      Comment::create(pool, &comment_form_2, Some(&inserted_comment_0.path)).await?;
-      .unwrap();
 
     let comment_form_3 = CommentInsertForm::builder()
       .content("Comment 3".into())
@@ -591,14 +584,11 @@ mod tests {
       .build();
 
     let _inserted_comment_3 =
-      Comment::create(pool, &comment_form_3, Some(&inserted_comment_1.path))
+      Comment::create(pool, &comment_form_3, Some(&inserted_comment_1.path)).await?;
-        .await
-        .unwrap();
 
     let polish_id = Language::read_id_from_code(pool, Some("pl"))
-      .await
+      .await?
-      .unwrap()
+      .ok_or(LemmyErrorType::LanguageNotAllowed)?;
-      .unwrap();
     let comment_form_4 = CommentInsertForm::builder()
       .content("Comment 4".into())
       .creator_id(inserted_timmy_person.id)
@@ -606,9 +596,8 @@ mod tests {
       .language_id(Some(polish_id))
       .build();
 
-    let inserted_comment_4 = Comment::create(pool, &comment_form_4, Some(&inserted_comment_1.path))
+    let inserted_comment_4 =
-      .await
+      Comment::create(pool, &comment_form_4, Some(&inserted_comment_1.path)).await?;
-      .unwrap();
 
     let comment_form_5 = CommentInsertForm::builder()
       .content("Comment 5".into())
@@ -617,18 +606,14 @@ mod tests {
       .build();
 
     let _inserted_comment_5 =
-      Comment::create(pool, &comment_form_5, Some(&inserted_comment_4.path))
+      Comment::create(pool, &comment_form_5, Some(&inserted_comment_4.path)).await?;
-        .await
-        .unwrap();
 
     let timmy_blocks_sara_form = PersonBlockForm {
       person_id: inserted_timmy_person.id,
       target_id: inserted_sara_person.id,
     };
 
-    let inserted_block = PersonBlock::block(pool, &timmy_blocks_sara_form)
+    let inserted_block = PersonBlock::block(pool, &timmy_blocks_sara_form).await?;
-      .await
-      .unwrap();
 
     let expected_block = PersonBlock {
       person_id: inserted_timmy_person.id,
@@ -644,7 +629,7 @@ mod tests {
       score: 1,
     };
 
-    let _inserted_comment_like = CommentLike::like(pool, &comment_like_form).await.unwrap();
+    let _inserted_comment_like = CommentLike::like(pool, &comment_like_form).await?;
 
     let timmy_local_user_view = LocalUserView {
       local_user: inserted_timmy_local_user.clone(),
@@ -652,7 +637,7 @@ mod tests {
       person: inserted_timmy_person.clone(),
       counts: Default::default(),
     };
-    Data {
+    Ok(Data {
       inserted_instance,
       inserted_comment_0,
       inserted_comment_1,
@@ -661,7 +646,7 @@ mod tests {
       timmy_local_user_view,
       inserted_sara_person,
       inserted_community,
-    }
+    })
   }
 
   #[tokio::test]
@@ -669,7 +654,7 @@ mod tests {
   async fn test_crud() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     let expected_comment_view_no_person = expected_comment_view(&data, pool).await?;
 
@@ -714,7 +699,7 @@ mod tests {
       Some(data.timmy_local_user_view.person.id),
     )
     .await?
-    .unwrap();
+    .ok_or(LemmyErrorType::CouldntFindComment)?;
 
     // Make sure block set the creator blocked
     assert!(read_comment_from_blocked_person.creator_blocked);
@@ -727,7 +712,7 @@ mod tests {
   async fn test_liked_only() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     // Unblock sara first
     let timmy_unblocks_sara_form = PersonBlockForm {
@@ -743,7 +728,7 @@ mod tests {
       person_id: data.timmy_local_user_view.person.id,
       score: 1,
     };
-    CommentLike::like(pool, &comment_like_form).await.unwrap();
+    CommentLike::like(pool, &comment_like_form).await?;
 
     let read_liked_comment_views = CommentQuery {
       local_user: (Some(&data.timmy_local_user_view)),
@@ -779,7 +764,7 @@ mod tests {
   async fn test_comment_tree() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     let top_path = data.inserted_comment_0.path.clone();
     let read_comment_views_top_path = CommentQuery {
@@ -852,7 +837,7 @@ mod tests {
   async fn test_languages() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     // by default, user has all languages enabled and should see all comments
     // (except from blocked user)
@@ -916,7 +901,7 @@ mod tests {
   async fn test_distinguished_first() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     let form = CommentUpdateForm {
       distinguished: Some(true),
@@ -941,7 +926,7 @@ mod tests {
   async fn test_creator_is_moderator() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     // Make one of the inserted persons a moderator
     let person_id = data.inserted_sara_person.id;
@@ -972,7 +957,7 @@ mod tests {
   async fn test_creator_is_admin() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     let comments = CommentQuery {
       sort: (Some(CommentSortType::Old)),
@@ -997,7 +982,7 @@ mod tests {
   async fn test_saved_order() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     // Save two comments
     let save_comment_0_form = CommentSavedForm {
@@ -1173,7 +1158,7 @@ mod tests {
   async fn local_only_instance() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     Community::update(
       pool,
@@ -1219,7 +1204,7 @@ mod tests {
   async fn comment_listing_local_user_banned_from_community() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     // Test that comment view shows if local user is blocked from community
     let banned_from_comm_person = PersonInsertForm::test_form(data.inserted_instance.id, "jill");
@@ -1262,7 +1247,7 @@ mod tests {
   async fn comment_listing_local_user_not_banned_from_community() -> LemmyResult<()> {
     let pool = &build_db_pool_for_tests().await;
     let pool = &mut pool.into();
-    let data = init_data(pool).await;
+    let data = init_data(pool).await?;
 
     let comment_view = CommentView::read(
       pool,

crates/db_views/src/structs.rs

@@ -216,6 +216,7 @@ pub struct VoteView {
   pub score: i16,
 }
 
+#[skip_serializing_none]
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[cfg_attr(feature = "full", derive(TS, Queryable))]
 #[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))]

crates/db_views_actor/src/comment_reply_view.rs

@@ -1,6 +1,6 @@
 use crate::structs::CommentReplyView;
 use diesel::{
-  dsl::exists,
+  dsl::{exists, not},
   pg::Pg,
   result::Error,
   sql_types,
@@ -217,6 +217,11 @@ fn queries<'a>() -> Queries<
       CommentSortType::Top => query.order_by(comment_aggregates::score.desc()),
     };
 
+    // Don't show replies from blocked persons
+    if let Some(my_person_id) = options.my_person_id {
+      query = query.filter(not(is_creator_blocked(my_person_id)));
+    }
+
     let (limit, offset) = limit_and_offset(options.page, options.limit)?;
 
     query
@@ -268,7 +273,7 @@ impl CommentReplyView {
   }
 }
 
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct CommentReplyQuery {
   pub my_person_id: Option<PersonId>,
   pub recipient_id: Option<PersonId>,
@@ -284,3 +289,141 @@ impl CommentReplyQuery {
     queries().list(pool, self).await
   }
 }
+
+#[cfg(test)]
+#[allow(clippy::indexing_slicing)]
+mod tests {
+
+  use crate::{comment_reply_view::CommentReplyQuery, structs::CommentReplyView};
+  use lemmy_db_schema::{
+    source::{
+      comment::{Comment, CommentInsertForm},
+      comment_reply::{CommentReply, CommentReplyInsertForm, CommentReplyUpdateForm},
+      community::{Community, CommunityInsertForm},
+      instance::Instance,
+      person::{Person, PersonInsertForm},
+      person_block::{PersonBlock, PersonBlockForm},
+      post::{Post, PostInsertForm},
+    },
+    traits::{Blockable, Crud},
+    utils::build_db_pool_for_tests,
+  };
+  use lemmy_utils::{error::LemmyResult, LemmyErrorType};
+  use pretty_assertions::assert_eq;
+  use serial_test::serial;
+
+  #[tokio::test]
+  #[serial]
+  async fn test_crud() -> LemmyResult<()> {
+    let pool = &build_db_pool_for_tests().await;
+    let pool = &mut pool.into();
+
+    let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?;
+
+    let terry_form = PersonInsertForm::builder()
+      .name("terrylake".into())
+      .public_key("pubkey".to_string())
+      .instance_id(inserted_instance.id)
+      .build();
+    let inserted_terry = Person::create(pool, &terry_form).await?;
+
+    let recipient_form = PersonInsertForm::builder()
+      .name("terrylakes recipient".into())
+      .public_key("pubkey".to_string())
+      .instance_id(inserted_instance.id)
+      .build();
+
+    let inserted_recipient = Person::create(pool, &recipient_form).await?;
+    let recipient_id = inserted_recipient.id;
+
+    let new_community = CommunityInsertForm::builder()
+      .name("test community lake".to_string())
+      .title("nada".to_owned())
+      .public_key("pubkey".to_string())
+      .instance_id(inserted_instance.id)
+      .build();
+
+    let inserted_community = Community::create(pool, &new_community).await?;
+
+    let new_post = PostInsertForm::builder()
+      .name("A test post".into())
+      .creator_id(inserted_terry.id)
+      .community_id(inserted_community.id)
+      .build();
+
+    let inserted_post = Post::create(pool, &new_post).await?;
+
+    let comment_form = CommentInsertForm::builder()
+      .content("A test comment".into())
+      .creator_id(inserted_terry.id)
+      .post_id(inserted_post.id)
+      .build();
+
+    let inserted_comment = Comment::create(pool, &comment_form, None).await?;
+
+    let comment_reply_form = CommentReplyInsertForm {
+      recipient_id: inserted_recipient.id,
+      comment_id: inserted_comment.id,
+      read: None,
+    };
+
+    let inserted_reply = CommentReply::create(pool, &comment_reply_form).await?;
+
+    let expected_reply = CommentReply {
+      id: inserted_reply.id,
+      recipient_id: inserted_reply.recipient_id,
+      comment_id: inserted_reply.comment_id,
+      read: false,
+      published: inserted_reply.published,
+    };
+
+    let read_reply = CommentReply::read(pool, inserted_reply.id)
+      .await?
+      .ok_or(LemmyErrorType::CouldntFindComment)?;
+
+    let comment_reply_update_form = CommentReplyUpdateForm { read: Some(false) };
+    let updated_reply =
+      CommentReply::update(pool, inserted_reply.id, &comment_reply_update_form).await?;
+
+    // Test to make sure counts and blocks work correctly
+    let unread_replies = CommentReplyView::get_unread_replies(pool, recipient_id).await?;
+
+    let query = CommentReplyQuery {
+      recipient_id: Some(recipient_id),
+      my_person_id: Some(recipient_id),
+      sort: None,
+      unread_only: false,
+      show_bot_accounts: true,
+      page: None,
+      limit: None,
+    };
+    let replies = query.clone().list(pool).await?;
+    assert_eq!(1, unread_replies);
+    assert_eq!(1, replies.len());
+
+    // Block the person, and make sure these counts are now empty
+    let block_form = PersonBlockForm {
+      person_id: recipient_id,
+      target_id: inserted_terry.id,
+    };
+    PersonBlock::block(pool, &block_form).await?;
+
+    let unread_replies_after_block =
+      CommentReplyView::get_unread_replies(pool, recipient_id).await?;
+    let replies_after_block = query.list(pool).await?;
+    assert_eq!(0, unread_replies_after_block);
+    assert_eq!(0, replies_after_block.len());
+
+    Comment::delete(pool, inserted_comment.id).await?;
+    Post::delete(pool, inserted_post.id).await?;
+    Community::delete(pool, inserted_community.id).await?;
+    Person::delete(pool, inserted_terry.id).await?;
+    Person::delete(pool, inserted_recipient.id).await?;
+    Instance::delete(pool, inserted_instance.id).await?;
+
+    assert_eq!(expected_reply, read_reply);
+    assert_eq!(expected_reply, inserted_reply);
+    assert_eq!(expected_reply, updated_reply);
+    Ok(())
+  }
+}

crates/db_views_actor/src/person_mention_view.rs

@@ -1,6 +1,6 @@
 use crate::structs::PersonMentionView;
 use diesel::{
-  dsl::exists,
+  dsl::{exists, not},
   pg::Pg,
   result::Error,
   sql_types,
@@ -216,6 +216,11 @@ fn queries<'a>() -> Queries<
       CommentSortType::Top => query.order_by(comment_aggregates::score.desc()),
     };
 
+    // Don't show mentions from blocked persons
+    if let Some(my_person_id) = options.my_person_id {
+      query = query.filter(not(is_creator_blocked(my_person_id)));
+    }
+
     let (limit, offset) = limit_and_offset(options.page, options.limit)?;
 
     query
@@ -249,6 +254,15 @@ impl PersonMentionView {
 
     person_mention::table
       .inner_join(comment::table)
+      .left_join(
+        person_block::table.on(
+          comment::creator_id
+            .eq(person_block::target_id)
+            .and(person_block::person_id.eq(my_person_id)),
+        ),
+      )
+      // Dont count replies from blocked users
+      .filter(person_block::person_id.is_null())
       .filter(person_mention::recipient_id.eq(my_person_id))
       .filter(person_mention::read.eq(false))
       .filter(comment::deleted.eq(false))
@@ -259,7 +273,7 @@ impl PersonMentionView {
   }
 }
 
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct PersonMentionQuery {
   pub my_person_id: Option<PersonId>,
   pub recipient_id: Option<PersonId>,
@@ -275,3 +289,143 @@ impl PersonMentionQuery {
     queries().list(pool, self).await
   }
 }
+
+#[cfg(test)]
+#[allow(clippy::indexing_slicing)]
+mod tests {
+
+  use crate::{person_mention_view::PersonMentionQuery, structs::PersonMentionView};
+  use lemmy_db_schema::{
+    source::{
+      comment::{Comment, CommentInsertForm},
+      community::{Community, CommunityInsertForm},
+      instance::Instance,
+      person::{Person, PersonInsertForm},
+      person_block::{PersonBlock, PersonBlockForm},
+      person_mention::{PersonMention, PersonMentionInsertForm, PersonMentionUpdateForm},
+      post::{Post, PostInsertForm},
+    },
+    traits::{Blockable, Crud},
+    utils::build_db_pool_for_tests,
+  };
+  use lemmy_utils::{error::LemmyResult, LemmyErrorType};
+  use pretty_assertions::assert_eq;
+  use serial_test::serial;
+
+  #[tokio::test]
+  #[serial]
+  async fn test_crud() -> LemmyResult<()> {
+    let pool = &build_db_pool_for_tests().await;
+    let pool = &mut pool.into();
+
+    let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?;
+
+    let new_person = PersonInsertForm::builder()
+      .name("terrylake".into())
+      .public_key("pubkey".to_string())
+      .instance_id(inserted_instance.id)
+      .build();
+
+    let inserted_person = Person::create(pool, &new_person).await?;
+
+    let recipient_form = PersonInsertForm::builder()
+      .name("terrylakes recipient".into())
+      .public_key("pubkey".to_string())
+      .instance_id(inserted_instance.id)
+      .build();
+
+    let inserted_recipient = Person::create(pool, &recipient_form).await?;
+    let recipient_id = inserted_recipient.id;
+
+    let new_community = CommunityInsertForm::builder()
+      .name("test community lake".to_string())
+      .title("nada".to_owned())
+      .public_key("pubkey".to_string())
+      .instance_id(inserted_instance.id)
+      .build();
+
+    let inserted_community = Community::create(pool, &new_community).await?;
+
+    let new_post = PostInsertForm::builder()
+      .name("A test post".into())
+      .creator_id(inserted_person.id)
+      .community_id(inserted_community.id)
+      .build();
+
+    let inserted_post = Post::create(pool, &new_post).await?;
+
+    let comment_form = CommentInsertForm::builder()
+      .content("A test comment".into())
+      .creator_id(inserted_person.id)
+      .post_id(inserted_post.id)
+      .build();
+
+    let inserted_comment = Comment::create(pool, &comment_form, None).await?;
+
+    let person_mention_form = PersonMentionInsertForm {
+      recipient_id: inserted_recipient.id,
+      comment_id: inserted_comment.id,
+      read: None,
+    };
+
+    let inserted_mention = PersonMention::create(pool, &person_mention_form).await?;
+
+    let expected_mention = PersonMention {
+      id: inserted_mention.id,
+      recipient_id: inserted_mention.recipient_id,
+      comment_id: inserted_mention.comment_id,
+      read: false,
+      published: inserted_mention.published,
+    };
+
+    let read_mention = PersonMention::read(pool, inserted_mention.id)
+      .await?
+      .ok_or(LemmyErrorType::CouldntFindComment)?;
+
+    let person_mention_update_form = PersonMentionUpdateForm { read: Some(false) };
+    let updated_mention =
+      PersonMention::update(pool, inserted_mention.id, &person_mention_update_form).await?;
+
+    // Test to make sure counts and blocks work correctly
+    let unread_mentions = PersonMentionView::get_unread_mentions(pool, recipient_id).await?;
+
+    let query = PersonMentionQuery {
+      recipient_id: Some(recipient_id),
+      my_person_id: Some(recipient_id),
+      sort: None,
+      unread_only: false,
+      show_bot_accounts: true,
+      page: None,
+      limit: None,
+    };
+    let mentions = query.clone().list(pool).await?;
+    assert_eq!(1, unread_mentions);
+    assert_eq!(1, mentions.len());
+
+    // Block the person, and make sure these counts are now empty
+    let block_form = PersonBlockForm {
+      person_id: recipient_id,
+      target_id: inserted_person.id,
+    };
+    PersonBlock::block(pool, &block_form).await?;
+
+    let unread_mentions_after_block =
+      PersonMentionView::get_unread_mentions(pool, recipient_id).await?;
+    let mentions_after_block = query.list(pool).await?;
+    assert_eq!(0, unread_mentions_after_block);
+    assert_eq!(0, mentions_after_block.len());
+
+    Comment::delete(pool, inserted_comment.id).await?;
+    Post::delete(pool, inserted_post.id).await?;
+    Community::delete(pool, inserted_community.id).await?;
+    Person::delete(pool, inserted_person.id).await?;
+    Person::delete(pool, inserted_recipient.id).await?;
+    Instance::delete(pool, inserted_instance.id).await?;
+
+    assert_eq!(expected_mention, read_mention);
+    assert_eq!(expected_mention, inserted_mention);
+    assert_eq!(expected_mention, updated_mention);
+
+    Ok(())
+  }
+}

crates/routes/src/images.rs

@@ -1,18 +1,16 @@
 use actix_web::{
   body::BodyStream,
-  error,
   http::{
     header::{HeaderName, ACCEPT_ENCODING, HOST},
     StatusCode,
   },
   web,
   web::Query,
-  Error,
   HttpRequest,
   HttpResponse,
 };
 use futures::stream::{Stream, StreamExt};
-use lemmy_api_common::context::LemmyContext;
+use lemmy_api_common::{context::LemmyContext, request::PictrsResponse};
 use lemmy_db_schema::source::{
   images::{LocalImage, LocalImageForm, RemoteImage},
   local_site::LocalSite,
@@ -21,7 +19,7 @@ use lemmy_db_views::structs::LocalUserView;
 use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell, REQWEST_TIMEOUT};
 use reqwest::Body;
 use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 use std::time::Duration;
 use url::Url;
 use urlencoding::decode;
@@ -43,20 +41,8 @@ pub fn config(
     .service(web::resource("/pictrs/image/delete/{token}/{filename}").route(web::get().to(delete)));
 }
 
-#[derive(Debug, Serialize, Deserialize)]
-struct Image {
-  file: String,
-  delete_token: String,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-struct Images {
-  msg: String,
-  files: Option<Vec<Image>>,
-}
-
 #[derive(Deserialize)]
-struct PictrsParams {
+struct PictrsGetParams {
   format: Option<String>,
   thumbnail: Option<i32>,
 }
@@ -92,7 +78,7 @@ async fn upload(
   local_user_view: LocalUserView,
   client: web::Data<ClientWithMiddleware>,
   context: web::Data<LemmyContext>,
-) -> Result<HttpResponse, Error> {
+) -> LemmyResult<HttpResponse> {
   // TODO: check rate limit here
   let pictrs_config = context.settings().pictrs_config()?;
   let image_url = format!("{}image", pictrs_config.url);
@@ -106,21 +92,18 @@ async fn upload(
     .timeout(Duration::from_secs(pictrs_config.upload_timeout))
     .body(Body::wrap_stream(make_send(body)))
     .send()
-    .await
+    .await?;
-    .map_err(error::ErrorBadRequest)?;
 
   let status = res.status();
-  let images = res.json::<Images>().await.map_err(error::ErrorBadRequest)?;
+  let images = res.json::<PictrsResponse>().await?;
   if let Some(images) = &images.files {
-    for uploaded_image in images {
+    for image in images {
       let form = LocalImageForm {
         local_user_id: Some(local_user_view.local_user.id),
-        pictrs_alias: uploaded_image.file.to_string(),
+        pictrs_alias: image.file.to_string(),
-        pictrs_delete_token: uploaded_image.delete_token.to_string(),
+        pictrs_delete_token: image.delete_token.to_string(),
       };
-      LocalImage::create(&mut context.pool(), &form)
+      LocalImage::create(&mut context.pool(), &form).await?;
-        .await
-        .map_err(error::ErrorBadRequest)?;
     }
   }
 
@@ -129,16 +112,14 @@ async fn upload(
 
 async fn full_res(
   filename: web::Path<String>,
-  web::Query(params): web::Query<PictrsParams>,
+  web::Query(params): web::Query<PictrsGetParams>,
   req: HttpRequest,
   client: web::Data<ClientWithMiddleware>,
   context: web::Data<LemmyContext>,
   local_user_view: Option<LocalUserView>,
-) -> Result<HttpResponse, Error> {
+) -> LemmyResult<HttpResponse> {
   // block access to images if instance is private and unauthorized, public
-  let local_site = LocalSite::read(&mut context.pool())
+  let local_site = LocalSite::read(&mut context.pool()).await?;
-    .await
-    .map_err(error::ErrorBadRequest)?;
   if local_site.private_instance && local_user_view.is_none() {
     return Ok(HttpResponse::Unauthorized().finish());
   }
@@ -169,7 +150,7 @@ async fn image(
   url: String,
   req: HttpRequest,
   client: &ClientWithMiddleware,
-) -> Result<HttpResponse, Error> {
+) -> LemmyResult<HttpResponse> {
   let mut client_req = adapt_request(&req, client, url);
 
   if let Some(addr) = req.head().peer_addr {
@@ -180,7 +161,7 @@ async fn image(
     client_req = client_req.header("X-Forwarded-For", addr.to_string());
   }
 
-  let res = client_req.send().await.map_err(error::ErrorBadRequest)?;
+  let res = client_req.send().await?;
 
   if res.status() == StatusCode::NOT_FOUND {
     return Ok(HttpResponse::NotFound().finish());
@@ -202,7 +183,7 @@ async fn delete(
   context: web::Data<LemmyContext>,
   // require login
   _local_user_view: LocalUserView,
-) -> Result<HttpResponse, Error> {
+) -> LemmyResult<HttpResponse> {
   let (token, file) = components.into_inner();
 
   let pictrs_config = context.settings().pictrs_config()?;
@@ -214,11 +195,9 @@ async fn delete(
     client_req = client_req.header("X-Forwarded-For", addr.to_string());
   }
 
-  let res = client_req.send().await.map_err(error::ErrorBadRequest)?;
+  let res = client_req.send().await?;
 
-  LocalImage::delete_by_alias(&mut context.pool(), &file)
+  LocalImage::delete_by_alias(&mut context.pool(), &file).await?;
-    .await
-    .map_err(error::ErrorBadRequest)?;
 
   Ok(HttpResponse::build(res.status()).body(BodyStream::new(res.bytes_stream())))
 }
@@ -230,6 +209,8 @@ pub struct ImageProxyParams {
 
 pub async fn image_proxy(
   Query(params): Query<ImageProxyParams>,
+  req: HttpRequest,
+  client: web::Data<ClientWithMiddleware>,
   context: web::Data<LemmyContext>,
 ) -> LemmyResult<HttpResponse> {
   let url = Url::parse(&decode(&params.url)?)?;
@@ -240,9 +221,8 @@ pub async fn image_proxy(
 
   let pictrs_config = context.settings().pictrs_config()?;
   let url = format!("{}image/original?proxy={}", pictrs_config.url, &params.url);
-  let image_response = context.client().get(url).send().await?;
 
-  Ok(HttpResponse::Ok().streaming(image_response.bytes_stream()))
+  image(url, req, &client).await
 }
 
 fn make_send<S>(mut stream: S) -> impl Stream<Item = S::Item> + Send + Unpin + 'static

crates/utils/translations

@@ -1 +1 @@
-Subproject commit f0ab81deea347c433277a90ae752b10f68473719
+Subproject commit 4e7c7ad4fcb4d8618d93ec17d72379367aa085b1

docker/docker-compose.yml

@@ -75,7 +75,7 @@ services:
     init: true
 
   pictrs:
-    image: asonix/pictrs:0.5.0-rc.2
+    image: asonix/pictrs:0.5.13
     # this needs to match the pictrs url in lemmy.hjson
     hostname: pictrs
     # we can set options to pictrs like this, here we set max. image size and forced format for conversion

docker/federation/docker-compose.yml

@@ -49,7 +49,7 @@ services:
 
   pictrs:
     restart: always
-    image: asonix/pictrs:0.5.0-rc.2
+    image: asonix/pictrs:0.5.13
     user: 991:991
     volumes:
       - ./volumes/pictrs_alpha:/mnt:Z

src/lib.rs

@@ -71,23 +71,25 @@ use url::Url;
   long_about = "A link aggregator for the fediverse.\n\nThis is the Lemmy backend API server. This will connect to a PostgreSQL database, run any pending migrations and start accepting API requests."
 )]
 #[command(args_conflicts_with_subcommands = true)]
+// TODO: Instead of defining individual env vars, only specify prefix once supported by clap.
+//       https://github.com/clap-rs/clap/issues/3221
 pub struct CmdArgs {
   /// Don't run scheduled tasks.
   ///
   /// If you are running multiple Lemmy server processes, you probably want to disable scheduled tasks on
   /// all but one of the processes, to avoid running the tasks more often than intended.
-  #[arg(long, default_value_t = false)]
+  #[arg(long, default_value_t = false, env = "LEMMY_DISABLE_SCHEDULED_TASKS")]
   disable_scheduled_tasks: bool,
   /// Disables the HTTP server.
   ///
   /// This can be used to run a Lemmy server process that only performs scheduled tasks or activity sending.
-  #[arg(long, default_value_t = false)]
+  #[arg(long, default_value_t = false, env = "LEMMY_DISABLE_HTTP_SERVER")]
   disable_http_server: bool,
   /// Disable sending outgoing ActivityPub messages.
   ///
   /// Only pass this for horizontally scaled setups.
   /// See https://join-lemmy.org/docs/administration/horizontal_scaling.html for details.
-  #[arg(long, default_value_t = false)]
+  #[arg(long, default_value_t = false, env = "LEMMY_DISABLE_ACTIVITY_SENDING")]
   disable_activity_sending: bool,
   /// The index of this outgoing federation process.
   ///
@@ -97,12 +99,12 @@ pub struct CmdArgs {
   /// Make you have exactly one server with each `i` running, otherwise federation will randomly send duplicates or nothing.
   ///
   /// See https://join-lemmy.org/docs/administration/horizontal_scaling.html for more detail.
-  #[arg(long, default_value_t = 1)]
+  #[arg(long, default_value_t = 1, env = "LEMMY_FEDERATE_PROCESS_INDEX")]
   federate_process_index: i32,
   /// How many outgoing federation processes you are starting in total.
   ///
   /// If set, make sure to set --federate-process-index differently for each.
-  #[arg(long, default_value_t = 1)]
+  #[arg(long, default_value_t = 1, env = "LEMMY_FEDERATE_PROCESS_COUNT")]
   federate_process_count: i32,
   #[command(subcommand)]
   subcommand: Option<CmdSubcommand>,