LemmyNet/lemmy
2
0
Fork 0
mirror of https://github.com/LemmyNet/lemmy.git synced 2025-01-15 14:35:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Image api rework (#5260)

Browse source 
* Split image endpoints into API v3 and v4

* Move into subfolders

* Upload avatar endpoint and other changes

* Various other changes

fixes #1772
fixes #4001

* clippy

* config options

* fix ts bindings

* fix api tests

* Add option to disable image upload (fixes #1118)

* split files into upload, download

* move sitemap to top level, not in api

* simplify code

* add upload user banner

* community icon/banner

* site icon/banner

* update js client

* wip

* add delete endpoints

* change comment

* optimization

Co-authored-by: dullbananas <dull.bananas0@gmail.com>

* move fn

* 1024px banner

* dont use static client

* fix api tests

* shear

* proxy pictrs in request.rs (fixes #5270)

* clippy

* try to fix api tests

* skip api tests

* create user

* debug

* dbg

* test

* image

* run another

* fixed?

* clippy

* fix

* fix health check

---------

Co-authored-by: dullbananas <dull.bananas0@gmail.com>
This commit is contained in:
Nutomic 2025-01-13 21:09:00 +00:00 committed by GitHub
parent c08e216ae8
commit a91a03a536
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 1187 additions and 889 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api_tests/package.json
View file

@ -28,7 +28,7 @@
"eslint": "^9.14.0", "eslint": "^9.14.0",
"eslint-plugin-prettier": "^5.1.3", "eslint-plugin-prettier": "^5.1.3",
"jest": "^29.5.0", "jest": "^29.5.0",
"lemmy-js-client": "0.20.0-reports-combined.3", "lemmy-js-client": "0.20.0-image-api-rework.8",
"prettier": "^3.2.5", "prettier": "^3.2.5",
"ts-jest": "^29.1.0", "ts-jest": "^29.1.0",
"typescript": "^5.5.4", "typescript": "^5.5.4",

10
api_tests/pnpm-lock.yaml
View file

@ -30,8 +30,8 @@ importers:
specifier: ^29.5.0 specifier: ^29.5.0
version: 29.7.0(@types/node@22.9.0) version: 29.7.0(@types/node@22.9.0)
lemmy-js-client: lemmy-js-client:
specifier: 0.20.0-reports-combined.3 specifier: 0.20.0-image-api-rework.8
version: 0.20.0-reports-combined.3 version: 0.20.0-image-api-rework.8
prettier: prettier:
specifier: ^3.2.5 specifier: ^3.2.5
version: 3.3.3 version: 3.3.3
@ -1167,8 +1167,8 @@ packages:
resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==}
engines: {node: '>=6'} engines: {node: '>=6'}
lemmy-js-client@0.20.0-reports-combined.3: lemmy-js-client@0.20.0-image-api-rework.8:
resolution: {integrity: sha512-0Z/9S41r6NM8f09Gkxerq9zYBE6UcywXfeWNxsYknkyh0ZnKbtNxjTkSxE6JpRbz7wokKFRSH9NpwgNloQY5uw==} resolution: {integrity: sha512-Ns/ayfCSm2lHbdAU1tGIZSx6kJ2ZeS7UiXlPuH0IzHQSi8Yuyzj3srDCyHpE6Td3pmXbQlt9N1ziPE4KeRJ3CA==}
leven@3.1.0: leven@3.1.0:
resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==}
@ -3077,7 +3077,7 @@ snapshots:
kleur@3.0.3: {} kleur@3.0.3: {}
lemmy-js-client@0.20.0-reports-combined.3: {} lemmy-js-client@0.20.0-image-api-rework.8: {}
leven@3.1.0: {} leven@3.1.0: {}

88
api_tests/src/image.spec.ts
View file

@ -2,9 +2,9 @@ jest.setTimeout(120000);
import { import {
UploadImage, UploadImage,
DeleteImage,
PurgePerson, PurgePerson,
PurgePost, PurgePost,
DeleteImageParams,
} from "lemmy-js-client"; } from "lemmy-js-client";
import { import {
alpha, alpha,
@ -41,8 +41,8 @@ afterAll(async () => {
}); });
test("Upload image and delete it", async () => { test("Upload image and delete it", async () => {
const healthz = await fetch(alphaUrl + "/pictrs/healthz"); const health = await alpha.imageHealth();
expect(healthz.status).toBe(200); expect(health.success).toBeTruthy();
// Before running this test, you need to delete all previous images in the DB // Before running this test, you need to delete all previous images in the DB
await deleteAllImages(alpha); await deleteAllImages(alpha);
@ -53,13 +53,12 @@ test("Upload image and delete it", async () => {
image: Buffer.from("test"), image: Buffer.from("test"),
}; };
const upload = await alphaImage.uploadImage(upload_form); const upload = await alphaImage.uploadImage(upload_form);
expect(upload.files![0].file).toBeDefined(); expect(upload.image_url).toBeDefined();
expect(upload.files![0].delete_token).toBeDefined(); expect(upload.filename).toBeDefined();
expect(upload.url).toBeDefined(); expect(upload.delete_token).toBeDefined();
expect(upload.delete_url).toBeDefined();
// ensure that image download is working. theres probably a better way to do this // ensure that image download is working. theres probably a better way to do this
const response = await fetch(upload.url ?? ""); const response = await fetch(upload.image_url ?? "");
const content = await response.text(); const content = await response.text();
expect(content.length).toBeGreaterThan(0); expect(content.length).toBeGreaterThan(0);
@ -76,26 +75,21 @@ test("Upload image and delete it", async () => {
const previousThumbnails = 1; const previousThumbnails = 1;
expect(listAllMediaRes.images.length).toBe(previousThumbnails); expect(listAllMediaRes.images.length).toBe(previousThumbnails);
// The deleteUrl is a combination of the endpoint, delete token, and alias
let firstImage = listMediaRes.images[0];
let deleteUrl = `${alphaUrl}/pictrs/image/delete/${firstImage.local_image.pictrs_delete_token}/${firstImage.local_image.pictrs_alias}`;
expect(deleteUrl).toBe(upload.delete_url);
// Make sure the uploader is correct // Make sure the uploader is correct
expect(firstImage.person.actor_id).toBe( expect(listMediaRes.images[0].person.actor_id).toBe(
`http://lemmy-alpha:8541/u/lemmy_alpha`, `http://lemmy-alpha:8541/u/lemmy_alpha`,
); );
// delete image // delete image
const delete_form: DeleteImage = { const delete_form: DeleteImageParams = {
token: upload.files![0].delete_token, token: upload.delete_token,
filename: upload.files![0].file, filename: upload.filename,
}; };
const delete_ = await alphaImage.deleteImage(delete_form); const delete_ = await alphaImage.deleteImage(delete_form);
expect(delete_).toBe(true); expect(delete_.success).toBe(true);
// ensure that image is deleted // ensure that image is deleted
const response2 = await fetch(upload.url ?? ""); const response2 = await fetch(upload.image_url ?? "");
const content2 = await response2.text(); const content2 = await response2.text();
expect(content2).toBe(""); expect(content2).toBe("");
@ -118,13 +112,12 @@ test("Purge user, uploaded image removed", async () => {
image: Buffer.from("test"), image: Buffer.from("test"),
}; };
const upload = await user.uploadImage(upload_form); const upload = await user.uploadImage(upload_form);
expect(upload.files![0].file).toBeDefined(); expect(upload.filename).toBeDefined();
expect(upload.files![0].delete_token).toBeDefined(); expect(upload.delete_token).toBeDefined();
expect(upload.url).toBeDefined(); expect(upload.image_url).toBeDefined();
expect(upload.delete_url).toBeDefined();
// ensure that image download is working. theres probably a better way to do this // ensure that image download is working. theres probably a better way to do this
const response = await fetch(upload.url ?? ""); const response = await fetch(upload.image_url ?? "");
const content = await response.text(); const content = await response.text();
expect(content.length).toBeGreaterThan(0); expect(content.length).toBeGreaterThan(0);
@ -137,7 +130,7 @@ test("Purge user, uploaded image removed", async () => {
expect(delete_.success).toBe(true); expect(delete_.success).toBe(true);
// ensure that image is deleted // ensure that image is deleted
const response2 = await fetch(upload.url ?? ""); const response2 = await fetch(upload.image_url ?? "");
const content2 = await response2.text(); const content2 = await response2.text();
expect(content2).toBe(""); expect(content2).toBe("");
}); });
@ -150,13 +143,12 @@ test("Purge post, linked image removed", async () => {
image: Buffer.from("test"), image: Buffer.from("test"),
}; };
const upload = await user.uploadImage(upload_form); const upload = await user.uploadImage(upload_form);
expect(upload.files![0].file).toBeDefined(); expect(upload.filename).toBeDefined();
expect(upload.files![0].delete_token).toBeDefined(); expect(upload.delete_token).toBeDefined();
expect(upload.url).toBeDefined(); expect(upload.image_url).toBeDefined();
expect(upload.delete_url).toBeDefined();
// ensure that image download is working. theres probably a better way to do this // ensure that image download is working. theres probably a better way to do this
const response = await fetch(upload.url ?? ""); const response = await fetch(upload.image_url ?? "");
const content = await response.text(); const content = await response.text();
expect(content.length).toBeGreaterThan(0); expect(content.length).toBeGreaterThan(0);
@ -164,9 +156,9 @@ test("Purge post, linked image removed", async () => {
let post = await createPost( let post = await createPost(
user, user,
community.community!.community.id, community.community!.community.id,
upload.url, upload.image_url,
); );
expect(post.post_view.post.url).toBe(upload.url); expect(post.post_view.post.url).toBe(upload.image_url);
expect(post.post_view.image_details).toBeDefined(); expect(post.post_view.image_details).toBeDefined();
// purge post // purge post
@ -177,7 +169,7 @@ test("Purge post, linked image removed", async () => {
expect(delete_.success).toBe(true); expect(delete_.success).toBe(true);
// ensure that image is deleted // ensure that image is deleted
const response2 = await fetch(upload.url ?? ""); const response2 = await fetch(upload.image_url ?? "");
const content2 = await response2.text(); const content2 = await response2.text();
expect(content2).toBe(""); expect(content2).toBe("");
}); });
@ -199,11 +191,11 @@ test("Images in remote image post are proxied if setting enabled", async () => {
// remote image gets proxied after upload // remote image gets proxied after upload
expect( expect(
post.thumbnail_url?.startsWith( post.thumbnail_url?.startsWith(
"http://lemmy-gamma:8561/api/v4/image_proxy?url", "http://lemmy-gamma:8561/api/v4/image/proxy?url",
), ),
).toBeTruthy(); ).toBeTruthy();
expect( expect(
post.body?.startsWith("![](http://lemmy-gamma:8561/api/v4/image_proxy?url"), post.body?.startsWith("![](http://lemmy-gamma:8561/api/v4/image/proxy?url"),
).toBeTruthy(); ).toBeTruthy();
// Make sure that it ends with jpg, to be sure its an image // Make sure that it ends with jpg, to be sure its an image
@ -222,12 +214,12 @@ test("Images in remote image post are proxied if setting enabled", async () => {
expect( expect(
epsilonPost.thumbnail_url?.startsWith( epsilonPost.thumbnail_url?.startsWith(
"http://lemmy-epsilon:8581/api/v4/image_proxy?url", "http://lemmy-epsilon:8581/api/v4/image/proxy?url",
), ),
).toBeTruthy(); ).toBeTruthy();
expect( expect(
epsilonPost.body?.startsWith( epsilonPost.body?.startsWith(
"![](http://lemmy-epsilon:8581/api/v4/image_proxy?url", "![](http://lemmy-epsilon:8581/api/v4/image/proxy?url",
), ),
).toBeTruthy(); ).toBeTruthy();
@ -249,7 +241,7 @@ test("Thumbnail of remote image link is proxied if setting enabled", async () =>
// remote image gets proxied after upload // remote image gets proxied after upload
expect( expect(
post.thumbnail_url?.startsWith( post.thumbnail_url?.startsWith(
"http://lemmy-gamma:8561/api/v4/image_proxy?url", "http://lemmy-gamma:8561/api/v4/image/proxy?url",
), ),
).toBeTruthy(); ).toBeTruthy();
@ -267,7 +259,7 @@ test("Thumbnail of remote image link is proxied if setting enabled", async () =>
expect( expect(
epsilonPost.thumbnail_url?.startsWith( epsilonPost.thumbnail_url?.startsWith(
"http://lemmy-epsilon:8581/api/v4/image_proxy?url", "http://lemmy-epsilon:8581/api/v4/image/proxy?url",
), ),
).toBeTruthy(); ).toBeTruthy();
@ -291,14 +283,14 @@ test("No image proxying if setting is disabled", async () => {
let post = await createPost( let post = await createPost(
alpha, alpha,
community.community_view.community.id, community.community_view.community.id,
upload.url, upload.image_url,
`![](${sampleImage})`, `![](${sampleImage})`,
); );
expect(post.post_view.post).toBeDefined(); expect(post.post_view.post).toBeDefined();
// remote image doesn't get proxied after upload // remote image doesn't get proxied after upload
expect( expect(
post.post_view.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"), post.post_view.post.url?.startsWith("http://lemmy-beta:8551/api/v4/image/"),
).toBeTruthy(); ).toBeTruthy();
expect(post.post_view.post.body).toBe(`![](${sampleImage})`); expect(post.post_view.post.body).toBe(`![](${sampleImage})`);
@ -311,7 +303,7 @@ test("No image proxying if setting is disabled", async () => {
// remote image doesn't get proxied after federation // remote image doesn't get proxied after federation
expect( expect(
betaPost.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"), betaPost.post.url?.startsWith("http://lemmy-beta:8551/api/v4/image/"),
).toBeTruthy(); ).toBeTruthy();
expect(betaPost.post.body).toBe(`![](${sampleImage})`); expect(betaPost.post.body).toBe(`![](${sampleImage})`);
// Make sure the alt text got federated // Make sure the alt text got federated
@ -333,7 +325,7 @@ test("Make regular post, and give it a custom thumbnail", async () => {
alphaImage, alphaImage,
community.community_view.community.id, community.community_view.community.id,
wikipediaUrl, wikipediaUrl,
upload1.url!, upload1.image_url!,
); );
// Wait for the metadata to get fetched, since this is backgrounded now // Wait for the metadata to get fetched, since this is backgrounded now
@ -343,7 +335,7 @@ test("Make regular post, and give it a custom thumbnail", async () => {
); );
expect(post.post_view.post.url).toBe(wikipediaUrl); expect(post.post_view.post.url).toBe(wikipediaUrl);
// Make sure it uses custom thumbnail // Make sure it uses custom thumbnail
expect(post.post_view.post.thumbnail_url).toBe(upload1.url); expect(post.post_view.post.thumbnail_url).toBe(upload1.image_url);
}); });
test("Create an image post, and make sure a custom thumbnail doesn't overwrite it", async () => { test("Create an image post, and make sure a custom thumbnail doesn't overwrite it", async () => {
@ -362,14 +354,14 @@ test("Create an image post, and make sure a custom thumbnail doesn't overwrite i
let post = await createPostWithThumbnail( let post = await createPostWithThumbnail(
alphaImage, alphaImage,
community.community_view.community.id, community.community_view.community.id,
upload1.url!, upload1.image_url!,
upload2.url!, upload2.image_url!,
); );
post = await waitUntil( post = await waitUntil(
() => getPost(alphaImage, post.post_view.post.id), () => getPost(alphaImage, post.post_view.post.id),
p => p.post_view.post.thumbnail_url != undefined, p => p.post_view.post.thumbnail_url != undefined,
); );
expect(post.post_view.post.url).toBe(upload1.url); expect(post.post_view.post.url).toBe(upload1.image_url);
// Make sure the custom thumbnail is ignored // Make sure the custom thumbnail is ignored
expect(post.post_view.post.thumbnail_url == upload2.url).toBe(false); expect(post.post_view.post.thumbnail_url == upload2.image_url).toBe(false);
}); });

8
api_tests/src/shared.ts
View file

@ -5,7 +5,6 @@ import {
CommunityId, CommunityId,
CommunityVisibility, CommunityVisibility,
CreatePrivateMessageReport, CreatePrivateMessageReport,
DeleteImage,
EditCommunity, EditCommunity,
GetCommunityPendingFollowsCountResponse, GetCommunityPendingFollowsCountResponse,
GetReplies, GetReplies,
@ -18,6 +17,7 @@ import {
ListReports, ListReports,
ListReportsResponse, ListReportsResponse,
MyUserInfo, MyUserInfo,
DeleteImageParams,
PersonId, PersonId,
PostView, PostView,
PrivateMessageReportResponse, PrivateMessageReportResponse,
@ -714,8 +714,6 @@ export async function saveUserSettingsBio(
export async function saveUserSettingsFederated( export async function saveUserSettingsFederated(
api: LemmyHttp, api: LemmyHttp,
): Promise<SuccessResponse> { ): Promise<SuccessResponse> {
let avatar = sampleImage;
let banner = sampleImage;
let bio = "a changed bio"; let bio = "a changed bio";
let form: SaveUserSettings = { let form: SaveUserSettings = {
show_nsfw: false, show_nsfw: false,
@ -723,8 +721,6 @@ export async function saveUserSettingsFederated(
default_post_sort_type: "Hot", default_post_sort_type: "Hot",
default_listing_type: "All", default_listing_type: "All",
interface_language: "", interface_language: "",
avatar,
banner,
display_name: "user321", display_name: "user321",
show_avatars: false, show_avatars: false,
send_notifications_to_email: false, send_notifications_to_email: false,
@ -939,7 +935,7 @@ export async function deleteAllImages(api: LemmyHttp) {
Promise.all( Promise.all(
imagesRes.images imagesRes.images
.map(image => { .map(image => {
const form: DeleteImage = { const form: DeleteImageParams = {
token: image.local_image.pictrs_delete_token, token: image.local_image.pictrs_delete_token,
filename: image.local_image.pictrs_alias, filename: image.local_image.pictrs_alias,
}; };

36
api_tests/src/user.spec.ts
View file

@ -21,7 +21,6 @@ import {
fetchFunction, fetchFunction,
alphaImage, alphaImage,
unfollows, unfollows,
saveUserSettingsBio,
getMyUser, getMyUser,
getPersonDetails, getPersonDetails,
} from "./shared"; } from "./shared";
@ -192,43 +191,36 @@ test("Set a new avatar, old avatar is deleted", async () => {
const upload_form1: UploadImage = { const upload_form1: UploadImage = {
image: Buffer.from("test1"), image: Buffer.from("test1"),
}; };
const upload1 = await alphaImage.uploadImage(upload_form1); await alpha.uploadUserAvatar(upload_form1);
expect(upload1.url).toBeDefined();
let form1 = {
avatar: upload1.url,
};
await saveUserSettings(alpha, form1);
const listMediaRes1 = await alphaImage.listMedia(); const listMediaRes1 = await alphaImage.listMedia();
expect(listMediaRes1.images.length).toBe(1); expect(listMediaRes1.images.length).toBe(1);
let my_user1 = await alpha.getMyUser();
expect(my_user1.local_user_view.person.avatar).toBeDefined();
const upload_form2: UploadImage = { const upload_form2: UploadImage = {
image: Buffer.from("test2"), image: Buffer.from("test2"),
}; };
const upload2 = await alphaImage.uploadImage(upload_form2); await alpha.uploadUserAvatar(upload_form2);
expect(upload2.url).toBeDefined();
let form2 = {
avatar: upload2.url,
};
await saveUserSettings(alpha, form2);
// make sure only the new avatar is kept // make sure only the new avatar is kept
const listMediaRes2 = await alphaImage.listMedia(); const listMediaRes2 = await alphaImage.listMedia();
expect(listMediaRes2.images.length).toBe(1); expect(listMediaRes2.images.length).toBe(1);
// Upload that same form2 avatar, make sure it isn't replaced / deleted // Upload that same form2 avatar, make sure it isn't replaced / deleted
await saveUserSettings(alpha, form2); await alpha.uploadUserAvatar(upload_form2);
// make sure only the new avatar is kept // make sure only the new avatar is kept
const listMediaRes3 = await alphaImage.listMedia(); const listMediaRes3 = await alphaImage.listMedia();
expect(listMediaRes3.images.length).toBe(1); expect(listMediaRes3.images.length).toBe(1);
// Now try to save a user settings, with the icon missing,
// and make sure it doesn't clear the data, or delete the image
await saveUserSettingsBio(alpha);
let my_user = await getMyUser(alpha);
expect(my_user.local_user_view.person.avatar).toBe(upload2.url);
// make sure only the new avatar is kept // make sure only the new avatar is kept
const listMediaRes4 = await alphaImage.listMedia(); const listMediaRes4 = await alphaImage.listMedia();
expect(listMediaRes4.images.length).toBe(1); expect(listMediaRes4.images.length).toBe(1);
// delete the avatar
await alpha.deleteUserAvatar();
// make sure only the new avatar is kept
const listMediaRes5 = await alphaImage.listMedia();
expect(listMediaRes5.images.length).toBe(0);
let my_user2 = await alpha.getMyUser();
expect(my_user2.local_user_view.person.avatar).toBeUndefined();
}); });

10
config/defaults.hjson
View file

@ -39,7 +39,7 @@
# or # or
# If enabled, all images from remote domains are rewritten to pass through # If enabled, all images from remote domains are rewritten to pass through
# `/api/v4/image_proxy`, including embedded images in markdown. Images are stored temporarily # `/api/v4/image/proxy`, including embedded images in markdown. Images are stored temporarily
# in pict-rs for caching. This improves privacy as users don't expose their IP to untrusted # in pict-rs for caching. This improves privacy as users don't expose their IP to untrusted
# servers, and decreases load on other servers. However it increases bandwidth use for the # servers, and decreases load on other servers. However it increases bandwidth use for the
# local server. # local server.
@ -59,6 +59,14 @@
upload_timeout: 30 upload_timeout: 30
# Resize post thumbnails to this maximum width/height. # Resize post thumbnails to this maximum width/height.
max_thumbnail_size: 512 max_thumbnail_size: 512
# Maximum size for user avatar, community icon and site icon.
max_avatar_size: 512
# Maximum size for user, community and site banner. Larger images are downscaled to fit
# into a square of this size.
max_banner_size: 1024
# Prevent users from uploading images for posts or embedding in markdown. Avatars, icons and
# banners can still be uploaded.
image_upload_disabled: false
} }
# Email sending configuration. All options except login/password are mandatory # Email sending configuration. All options except login/password are mandatory
email: { email: {

14
crates/api/src/local_user/save_settings.rs
View file

@ -3,12 +3,10 @@ use actix_web::web::Json;
use lemmy_api_common::{ use lemmy_api_common::{
context::LemmyContext, context::LemmyContext,
person::SaveUserSettings, person::SaveUserSettings,
request::replace_image,
utils::{ utils::{
get_url_blocklist, get_url_blocklist,
local_site_to_slur_regex, local_site_to_slur_regex,
process_markdown_opt, process_markdown_opt,
proxy_image_link_opt_api,
send_verification_email, send_verification_email,
}, },
SuccessResponse, SuccessResponse,
@ -21,7 +19,7 @@ use lemmy_db_schema::{
person::{Person, PersonUpdateForm}, person::{Person, PersonUpdateForm},
}, },
traits::Crud, traits::Crud,
utils::{diesel_string_update, diesel_url_update}, utils::diesel_string_update,
}; };
use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_db_views::structs::{LocalUserView, SiteView};
use lemmy_utils::{ use lemmy_utils::{
@ -46,14 +44,6 @@ pub async fn save_user_settings(
.as_deref(), .as_deref(),
); );
let avatar = diesel_url_update(data.avatar.as_deref())?;
replace_image(&avatar, &local_user_view.person.avatar, &context).await?;
let avatar = proxy_image_link_opt_api(avatar, &context).await?;
let banner = diesel_url_update(data.banner.as_deref())?;
replace_image(&banner, &local_user_view.person.banner, &context).await?;
let banner = proxy_image_link_opt_api(banner, &context).await?;
let display_name = diesel_string_update(data.display_name.as_deref()); let display_name = diesel_string_update(data.display_name.as_deref());
let matrix_user_id = diesel_string_update(data.matrix_user_id.as_deref()); let matrix_user_id = diesel_string_update(data.matrix_user_id.as_deref());
let email_deref = data.email.as_deref().map(str::to_lowercase); let email_deref = data.email.as_deref().map(str::to_lowercase);
@ -108,8 +98,6 @@ pub async fn save_user_settings(
bio, bio,
matrix_user_id, matrix_user_id,
bot_account: data.bot_account, bot_account: data.bot_account,
avatar,
banner,
..Default::default() ..Default::default()
}; };

1
crates/api/src/site/leave_admin.rs
View file

@ -76,5 +76,6 @@ pub async fn leave_admin(
blocked_urls, blocked_urls,
tagline, tagline,
my_user: None, my_user: None,
image_upload_disabled: context.settings().pictrs()?.image_upload_disabled,
})) }))
} }

17
crates/api_common/src/claims.rs
View file

@ -78,29 +78,18 @@ mod tests {
instance::Instance, instance::Instance,
local_user::{LocalUser, LocalUserInsertForm}, local_user::{LocalUser, LocalUserInsertForm},
person::{Person, PersonInsertForm}, person::{Person, PersonInsertForm},
secret::Secret,
}, },
traits::Crud, traits::Crud,
utils::build_db_pool_for_tests,
}; };
use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell}; use lemmy_utils::error::LemmyResult;
use pretty_assertions::assert_eq; use pretty_assertions::assert_eq;
use reqwest::Client;
use reqwest_middleware::ClientBuilder;
use serial_test::serial; use serial_test::serial;
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn test_should_not_validate_user_token_after_password_change() -> LemmyResult<()> { async fn test_should_not_validate_user_token_after_password_change() -> LemmyResult<()> {
let pool_ = build_db_pool_for_tests(); let context = LemmyContext::init_test_context().await;
let pool = &mut (&pool_).into(); let pool = &mut context.pool();
let secret = Secret::init(pool).await?;
let context = LemmyContext::create(
pool_.clone(),
ClientBuilder::new(Client::default()).build(),
secret,
RateLimitCell::with_test_config(),
);
let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?;

6
crates/api_common/src/community.rs
View file

@ -177,12 +177,6 @@ pub struct EditCommunity {
/// A shorter, one line description of your community. /// A shorter, one line description of your community.
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub description: Option<String>, pub description: Option<String>,
/// An icon URL.
#[cfg_attr(feature = "full", ts(optional))]
pub icon: Option<String>,
/// A banner URL.
#[cfg_attr(feature = "full", ts(optional))]
pub banner: Option<String>,
/// Whether its an NSFW community. /// Whether its an NSFW community.
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub nsfw: Option<bool>, pub nsfw: Option<bool>,

16
crates/api_common/src/context.rs
View file

@ -15,6 +15,9 @@ use std::sync::Arc;
pub struct LemmyContext { pub struct LemmyContext {
pool: ActualDbPool, pool: ActualDbPool,
client: Arc<ClientWithMiddleware>, client: Arc<ClientWithMiddleware>,
/// Pictrs requests must bypass proxy. Unfortunately no_proxy can only be set on ClientBuilder
/// and not on RequestBuilder, so we need a separate client here.
pictrs_client: Arc<ClientWithMiddleware>,
secret: Arc<Secret>, secret: Arc<Secret>,
rate_limit_cell: RateLimitCell, rate_limit_cell: RateLimitCell,
} }
@ -23,12 +26,14 @@ impl LemmyContext {
pub fn create( pub fn create(
pool: ActualDbPool, pool: ActualDbPool,
client: ClientWithMiddleware, client: ClientWithMiddleware,
pictrs_client: ClientWithMiddleware,
secret: Secret, secret: Secret,
rate_limit_cell: RateLimitCell, rate_limit_cell: RateLimitCell,
) -> LemmyContext { ) -> LemmyContext {
LemmyContext { LemmyContext {
pool, pool,
client: Arc::new(client), client: Arc::new(client),
pictrs_client: Arc::new(pictrs_client),
secret: Arc::new(secret), secret: Arc::new(secret),
rate_limit_cell, rate_limit_cell,
} }
@ -42,6 +47,9 @@ impl LemmyContext {
pub fn client(&self) -> &ClientWithMiddleware { pub fn client(&self) -> &ClientWithMiddleware {
&self.client &self.client
} }
pub fn pictrs_client(&self) -> &ClientWithMiddleware {
&self.pictrs_client
}
pub fn settings(&self) -> &'static Settings { pub fn settings(&self) -> &'static Settings {
&SETTINGS &SETTINGS
} }
@ -70,7 +78,13 @@ impl LemmyContext {
let rate_limit_cell = RateLimitCell::with_test_config(); let rate_limit_cell = RateLimitCell::with_test_config();
let context = LemmyContext::create(pool, client, secret, rate_limit_cell.clone()); let context = LemmyContext::create(
pool,
client.clone(),
client,
secret,
rate_limit_cell.clone(),
);
FederationConfig::builder() FederationConfig::builder()
.domain(context.settings().hostname.clone()) .domain(context.settings().hostname.clone())

54
crates/api_common/src/image.rs Normal file
View file

@ -0,0 +1,54 @@
use lemmy_db_schema::newtypes::CommunityId;
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
#[cfg(feature = "full")]
use ts_rs::TS;
use url::Url;
#[skip_serializing_none]
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "full", derive(TS))]
#[cfg_attr(feature = "full", ts(export))]
pub struct ImageGetParams {
#[cfg_attr(feature = "full", ts(optional))]
pub file_type: Option<String>,
#[cfg_attr(feature = "full", ts(optional))]
pub max_size: Option<i32>,
}
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "full", derive(TS))]
#[cfg_attr(feature = "full", ts(export))]
pub struct DeleteImageParams {
pub filename: String,
pub token: String,
}
#[skip_serializing_none]
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "full", derive(TS))]
#[cfg_attr(feature = "full", ts(export))]
pub struct ImageProxyParams {
pub url: String,
#[cfg_attr(feature = "full", ts(optional))]
pub file_type: Option<String>,
#[cfg_attr(feature = "full", ts(optional))]
pub max_size: Option<i32>,
}
#[skip_serializing_none]
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "full", derive(TS))]
#[cfg_attr(feature = "full", ts(export))]
pub struct UploadImageResponse {
pub image_url: Url,
pub filename: String,
pub delete_token: String,
}
/// Parameter for setting community icon or banner. Can't use POST data here as it already contains
/// the image data.
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq, Hash)]
pub struct CommunityIdQuery {
pub id: CommunityId,
}

1
crates/api_common/src/lib.rs
View file

@ -7,6 +7,7 @@ pub mod community;
#[cfg(feature = "full")] #[cfg(feature = "full")]
pub mod context; pub mod context;
pub mod custom_emoji; pub mod custom_emoji;
pub mod image;
pub mod oauth_provider; pub mod oauth_provider;
pub mod person; pub mod person;
pub mod post; pub mod post;

6
crates/api_common/src/person.rs
View file

@ -120,12 +120,6 @@ pub struct SaveUserSettings {
/// The language of the lemmy interface /// The language of the lemmy interface
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub interface_language: Option<String>, pub interface_language: Option<String>,
/// A URL for your avatar.
#[cfg_attr(feature = "full", ts(optional))]
pub avatar: Option<String>,
/// A URL for your banner.
#[cfg_attr(feature = "full", ts(optional))]
pub banner: Option<String>,
/// Your display name, which can contain strange characters, and does not need to be unique. /// Your display name, which can contain strange characters, and does not need to be unique.
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub display_name: Option<String>, pub display_name: Option<String>,

87
crates/api_common/src/request.rs
View file

@ -9,13 +9,10 @@ use activitypub_federation::config::Data;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use encoding_rs::{Encoding, UTF_8}; use encoding_rs::{Encoding, UTF_8};
use futures::StreamExt; use futures::StreamExt;
use lemmy_db_schema::{ use lemmy_db_schema::source::{
newtypes::DbUrl, images::{ImageDetailsForm, LocalImage, LocalImageForm},
source::{ post::{Post, PostUpdateForm},
images::{ImageDetailsForm, LocalImage, LocalImageForm}, site::Site,
post::{Post, PostUpdateForm},
site::Site,
},
}; };
use lemmy_utils::{ use lemmy_utils::{
error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult}, error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult},
@ -260,7 +257,8 @@ fn extract_opengraph_data(html_bytes: &[u8], url: &Url) -> LemmyResult<OpenGraph
#[derive(Deserialize, Serialize, Debug)] #[derive(Deserialize, Serialize, Debug)]
pub struct PictrsResponse { pub struct PictrsResponse {
pub files: Option<Vec<PictrsFile>>, #[serde(default)]
pub files: Vec<PictrsFile>,
pub msg: String, pub msg: String,
} }
@ -272,9 +270,15 @@ pub struct PictrsFile {
} }
impl PictrsFile { impl PictrsFile {
pub fn thumbnail_url(&self, protocol_and_hostname: &str) -> Result<Url, url::ParseError> { pub fn image_url(&self, protocol_and_hostname: &str) -> Result<Url, url::ParseError> {
Url::parse(&format!( Url::parse(&format!(
"{protocol_and_hostname}/pictrs/image/{}", "{protocol_and_hostname}/api/v4/image/{}",
self.file
))
}
pub fn delete_url(&self, protocol_and_hostname: &str) -> Result<Url, url::ParseError> {
Url::parse(&format!(
"{protocol_and_hostname}/api/v4/image/{}",
self.file self.file
)) ))
} }
@ -315,7 +319,7 @@ struct PictrsPurgeResponse {
/// - It might not be an image /// - It might not be an image
/// - Pictrs might not be set up /// - Pictrs might not be set up
pub async fn purge_image_from_pictrs(image_url: &Url, context: &LemmyContext) -> LemmyResult<()> { pub async fn purge_image_from_pictrs(image_url: &Url, context: &LemmyContext) -> LemmyResult<()> {
is_image_content_type(context.client(), image_url).await?; is_image_content_type(context.pictrs_client(), image_url).await?;
let alias = image_url let alias = image_url
.path_segments() .path_segments()
@ -323,14 +327,19 @@ pub async fn purge_image_from_pictrs(image_url: &Url, context: &LemmyContext) ->
.next_back() .next_back()
.ok_or(LemmyErrorType::ImageUrlMissingLastPathSegment)?; .ok_or(LemmyErrorType::ImageUrlMissingLastPathSegment)?;
let pictrs_config = context.settings().pictrs_config()?; // Delete db row if any (old Lemmy versions didnt generate this).
LocalImage::delete_by_alias(&mut context.pool(), alias)
.await
.ok();
let pictrs_config = context.settings().pictrs()?;
let purge_url = format!("{}internal/purge?alias={}", pictrs_config.url, alias); let purge_url = format!("{}internal/purge?alias={}", pictrs_config.url, alias);
let pictrs_api_key = pictrs_config let pictrs_api_key = pictrs_config
.api_key .api_key
.ok_or(LemmyErrorType::PictrsApiKeyNotProvided)?; .ok_or(LemmyErrorType::PictrsApiKeyNotProvided)?;
let response = context let response = context
.client() .pictrs_client()
.post(&purge_url) .post(&purge_url)
.timeout(REQWEST_TIMEOUT) .timeout(REQWEST_TIMEOUT)
.header("x-api-token", pictrs_api_key) .header("x-api-token", pictrs_api_key)
@ -351,13 +360,18 @@ pub async fn delete_image_from_pictrs(
delete_token: &str, delete_token: &str,
context: &LemmyContext, context: &LemmyContext,
) -> LemmyResult<()> { ) -> LemmyResult<()> {
let pictrs_config = context.settings().pictrs_config()?; // Delete db row if any (old Lemmy versions didnt generate this).
LocalImage::delete_by_alias(&mut context.pool(), alias)
.await
.ok();
let pictrs_config = context.settings().pictrs()?;
let url = format!( let url = format!(
"{}image/delete/{}/{}", "{}image/delete/{}/{}",
pictrs_config.url, &delete_token, &alias pictrs_config.url, &delete_token, &alias
); );
context context
.client() .pictrs_client()
.delete(&url) .delete(&url)
.timeout(REQWEST_TIMEOUT) .timeout(REQWEST_TIMEOUT)
.send() .send()
@ -369,7 +383,7 @@ pub async fn delete_image_from_pictrs(
/// Retrieves the image with local pict-rs and generates a thumbnail. Returns the thumbnail url. /// Retrieves the image with local pict-rs and generates a thumbnail. Returns the thumbnail url.
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> LemmyResult<Url> { async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> LemmyResult<Url> {
let pictrs_config = context.settings().pictrs_config()?; let pictrs_config = context.settings().pictrs()?;
match pictrs_config.image_mode { match pictrs_config.image_mode {
PictrsImageMode::None => return Ok(image_url.clone()), PictrsImageMode::None => return Ok(image_url.clone()),
@ -380,16 +394,15 @@ async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> L
}; };
// fetch remote non-pictrs images for persistent thumbnail link // fetch remote non-pictrs images for persistent thumbnail link
// TODO: should limit size once supported by pictrs
let fetch_url = format!( let fetch_url = format!(
"{}image/download?url={}&resize={}", "{}image/download?url={}&resize={}",
pictrs_config.url, pictrs_config.url,
encode(image_url.as_str()), encode(image_url.as_str()),
context.settings().pictrs_config()?.max_thumbnail_size context.settings().pictrs()?.max_thumbnail_size
); );
let res = context let res = context
.client() .pictrs_client()
.get(&fetch_url) .get(&fetch_url)
.timeout(REQWEST_TIMEOUT) .timeout(REQWEST_TIMEOUT)
.send() .send()
@ -398,9 +411,8 @@ async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> L
.json::<PictrsResponse>() .json::<PictrsResponse>()
.await?; .await?;
let files = res.files.unwrap_or_default(); let image = res
.files
let image = files
.first() .first()
.ok_or(LemmyErrorType::PictrsResponseError(res.msg))?; .ok_or(LemmyErrorType::PictrsResponseError(res.msg))?;
@ -412,7 +424,7 @@ async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> L
pictrs_delete_token: image.delete_token.clone(), pictrs_delete_token: image.delete_token.clone(),
}; };
let protocol_and_hostname = context.settings().get_protocol_and_hostname(); let protocol_and_hostname = context.settings().get_protocol_and_hostname();
let thumbnail_url = image.thumbnail_url(&protocol_and_hostname)?; let thumbnail_url = image.image_url(&protocol_and_hostname)?;
// Also store the details for the image // Also store the details for the image
let details_form = image.details.build_image_details_form(&thumbnail_url); let details_form = image.details.build_image_details_form(&thumbnail_url);
@ -429,14 +441,14 @@ pub async fn fetch_pictrs_proxied_image_details(
image_url: &Url, image_url: &Url,
context: &LemmyContext, context: &LemmyContext,
) -> LemmyResult<PictrsFileDetails> { ) -> LemmyResult<PictrsFileDetails> {
let pictrs_url = context.settings().pictrs_config()?.url; let pictrs_url = context.settings().pictrs()?.url;
let encoded_image_url = encode(image_url.as_str()); let encoded_image_url = encode(image_url.as_str());
// Pictrs needs you to fetch the proxied image before you can fetch the details // Pictrs needs you to fetch the proxied image before you can fetch the details
let proxy_url = format!("{pictrs_url}image/original?proxy={encoded_image_url}"); let proxy_url = format!("{pictrs_url}image/original?proxy={encoded_image_url}");
context context
.client() .pictrs_client()
.get(&proxy_url) .get(&proxy_url)
.timeout(REQWEST_TIMEOUT) .timeout(REQWEST_TIMEOUT)
.send() .send()
@ -447,7 +459,7 @@ pub async fn fetch_pictrs_proxied_image_details(
let details_url = format!("{pictrs_url}image/details/original?proxy={encoded_image_url}"); let details_url = format!("{pictrs_url}image/details/original?proxy={encoded_image_url}");
let res = context let res = context
.client() .pictrs_client()
.get(&details_url) .get(&details_url)
.timeout(REQWEST_TIMEOUT) .timeout(REQWEST_TIMEOUT)
.send() .send()
@ -476,29 +488,6 @@ async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> Lemm
} }
} }
/// When adding a new avatar, banner or similar image, delete the old one.
pub async fn replace_image(
new_image: &Option<Option<DbUrl>>,
old_image: &Option<DbUrl>,
context: &Data<LemmyContext>,
) -> LemmyResult<()> {
if let (Some(Some(new_image)), Some(old_image)) = (new_image, old_image) {
// Note: Oftentimes front ends will include the current image in the form.
// In this case, deleting `old_image` would also be deletion of `new_image`,
// so the deletion must be skipped for the image to be kept.
if new_image != old_image {
// Ignore errors because image may be stored externally.
let image = LocalImage::delete_by_url(&mut context.pool(), old_image)
.await
.ok();
if let Some(image) = image {
delete_image_from_pictrs(&image.pictrs_alias, &image.pictrs_delete_token, context).await?;
}
}
}
Ok(())
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {

12
crates/api_common/src/site.rs
View file

@ -199,9 +199,6 @@ pub struct CreateSite {
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub description: Option<String>, pub description: Option<String>,
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub icon: Option<String>,
#[cfg_attr(feature = "full", ts(optional))]
pub banner: Option<String>,
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub community_creation_admin_only: Option<bool>, pub community_creation_admin_only: Option<bool>,
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
@ -292,12 +289,6 @@ pub struct EditSite {
/// A shorter, one line description of your site. /// A shorter, one line description of your site.
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub description: Option<String>, pub description: Option<String>,
/// A url for your site's icon.
#[cfg_attr(feature = "full", ts(optional))]
pub icon: Option<String>,
/// A url for your site's banner.
#[cfg_attr(feature = "full", ts(optional))]
pub banner: Option<String>,
/// Limits community creation to admins only. /// Limits community creation to admins only.
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub community_creation_admin_only: Option<bool>, pub community_creation_admin_only: Option<bool>,
@ -443,6 +434,9 @@ pub struct GetSiteResponse {
#[cfg_attr(feature = "full", ts(optional))] #[cfg_attr(feature = "full", ts(optional))]
pub admin_oauth_providers: Option<Vec<OAuthProvider>>, pub admin_oauth_providers: Option<Vec<OAuthProvider>>,
pub blocked_urls: Vec<LocalSiteUrlBlocklist>, pub blocked_urls: Vec<LocalSiteUrlBlocklist>,
// If true then uploads for post images or markdown images are disabled. Only avatars, icons and
// banners can be set.
pub image_upload_disabled: bool,
} }
#[skip_serializing_none] #[skip_serializing_none]

38
crates/api_common/src/utils.rs
View file

@ -1060,7 +1060,7 @@ pub async fn process_markdown(
markdown_check_for_blocked_urls(&text, url_blocklist)?; markdown_check_for_blocked_urls(&text, url_blocklist)?;
if context.settings().pictrs_config()?.image_mode == PictrsImageMode::ProxyAllImages { if context.settings().pictrs()?.image_mode == PictrsImageMode::ProxyAllImages {
let (text, links) = markdown_rewrite_image_links(text); let (text, links) = markdown_rewrite_image_links(text);
RemoteImage::create(&mut context.pool(), links.clone()).await?; RemoteImage::create(&mut context.pool(), links.clone()).await?;
@ -1128,37 +1128,7 @@ async fn proxy_image_link_internal(
/// Rewrite a link to go through `/api/v4/image_proxy` endpoint. This is only for remote urls and /// Rewrite a link to go through `/api/v4/image_proxy` endpoint. This is only for remote urls and
/// if image_proxy setting is enabled. /// if image_proxy setting is enabled.
pub async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult<DbUrl> { pub async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult<DbUrl> {
proxy_image_link_internal( proxy_image_link_internal(link, context.settings().pictrs()?.image_mode, context).await
link,
context.settings().pictrs_config()?.image_mode,
context,
)
.await
}
pub async fn proxy_image_link_opt_api(
link: Option<Option<DbUrl>>,
context: &LemmyContext,
) -> LemmyResult<Option<Option<DbUrl>>> {
if let Some(Some(link)) = link {
proxy_image_link(link.into(), context)
.await
.map(Some)
.map(Some)
} else {
Ok(link)
}
}
pub async fn proxy_image_link_api(
link: Option<DbUrl>,
context: &LemmyContext,
) -> LemmyResult<Option<DbUrl>> {
if let Some(link) = link {
proxy_image_link(link.into(), context).await.map(Some)
} else {
Ok(link)
}
} }
pub async fn proxy_image_link_opt_apub( pub async fn proxy_image_link_opt_apub(
@ -1177,7 +1147,7 @@ fn build_proxied_image_url(
protocol_and_hostname: &str, protocol_and_hostname: &str,
) -> Result<Url, url::ParseError> { ) -> Result<Url, url::ParseError> {
Url::parse(&format!( Url::parse(&format!(
"{}/api/v4/image_proxy?url={}", "{}/api/v4/image/proxy?url={}",
protocol_and_hostname, protocol_and_hostname,
encode(link.as_str()) encode(link.as_str())
)) ))
@ -1256,7 +1226,7 @@ mod tests {
) )
.await?; .await?;
assert_eq!( assert_eq!(
"https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Flemmy-beta%2Fimage.png", "https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Flemmy-beta%2Fimage.png",
proxied.as_str() proxied.as_str()
); );

10
crates/api_crud/src/community/create.rs
View file

@ -13,7 +13,6 @@ use lemmy_api_common::{
is_admin, is_admin,
local_site_to_slur_regex, local_site_to_slur_regex,
process_markdown_opt, process_markdown_opt,
proxy_image_link_api,
EndpointType, EndpointType,
}, },
}; };
@ -31,7 +30,6 @@ use lemmy_db_schema::{
}, },
}, },
traits::{ApubActor, Crud, Followable, Joinable}, traits::{ApubActor, Crud, Followable, Joinable},
utils::diesel_url_create,
}; };
use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_db_views::structs::{LocalUserView, SiteView};
use lemmy_utils::{ use lemmy_utils::{
@ -76,12 +74,6 @@ pub async fn create_community(
check_slurs(desc, &slur_regex)?; check_slurs(desc, &slur_regex)?;
} }
let icon = diesel_url_create(data.icon.as_deref())?;
let icon = proxy_image_link_api(icon, &context).await?;
let banner = diesel_url_create(data.banner.as_deref())?;
let banner = proxy_image_link_api(banner, &context).await?;
is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?; is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?;
if let Some(desc) = &data.description { if let Some(desc) = &data.description {
@ -108,8 +100,6 @@ pub async fn create_community(
let community_form = CommunityInsertForm { let community_form = CommunityInsertForm {
sidebar, sidebar,
description, description,
icon,
banner,
nsfw: data.nsfw, nsfw: data.nsfw,
actor_id: Some(community_actor_id.clone()), actor_id: Some(community_actor_id.clone()),
private_key: Some(keypair.private_key), private_key: Some(keypair.private_key),

14
crates/api_crud/src/community/update.rs
View file

@ -6,14 +6,12 @@ use lemmy_api_common::{
build_response::build_community_response, build_response::build_community_response,
community::{CommunityResponse, EditCommunity}, community::{CommunityResponse, EditCommunity},
context::LemmyContext, context::LemmyContext,
request::replace_image,
send_activity::{ActivityChannel, SendActivityData}, send_activity::{ActivityChannel, SendActivityData},
utils::{ utils::{
check_community_mod_action, check_community_mod_action,
get_url_blocklist, get_url_blocklist,
local_site_to_slur_regex, local_site_to_slur_regex,
process_markdown_opt, process_markdown_opt,
proxy_image_link_opt_api,
}, },
}; };
use lemmy_db_schema::{ use lemmy_db_schema::{
@ -23,7 +21,7 @@ use lemmy_db_schema::{
local_site::LocalSite, local_site::LocalSite,
}, },
traits::Crud, traits::Crud,
utils::{diesel_string_update, diesel_url_update}, utils::diesel_string_update,
}; };
use lemmy_db_views::structs::LocalUserView; use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::{ use lemmy_utils::{
@ -58,14 +56,6 @@ pub async fn update_community(
let old_community = Community::read(&mut context.pool(), data.community_id).await?; let old_community = Community::read(&mut context.pool(), data.community_id).await?;
let icon = diesel_url_update(data.icon.as_deref())?;
replace_image(&icon, &old_community.icon, &context).await?;
let icon = proxy_image_link_opt_api(icon, &context).await?;
let banner = diesel_url_update(data.banner.as_deref())?;
replace_image(&banner, &old_community.banner, &context).await?;
let banner = proxy_image_link_opt_api(banner, &context).await?;
// Verify its a mod (only mods can edit it) // Verify its a mod (only mods can edit it)
check_community_mod_action( check_community_mod_action(
&local_user_view.person, &local_user_view.person,
@ -91,8 +81,6 @@ pub async fn update_community(
title: data.title.clone(), title: data.title.clone(),
sidebar, sidebar,
description, description,
icon,
banner,
nsfw: data.nsfw, nsfw: data.nsfw,
posting_restricted_to_mods: data.posting_restricted_to_mods, posting_restricted_to_mods: data.posting_restricted_to_mods,
visibility: data.visibility, visibility: data.visibility,

11
crates/api_crud/src/site/create.rs
View file

@ -13,7 +13,6 @@ use lemmy_api_common::{
local_site_rate_limit_to_rate_limit_config, local_site_rate_limit_to_rate_limit_config,
local_site_to_slur_regex, local_site_to_slur_regex,
process_markdown_opt, process_markdown_opt,
proxy_image_link_api,
}, },
}; };
use lemmy_db_schema::{ use lemmy_db_schema::{
@ -24,7 +23,7 @@ use lemmy_db_schema::{
site::{Site, SiteUpdateForm}, site::{Site, SiteUpdateForm},
}, },
traits::Crud, traits::Crud,
utils::{diesel_string_update, diesel_url_create}, utils::diesel_string_update,
}; };
use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_db_views::structs::{LocalUserView, SiteView};
use lemmy_utils::{ use lemmy_utils::{
@ -63,18 +62,10 @@ pub async fn create_site(
let url_blocklist = get_url_blocklist(&context).await?; let url_blocklist = get_url_blocklist(&context).await?;
let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &url_blocklist, &context).await?; let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &url_blocklist, &context).await?;
let icon = diesel_url_create(data.icon.as_deref())?;
let icon = proxy_image_link_api(icon, &context).await?;
let banner = diesel_url_create(data.banner.as_deref())?;
let banner = proxy_image_link_api(banner, &context).await?;
let site_form = SiteUpdateForm { let site_form = SiteUpdateForm {
name: Some(data.name.clone()), name: Some(data.name.clone()),
sidebar: diesel_string_update(sidebar.as_deref()), sidebar: diesel_string_update(sidebar.as_deref()),
description: diesel_string_update(data.description.as_deref()), description: diesel_string_update(data.description.as_deref()),
icon: Some(icon),
banner: Some(banner),
actor_id: Some(actor_id), actor_id: Some(actor_id),
last_refreshed_at: Some(Utc::now()), last_refreshed_at: Some(Utc::now()),
inbox_url, inbox_url,

1
crates/api_crud/src/site/read.rs
View file

@ -69,5 +69,6 @@ async fn read_site(context: &LemmyContext) -> LemmyResult<GetSiteResponse> {
tagline, tagline,
oauth_providers: Some(oauth_providers), oauth_providers: Some(oauth_providers),
admin_oauth_providers: Some(admin_oauth_providers), admin_oauth_providers: Some(admin_oauth_providers),
image_upload_disabled: context.settings().pictrs()?.image_upload_disabled,
}) })
} }

14
crates/api_crud/src/site/update.rs
View file

@ -5,7 +5,6 @@ use actix_web::web::Json;
use chrono::Utc; use chrono::Utc;
use lemmy_api_common::{ use lemmy_api_common::{
context::LemmyContext, context::LemmyContext,
request::replace_image,
site::{EditSite, SiteResponse}, site::{EditSite, SiteResponse},
utils::{ utils::{
get_url_blocklist, get_url_blocklist,
@ -13,7 +12,6 @@ use lemmy_api_common::{
local_site_rate_limit_to_rate_limit_config, local_site_rate_limit_to_rate_limit_config,
local_site_to_slur_regex, local_site_to_slur_regex,
process_markdown_opt, process_markdown_opt,
proxy_image_link_opt_api,
}, },
}; };
use lemmy_db_schema::{ use lemmy_db_schema::{
@ -26,7 +24,7 @@ use lemmy_db_schema::{
site::{Site, SiteUpdateForm}, site::{Site, SiteUpdateForm},
}, },
traits::Crud, traits::Crud,
utils::{diesel_string_update, diesel_url_update}, utils::diesel_string_update,
RegistrationMode, RegistrationMode,
}; };
use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_db_views::structs::{LocalUserView, SiteView};
@ -72,20 +70,10 @@ pub async fn update_site(
.as_deref(), .as_deref(),
); );
let icon = diesel_url_update(data.icon.as_deref())?;
replace_image(&icon, &site.icon, &context).await?;
let icon = proxy_image_link_opt_api(icon, &context).await?;
let banner = diesel_url_update(data.banner.as_deref())?;
replace_image(&banner, &site.banner, &context).await?;
let banner = proxy_image_link_opt_api(banner, &context).await?;
let site_form = SiteUpdateForm { let site_form = SiteUpdateForm {
name: data.name.clone(), name: data.name.clone(),
sidebar, sidebar,
description: diesel_string_update(data.description.as_deref()), description: diesel_string_update(data.description.as_deref()),
icon,
banner,
content_warning: diesel_string_update(data.content_warning.as_deref()), content_warning: diesel_string_update(data.content_warning.as_deref()),
updated: Some(Some(Utc::now())), updated: Some(Some(Utc::now())),
..Default::default() ..Default::default()

353
crates/routes/src/images.rs
View file

@ -1,353 +0,0 @@
use actix_web::{
body::{BodyStream, BoxBody},
http::{
header::{HeaderName, ACCEPT_ENCODING, HOST},
Method,
StatusCode,
},
web::*,
HttpRequest,
HttpResponse,
Responder,
};
use futures::stream::{Stream, StreamExt};
use http::HeaderValue;
use lemmy_api_common::{context::LemmyContext, request::PictrsResponse};
use lemmy_db_schema::source::{
images::{LocalImage, LocalImageForm, RemoteImage},
local_site::LocalSite,
};
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell, REQWEST_TIMEOUT};
use reqwest::Body;
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
use serde::Deserialize;
use std::time::Duration;
use url::Url;
pub fn config(cfg: &mut ServiceConfig, client: ClientWithMiddleware, rate_limit: &RateLimitCell) {
cfg
.app_data(Data::new(client))
.service(
resource("/pictrs/image")
.wrap(rate_limit.image())
.route(post().to(upload)),
)
// This has optional query params: /image/{filename}?format=jpg&thumbnail=256
.service(resource("/pictrs/image/{filename}").route(get().to(full_res)))
.service(resource("/pictrs/image/delete/{token}/{filename}").route(get().to(delete)))
.service(resource("/pictrs/healthz").route(get().to(healthz)));
}
trait ProcessUrl {
/// If thumbnail or format is given, this uses the pictrs process endpoint.
/// Otherwise, it uses the normal pictrs url (IE image/original).
fn process_url(&self, image_url: &str, pictrs_url: &Url) -> String;
}
#[derive(Deserialize, Clone)]
struct PictrsGetParams {
format: Option<String>,
thumbnail: Option<i32>,
}
impl ProcessUrl for PictrsGetParams {
fn process_url(&self, src: &str, pictrs_url: &Url) -> String {
if self.format.is_none() && self.thumbnail.is_none() {
format!("{}image/original/{}", pictrs_url, src)
} else {
// Take file type from name, or jpg if nothing is given
let format = self
.clone()
.format
.unwrap_or_else(|| src.split('.').last().unwrap_or("jpg").to_string());
let mut url = format!("{}image/process.{}?src={}", pictrs_url, format, src);
if let Some(size) = self.thumbnail {
url = format!("{url}&thumbnail={size}",);
}
url
}
}
}
#[derive(Deserialize, Clone)]
pub struct ImageProxyParams {
url: String,
format: Option<String>,
thumbnail: Option<i32>,
}
impl ProcessUrl for ImageProxyParams {
fn process_url(&self, proxy_url: &str, pictrs_url: &Url) -> String {
if self.format.is_none() && self.thumbnail.is_none() {
format!("{}image/original?proxy={}", pictrs_url, proxy_url)
} else {
// Take file type from name, or jpg if nothing is given
let format = self
.clone()
.format
.unwrap_or_else(|| proxy_url.split('.').last().unwrap_or("jpg").to_string());
let mut url = format!("{}image/process.{}?proxy={}", pictrs_url, format, proxy_url);
if let Some(size) = self.thumbnail {
url = format!("{url}&thumbnail={size}",);
}
url
}
}
}
fn adapt_request(
request: &HttpRequest,
client: &ClientWithMiddleware,
url: String,
) -> RequestBuilder {
// remove accept-encoding header so that pictrs doesn't compress the response
const INVALID_HEADERS: &[HeaderName] = &[ACCEPT_ENCODING, HOST];
let client_request = client
.request(convert_method(request.method()), url)
.timeout(REQWEST_TIMEOUT);
request
.headers()
.iter()
.fold(client_request, |client_req, (key, value)| {
if INVALID_HEADERS.contains(key) {
client_req
} else {
// TODO: remove as_str and as_bytes conversions after actix-web upgrades to http 1.0
client_req.header(key.as_str(), value.as_bytes())
}
})
}
async fn upload(
req: HttpRequest,
body: Payload,
// require login
local_user_view: LocalUserView,
client: Data<ClientWithMiddleware>,
context: Data<LemmyContext>,
) -> LemmyResult<HttpResponse> {
// TODO: check rate limit here
let pictrs_config = context.settings().pictrs_config()?;
let image_url = format!("{}image", pictrs_config.url);
let mut client_req = adapt_request(&req, &client, image_url);
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string())
};
let res = client_req
.timeout(Duration::from_secs(pictrs_config.upload_timeout))
.body(Body::wrap_stream(make_send(body)))
.send()
.await?;
let status = res.status();
let images = res.json::<PictrsResponse>().await?;
if let Some(images) = &images.files {
for image in images {
let form = LocalImageForm {
local_user_id: Some(local_user_view.local_user.id),
pictrs_alias: image.file.to_string(),
pictrs_delete_token: image.delete_token.to_string(),
};
let protocol_and_hostname = context.settings().get_protocol_and_hostname();
let thumbnail_url = image.thumbnail_url(&protocol_and_hostname)?;
// Also store the details for the image
let details_form = image.details.build_image_details_form(&thumbnail_url);
LocalImage::create(&mut context.pool(), &form, &details_form).await?;
}
}
Ok(HttpResponse::build(convert_status(status)).json(images))
}
async fn full_res(
filename: Path<String>,
Query(params): Query<PictrsGetParams>,
req: HttpRequest,
client: Data<ClientWithMiddleware>,
context: Data<LemmyContext>,
local_user_view: Option<LocalUserView>,
) -> LemmyResult<HttpResponse> {
// block access to images if instance is private and unauthorized, public
let local_site = LocalSite::read(&mut context.pool()).await?;
if local_site.private_instance && local_user_view.is_none() {
return Ok(HttpResponse::Unauthorized().finish());
}
let name = &filename.into_inner();
// If there are no query params, the URL is original
let pictrs_config = context.settings().pictrs_config()?;
let processed_url = params.process_url(name, &pictrs_config.url);
image(processed_url, req, &client).await
}
async fn image(
url: String,
req: HttpRequest,
client: &ClientWithMiddleware,
) -> LemmyResult<HttpResponse> {
let mut client_req = adapt_request(&req, client, url);
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string());
}
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string());
}
let res = client_req.send().await?;
if res.status() == http::StatusCode::NOT_FOUND {
return Ok(HttpResponse::NotFound().finish());
}
let mut client_res = HttpResponse::build(StatusCode::from_u16(res.status().as_u16())?);
for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") {
client_res.insert_header(convert_header(name, value));
}
Ok(client_res.body(BodyStream::new(res.bytes_stream())))
}
async fn delete(
components: Path<(String, String)>,
req: HttpRequest,
client: Data<ClientWithMiddleware>,
context: Data<LemmyContext>,
// require login
_local_user_view: LocalUserView,
) -> LemmyResult<HttpResponse> {
let (token, file) = components.into_inner();
let pictrs_config = context.settings().pictrs_config()?;
let url = format!("{}image/delete/{}/{}", pictrs_config.url, &token, &file);
let mut client_req = adapt_request(&req, &client, url);
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string());
}
let res = client_req.send().await?;
LocalImage::delete_by_alias(&mut context.pool(), &file).await?;
Ok(HttpResponse::build(convert_status(res.status())).body(BodyStream::new(res.bytes_stream())))
}
async fn healthz(
req: HttpRequest,
client: Data<ClientWithMiddleware>,
context: Data<LemmyContext>,
) -> LemmyResult<HttpResponse> {
let pictrs_config = context.settings().pictrs_config()?;
let url = format!("{}healthz", pictrs_config.url);
let mut client_req = adapt_request(&req, &client, url);
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string());
}
let res = client_req.send().await?;
Ok(HttpResponse::build(convert_status(res.status())).body(BodyStream::new(res.bytes_stream())))
}
pub async fn image_proxy(
Query(params): Query<ImageProxyParams>,
req: HttpRequest,
client: Data<ClientWithMiddleware>,
context: Data<LemmyContext>,
) -> LemmyResult<Either<HttpResponse<()>, HttpResponse<BoxBody>>> {
let url = Url::parse(&params.url)?;
// Check that url corresponds to a federated image so that this can't be abused as a proxy
// for arbitrary purposes.
RemoteImage::validate(&mut context.pool(), url.clone().into()).await?;
let pictrs_config = context.settings().pictrs_config()?;
let processed_url = params.process_url(&params.url, &pictrs_config.url);
let bypass_proxy = pictrs_config
.proxy_bypass_domains
.iter()
.any(|s| url.domain().is_some_and(|d| d == s));
if bypass_proxy {
// Bypass proxy and redirect user to original image
Ok(Either::Left(Redirect::to(url.to_string()).respond_to(&req)))
} else {
// Proxy the image data through Lemmy
Ok(Either::Right(image(processed_url, req, &client).await?))
}
}
fn make_send<S>(mut stream: S) -> impl Stream<Item = S::Item> + Send + Unpin + 'static
where
S: Stream + Unpin + 'static,
S::Item: Send,
{
// NOTE: the 8 here is arbitrary
let (tx, rx) = tokio::sync::mpsc::channel(8);
// NOTE: spawning stream into a new task can potentially hit this bug:
// - https://github.com/actix/actix-web/issues/1679
//
// Since 4.0.0-beta.2 this issue is incredibly less frequent. I have not personally reproduced it.
// That said, it is still technically possible to encounter.
actix_web::rt::spawn(async move {
while let Some(res) = stream.next().await {
if tx.send(res).await.is_err() {
break;
}
}
});
SendStream { rx }
}
struct SendStream<T> {
rx: tokio::sync::mpsc::Receiver<T>,
}
impl<T> Stream for SendStream<T>
where
T: Send,
{
type Item = T;
fn poll_next(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
std::pin::Pin::new(&mut self.rx).poll_recv(cx)
}
}
// TODO: remove these conversions after actix-web upgrades to http 1.0
#[allow(clippy::expect_used)]
fn convert_status(status: http::StatusCode) -> StatusCode {
StatusCode::from_u16(status.as_u16()).expect("status can be converted")
}
#[allow(clippy::expect_used)]
fn convert_method(method: &Method) -> http::Method {
http::Method::from_bytes(method.as_str().as_bytes()).expect("method can be converted")
}
fn convert_header<'a>(name: &'a http::HeaderName, value: &'a HeaderValue) -> (&'a str, &'a [u8]) {
(name.as_str(), value.as_bytes())
}

147
crates/routes/src/images/delete.rs Normal file
View file

@ -0,0 +1,147 @@
use super::utils::delete_old_image;
use actix_web::web::*;
use lemmy_api_common::{
context::LemmyContext,
image::{CommunityIdQuery, DeleteImageParams},
utils::{is_admin, is_mod_or_admin},
SuccessResponse,
};
use lemmy_db_schema::{
source::{
community::{Community, CommunityUpdateForm},
images::LocalImage,
person::{Person, PersonUpdateForm},
site::{Site, SiteUpdateForm},
},
traits::Crud,
};
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::error::LemmyResult;
pub async fn delete_site_icon(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<SuccessResponse>> {
let site = Site::read_local(&mut context.pool()).await?;
is_admin(&local_user_view)?;
delete_old_image(&site.icon, &context).await?;
let form = SiteUpdateForm {
icon: Some(None),
..Default::default()
};
Site::update(&mut context.pool(), site.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn delete_site_banner(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<SuccessResponse>> {
let site = Site::read_local(&mut context.pool()).await?;
is_admin(&local_user_view)?;
delete_old_image(&site.banner, &context).await?;
let form = SiteUpdateForm {
banner: Some(None),
..Default::default()
};
Site::update(&mut context.pool(), site.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn delete_community_icon(
data: Json<CommunityIdQuery>,
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<SuccessResponse>> {
let community = Community::read(&mut context.pool(), data.id).await?;
is_mod_or_admin(&mut context.pool(), &local_user_view.person, community.id).await?;
delete_old_image(&community.icon, &context).await?;
let form = CommunityUpdateForm {
icon: Some(None),
..Default::default()
};
Community::update(&mut context.pool(), community.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn delete_community_banner(
data: Json<CommunityIdQuery>,
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<SuccessResponse>> {
let community = Community::read(&mut context.pool(), data.id).await?;
is_mod_or_admin(&mut context.pool(), &local_user_view.person, community.id).await?;
delete_old_image(&community.icon, &context).await?;
let form = CommunityUpdateForm {
icon: Some(None),
..Default::default()
};
Community::update(&mut context.pool(), community.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn delete_user_avatar(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<SuccessResponse>> {
delete_old_image(&local_user_view.person.avatar, &context).await?;
let form = PersonUpdateForm {
avatar: Some(None),
..Default::default()
};
Person::update(&mut context.pool(), local_user_view.person.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn delete_user_banner(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<SuccessResponse>> {
delete_old_image(&local_user_view.person.banner, &context).await?;
let form = PersonUpdateForm {
banner: Some(None),
..Default::default()
};
Person::update(&mut context.pool(), local_user_view.person.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
// TODO: get rid of delete tokens and allow deletion by admin or uploader
pub async fn delete_image(
data: Json<DeleteImageParams>,
context: Data<LemmyContext>,
// require login
_local_user_view: LocalUserView,
) -> LemmyResult<Json<SuccessResponse>> {
let pictrs_config = context.settings().pictrs()?;
let url = format!(
"{}image/delete/{}/{}",
pictrs_config.url, &data.token, &data.filename
);
context
.pictrs_client()
.delete(url)
.send()
.await?
.error_for_status()?;
LocalImage::delete_by_alias(&mut context.pool(), &data.filename).await?;
Ok(Json(SuccessResponse::default()))
}

129
crates/routes/src/images/download.rs Normal file
View file

@ -0,0 +1,129 @@
use super::utils::{adapt_request, convert_header};
use actix_web::{
body::{BodyStream, BoxBody},
http::StatusCode,
web::{Data, *},
HttpRequest,
HttpResponse,
Responder,
};
use lemmy_api_common::{
context::LemmyContext,
image::{ImageGetParams, ImageProxyParams},
};
use lemmy_db_schema::source::{images::RemoteImage, local_site::LocalSite};
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::error::LemmyResult;
use url::Url;
pub async fn get_image(
filename: Path<String>,
Query(params): Query<ImageGetParams>,
req: HttpRequest,
local_user_view: Option<LocalUserView>,
context: Data<LemmyContext>,
) -> LemmyResult<HttpResponse> {
// block access to images if instance is private
if local_user_view.is_none() {
let local_site = LocalSite::read(&mut context.pool()).await?;
if local_site.private_instance {
return Ok(HttpResponse::Unauthorized().finish());
}
}
let name = &filename.into_inner();
// If there are no query params, the URL is original
let pictrs_url = context.settings().pictrs()?.url;
let processed_url = if params.file_type.is_none() && params.max_size.is_none() {
format!("{}image/original/{}", pictrs_url, name)
} else {
let file_type = file_type(params.file_type, name);
let mut url = format!("{}image/process.{}?src={}", pictrs_url, file_type, name);
if let Some(size) = params.max_size {
url = format!("{url}&thumbnail={size}",);
}
url
};
do_get_image(processed_url, req, &context).await
}
pub async fn image_proxy(
Query(params): Query<ImageProxyParams>,
req: HttpRequest,
context: Data<LemmyContext>,
) -> LemmyResult<Either<HttpResponse<()>, HttpResponse<BoxBody>>> {
let url = Url::parse(&params.url)?;
// Check that url corresponds to a federated image so that this can't be abused as a proxy
// for arbitrary purposes.
RemoteImage::validate(&mut context.pool(), url.clone().into()).await?;
let pictrs_config = context.settings().pictrs()?;
let processed_url = if params.file_type.is_none() && params.max_size.is_none() {
format!("{}image/original?proxy={}", pictrs_config.url, params.url)
} else {
let file_type = file_type(params.file_type, url.as_str());
let mut url = format!(
"{}image/process.{}?proxy={}",
pictrs_config.url, file_type, url
);
if let Some(size) = params.max_size {
url = format!("{url}&thumbnail={size}",);
}
url
};
let bypass_proxy = pictrs_config
.proxy_bypass_domains
.iter()
.any(|s| url.domain().is_some_and(|d| d == s));
if bypass_proxy {
// Bypass proxy and redirect user to original image
Ok(Either::Left(Redirect::to(url.to_string()).respond_to(&req)))
} else {
// Proxy the image data through Lemmy
Ok(Either::Right(
do_get_image(processed_url, req, &context).await?,
))
}
}
pub(super) async fn do_get_image(
url: String,
req: HttpRequest,
context: &LemmyContext,
) -> LemmyResult<HttpResponse> {
let mut client_req = adapt_request(&req, url, context);
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string());
}
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string());
}
let res = client_req.send().await?;
if res.status() == http::StatusCode::NOT_FOUND {
return Ok(HttpResponse::NotFound().finish());
}
let mut client_res = HttpResponse::build(StatusCode::from_u16(res.status().as_u16())?);
for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") {
client_res.insert_header(convert_header(name, value));
}
Ok(client_res.body(BodyStream::new(res.bytes_stream())))
}
/// Take file type from param, name, or use jpg if nothing is given
pub(super) fn file_type(file_type: Option<String>, name: &str) -> String {
file_type
.clone()
.unwrap_or_else(|| name.split('.').last().unwrap_or("jpg").to_string())
}

22
crates/routes/src/images/mod.rs Normal file
View file

@ -0,0 +1,22 @@
use actix_web::web::*;
use lemmy_api_common::{context::LemmyContext, SuccessResponse};
use lemmy_utils::error::LemmyResult;
pub mod delete;
pub mod download;
pub mod upload;
mod utils;
pub async fn pictrs_health(context: Data<LemmyContext>) -> LemmyResult<Json<SuccessResponse>> {
let pictrs_config = context.settings().pictrs()?;
let url = format!("{}healthz", pictrs_config.url);
context
.pictrs_client()
.get(url)
.send()
.await?
.error_for_status()?;
Ok(Json(SuccessResponse::default()))
}

239
crates/routes/src/images/upload.rs Normal file
View file

@ -0,0 +1,239 @@
use super::utils::{adapt_request, delete_old_image, make_send};
use actix_web::{self, web::*, HttpRequest};
use lemmy_api_common::{
context::LemmyContext,
image::{CommunityIdQuery, UploadImageResponse},
request::PictrsResponse,
utils::{is_admin, is_mod_or_admin},
LemmyErrorType,
SuccessResponse,
};
use lemmy_db_schema::{
source::{
community::{Community, CommunityUpdateForm},
images::{LocalImage, LocalImageForm},
person::{Person, PersonUpdateForm},
site::{Site, SiteUpdateForm},
},
traits::Crud,
};
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::error::LemmyResult;
use reqwest::Body;
use std::time::Duration;
use UploadType::*;
pub enum UploadType {
Avatar,
Banner,
Other,
}
pub async fn upload_image(
req: HttpRequest,
body: Payload,
local_user_view: LocalUserView,
context: Data<LemmyContext>,
) -> LemmyResult<Json<UploadImageResponse>> {
if context.settings().pictrs()?.image_upload_disabled {
return Err(LemmyErrorType::ImageUploadDisabled.into());
}
Ok(Json(
do_upload_image(req, body, Other, &local_user_view, &context).await?,
))
}
pub async fn upload_user_avatar(
req: HttpRequest,
body: Payload,
local_user_view: LocalUserView,
context: Data<LemmyContext>,
) -> LemmyResult<Json<SuccessResponse>> {
let image = do_upload_image(req, body, Avatar, &local_user_view, &context).await?;
delete_old_image(&local_user_view.person.avatar, &context).await?;
let form = PersonUpdateForm {
avatar: Some(Some(image.image_url.into())),
..Default::default()
};
Person::update(&mut context.pool(), local_user_view.person.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn upload_user_banner(
req: HttpRequest,
body: Payload,
local_user_view: LocalUserView,
context: Data<LemmyContext>,
) -> LemmyResult<Json<SuccessResponse>> {
let image = do_upload_image(req, body, Banner, &local_user_view, &context).await?;
delete_old_image(&local_user_view.person.banner, &context).await?;
let form = PersonUpdateForm {
banner: Some(Some(image.image_url.into())),
..Default::default()
};
Person::update(&mut context.pool(), local_user_view.person.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn upload_community_icon(
req: HttpRequest,
query: Query<CommunityIdQuery>,
body: Payload,
local_user_view: LocalUserView,
context: Data<LemmyContext>,
) -> LemmyResult<Json<SuccessResponse>> {
let community: Community = Community::read(&mut context.pool(), query.id).await?;
is_mod_or_admin(&mut context.pool(), &local_user_view.person, community.id).await?;
let image = do_upload_image(req, body, Avatar, &local_user_view, &context).await?;
delete_old_image(&community.icon, &context).await?;
let form = CommunityUpdateForm {
icon: Some(Some(image.image_url.into())),
..Default::default()
};
Community::update(&mut context.pool(), community.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn upload_community_banner(
req: HttpRequest,
query: Query<CommunityIdQuery>,
body: Payload,
local_user_view: LocalUserView,
context: Data<LemmyContext>,
) -> LemmyResult<Json<SuccessResponse>> {
let community: Community = Community::read(&mut context.pool(), query.id).await?;
is_mod_or_admin(&mut context.pool(), &local_user_view.person, community.id).await?;
let image = do_upload_image(req, body, Banner, &local_user_view, &context).await?;
delete_old_image(&community.banner, &context).await?;
let form = CommunityUpdateForm {
banner: Some(Some(image.image_url.into())),
..Default::default()
};
Community::update(&mut context.pool(), community.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn upload_site_icon(
req: HttpRequest,
body: Payload,
local_user_view: LocalUserView,
context: Data<LemmyContext>,
) -> LemmyResult<Json<SuccessResponse>> {
is_admin(&local_user_view)?;
let site = Site::read_local(&mut context.pool()).await?;
let image = do_upload_image(req, body, Avatar, &local_user_view, &context).await?;
delete_old_image(&site.icon, &context).await?;
let form = SiteUpdateForm {
icon: Some(Some(image.image_url.into())),
..Default::default()
};
Site::update(&mut context.pool(), site.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn upload_site_banner(
req: HttpRequest,
body: Payload,
local_user_view: LocalUserView,
context: Data<LemmyContext>,
) -> LemmyResult<Json<SuccessResponse>> {
is_admin(&local_user_view)?;
let site = Site::read_local(&mut context.pool()).await?;
let image = do_upload_image(req, body, Banner, &local_user_view, &context).await?;
delete_old_image(&site.banner, &context).await?;
let form = SiteUpdateForm {
banner: Some(Some(image.image_url.into())),
..Default::default()
};
Site::update(&mut context.pool(), site.id, &form).await?;
Ok(Json(SuccessResponse::default()))
}
pub async fn do_upload_image(
req: HttpRequest,
body: Payload,
upload_type: UploadType,
local_user_view: &LocalUserView,
context: &Data<LemmyContext>,
) -> LemmyResult<UploadImageResponse> {
let pictrs = context.settings().pictrs()?;
let image_url = format!("{}image", pictrs.url);
let mut client_req = adapt_request(&req, image_url, context);
client_req = match upload_type {
Avatar => {
let max_size = pictrs.max_avatar_size.to_string();
client_req.query(&[
("resize", max_size.as_ref()),
("allow_animation", "false"),
("allow_video", "false"),
])
}
Banner => {
let max_size = pictrs.max_banner_size.to_string();
client_req.query(&[
("resize", max_size.as_ref()),
("allow_animation", "false"),
("allow_video", "false"),
])
}
_ => client_req,
};
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string())
};
let res = client_req
.timeout(Duration::from_secs(pictrs.upload_timeout))
.body(Body::wrap_stream(make_send(body)))
.send()
.await?
.error_for_status()?;
let mut images = res.json::<PictrsResponse>().await?;
for image in &images.files {
// Pictrs allows uploading multiple images in a single request. Lemmy doesnt need this,
// but still a user may upload multiple and so we need to store all links in db for
// to allow deletion via web ui.
let form = LocalImageForm {
local_user_id: Some(local_user_view.local_user.id),
pictrs_alias: image.file.to_string(),
pictrs_delete_token: image.delete_token.to_string(),
};
let protocol_and_hostname = context.settings().get_protocol_and_hostname();
let thumbnail_url = image.image_url(&protocol_and_hostname)?;
// Also store the details for the image
let details_form = image.details.build_image_details_form(&thumbnail_url);
LocalImage::create(&mut context.pool(), &form, &details_form).await?;
}
let image = images
.files
.pop()
.ok_or(LemmyErrorType::InvalidImageUpload)?;
let url = image.image_url(&context.settings().get_protocol_and_hostname())?;
Ok(UploadImageResponse {
image_url: url,
filename: image.file,
delete_token: image.delete_token,
})
}

111
crates/routes/src/images/utils.rs Normal file
View file

@ -0,0 +1,111 @@
use actix_web::{
http::{
header::{HeaderName, ACCEPT_ENCODING, HOST},
Method,
},
web::Data,
HttpRequest,
};
use futures::stream::{Stream, StreamExt};
use http::HeaderValue;
use lemmy_api_common::{context::LemmyContext, request::delete_image_from_pictrs};
use lemmy_db_schema::{newtypes::DbUrl, source::images::LocalImage};
use lemmy_utils::{error::LemmyResult, REQWEST_TIMEOUT};
use reqwest_middleware::RequestBuilder;
pub(super) fn adapt_request(
request: &HttpRequest,
url: String,
context: &LemmyContext,
) -> RequestBuilder {
// remove accept-encoding header so that pictrs doesn't compress the response
const INVALID_HEADERS: &[HeaderName] = &[ACCEPT_ENCODING, HOST];
let client_request = context
.pictrs_client()
.request(convert_method(request.method()), url)
.timeout(REQWEST_TIMEOUT);
request
.headers()
.iter()
.fold(client_request, |client_req, (key, value)| {
if INVALID_HEADERS.contains(key) {
client_req
} else {
// TODO: remove as_str and as_bytes conversions after actix-web upgrades to http 1.0
client_req.header(key.as_str(), value.as_bytes())
}
})
}
pub(super) fn make_send<S>(mut stream: S) -> impl Stream<Item = S::Item> + Send + Unpin + 'static
where
S: Stream + Unpin + 'static,
S::Item: Send,
{
// NOTE: the 8 here is arbitrary
let (tx, rx) = tokio::sync::mpsc::channel(8);
// NOTE: spawning stream into a new task can potentially hit this bug:
// - https://github.com/actix/actix-web/issues/1679
//
// Since 4.0.0-beta.2 this issue is incredibly less frequent. I have not personally reproduced it.
// That said, it is still technically possible to encounter.
actix_web::rt::spawn(async move {
while let Some(res) = stream.next().await {
if tx.send(res).await.is_err() {
break;
}
}
});
SendStream { rx }
}
struct SendStream<T> {
rx: tokio::sync::mpsc::Receiver<T>,
}
impl<T> Stream for SendStream<T>
where
T: Send,
{
type Item = T;
fn poll_next(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
std::pin::Pin::new(&mut self.rx).poll_recv(cx)
}
}
// TODO: remove these conversions after actix-web upgrades to http 1.0
#[allow(clippy::expect_used)]
pub(super) fn convert_method(method: &Method) -> http::Method {
http::Method::from_bytes(method.as_str().as_bytes()).expect("method can be converted")
}
pub(super) fn convert_header<'a>(
name: &'a http::HeaderName,
value: &'a HeaderValue,
) -> (&'a str, &'a [u8]) {
(name.as_str(), value.as_bytes())
}
/// When adding a new avatar, banner or similar image, delete the old one.
pub(super) async fn delete_old_image(
old_image: &Option<DbUrl>,
context: &Data<LemmyContext>,
) -> LemmyResult<()> {
if let Some(old_image) = old_image {
let image = LocalImage::delete_by_url(&mut context.pool(), old_image)
.await
.ok();
if let Some(image) = image {
delete_image_from_pictrs(&image.pictrs_alias, &image.pictrs_delete_token, context).await?;
}
}
Ok(())
}

3
crates/utils/src/error.rs
View file

@ -23,7 +23,6 @@ pub enum LemmyErrorType {
CouldntUpdateComment, CouldntUpdateComment,
CouldntUpdatePrivateMessage, CouldntUpdatePrivateMessage,
CannotLeaveAdmin, CannotLeaveAdmin,
// TODO: also remove the translations of unused errors
PictrsResponseError(String), PictrsResponseError(String),
PictrsPurgeResponseError(String), PictrsPurgeResponseError(String),
ImageUrlMissingPathSegments, ImageUrlMissingPathSegments,
@ -31,6 +30,8 @@ pub enum LemmyErrorType {
PictrsApiKeyNotProvided, PictrsApiKeyNotProvided,
NoContentTypeHeader, NoContentTypeHeader,
NotAnImageType, NotAnImageType,
InvalidImageUpload,
ImageUploadDisabled,
NotAModOrAdmin, NotAModOrAdmin,
NotTopMod, NotTopMod,
NotLoggedIn, NotLoggedIn,

2
crates/utils/src/settings/mod.rs
View file

@ -97,7 +97,7 @@ impl Settings {
WEBFINGER_REGEX.clone() WEBFINGER_REGEX.clone()
} }
pub fn pictrs_config(&self) -> LemmyResult<PictrsConfig> { pub fn pictrs(&self) -> LemmyResult<PictrsConfig> {
self self
.pictrs .pictrs
.clone() .clone()

16
crates/utils/src/settings/structs.rs
View file

@ -98,6 +98,20 @@ pub struct PictrsConfig {
/// Resize post thumbnails to this maximum width/height. /// Resize post thumbnails to this maximum width/height.
#[default(512)] #[default(512)]
pub max_thumbnail_size: u32, pub max_thumbnail_size: u32,
/// Maximum size for user avatar, community icon and site icon.
#[default(512)]
pub max_avatar_size: u32,
/// Maximum size for user, community and site banner. Larger images are downscaled to fit
/// into a square of this size.
#[default(1024)]
pub max_banner_size: u32,
/// Prevent users from uploading images for posts or embedding in markdown. Avatars, icons and
/// banners can still be uploaded.
#[default(false)]
pub image_upload_disabled: bool,
} }
#[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document, PartialEq)] #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document, PartialEq)]
@ -113,7 +127,7 @@ pub enum PictrsImageMode {
/// This behaviour matches Lemmy 0.18. /// This behaviour matches Lemmy 0.18.
StoreLinkPreviews, StoreLinkPreviews,
/// If enabled, all images from remote domains are rewritten to pass through /// If enabled, all images from remote domains are rewritten to pass through
/// `/api/v4/image_proxy`, including embedded images in markdown. Images are stored temporarily /// `/api/v4/image/proxy`, including embedded images in markdown. Images are stored temporarily
/// in pict-rs for caching. This improves privacy as users don't expose their IP to untrusted /// in pict-rs for caching. This improves privacy as users don't expose their IP to untrusted
/// servers, and decreases load on other servers. However it increases bandwidth use for the /// servers, and decreases load on other servers. However it increases bandwidth use for the
/// local server. /// local server.

12
crates/utils/src/utils/markdown/image_links.rs
View file

@ -18,7 +18,7 @@ pub fn markdown_rewrite_image_links(mut src: String) -> (String, Vec<Url>) {
// If link points to remote domain, replace with proxied link // If link points to remote domain, replace with proxied link
if parsed.domain() != Some(&SETTINGS.hostname) { if parsed.domain() != Some(&SETTINGS.hostname) {
let mut proxied = format!( let mut proxied = format!(
"{}/api/v4/image_proxy?url={}", "{}/api/v4/image/proxy?url={}",
SETTINGS.get_protocol_and_hostname(), SETTINGS.get_protocol_and_hostname(),
encode(url), encode(url),
); );
@ -116,7 +116,7 @@ mod tests {
( (
"remote image proxied", "remote image proxied",
"![link](http://example.com/image.jpg)", "![link](http://example.com/image.jpg)",
"![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)", "![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)",
), ),
( (
"local image unproxied", "local image unproxied",
@ -126,7 +126,7 @@ mod tests {
( (
"multiple image links", "multiple image links",
"![link](http://example.com/image1.jpg) ![link](http://example.com/image2.jpg)", "![link](http://example.com/image1.jpg) ![link](http://example.com/image2.jpg)",
"![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)", "![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)",
), ),
( (
"empty link handled", "empty link handled",
@ -136,7 +136,7 @@ mod tests {
( (
"empty label handled", "empty label handled",
"![](http://example.com/image.jpg)", "![](http://example.com/image.jpg)",
"![](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" "![](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)"
), ),
( (
"invalid image link removed", "invalid image link removed",
@ -146,12 +146,12 @@ mod tests {
( (
"label with nested markdown handled", "label with nested markdown handled",
"![a *b* c](http://example.com/image.jpg)", "![a *b* c](http://example.com/image.jpg)",
"![a *b* c](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" "![a *b* c](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)"
), ),
( (
"custom emoji support", "custom emoji support",
r#"![party-blob](https://www.hexbear.net/pictrs/image/83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#, r#"![party-blob](https://www.hexbear.net/pictrs/image/83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#,
r#"![party-blob](https://lemmy-alpha/api/v4/image_proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"# r#"![party-blob](https://lemmy-alpha/api/v4/image/proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#
) )
]; ];

10
crates/utils/src/utils/markdown/mod.rs
View file

@ -155,7 +155,7 @@ mod tests {
( (
"remote image proxied", "remote image proxied",
"![link](http://example.com/image.jpg)", "![link](http://example.com/image.jpg)",
"![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)", "![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)",
), ),
( (
"local image unproxied", "local image unproxied",
@ -165,7 +165,7 @@ mod tests {
( (
"multiple image links", "multiple image links",
"![link](http://example.com/image1.jpg) ![link](http://example.com/image2.jpg)", "![link](http://example.com/image1.jpg) ![link](http://example.com/image2.jpg)",
"![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)", "![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)",
), ),
( (
"empty link handled", "empty link handled",
@ -175,7 +175,7 @@ mod tests {
( (
"empty label handled", "empty label handled",
"![](http://example.com/image.jpg)", "![](http://example.com/image.jpg)",
"![](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" "![](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)"
), ),
( (
"invalid image link removed", "invalid image link removed",
@ -185,12 +185,12 @@ mod tests {
( (
"label with nested markdown handled", "label with nested markdown handled",
"![a *b* c](http://example.com/image.jpg)", "![a *b* c](http://example.com/image.jpg)",
"![a *b* c](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" "![a *b* c](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)"
), ),
( (
"custom emoji support", "custom emoji support",
r#"![party-blob](https://www.hexbear.net/pictrs/image/83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#, r#"![party-blob](https://www.hexbear.net/pictrs/image/83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#,
r#"![party-blob](https://lemmy-alpha/api/v4/image_proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"# r#"![party-blob](https://lemmy-alpha/api/v4/image/proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#
) )
]; ];

498
src/api_routes_v3.rs
View file

@ -77,7 +77,6 @@ use lemmy_api::{
unread_count::get_unread_registration_application_count, unread_count::get_unread_registration_application_count,
}, },
}, },
sitemap::get_sitemap,
}; };
use lemmy_api_crud::{ use lemmy_api_crud::{
comment::{ comment::{
@ -124,252 +123,261 @@ use lemmy_apub::api::{
search::search, search::search,
user_settings_backup::{export_settings, import_settings}, user_settings_backup::{export_settings, import_settings},
}; };
use lemmy_routes::images::image_proxy; use lemmy_routes::images::{
delete::delete_image,
download::{get_image, image_proxy},
pictrs_health,
upload::upload_image,
};
use lemmy_utils::rate_limit::RateLimitCell; use lemmy_utils::rate_limit::RateLimitCell;
// Deprecated, use api v4 instead. // Deprecated, use api v4 instead.
// When removing api v3, we also need to rewrite all links in database with // When removing api v3, make sure to keep `/api/v3/image_proxy` as it is still used in old posts.
// `/api/v3/image_proxy` to use `/api/v4/image_proxy` instead.
pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) { pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) {
cfg.service( cfg
scope("/api/v3") .service(
.route("/image_proxy", get().to(image_proxy)) resource("/pictrs/image")
// Site .wrap(rate_limit.image())
.service( .route(post().to(upload_image)),
scope("/site") )
.wrap(rate_limit.message()) .service(resource("/pictrs/image/{filename}").route(get().to(get_image)))
.route("", get().to(get_site_v3)) .service(resource("/pictrs/image/delete/{token}/{filename}").route(get().to(delete_image)))
// Admin Actions .service(resource("/pictrs/healthz").route(get().to(pictrs_health)))
.route("", post().to(create_site)) .service(
.route("", put().to(update_site)) scope("/api/v3")
.route("/block", post().to(user_block_instance)), .route("/image_proxy", get().to(image_proxy))
) // Site
.service( .service(
resource("/modlog") scope("/site")
.wrap(rate_limit.message()) .wrap(rate_limit.message())
.route(get().to(get_mod_log)), .route("", get().to(get_site_v3))
) // Admin Actions
.service( .route("", post().to(create_site))
resource("/search") .route("", put().to(update_site))
.wrap(rate_limit.search()) .route("/block", post().to(user_block_instance)),
.route(get().to(search)), )
) .service(
.service( resource("/modlog")
resource("/resolve_object") .wrap(rate_limit.message())
.wrap(rate_limit.message()) .route(get().to(get_mod_log)),
.route(get().to(resolve_object)), )
) .service(
// Community resource("/search")
.service( .wrap(rate_limit.search())
resource("/community") .route(get().to(search)),
.guard(guard::Post()) )
.wrap(rate_limit.register()) .service(
.route(post().to(create_community)), resource("/resolve_object")
) .wrap(rate_limit.message())
.service( .route(get().to(resolve_object)),
scope("/community") )
.wrap(rate_limit.message()) // Community
.route("", get().to(get_community)) .service(
.route("", put().to(update_community)) resource("/community")
.route("/hide", put().to(hide_community)) .guard(guard::Post())
.route("/list", get().to(list_communities)) .wrap(rate_limit.register())
.route("/follow", post().to(follow_community)) .route(post().to(create_community)),
.route("/block", post().to(user_block_community)) )
.route("/delete", post().to(delete_community)) .service(
// Mod Actions scope("/community")
.route("/remove", post().to(remove_community)) .wrap(rate_limit.message())
.route("/transfer", post().to(transfer_community)) .route("", get().to(get_community))
.route("/ban_user", post().to(ban_from_community)) .route("", put().to(update_community))
.route("/mod", post().to(add_mod_to_community)), .route("/hide", put().to(hide_community))
) .route("/list", get().to(list_communities))
.service( .route("/follow", post().to(follow_community))
scope("/federated_instances") .route("/block", post().to(user_block_community))
.wrap(rate_limit.message()) .route("/delete", post().to(delete_community))
.route("", get().to(get_federated_instances)), // Mod Actions
) .route("/remove", post().to(remove_community))
// Post .route("/transfer", post().to(transfer_community))
.service( .route("/ban_user", post().to(ban_from_community))
// Handle POST to /post separately to add the post() rate limitter .route("/mod", post().to(add_mod_to_community)),
resource("/post") )
.guard(guard::Post()) .service(
.wrap(rate_limit.post()) scope("/federated_instances")
.route(post().to(create_post)), .wrap(rate_limit.message())
) .route("", get().to(get_federated_instances)),
.service( )
scope("/post") // Post
.wrap(rate_limit.message()) .service(
.route("", get().to(get_post)) // Handle POST to /post separately to add the post() rate limitter
.route("", put().to(update_post)) resource("/post")
.route("/delete", post().to(delete_post)) .guard(guard::Post())
.route("/remove", post().to(remove_post)) .wrap(rate_limit.post())
.route("/mark_as_read", post().to(mark_post_as_read)) .route(post().to(create_post)),
.route("/hide", post().to(hide_post)) )
.route("/lock", post().to(lock_post)) .service(
.route("/feature", post().to(feature_post)) scope("/post")
.route("/list", get().to(list_posts)) .wrap(rate_limit.message())
.route("/like", post().to(like_post)) .route("", get().to(get_post))
.route("/like/list", get().to(list_post_likes)) .route("", put().to(update_post))
.route("/save", put().to(save_post)) .route("/delete", post().to(delete_post))
.route("/report", post().to(create_post_report)) .route("/remove", post().to(remove_post))
.route("/report/resolve", put().to(resolve_post_report)) .route("/mark_as_read", post().to(mark_post_as_read))
.route("/site_metadata", get().to(get_link_metadata)), .route("/hide", post().to(hide_post))
) .route("/lock", post().to(lock_post))
// Comment .route("/feature", post().to(feature_post))
.service( .route("/list", get().to(list_posts))
// Handle POST to /comment separately to add the comment() rate limitter .route("/like", post().to(like_post))
resource("/comment") .route("/like/list", get().to(list_post_likes))
.guard(guard::Post()) .route("/save", put().to(save_post))
.wrap(rate_limit.comment()) .route("/report", post().to(create_post_report))
.route(post().to(create_comment)), .route("/report/resolve", put().to(resolve_post_report))
) .route("/site_metadata", get().to(get_link_metadata)),
.service( )
scope("/comment") // Comment
.wrap(rate_limit.message()) .service(
.route("", get().to(get_comment)) // Handle POST to /comment separately to add the comment() rate limitter
.route("", put().to(update_comment)) resource("/comment")
.route("/delete", post().to(delete_comment)) .guard(guard::Post())
.route("/remove", post().to(remove_comment)) .wrap(rate_limit.comment())
.route("/mark_as_read", post().to(mark_reply_as_read)) .route(post().to(create_comment)),
.route("/distinguish", post().to(distinguish_comment)) )
.route("/like", post().to(like_comment)) .service(
.route("/like/list", get().to(list_comment_likes)) scope("/comment")
.route("/save", put().to(save_comment)) .wrap(rate_limit.message())
.route("/list", get().to(list_comments)) .route("", get().to(get_comment))
.route("/report", post().to(create_comment_report)) .route("", put().to(update_comment))
.route("/report/resolve", put().to(resolve_comment_report)), .route("/delete", post().to(delete_comment))
) .route("/remove", post().to(remove_comment))
// Private Message .route("/mark_as_read", post().to(mark_reply_as_read))
.service( .route("/distinguish", post().to(distinguish_comment))
scope("/private_message") .route("/like", post().to(like_comment))
.wrap(rate_limit.message()) .route("/like/list", get().to(list_comment_likes))
.route("/list", get().to(get_private_message)) .route("/save", put().to(save_comment))
.route("", post().to(create_private_message)) .route("/list", get().to(list_comments))
.route("", put().to(update_private_message)) .route("/report", post().to(create_comment_report))
.route("/delete", post().to(delete_private_message)) .route("/report/resolve", put().to(resolve_comment_report)),
.route("/mark_as_read", post().to(mark_pm_as_read)) )
.route("/report", post().to(create_pm_report)) // Private Message
.route("/report/resolve", put().to(resolve_pm_report)), .service(
) scope("/private_message")
// User .wrap(rate_limit.message())
.service( .route("/list", get().to(get_private_message))
// Account action, I don't like that it's in /user maybe /accounts .route("", post().to(create_private_message))
// Handle /user/register separately to add the register() rate limiter .route("", put().to(update_private_message))
resource("/user/register") .route("/delete", post().to(delete_private_message))
.guard(guard::Post()) .route("/mark_as_read", post().to(mark_pm_as_read))
.wrap(rate_limit.register()) .route("/report", post().to(create_pm_report))
.route(post().to(register)), .route("/report/resolve", put().to(resolve_pm_report)),
) )
// User // User
.service( .service(
// Handle /user/login separately to add the register() rate limiter // Account action, I don't like that it's in /user maybe /accounts
// TODO: pretty annoying way to apply rate limits for register and login, we should // Handle /user/register separately to add the register() rate limiter
// group them under a common path so that rate limit is only applied once (eg under resource("/user/register")
// /account). .guard(guard::Post())
resource("/user/login") .wrap(rate_limit.register())
.guard(guard::Post()) .route(post().to(register)),
.wrap(rate_limit.register()) )
.route(post().to(login)), // User
) .service(
.service( // Handle /user/login separately to add the register() rate limiter
resource("/user/password_reset") // TODO: pretty annoying way to apply rate limits for register and login, we should
.wrap(rate_limit.register()) // group them under a common path so that rate limit is only applied once (eg under
.route(post().to(reset_password)), // /account).
) resource("/user/login")
.service( .guard(guard::Post())
// Handle captcha separately .wrap(rate_limit.register())
resource("/user/get_captcha") .route(post().to(login)),
.wrap(rate_limit.post()) )
.route(get().to(get_captcha)), .service(
) resource("/user/password_reset")
.service( .wrap(rate_limit.register())
resource("/user/export_settings") .route(post().to(reset_password)),
.wrap(rate_limit.import_user_settings()) )
.route(get().to(export_settings)), .service(
) // Handle captcha separately
.service( resource("/user/get_captcha")
resource("/user/import_settings") .wrap(rate_limit.post())
.wrap(rate_limit.import_user_settings()) .route(get().to(get_captcha)),
.route(post().to(import_settings)), )
) .service(
// TODO, all the current account related actions under /user need to get moved here eventually resource("/user/export_settings")
.service( .wrap(rate_limit.import_user_settings())
scope("/account") .route(get().to(export_settings)),
.wrap(rate_limit.message()) )
.route("/list_media", get().to(list_media)), .service(
) resource("/user/import_settings")
// User actions .wrap(rate_limit.import_user_settings())
.service( .route(post().to(import_settings)),
scope("/user") )
.wrap(rate_limit.message()) // TODO, all the current account related actions under /user need to get moved here
.route("", get().to(read_person)) // eventually
.route("/mention", get().to(list_mentions)) .service(
.route( scope("/account")
"/mention/mark_as_read", .wrap(rate_limit.message())
post().to(mark_person_mention_as_read), .route("/list_media", get().to(list_media)),
) )
.route("/replies", get().to(list_replies)) // User actions
// Admin action. I don't like that it's in /user .service(
.route("/ban", post().to(ban_from_site)) scope("/user")
.route("/banned", get().to(list_banned_users)) .wrap(rate_limit.message())
.route("/block", post().to(user_block_person)) .route("", get().to(read_person))
// TODO Account actions. I don't like that they're in /user maybe /accounts .route("/mention", get().to(list_mentions))
.route("/logout", post().to(logout)) .route(
.route("/delete_account", post().to(delete_account)) "/mention/mark_as_read",
.route("/password_change", post().to(change_password_after_reset)) post().to(mark_person_mention_as_read),
// TODO mark_all_as_read feels off being in this section as well )
.route("/mark_all_as_read", post().to(mark_all_notifications_read)) .route("/replies", get().to(list_replies))
.route("/save_user_settings", put().to(save_user_settings)) // Admin action. I don't like that it's in /user
.route("/change_password", put().to(change_password)) .route("/ban", post().to(ban_from_site))
.route("/report_count", get().to(report_count)) .route("/banned", get().to(list_banned_users))
.route("/unread_count", get().to(unread_count)) .route("/block", post().to(user_block_person))
.route("/verify_email", post().to(verify_email)) // TODO Account actions. I don't like that they're in /user maybe /accounts
.route("/leave_admin", post().to(leave_admin)) .route("/logout", post().to(logout))
.route("/totp/generate", post().to(generate_totp_secret)) .route("/delete_account", post().to(delete_account))
.route("/totp/update", post().to(update_totp)) .route("/password_change", post().to(change_password_after_reset))
.route("/list_logins", get().to(list_logins)) // TODO mark_all_as_read feels off being in this section as well
.route("/validate_auth", get().to(validate_auth)), .route("/mark_all_as_read", post().to(mark_all_notifications_read))
) .route("/save_user_settings", put().to(save_user_settings))
// Admin Actions .route("/change_password", put().to(change_password))
.service( .route("/report_count", get().to(report_count))
scope("/admin") .route("/unread_count", get().to(unread_count))
.wrap(rate_limit.message()) .route("/verify_email", post().to(verify_email))
.route("/add", post().to(add_admin)) .route("/leave_admin", post().to(leave_admin))
.route( .route("/totp/generate", post().to(generate_totp_secret))
"/registration_application/count", .route("/totp/update", post().to(update_totp))
get().to(get_unread_registration_application_count), .route("/list_logins", get().to(list_logins))
) .route("/validate_auth", get().to(validate_auth)),
.route( )
"/registration_application/list", // Admin Actions
get().to(list_registration_applications), .service(
) scope("/admin")
.route( .wrap(rate_limit.message())
"/registration_application/approve", .route("/add", post().to(add_admin))
put().to(approve_registration_application), .route(
) "/registration_application/count",
.route( get().to(get_unread_registration_application_count),
"/registration_application", )
get().to(get_registration_application), .route(
) "/registration_application/list",
.route("/list_all_media", get().to(list_all_media)) get().to(list_registration_applications),
.service( )
scope("/purge") .route(
.route("/person", post().to(purge_person)) "/registration_application/approve",
.route("/community", post().to(purge_community)) put().to(approve_registration_application),
.route("/post", post().to(purge_post)) )
.route("/comment", post().to(purge_comment)), .route(
), "/registration_application",
) get().to(get_registration_application),
.service( )
scope("/custom_emoji") .route("/list_all_media", get().to(list_all_media))
.wrap(rate_limit.message()) .service(
.route("", post().to(create_custom_emoji)) scope("/purge")
.route("", put().to(update_custom_emoji)) .route("/person", post().to(purge_person))
.route("/delete", post().to(delete_custom_emoji)), .route("/community", post().to(purge_community))
), .route("/post", post().to(purge_post))
); .route("/comment", post().to(purge_comment)),
cfg.service( ),
scope("/sitemap.xml") )
.wrap(rate_limit.message()) .service(
.route("", get().to(get_sitemap)), scope("/custom_emoji")
); .wrap(rate_limit.message())
.route("", post().to(create_custom_emoji))
.route("", put().to(update_custom_emoji))
.route("/delete", post().to(delete_custom_emoji)),
),
);
} }

52
src/api_routes_v4.rs
View file

@ -88,7 +88,6 @@ use lemmy_api::{
unread_count::get_unread_registration_application_count, unread_count::get_unread_registration_application_count,
}, },
}, },
sitemap::get_sitemap,
}; };
use lemmy_api_crud::{ use lemmy_api_crud::{
comment::{ comment::{
@ -152,20 +151,44 @@ use lemmy_apub::api::{
search::search, search::search,
user_settings_backup::{export_settings, import_settings}, user_settings_backup::{export_settings, import_settings},
}; };
use lemmy_routes::images::image_proxy; use lemmy_routes::images::{
delete::{
delete_community_banner,
delete_community_icon,
delete_image,
delete_site_banner,
delete_site_icon,
delete_user_avatar,
delete_user_banner,
},
download::{get_image, image_proxy},
pictrs_health,
upload::{
upload_community_banner,
upload_community_icon,
upload_image,
upload_site_banner,
upload_site_icon,
upload_user_avatar,
upload_user_banner,
},
};
use lemmy_utils::rate_limit::RateLimitCell; use lemmy_utils::rate_limit::RateLimitCell;
pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) { pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) {
cfg.service( cfg.service(
scope("/api/v4") scope("/api/v4")
.wrap(rate_limit.message()) .wrap(rate_limit.message())
.route("/image_proxy", get().to(image_proxy))
// Site // Site
.service( .service(
scope("/site") scope("/site")
.route("", get().to(get_site_v4)) .route("", get().to(get_site_v4))
.route("", post().to(create_site)) .route("", post().to(create_site))
.route("", put().to(update_site)), .route("", put().to(update_site))
.route("/icon", post().to(upload_site_icon))
.route("/icon", delete().to(delete_site_icon))
.route("/banner", post().to(upload_site_banner))
.route("/banner", delete().to(delete_site_banner)),
) )
.route("/modlog", get().to(get_mod_log)) .route("/modlog", get().to(get_mod_log))
.service( .service(
@ -195,6 +218,10 @@ pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) {
.route("/transfer", post().to(transfer_community)) .route("/transfer", post().to(transfer_community))
.route("/ban_user", post().to(ban_from_community)) .route("/ban_user", post().to(ban_from_community))
.route("/mod", post().to(add_mod_to_community)) .route("/mod", post().to(add_mod_to_community))
.route("/icon", post().to(upload_community_icon))
.route("/icon", delete().to(delete_community_icon))
.route("/banner", post().to(upload_community_banner))
.route("/banner", delete().to(delete_community_banner))
.service( .service(
scope("/pending_follows") scope("/pending_follows")
.route("/count", get().to(get_pending_follows_count)) .route("/count", get().to(get_pending_follows_count))
@ -313,6 +340,10 @@ pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) {
.route("/unread_count", get().to(unread_count)) .route("/unread_count", get().to(unread_count))
.route("/list_logins", get().to(list_logins)) .route("/list_logins", get().to(list_logins))
.route("/validate_auth", get().to(validate_auth)) .route("/validate_auth", get().to(validate_auth))
.route("/avatar", post().to(upload_user_avatar))
.route("/avatar", delete().to(delete_user_avatar))
.route("/banner", post().to(upload_user_banner))
.route("/banner", delete().to(delete_user_banner))
.service( .service(
scope("/block") scope("/block")
.route("/person", post().to(user_block_person)) .route("/person", post().to(user_block_person))
@ -388,6 +419,17 @@ pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) {
.wrap(rate_limit.register()) .wrap(rate_limit.register())
.route("/authenticate", post().to(authenticate_with_oauth)), .route("/authenticate", post().to(authenticate_with_oauth)),
) )
.route("/sitemap.xml", get().to(get_sitemap)), .service(
scope("/image")
.service(
resource("")
.wrap(rate_limit.image())
.route(post().to(upload_image))
.route(delete().to(delete_image)),
)
.route("/proxy", get().to(image_proxy))
.route("/health", get().to(pictrs_health))
.route("/{filename}", get().to(get_image)),
),
); );
} }

26
src/lib.rs
View file

@ -11,13 +11,14 @@ use actix_cors::Cors;
use actix_web::{ use actix_web::{
dev::{ServerHandle, ServiceResponse}, dev::{ServerHandle, ServiceResponse},
middleware::{self, Condition, ErrorHandlerResponse, ErrorHandlers}, middleware::{self, Condition, ErrorHandlerResponse, ErrorHandlers},
web::Data, web::{get, scope, Data},
App, App,
HttpResponse, HttpResponse,
HttpServer, HttpServer,
}; };
use actix_web_prom::PrometheusMetricsBuilder; use actix_web_prom::PrometheusMetricsBuilder;
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
use lemmy_api::sitemap::get_sitemap;
use lemmy_api_common::{ use lemmy_api_common::{
context::LemmyContext, context::LemmyContext,
lemmy_db_views::structs::SiteView, lemmy_db_views::structs::SiteView,
@ -36,7 +37,7 @@ use lemmy_apub::{
}; };
use lemmy_db_schema::{schema_setup, source::secret::Secret, utils::build_db_pool}; use lemmy_db_schema::{schema_setup, source::secret::Secret, utils::build_db_pool};
use lemmy_federate::{Opts, SendManager}; use lemmy_federate::{Opts, SendManager};
use lemmy_routes::{feeds, images, nodeinfo, webfinger}; use lemmy_routes::{feeds, nodeinfo, webfinger};
use lemmy_utils::{ use lemmy_utils::{
error::{LemmyErrorType, LemmyResult}, error::{LemmyErrorType, LemmyResult},
rate_limit::RateLimitCell, rate_limit::RateLimitCell,
@ -194,9 +195,13 @@ pub async fn start_lemmy_server(args: CmdArgs) -> LemmyResult<()> {
let client = ClientBuilder::new(client_builder(&SETTINGS).build()?) let client = ClientBuilder::new(client_builder(&SETTINGS).build()?)
.with(TracingMiddleware::default()) .with(TracingMiddleware::default())
.build(); .build();
let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?)
.with(TracingMiddleware::default())
.build();
let context = LemmyContext::create( let context = LemmyContext::create(
pool.clone(), pool.clone(),
client.clone(), client.clone(),
pictrs_client,
secret.clone(), secret.clone(),
rate_limit_cell.clone(), rate_limit_cell.clone(),
); );
@ -329,17 +334,12 @@ fn create_http_server(
.build() .build()
.map_err(|e| LemmyErrorType::Unknown(format!("Should always be buildable: {e}")))?; .map_err(|e| LemmyErrorType::Unknown(format!("Should always be buildable: {e}")))?;
let context: LemmyContext = federation_config.deref().clone();
let rate_limit_cell = federation_config.rate_limit_cell().clone();
// Pictrs cannot use proxy
let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?)
.with(TracingMiddleware::default())
.build();
// Create Http server // Create Http server
let bind = (settings.bind, settings.port); let bind = (settings.bind, settings.port);
let server = HttpServer::new(move || { let server = HttpServer::new(move || {
let context: LemmyContext = federation_config.deref().clone();
let rate_limit_cell = federation_config.rate_limit_cell().clone();
let cors_config = cors_config(&settings); let cors_config = cors_config(&settings);
let app = App::new() let app = App::new()
.wrap(middleware::Logger::new( .wrap(middleware::Logger::new(
@ -372,8 +372,12 @@ fn create_http_server(
} }
}) })
.configure(feeds::config) .configure(feeds::config)
.configure(|cfg| images::config(cfg, pictrs_client.clone(), &rate_limit_cell))
.configure(nodeinfo::config) .configure(nodeinfo::config)
.service(
scope("/sitemap.xml")
.wrap(rate_limit_cell.message())
.route("", get().to(get_sitemap)),
)
}) })
.disable_signals() .disable_signals()
.bind(bind)? .bind(bind)?