From a91a03a5361d84d07e12e39c04a9138e6a6ad53c Mon Sep 17 00:00:00 2001 From: Nutomic Date: Mon, 13 Jan 2025 21:09:00 +0000 Subject: [PATCH] Image api rework (#5260) * Split image endpoints into API v3 and v4 * Move into subfolders * Upload avatar endpoint and other changes * Various other changes fixes #1772 fixes #4001 * clippy * config options * fix ts bindings * fix api tests * Add option to disable image upload (fixes #1118) * split files into upload, download * move sitemap to top level, not in api * simplify code * add upload user banner * community icon/banner * site icon/banner * update js client * wip * add delete endpoints * change comment * optimization Co-authored-by: dullbananas * move fn * 1024px banner * dont use static client * fix api tests * shear * proxy pictrs in request.rs (fixes #5270) * clippy * try to fix api tests * skip api tests * create user * debug * dbg * test * image * run another * fixed? * clippy * fix * fix health check --------- Co-authored-by: dullbananas --- api_tests/package.json | 2 +- api_tests/pnpm-lock.yaml | 10 +- api_tests/src/image.spec.ts | 88 ++-- api_tests/src/shared.ts | 8 +- api_tests/src/user.spec.ts | 36 +- config/defaults.hjson | 10 +- crates/api/src/local_user/save_settings.rs | 14 +- crates/api/src/site/leave_admin.rs | 1 + crates/api_common/src/claims.rs | 17 +- crates/api_common/src/community.rs | 6 - crates/api_common/src/context.rs | 16 +- crates/api_common/src/image.rs | 54 ++ crates/api_common/src/lib.rs | 1 + crates/api_common/src/person.rs | 6 - crates/api_common/src/request.rs | 87 ++- crates/api_common/src/site.rs | 12 +- crates/api_common/src/utils.rs | 38 +- crates/api_crud/src/community/create.rs | 10 - crates/api_crud/src/community/update.rs | 14 +- crates/api_crud/src/site/create.rs | 11 +- crates/api_crud/src/site/read.rs | 1 + crates/api_crud/src/site/update.rs | 14 +- crates/routes/src/images.rs | 353 ------------- crates/routes/src/images/delete.rs | 147 ++++++ crates/routes/src/images/download.rs | 129 +++++ crates/routes/src/images/mod.rs | 22 + crates/routes/src/images/upload.rs | 239 +++++++++ crates/routes/src/images/utils.rs | 111 ++++ crates/utils/src/error.rs | 3 +- crates/utils/src/settings/mod.rs | 2 +- crates/utils/src/settings/structs.rs | 16 +- .../utils/src/utils/markdown/image_links.rs | 12 +- crates/utils/src/utils/markdown/mod.rs | 10 +- src/api_routes_v3.rs | 498 +++++++++--------- src/api_routes_v4.rs | 52 +- src/lib.rs | 26 +- 36 files changed, 1187 insertions(+), 889 deletions(-) create mode 100644 crates/api_common/src/image.rs delete mode 100644 crates/routes/src/images.rs create mode 100644 crates/routes/src/images/delete.rs create mode 100644 crates/routes/src/images/download.rs create mode 100644 crates/routes/src/images/mod.rs create mode 100644 crates/routes/src/images/upload.rs create mode 100644 crates/routes/src/images/utils.rs diff --git a/api_tests/package.json b/api_tests/package.json index 965388625..39deef22f 100644 --- a/api_tests/package.json +++ b/api_tests/package.json @@ -28,7 +28,7 @@ "eslint": "^9.14.0", "eslint-plugin-prettier": "^5.1.3", "jest": "^29.5.0", - "lemmy-js-client": "0.20.0-reports-combined.3", + "lemmy-js-client": "0.20.0-image-api-rework.8", "prettier": "^3.2.5", "ts-jest": "^29.1.0", "typescript": "^5.5.4", diff --git a/api_tests/pnpm-lock.yaml b/api_tests/pnpm-lock.yaml index 198062652..5aaa59d2c 100644 --- a/api_tests/pnpm-lock.yaml +++ b/api_tests/pnpm-lock.yaml @@ -30,8 +30,8 @@ importers: specifier: ^29.5.0 version: 29.7.0(@types/node@22.9.0) lemmy-js-client: - specifier: 0.20.0-reports-combined.3 - version: 0.20.0-reports-combined.3 + specifier: 0.20.0-image-api-rework.8 + version: 0.20.0-image-api-rework.8 prettier: specifier: ^3.2.5 version: 3.3.3 @@ -1167,8 +1167,8 @@ packages: resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} engines: {node: '>=6'} - lemmy-js-client@0.20.0-reports-combined.3: - resolution: {integrity: sha512-0Z/9S41r6NM8f09Gkxerq9zYBE6UcywXfeWNxsYknkyh0ZnKbtNxjTkSxE6JpRbz7wokKFRSH9NpwgNloQY5uw==} + lemmy-js-client@0.20.0-image-api-rework.8: + resolution: {integrity: sha512-Ns/ayfCSm2lHbdAU1tGIZSx6kJ2ZeS7UiXlPuH0IzHQSi8Yuyzj3srDCyHpE6Td3pmXbQlt9N1ziPE4KeRJ3CA==} leven@3.1.0: resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} @@ -3077,7 +3077,7 @@ snapshots: kleur@3.0.3: {} - lemmy-js-client@0.20.0-reports-combined.3: {} + lemmy-js-client@0.20.0-image-api-rework.8: {} leven@3.1.0: {} diff --git a/api_tests/src/image.spec.ts b/api_tests/src/image.spec.ts index 4d1abbdfd..1a686879b 100644 --- a/api_tests/src/image.spec.ts +++ b/api_tests/src/image.spec.ts @@ -2,9 +2,9 @@ jest.setTimeout(120000); import { UploadImage, - DeleteImage, PurgePerson, PurgePost, + DeleteImageParams, } from "lemmy-js-client"; import { alpha, @@ -41,8 +41,8 @@ afterAll(async () => { }); test("Upload image and delete it", async () => { - const healthz = await fetch(alphaUrl + "/pictrs/healthz"); - expect(healthz.status).toBe(200); + const health = await alpha.imageHealth(); + expect(health.success).toBeTruthy(); // Before running this test, you need to delete all previous images in the DB await deleteAllImages(alpha); @@ -53,13 +53,12 @@ test("Upload image and delete it", async () => { image: Buffer.from("test"), }; const upload = await alphaImage.uploadImage(upload_form); - expect(upload.files![0].file).toBeDefined(); - expect(upload.files![0].delete_token).toBeDefined(); - expect(upload.url).toBeDefined(); - expect(upload.delete_url).toBeDefined(); + expect(upload.image_url).toBeDefined(); + expect(upload.filename).toBeDefined(); + expect(upload.delete_token).toBeDefined(); // ensure that image download is working. theres probably a better way to do this - const response = await fetch(upload.url ?? ""); + const response = await fetch(upload.image_url ?? ""); const content = await response.text(); expect(content.length).toBeGreaterThan(0); @@ -76,26 +75,21 @@ test("Upload image and delete it", async () => { const previousThumbnails = 1; expect(listAllMediaRes.images.length).toBe(previousThumbnails); - // The deleteUrl is a combination of the endpoint, delete token, and alias - let firstImage = listMediaRes.images[0]; - let deleteUrl = `${alphaUrl}/pictrs/image/delete/${firstImage.local_image.pictrs_delete_token}/${firstImage.local_image.pictrs_alias}`; - expect(deleteUrl).toBe(upload.delete_url); - // Make sure the uploader is correct - expect(firstImage.person.actor_id).toBe( + expect(listMediaRes.images[0].person.actor_id).toBe( `http://lemmy-alpha:8541/u/lemmy_alpha`, ); // delete image - const delete_form: DeleteImage = { - token: upload.files![0].delete_token, - filename: upload.files![0].file, + const delete_form: DeleteImageParams = { + token: upload.delete_token, + filename: upload.filename, }; const delete_ = await alphaImage.deleteImage(delete_form); - expect(delete_).toBe(true); + expect(delete_.success).toBe(true); // ensure that image is deleted - const response2 = await fetch(upload.url ?? ""); + const response2 = await fetch(upload.image_url ?? ""); const content2 = await response2.text(); expect(content2).toBe(""); @@ -118,13 +112,12 @@ test("Purge user, uploaded image removed", async () => { image: Buffer.from("test"), }; const upload = await user.uploadImage(upload_form); - expect(upload.files![0].file).toBeDefined(); - expect(upload.files![0].delete_token).toBeDefined(); - expect(upload.url).toBeDefined(); - expect(upload.delete_url).toBeDefined(); + expect(upload.filename).toBeDefined(); + expect(upload.delete_token).toBeDefined(); + expect(upload.image_url).toBeDefined(); // ensure that image download is working. theres probably a better way to do this - const response = await fetch(upload.url ?? ""); + const response = await fetch(upload.image_url ?? ""); const content = await response.text(); expect(content.length).toBeGreaterThan(0); @@ -137,7 +130,7 @@ test("Purge user, uploaded image removed", async () => { expect(delete_.success).toBe(true); // ensure that image is deleted - const response2 = await fetch(upload.url ?? ""); + const response2 = await fetch(upload.image_url ?? ""); const content2 = await response2.text(); expect(content2).toBe(""); }); @@ -150,13 +143,12 @@ test("Purge post, linked image removed", async () => { image: Buffer.from("test"), }; const upload = await user.uploadImage(upload_form); - expect(upload.files![0].file).toBeDefined(); - expect(upload.files![0].delete_token).toBeDefined(); - expect(upload.url).toBeDefined(); - expect(upload.delete_url).toBeDefined(); + expect(upload.filename).toBeDefined(); + expect(upload.delete_token).toBeDefined(); + expect(upload.image_url).toBeDefined(); // ensure that image download is working. theres probably a better way to do this - const response = await fetch(upload.url ?? ""); + const response = await fetch(upload.image_url ?? ""); const content = await response.text(); expect(content.length).toBeGreaterThan(0); @@ -164,9 +156,9 @@ test("Purge post, linked image removed", async () => { let post = await createPost( user, community.community!.community.id, - upload.url, + upload.image_url, ); - expect(post.post_view.post.url).toBe(upload.url); + expect(post.post_view.post.url).toBe(upload.image_url); expect(post.post_view.image_details).toBeDefined(); // purge post @@ -177,7 +169,7 @@ test("Purge post, linked image removed", async () => { expect(delete_.success).toBe(true); // ensure that image is deleted - const response2 = await fetch(upload.url ?? ""); + const response2 = await fetch(upload.image_url ?? ""); const content2 = await response2.text(); expect(content2).toBe(""); }); @@ -199,11 +191,11 @@ test("Images in remote image post are proxied if setting enabled", async () => { // remote image gets proxied after upload expect( post.thumbnail_url?.startsWith( - "http://lemmy-gamma:8561/api/v4/image_proxy?url", + "http://lemmy-gamma:8561/api/v4/image/proxy?url", ), ).toBeTruthy(); expect( - post.body?.startsWith("![](http://lemmy-gamma:8561/api/v4/image_proxy?url"), + post.body?.startsWith("![](http://lemmy-gamma:8561/api/v4/image/proxy?url"), ).toBeTruthy(); // Make sure that it ends with jpg, to be sure its an image @@ -222,12 +214,12 @@ test("Images in remote image post are proxied if setting enabled", async () => { expect( epsilonPost.thumbnail_url?.startsWith( - "http://lemmy-epsilon:8581/api/v4/image_proxy?url", + "http://lemmy-epsilon:8581/api/v4/image/proxy?url", ), ).toBeTruthy(); expect( epsilonPost.body?.startsWith( - "![](http://lemmy-epsilon:8581/api/v4/image_proxy?url", + "![](http://lemmy-epsilon:8581/api/v4/image/proxy?url", ), ).toBeTruthy(); @@ -249,7 +241,7 @@ test("Thumbnail of remote image link is proxied if setting enabled", async () => // remote image gets proxied after upload expect( post.thumbnail_url?.startsWith( - "http://lemmy-gamma:8561/api/v4/image_proxy?url", + "http://lemmy-gamma:8561/api/v4/image/proxy?url", ), ).toBeTruthy(); @@ -267,7 +259,7 @@ test("Thumbnail of remote image link is proxied if setting enabled", async () => expect( epsilonPost.thumbnail_url?.startsWith( - "http://lemmy-epsilon:8581/api/v4/image_proxy?url", + "http://lemmy-epsilon:8581/api/v4/image/proxy?url", ), ).toBeTruthy(); @@ -291,14 +283,14 @@ test("No image proxying if setting is disabled", async () => { let post = await createPost( alpha, community.community_view.community.id, - upload.url, + upload.image_url, `![](${sampleImage})`, ); expect(post.post_view.post).toBeDefined(); // remote image doesn't get proxied after upload expect( - post.post_view.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"), + post.post_view.post.url?.startsWith("http://lemmy-beta:8551/api/v4/image/"), ).toBeTruthy(); expect(post.post_view.post.body).toBe(`![](${sampleImage})`); @@ -311,7 +303,7 @@ test("No image proxying if setting is disabled", async () => { // remote image doesn't get proxied after federation expect( - betaPost.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"), + betaPost.post.url?.startsWith("http://lemmy-beta:8551/api/v4/image/"), ).toBeTruthy(); expect(betaPost.post.body).toBe(`![](${sampleImage})`); // Make sure the alt text got federated @@ -333,7 +325,7 @@ test("Make regular post, and give it a custom thumbnail", async () => { alphaImage, community.community_view.community.id, wikipediaUrl, - upload1.url!, + upload1.image_url!, ); // Wait for the metadata to get fetched, since this is backgrounded now @@ -343,7 +335,7 @@ test("Make regular post, and give it a custom thumbnail", async () => { ); expect(post.post_view.post.url).toBe(wikipediaUrl); // Make sure it uses custom thumbnail - expect(post.post_view.post.thumbnail_url).toBe(upload1.url); + expect(post.post_view.post.thumbnail_url).toBe(upload1.image_url); }); test("Create an image post, and make sure a custom thumbnail doesn't overwrite it", async () => { @@ -362,14 +354,14 @@ test("Create an image post, and make sure a custom thumbnail doesn't overwrite i let post = await createPostWithThumbnail( alphaImage, community.community_view.community.id, - upload1.url!, - upload2.url!, + upload1.image_url!, + upload2.image_url!, ); post = await waitUntil( () => getPost(alphaImage, post.post_view.post.id), p => p.post_view.post.thumbnail_url != undefined, ); - expect(post.post_view.post.url).toBe(upload1.url); + expect(post.post_view.post.url).toBe(upload1.image_url); // Make sure the custom thumbnail is ignored - expect(post.post_view.post.thumbnail_url == upload2.url).toBe(false); + expect(post.post_view.post.thumbnail_url == upload2.image_url).toBe(false); }); diff --git a/api_tests/src/shared.ts b/api_tests/src/shared.ts index 4cad739f4..daa2203bf 100644 --- a/api_tests/src/shared.ts +++ b/api_tests/src/shared.ts @@ -5,7 +5,6 @@ import { CommunityId, CommunityVisibility, CreatePrivateMessageReport, - DeleteImage, EditCommunity, GetCommunityPendingFollowsCountResponse, GetReplies, @@ -18,6 +17,7 @@ import { ListReports, ListReportsResponse, MyUserInfo, + DeleteImageParams, PersonId, PostView, PrivateMessageReportResponse, @@ -714,8 +714,6 @@ export async function saveUserSettingsBio( export async function saveUserSettingsFederated( api: LemmyHttp, ): Promise { - let avatar = sampleImage; - let banner = sampleImage; let bio = "a changed bio"; let form: SaveUserSettings = { show_nsfw: false, @@ -723,8 +721,6 @@ export async function saveUserSettingsFederated( default_post_sort_type: "Hot", default_listing_type: "All", interface_language: "", - avatar, - banner, display_name: "user321", show_avatars: false, send_notifications_to_email: false, @@ -939,7 +935,7 @@ export async function deleteAllImages(api: LemmyHttp) { Promise.all( imagesRes.images .map(image => { - const form: DeleteImage = { + const form: DeleteImageParams = { token: image.local_image.pictrs_delete_token, filename: image.local_image.pictrs_alias, }; diff --git a/api_tests/src/user.spec.ts b/api_tests/src/user.spec.ts index d1d6144f5..551677262 100644 --- a/api_tests/src/user.spec.ts +++ b/api_tests/src/user.spec.ts @@ -21,7 +21,6 @@ import { fetchFunction, alphaImage, unfollows, - saveUserSettingsBio, getMyUser, getPersonDetails, } from "./shared"; @@ -192,43 +191,36 @@ test("Set a new avatar, old avatar is deleted", async () => { const upload_form1: UploadImage = { image: Buffer.from("test1"), }; - const upload1 = await alphaImage.uploadImage(upload_form1); - expect(upload1.url).toBeDefined(); - - let form1 = { - avatar: upload1.url, - }; - await saveUserSettings(alpha, form1); + await alpha.uploadUserAvatar(upload_form1); const listMediaRes1 = await alphaImage.listMedia(); expect(listMediaRes1.images.length).toBe(1); + let my_user1 = await alpha.getMyUser(); + expect(my_user1.local_user_view.person.avatar).toBeDefined(); + const upload_form2: UploadImage = { image: Buffer.from("test2"), }; - const upload2 = await alphaImage.uploadImage(upload_form2); - expect(upload2.url).toBeDefined(); - - let form2 = { - avatar: upload2.url, - }; - await saveUserSettings(alpha, form2); + await alpha.uploadUserAvatar(upload_form2); // make sure only the new avatar is kept const listMediaRes2 = await alphaImage.listMedia(); expect(listMediaRes2.images.length).toBe(1); // Upload that same form2 avatar, make sure it isn't replaced / deleted - await saveUserSettings(alpha, form2); + await alpha.uploadUserAvatar(upload_form2); // make sure only the new avatar is kept const listMediaRes3 = await alphaImage.listMedia(); expect(listMediaRes3.images.length).toBe(1); - // Now try to save a user settings, with the icon missing, - // and make sure it doesn't clear the data, or delete the image - await saveUserSettingsBio(alpha); - let my_user = await getMyUser(alpha); - expect(my_user.local_user_view.person.avatar).toBe(upload2.url); - // make sure only the new avatar is kept const listMediaRes4 = await alphaImage.listMedia(); expect(listMediaRes4.images.length).toBe(1); + + // delete the avatar + await alpha.deleteUserAvatar(); + // make sure only the new avatar is kept + const listMediaRes5 = await alphaImage.listMedia(); + expect(listMediaRes5.images.length).toBe(0); + let my_user2 = await alpha.getMyUser(); + expect(my_user2.local_user_view.person.avatar).toBeUndefined(); }); diff --git a/config/defaults.hjson b/config/defaults.hjson index ad9d6c15f..b5d3b1004 100644 --- a/config/defaults.hjson +++ b/config/defaults.hjson @@ -39,7 +39,7 @@ # or # If enabled, all images from remote domains are rewritten to pass through - # `/api/v4/image_proxy`, including embedded images in markdown. Images are stored temporarily + # `/api/v4/image/proxy`, including embedded images in markdown. Images are stored temporarily # in pict-rs for caching. This improves privacy as users don't expose their IP to untrusted # servers, and decreases load on other servers. However it increases bandwidth use for the # local server. @@ -59,6 +59,14 @@ upload_timeout: 30 # Resize post thumbnails to this maximum width/height. max_thumbnail_size: 512 + # Maximum size for user avatar, community icon and site icon. + max_avatar_size: 512 + # Maximum size for user, community and site banner. Larger images are downscaled to fit + # into a square of this size. + max_banner_size: 1024 + # Prevent users from uploading images for posts or embedding in markdown. Avatars, icons and + # banners can still be uploaded. + image_upload_disabled: false } # Email sending configuration. All options except login/password are mandatory email: { diff --git a/crates/api/src/local_user/save_settings.rs b/crates/api/src/local_user/save_settings.rs index 992fea163..930f6bf48 100644 --- a/crates/api/src/local_user/save_settings.rs +++ b/crates/api/src/local_user/save_settings.rs @@ -3,12 +3,10 @@ use actix_web::web::Json; use lemmy_api_common::{ context::LemmyContext, person::SaveUserSettings, - request::replace_image, utils::{ get_url_blocklist, local_site_to_slur_regex, process_markdown_opt, - proxy_image_link_opt_api, send_verification_email, }, SuccessResponse, @@ -21,7 +19,7 @@ use lemmy_db_schema::{ person::{Person, PersonUpdateForm}, }, traits::Crud, - utils::{diesel_string_update, diesel_url_update}, + utils::diesel_string_update, }; use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_utils::{ @@ -46,14 +44,6 @@ pub async fn save_user_settings( .as_deref(), ); - let avatar = diesel_url_update(data.avatar.as_deref())?; - replace_image(&avatar, &local_user_view.person.avatar, &context).await?; - let avatar = proxy_image_link_opt_api(avatar, &context).await?; - - let banner = diesel_url_update(data.banner.as_deref())?; - replace_image(&banner, &local_user_view.person.banner, &context).await?; - let banner = proxy_image_link_opt_api(banner, &context).await?; - let display_name = diesel_string_update(data.display_name.as_deref()); let matrix_user_id = diesel_string_update(data.matrix_user_id.as_deref()); let email_deref = data.email.as_deref().map(str::to_lowercase); @@ -108,8 +98,6 @@ pub async fn save_user_settings( bio, matrix_user_id, bot_account: data.bot_account, - avatar, - banner, ..Default::default() }; diff --git a/crates/api/src/site/leave_admin.rs b/crates/api/src/site/leave_admin.rs index fde258dd2..042009d24 100644 --- a/crates/api/src/site/leave_admin.rs +++ b/crates/api/src/site/leave_admin.rs @@ -76,5 +76,6 @@ pub async fn leave_admin( blocked_urls, tagline, my_user: None, + image_upload_disabled: context.settings().pictrs()?.image_upload_disabled, })) } diff --git a/crates/api_common/src/claims.rs b/crates/api_common/src/claims.rs index 759673f4b..5063ea8fe 100644 --- a/crates/api_common/src/claims.rs +++ b/crates/api_common/src/claims.rs @@ -78,29 +78,18 @@ mod tests { instance::Instance, local_user::{LocalUser, LocalUserInsertForm}, person::{Person, PersonInsertForm}, - secret::Secret, }, traits::Crud, - utils::build_db_pool_for_tests, }; - use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell}; + use lemmy_utils::error::LemmyResult; use pretty_assertions::assert_eq; - use reqwest::Client; - use reqwest_middleware::ClientBuilder; use serial_test::serial; #[tokio::test] #[serial] async fn test_should_not_validate_user_token_after_password_change() -> LemmyResult<()> { - let pool_ = build_db_pool_for_tests(); - let pool = &mut (&pool_).into(); - let secret = Secret::init(pool).await?; - let context = LemmyContext::create( - pool_.clone(), - ClientBuilder::new(Client::default()).build(), - secret, - RateLimitCell::with_test_config(), - ); + let context = LemmyContext::init_test_context().await; + let pool = &mut context.pool(); let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; diff --git a/crates/api_common/src/community.rs b/crates/api_common/src/community.rs index 898767b34..9105c6f84 100644 --- a/crates/api_common/src/community.rs +++ b/crates/api_common/src/community.rs @@ -177,12 +177,6 @@ pub struct EditCommunity { /// A shorter, one line description of your community. #[cfg_attr(feature = "full", ts(optional))] pub description: Option, - /// An icon URL. - #[cfg_attr(feature = "full", ts(optional))] - pub icon: Option, - /// A banner URL. - #[cfg_attr(feature = "full", ts(optional))] - pub banner: Option, /// Whether its an NSFW community. #[cfg_attr(feature = "full", ts(optional))] pub nsfw: Option, diff --git a/crates/api_common/src/context.rs b/crates/api_common/src/context.rs index b578914d1..1c423d156 100644 --- a/crates/api_common/src/context.rs +++ b/crates/api_common/src/context.rs @@ -15,6 +15,9 @@ use std::sync::Arc; pub struct LemmyContext { pool: ActualDbPool, client: Arc, + /// Pictrs requests must bypass proxy. Unfortunately no_proxy can only be set on ClientBuilder + /// and not on RequestBuilder, so we need a separate client here. + pictrs_client: Arc, secret: Arc, rate_limit_cell: RateLimitCell, } @@ -23,12 +26,14 @@ impl LemmyContext { pub fn create( pool: ActualDbPool, client: ClientWithMiddleware, + pictrs_client: ClientWithMiddleware, secret: Secret, rate_limit_cell: RateLimitCell, ) -> LemmyContext { LemmyContext { pool, client: Arc::new(client), + pictrs_client: Arc::new(pictrs_client), secret: Arc::new(secret), rate_limit_cell, } @@ -42,6 +47,9 @@ impl LemmyContext { pub fn client(&self) -> &ClientWithMiddleware { &self.client } + pub fn pictrs_client(&self) -> &ClientWithMiddleware { + &self.pictrs_client + } pub fn settings(&self) -> &'static Settings { &SETTINGS } @@ -70,7 +78,13 @@ impl LemmyContext { let rate_limit_cell = RateLimitCell::with_test_config(); - let context = LemmyContext::create(pool, client, secret, rate_limit_cell.clone()); + let context = LemmyContext::create( + pool, + client.clone(), + client, + secret, + rate_limit_cell.clone(), + ); FederationConfig::builder() .domain(context.settings().hostname.clone()) diff --git a/crates/api_common/src/image.rs b/crates/api_common/src/image.rs new file mode 100644 index 000000000..e93b367f6 --- /dev/null +++ b/crates/api_common/src/image.rs @@ -0,0 +1,54 @@ +use lemmy_db_schema::newtypes::CommunityId; +use serde::{Deserialize, Serialize}; +use serde_with::skip_serializing_none; +#[cfg(feature = "full")] +use ts_rs::TS; +use url::Url; + +#[skip_serializing_none] +#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "full", derive(TS))] +#[cfg_attr(feature = "full", ts(export))] +pub struct ImageGetParams { + #[cfg_attr(feature = "full", ts(optional))] + pub file_type: Option, + #[cfg_attr(feature = "full", ts(optional))] + pub max_size: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "full", derive(TS))] +#[cfg_attr(feature = "full", ts(export))] +pub struct DeleteImageParams { + pub filename: String, + pub token: String, +} + +#[skip_serializing_none] +#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "full", derive(TS))] +#[cfg_attr(feature = "full", ts(export))] +pub struct ImageProxyParams { + pub url: String, + #[cfg_attr(feature = "full", ts(optional))] + pub file_type: Option, + #[cfg_attr(feature = "full", ts(optional))] + pub max_size: Option, +} + +#[skip_serializing_none] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "full", derive(TS))] +#[cfg_attr(feature = "full", ts(export))] +pub struct UploadImageResponse { + pub image_url: Url, + pub filename: String, + pub delete_token: String, +} + +/// Parameter for setting community icon or banner. Can't use POST data here as it already contains +/// the image data. +#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq, Hash)] +pub struct CommunityIdQuery { + pub id: CommunityId, +} diff --git a/crates/api_common/src/lib.rs b/crates/api_common/src/lib.rs index 8af1dec25..dd1c0a68a 100644 --- a/crates/api_common/src/lib.rs +++ b/crates/api_common/src/lib.rs @@ -7,6 +7,7 @@ pub mod community; #[cfg(feature = "full")] pub mod context; pub mod custom_emoji; +pub mod image; pub mod oauth_provider; pub mod person; pub mod post; diff --git a/crates/api_common/src/person.rs b/crates/api_common/src/person.rs index 4781683d4..2df3a7162 100644 --- a/crates/api_common/src/person.rs +++ b/crates/api_common/src/person.rs @@ -120,12 +120,6 @@ pub struct SaveUserSettings { /// The language of the lemmy interface #[cfg_attr(feature = "full", ts(optional))] pub interface_language: Option, - /// A URL for your avatar. - #[cfg_attr(feature = "full", ts(optional))] - pub avatar: Option, - /// A URL for your banner. - #[cfg_attr(feature = "full", ts(optional))] - pub banner: Option, /// Your display name, which can contain strange characters, and does not need to be unique. #[cfg_attr(feature = "full", ts(optional))] pub display_name: Option, diff --git a/crates/api_common/src/request.rs b/crates/api_common/src/request.rs index a0cc38b82..b0e37915b 100644 --- a/crates/api_common/src/request.rs +++ b/crates/api_common/src/request.rs @@ -9,13 +9,10 @@ use activitypub_federation::config::Data; use chrono::{DateTime, Utc}; use encoding_rs::{Encoding, UTF_8}; use futures::StreamExt; -use lemmy_db_schema::{ - newtypes::DbUrl, - source::{ - images::{ImageDetailsForm, LocalImage, LocalImageForm}, - post::{Post, PostUpdateForm}, - site::Site, - }, +use lemmy_db_schema::source::{ + images::{ImageDetailsForm, LocalImage, LocalImageForm}, + post::{Post, PostUpdateForm}, + site::Site, }; use lemmy_utils::{ error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult}, @@ -260,7 +257,8 @@ fn extract_opengraph_data(html_bytes: &[u8], url: &Url) -> LemmyResult>, + #[serde(default)] + pub files: Vec, pub msg: String, } @@ -272,9 +270,15 @@ pub struct PictrsFile { } impl PictrsFile { - pub fn thumbnail_url(&self, protocol_and_hostname: &str) -> Result { + pub fn image_url(&self, protocol_and_hostname: &str) -> Result { Url::parse(&format!( - "{protocol_and_hostname}/pictrs/image/{}", + "{protocol_and_hostname}/api/v4/image/{}", + self.file + )) + } + pub fn delete_url(&self, protocol_and_hostname: &str) -> Result { + Url::parse(&format!( + "{protocol_and_hostname}/api/v4/image/{}", self.file )) } @@ -315,7 +319,7 @@ struct PictrsPurgeResponse { /// - It might not be an image /// - Pictrs might not be set up pub async fn purge_image_from_pictrs(image_url: &Url, context: &LemmyContext) -> LemmyResult<()> { - is_image_content_type(context.client(), image_url).await?; + is_image_content_type(context.pictrs_client(), image_url).await?; let alias = image_url .path_segments() @@ -323,14 +327,19 @@ pub async fn purge_image_from_pictrs(image_url: &Url, context: &LemmyContext) -> .next_back() .ok_or(LemmyErrorType::ImageUrlMissingLastPathSegment)?; - let pictrs_config = context.settings().pictrs_config()?; + // Delete db row if any (old Lemmy versions didnt generate this). + LocalImage::delete_by_alias(&mut context.pool(), alias) + .await + .ok(); + + let pictrs_config = context.settings().pictrs()?; let purge_url = format!("{}internal/purge?alias={}", pictrs_config.url, alias); let pictrs_api_key = pictrs_config .api_key .ok_or(LemmyErrorType::PictrsApiKeyNotProvided)?; let response = context - .client() + .pictrs_client() .post(&purge_url) .timeout(REQWEST_TIMEOUT) .header("x-api-token", pictrs_api_key) @@ -351,13 +360,18 @@ pub async fn delete_image_from_pictrs( delete_token: &str, context: &LemmyContext, ) -> LemmyResult<()> { - let pictrs_config = context.settings().pictrs_config()?; + // Delete db row if any (old Lemmy versions didnt generate this). + LocalImage::delete_by_alias(&mut context.pool(), alias) + .await + .ok(); + + let pictrs_config = context.settings().pictrs()?; let url = format!( "{}image/delete/{}/{}", pictrs_config.url, &delete_token, &alias ); context - .client() + .pictrs_client() .delete(&url) .timeout(REQWEST_TIMEOUT) .send() @@ -369,7 +383,7 @@ pub async fn delete_image_from_pictrs( /// Retrieves the image with local pict-rs and generates a thumbnail. Returns the thumbnail url. #[tracing::instrument(skip_all)] async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> LemmyResult { - let pictrs_config = context.settings().pictrs_config()?; + let pictrs_config = context.settings().pictrs()?; match pictrs_config.image_mode { PictrsImageMode::None => return Ok(image_url.clone()), @@ -380,16 +394,15 @@ async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> L }; // fetch remote non-pictrs images for persistent thumbnail link - // TODO: should limit size once supported by pictrs let fetch_url = format!( "{}image/download?url={}&resize={}", pictrs_config.url, encode(image_url.as_str()), - context.settings().pictrs_config()?.max_thumbnail_size + context.settings().pictrs()?.max_thumbnail_size ); let res = context - .client() + .pictrs_client() .get(&fetch_url) .timeout(REQWEST_TIMEOUT) .send() @@ -398,9 +411,8 @@ async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> L .json::() .await?; - let files = res.files.unwrap_or_default(); - - let image = files + let image = res + .files .first() .ok_or(LemmyErrorType::PictrsResponseError(res.msg))?; @@ -412,7 +424,7 @@ async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> L pictrs_delete_token: image.delete_token.clone(), }; let protocol_and_hostname = context.settings().get_protocol_and_hostname(); - let thumbnail_url = image.thumbnail_url(&protocol_and_hostname)?; + let thumbnail_url = image.image_url(&protocol_and_hostname)?; // Also store the details for the image let details_form = image.details.build_image_details_form(&thumbnail_url); @@ -429,14 +441,14 @@ pub async fn fetch_pictrs_proxied_image_details( image_url: &Url, context: &LemmyContext, ) -> LemmyResult { - let pictrs_url = context.settings().pictrs_config()?.url; + let pictrs_url = context.settings().pictrs()?.url; let encoded_image_url = encode(image_url.as_str()); // Pictrs needs you to fetch the proxied image before you can fetch the details let proxy_url = format!("{pictrs_url}image/original?proxy={encoded_image_url}"); context - .client() + .pictrs_client() .get(&proxy_url) .timeout(REQWEST_TIMEOUT) .send() @@ -447,7 +459,7 @@ pub async fn fetch_pictrs_proxied_image_details( let details_url = format!("{pictrs_url}image/details/original?proxy={encoded_image_url}"); let res = context - .client() + .pictrs_client() .get(&details_url) .timeout(REQWEST_TIMEOUT) .send() @@ -476,29 +488,6 @@ async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> Lemm } } -/// When adding a new avatar, banner or similar image, delete the old one. -pub async fn replace_image( - new_image: &Option>, - old_image: &Option, - context: &Data, -) -> LemmyResult<()> { - if let (Some(Some(new_image)), Some(old_image)) = (new_image, old_image) { - // Note: Oftentimes front ends will include the current image in the form. - // In this case, deleting `old_image` would also be deletion of `new_image`, - // so the deletion must be skipped for the image to be kept. - if new_image != old_image { - // Ignore errors because image may be stored externally. - let image = LocalImage::delete_by_url(&mut context.pool(), old_image) - .await - .ok(); - if let Some(image) = image { - delete_image_from_pictrs(&image.pictrs_alias, &image.pictrs_delete_token, context).await?; - } - } - } - Ok(()) -} - #[cfg(test)] mod tests { diff --git a/crates/api_common/src/site.rs b/crates/api_common/src/site.rs index eeddb34cc..c0ef01257 100644 --- a/crates/api_common/src/site.rs +++ b/crates/api_common/src/site.rs @@ -199,9 +199,6 @@ pub struct CreateSite { #[cfg_attr(feature = "full", ts(optional))] pub description: Option, #[cfg_attr(feature = "full", ts(optional))] - pub icon: Option, - #[cfg_attr(feature = "full", ts(optional))] - pub banner: Option, #[cfg_attr(feature = "full", ts(optional))] pub community_creation_admin_only: Option, #[cfg_attr(feature = "full", ts(optional))] @@ -292,12 +289,6 @@ pub struct EditSite { /// A shorter, one line description of your site. #[cfg_attr(feature = "full", ts(optional))] pub description: Option, - /// A url for your site's icon. - #[cfg_attr(feature = "full", ts(optional))] - pub icon: Option, - /// A url for your site's banner. - #[cfg_attr(feature = "full", ts(optional))] - pub banner: Option, /// Limits community creation to admins only. #[cfg_attr(feature = "full", ts(optional))] pub community_creation_admin_only: Option, @@ -443,6 +434,9 @@ pub struct GetSiteResponse { #[cfg_attr(feature = "full", ts(optional))] pub admin_oauth_providers: Option>, pub blocked_urls: Vec, + // If true then uploads for post images or markdown images are disabled. Only avatars, icons and + // banners can be set. + pub image_upload_disabled: bool, } #[skip_serializing_none] diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 116bd5e0e..91acde0a9 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -1060,7 +1060,7 @@ pub async fn process_markdown( markdown_check_for_blocked_urls(&text, url_blocklist)?; - if context.settings().pictrs_config()?.image_mode == PictrsImageMode::ProxyAllImages { + if context.settings().pictrs()?.image_mode == PictrsImageMode::ProxyAllImages { let (text, links) = markdown_rewrite_image_links(text); RemoteImage::create(&mut context.pool(), links.clone()).await?; @@ -1128,37 +1128,7 @@ async fn proxy_image_link_internal( /// Rewrite a link to go through `/api/v4/image_proxy` endpoint. This is only for remote urls and /// if image_proxy setting is enabled. pub async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult { - proxy_image_link_internal( - link, - context.settings().pictrs_config()?.image_mode, - context, - ) - .await -} - -pub async fn proxy_image_link_opt_api( - link: Option>, - context: &LemmyContext, -) -> LemmyResult>> { - if let Some(Some(link)) = link { - proxy_image_link(link.into(), context) - .await - .map(Some) - .map(Some) - } else { - Ok(link) - } -} - -pub async fn proxy_image_link_api( - link: Option, - context: &LemmyContext, -) -> LemmyResult> { - if let Some(link) = link { - proxy_image_link(link.into(), context).await.map(Some) - } else { - Ok(link) - } + proxy_image_link_internal(link, context.settings().pictrs()?.image_mode, context).await } pub async fn proxy_image_link_opt_apub( @@ -1177,7 +1147,7 @@ fn build_proxied_image_url( protocol_and_hostname: &str, ) -> Result { Url::parse(&format!( - "{}/api/v4/image_proxy?url={}", + "{}/api/v4/image/proxy?url={}", protocol_and_hostname, encode(link.as_str()) )) @@ -1256,7 +1226,7 @@ mod tests { ) .await?; assert_eq!( - "https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Flemmy-beta%2Fimage.png", + "https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Flemmy-beta%2Fimage.png", proxied.as_str() ); diff --git a/crates/api_crud/src/community/create.rs b/crates/api_crud/src/community/create.rs index c81157950..0437396cc 100644 --- a/crates/api_crud/src/community/create.rs +++ b/crates/api_crud/src/community/create.rs @@ -13,7 +13,6 @@ use lemmy_api_common::{ is_admin, local_site_to_slur_regex, process_markdown_opt, - proxy_image_link_api, EndpointType, }, }; @@ -31,7 +30,6 @@ use lemmy_db_schema::{ }, }, traits::{ApubActor, Crud, Followable, Joinable}, - utils::diesel_url_create, }; use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_utils::{ @@ -76,12 +74,6 @@ pub async fn create_community( check_slurs(desc, &slur_regex)?; } - let icon = diesel_url_create(data.icon.as_deref())?; - let icon = proxy_image_link_api(icon, &context).await?; - - let banner = diesel_url_create(data.banner.as_deref())?; - let banner = proxy_image_link_api(banner, &context).await?; - is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?; if let Some(desc) = &data.description { @@ -108,8 +100,6 @@ pub async fn create_community( let community_form = CommunityInsertForm { sidebar, description, - icon, - banner, nsfw: data.nsfw, actor_id: Some(community_actor_id.clone()), private_key: Some(keypair.private_key), diff --git a/crates/api_crud/src/community/update.rs b/crates/api_crud/src/community/update.rs index d9c062c53..944f5bade 100644 --- a/crates/api_crud/src/community/update.rs +++ b/crates/api_crud/src/community/update.rs @@ -6,14 +6,12 @@ use lemmy_api_common::{ build_response::build_community_response, community::{CommunityResponse, EditCommunity}, context::LemmyContext, - request::replace_image, send_activity::{ActivityChannel, SendActivityData}, utils::{ check_community_mod_action, get_url_blocklist, local_site_to_slur_regex, process_markdown_opt, - proxy_image_link_opt_api, }, }; use lemmy_db_schema::{ @@ -23,7 +21,7 @@ use lemmy_db_schema::{ local_site::LocalSite, }, traits::Crud, - utils::{diesel_string_update, diesel_url_update}, + utils::diesel_string_update, }; use lemmy_db_views::structs::LocalUserView; use lemmy_utils::{ @@ -58,14 +56,6 @@ pub async fn update_community( let old_community = Community::read(&mut context.pool(), data.community_id).await?; - let icon = diesel_url_update(data.icon.as_deref())?; - replace_image(&icon, &old_community.icon, &context).await?; - let icon = proxy_image_link_opt_api(icon, &context).await?; - - let banner = diesel_url_update(data.banner.as_deref())?; - replace_image(&banner, &old_community.banner, &context).await?; - let banner = proxy_image_link_opt_api(banner, &context).await?; - // Verify its a mod (only mods can edit it) check_community_mod_action( &local_user_view.person, @@ -91,8 +81,6 @@ pub async fn update_community( title: data.title.clone(), sidebar, description, - icon, - banner, nsfw: data.nsfw, posting_restricted_to_mods: data.posting_restricted_to_mods, visibility: data.visibility, diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index c8140cc28..34965742d 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -13,7 +13,6 @@ use lemmy_api_common::{ local_site_rate_limit_to_rate_limit_config, local_site_to_slur_regex, process_markdown_opt, - proxy_image_link_api, }, }; use lemmy_db_schema::{ @@ -24,7 +23,7 @@ use lemmy_db_schema::{ site::{Site, SiteUpdateForm}, }, traits::Crud, - utils::{diesel_string_update, diesel_url_create}, + utils::diesel_string_update, }; use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_utils::{ @@ -63,18 +62,10 @@ pub async fn create_site( let url_blocklist = get_url_blocklist(&context).await?; let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &url_blocklist, &context).await?; - let icon = diesel_url_create(data.icon.as_deref())?; - let icon = proxy_image_link_api(icon, &context).await?; - - let banner = diesel_url_create(data.banner.as_deref())?; - let banner = proxy_image_link_api(banner, &context).await?; - let site_form = SiteUpdateForm { name: Some(data.name.clone()), sidebar: diesel_string_update(sidebar.as_deref()), description: diesel_string_update(data.description.as_deref()), - icon: Some(icon), - banner: Some(banner), actor_id: Some(actor_id), last_refreshed_at: Some(Utc::now()), inbox_url, diff --git a/crates/api_crud/src/site/read.rs b/crates/api_crud/src/site/read.rs index 220fe1bd5..64d2237a0 100644 --- a/crates/api_crud/src/site/read.rs +++ b/crates/api_crud/src/site/read.rs @@ -69,5 +69,6 @@ async fn read_site(context: &LemmyContext) -> LemmyResult { tagline, oauth_providers: Some(oauth_providers), admin_oauth_providers: Some(admin_oauth_providers), + image_upload_disabled: context.settings().pictrs()?.image_upload_disabled, }) } diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index 8b0dfe0c5..959c07ff0 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -5,7 +5,6 @@ use actix_web::web::Json; use chrono::Utc; use lemmy_api_common::{ context::LemmyContext, - request::replace_image, site::{EditSite, SiteResponse}, utils::{ get_url_blocklist, @@ -13,7 +12,6 @@ use lemmy_api_common::{ local_site_rate_limit_to_rate_limit_config, local_site_to_slur_regex, process_markdown_opt, - proxy_image_link_opt_api, }, }; use lemmy_db_schema::{ @@ -26,7 +24,7 @@ use lemmy_db_schema::{ site::{Site, SiteUpdateForm}, }, traits::Crud, - utils::{diesel_string_update, diesel_url_update}, + utils::diesel_string_update, RegistrationMode, }; use lemmy_db_views::structs::{LocalUserView, SiteView}; @@ -72,20 +70,10 @@ pub async fn update_site( .as_deref(), ); - let icon = diesel_url_update(data.icon.as_deref())?; - replace_image(&icon, &site.icon, &context).await?; - let icon = proxy_image_link_opt_api(icon, &context).await?; - - let banner = diesel_url_update(data.banner.as_deref())?; - replace_image(&banner, &site.banner, &context).await?; - let banner = proxy_image_link_opt_api(banner, &context).await?; - let site_form = SiteUpdateForm { name: data.name.clone(), sidebar, description: diesel_string_update(data.description.as_deref()), - icon, - banner, content_warning: diesel_string_update(data.content_warning.as_deref()), updated: Some(Some(Utc::now())), ..Default::default() diff --git a/crates/routes/src/images.rs b/crates/routes/src/images.rs deleted file mode 100644 index 50897b95d..000000000 --- a/crates/routes/src/images.rs +++ /dev/null @@ -1,353 +0,0 @@ -use actix_web::{ - body::{BodyStream, BoxBody}, - http::{ - header::{HeaderName, ACCEPT_ENCODING, HOST}, - Method, - StatusCode, - }, - web::*, - HttpRequest, - HttpResponse, - Responder, -}; -use futures::stream::{Stream, StreamExt}; -use http::HeaderValue; -use lemmy_api_common::{context::LemmyContext, request::PictrsResponse}; -use lemmy_db_schema::source::{ - images::{LocalImage, LocalImageForm, RemoteImage}, - local_site::LocalSite, -}; -use lemmy_db_views::structs::LocalUserView; -use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell, REQWEST_TIMEOUT}; -use reqwest::Body; -use reqwest_middleware::{ClientWithMiddleware, RequestBuilder}; -use serde::Deserialize; -use std::time::Duration; -use url::Url; - -pub fn config(cfg: &mut ServiceConfig, client: ClientWithMiddleware, rate_limit: &RateLimitCell) { - cfg - .app_data(Data::new(client)) - .service( - resource("/pictrs/image") - .wrap(rate_limit.image()) - .route(post().to(upload)), - ) - // This has optional query params: /image/{filename}?format=jpg&thumbnail=256 - .service(resource("/pictrs/image/{filename}").route(get().to(full_res))) - .service(resource("/pictrs/image/delete/{token}/{filename}").route(get().to(delete))) - .service(resource("/pictrs/healthz").route(get().to(healthz))); -} - -trait ProcessUrl { - /// If thumbnail or format is given, this uses the pictrs process endpoint. - /// Otherwise, it uses the normal pictrs url (IE image/original). - fn process_url(&self, image_url: &str, pictrs_url: &Url) -> String; -} - -#[derive(Deserialize, Clone)] -struct PictrsGetParams { - format: Option, - thumbnail: Option, -} - -impl ProcessUrl for PictrsGetParams { - fn process_url(&self, src: &str, pictrs_url: &Url) -> String { - if self.format.is_none() && self.thumbnail.is_none() { - format!("{}image/original/{}", pictrs_url, src) - } else { - // Take file type from name, or jpg if nothing is given - let format = self - .clone() - .format - .unwrap_or_else(|| src.split('.').last().unwrap_or("jpg").to_string()); - - let mut url = format!("{}image/process.{}?src={}", pictrs_url, format, src); - - if let Some(size) = self.thumbnail { - url = format!("{url}&thumbnail={size}",); - } - url - } - } -} - -#[derive(Deserialize, Clone)] -pub struct ImageProxyParams { - url: String, - format: Option, - thumbnail: Option, -} - -impl ProcessUrl for ImageProxyParams { - fn process_url(&self, proxy_url: &str, pictrs_url: &Url) -> String { - if self.format.is_none() && self.thumbnail.is_none() { - format!("{}image/original?proxy={}", pictrs_url, proxy_url) - } else { - // Take file type from name, or jpg if nothing is given - let format = self - .clone() - .format - .unwrap_or_else(|| proxy_url.split('.').last().unwrap_or("jpg").to_string()); - - let mut url = format!("{}image/process.{}?proxy={}", pictrs_url, format, proxy_url); - - if let Some(size) = self.thumbnail { - url = format!("{url}&thumbnail={size}",); - } - url - } - } -} -fn adapt_request( - request: &HttpRequest, - client: &ClientWithMiddleware, - url: String, -) -> RequestBuilder { - // remove accept-encoding header so that pictrs doesn't compress the response - const INVALID_HEADERS: &[HeaderName] = &[ACCEPT_ENCODING, HOST]; - - let client_request = client - .request(convert_method(request.method()), url) - .timeout(REQWEST_TIMEOUT); - - request - .headers() - .iter() - .fold(client_request, |client_req, (key, value)| { - if INVALID_HEADERS.contains(key) { - client_req - } else { - // TODO: remove as_str and as_bytes conversions after actix-web upgrades to http 1.0 - client_req.header(key.as_str(), value.as_bytes()) - } - }) -} - -async fn upload( - req: HttpRequest, - body: Payload, - // require login - local_user_view: LocalUserView, - client: Data, - context: Data, -) -> LemmyResult { - // TODO: check rate limit here - let pictrs_config = context.settings().pictrs_config()?; - let image_url = format!("{}image", pictrs_config.url); - - let mut client_req = adapt_request(&req, &client, image_url); - - if let Some(addr) = req.head().peer_addr { - client_req = client_req.header("X-Forwarded-For", addr.to_string()) - }; - let res = client_req - .timeout(Duration::from_secs(pictrs_config.upload_timeout)) - .body(Body::wrap_stream(make_send(body))) - .send() - .await?; - - let status = res.status(); - let images = res.json::().await?; - if let Some(images) = &images.files { - for image in images { - let form = LocalImageForm { - local_user_id: Some(local_user_view.local_user.id), - pictrs_alias: image.file.to_string(), - pictrs_delete_token: image.delete_token.to_string(), - }; - - let protocol_and_hostname = context.settings().get_protocol_and_hostname(); - let thumbnail_url = image.thumbnail_url(&protocol_and_hostname)?; - - // Also store the details for the image - let details_form = image.details.build_image_details_form(&thumbnail_url); - LocalImage::create(&mut context.pool(), &form, &details_form).await?; - } - } - - Ok(HttpResponse::build(convert_status(status)).json(images)) -} - -async fn full_res( - filename: Path, - Query(params): Query, - req: HttpRequest, - client: Data, - context: Data, - local_user_view: Option, -) -> LemmyResult { - // block access to images if instance is private and unauthorized, public - let local_site = LocalSite::read(&mut context.pool()).await?; - if local_site.private_instance && local_user_view.is_none() { - return Ok(HttpResponse::Unauthorized().finish()); - } - let name = &filename.into_inner(); - - // If there are no query params, the URL is original - let pictrs_config = context.settings().pictrs_config()?; - - let processed_url = params.process_url(name, &pictrs_config.url); - - image(processed_url, req, &client).await -} - -async fn image( - url: String, - req: HttpRequest, - client: &ClientWithMiddleware, -) -> LemmyResult { - let mut client_req = adapt_request(&req, client, url); - - if let Some(addr) = req.head().peer_addr { - client_req = client_req.header("X-Forwarded-For", addr.to_string()); - } - - if let Some(addr) = req.head().peer_addr { - client_req = client_req.header("X-Forwarded-For", addr.to_string()); - } - - let res = client_req.send().await?; - - if res.status() == http::StatusCode::NOT_FOUND { - return Ok(HttpResponse::NotFound().finish()); - } - - let mut client_res = HttpResponse::build(StatusCode::from_u16(res.status().as_u16())?); - - for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") { - client_res.insert_header(convert_header(name, value)); - } - - Ok(client_res.body(BodyStream::new(res.bytes_stream()))) -} - -async fn delete( - components: Path<(String, String)>, - req: HttpRequest, - client: Data, - context: Data, - // require login - _local_user_view: LocalUserView, -) -> LemmyResult { - let (token, file) = components.into_inner(); - - let pictrs_config = context.settings().pictrs_config()?; - let url = format!("{}image/delete/{}/{}", pictrs_config.url, &token, &file); - - let mut client_req = adapt_request(&req, &client, url); - - if let Some(addr) = req.head().peer_addr { - client_req = client_req.header("X-Forwarded-For", addr.to_string()); - } - - let res = client_req.send().await?; - - LocalImage::delete_by_alias(&mut context.pool(), &file).await?; - - Ok(HttpResponse::build(convert_status(res.status())).body(BodyStream::new(res.bytes_stream()))) -} - -async fn healthz( - req: HttpRequest, - client: Data, - context: Data, -) -> LemmyResult { - let pictrs_config = context.settings().pictrs_config()?; - let url = format!("{}healthz", pictrs_config.url); - - let mut client_req = adapt_request(&req, &client, url); - - if let Some(addr) = req.head().peer_addr { - client_req = client_req.header("X-Forwarded-For", addr.to_string()); - } - - let res = client_req.send().await?; - - Ok(HttpResponse::build(convert_status(res.status())).body(BodyStream::new(res.bytes_stream()))) -} - -pub async fn image_proxy( - Query(params): Query, - req: HttpRequest, - client: Data, - context: Data, -) -> LemmyResult, HttpResponse>> { - let url = Url::parse(¶ms.url)?; - - // Check that url corresponds to a federated image so that this can't be abused as a proxy - // for arbitrary purposes. - RemoteImage::validate(&mut context.pool(), url.clone().into()).await?; - - let pictrs_config = context.settings().pictrs_config()?; - let processed_url = params.process_url(¶ms.url, &pictrs_config.url); - - let bypass_proxy = pictrs_config - .proxy_bypass_domains - .iter() - .any(|s| url.domain().is_some_and(|d| d == s)); - if bypass_proxy { - // Bypass proxy and redirect user to original image - Ok(Either::Left(Redirect::to(url.to_string()).respond_to(&req))) - } else { - // Proxy the image data through Lemmy - Ok(Either::Right(image(processed_url, req, &client).await?)) - } -} - -fn make_send(mut stream: S) -> impl Stream + Send + Unpin + 'static -where - S: Stream + Unpin + 'static, - S::Item: Send, -{ - // NOTE: the 8 here is arbitrary - let (tx, rx) = tokio::sync::mpsc::channel(8); - - // NOTE: spawning stream into a new task can potentially hit this bug: - // - https://github.com/actix/actix-web/issues/1679 - // - // Since 4.0.0-beta.2 this issue is incredibly less frequent. I have not personally reproduced it. - // That said, it is still technically possible to encounter. - actix_web::rt::spawn(async move { - while let Some(res) = stream.next().await { - if tx.send(res).await.is_err() { - break; - } - } - }); - - SendStream { rx } -} - -struct SendStream { - rx: tokio::sync::mpsc::Receiver, -} - -impl Stream for SendStream -where - T: Send, -{ - type Item = T; - - fn poll_next( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::pin::Pin::new(&mut self.rx).poll_recv(cx) - } -} - -// TODO: remove these conversions after actix-web upgrades to http 1.0 -#[allow(clippy::expect_used)] -fn convert_status(status: http::StatusCode) -> StatusCode { - StatusCode::from_u16(status.as_u16()).expect("status can be converted") -} - -#[allow(clippy::expect_used)] -fn convert_method(method: &Method) -> http::Method { - http::Method::from_bytes(method.as_str().as_bytes()).expect("method can be converted") -} - -fn convert_header<'a>(name: &'a http::HeaderName, value: &'a HeaderValue) -> (&'a str, &'a [u8]) { - (name.as_str(), value.as_bytes()) -} diff --git a/crates/routes/src/images/delete.rs b/crates/routes/src/images/delete.rs new file mode 100644 index 000000000..b28c87c6c --- /dev/null +++ b/crates/routes/src/images/delete.rs @@ -0,0 +1,147 @@ +use super::utils::delete_old_image; +use actix_web::web::*; +use lemmy_api_common::{ + context::LemmyContext, + image::{CommunityIdQuery, DeleteImageParams}, + utils::{is_admin, is_mod_or_admin}, + SuccessResponse, +}; +use lemmy_db_schema::{ + source::{ + community::{Community, CommunityUpdateForm}, + images::LocalImage, + person::{Person, PersonUpdateForm}, + site::{Site, SiteUpdateForm}, + }, + traits::Crud, +}; +use lemmy_db_views::structs::LocalUserView; +use lemmy_utils::error::LemmyResult; + +pub async fn delete_site_icon( + context: Data, + local_user_view: LocalUserView, +) -> LemmyResult> { + let site = Site::read_local(&mut context.pool()).await?; + is_admin(&local_user_view)?; + + delete_old_image(&site.icon, &context).await?; + + let form = SiteUpdateForm { + icon: Some(None), + ..Default::default() + }; + Site::update(&mut context.pool(), site.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} +pub async fn delete_site_banner( + context: Data, + local_user_view: LocalUserView, +) -> LemmyResult> { + let site = Site::read_local(&mut context.pool()).await?; + is_admin(&local_user_view)?; + + delete_old_image(&site.banner, &context).await?; + + let form = SiteUpdateForm { + banner: Some(None), + ..Default::default() + }; + Site::update(&mut context.pool(), site.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn delete_community_icon( + data: Json, + context: Data, + local_user_view: LocalUserView, +) -> LemmyResult> { + let community = Community::read(&mut context.pool(), data.id).await?; + is_mod_or_admin(&mut context.pool(), &local_user_view.person, community.id).await?; + + delete_old_image(&community.icon, &context).await?; + + let form = CommunityUpdateForm { + icon: Some(None), + ..Default::default() + }; + Community::update(&mut context.pool(), community.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn delete_community_banner( + data: Json, + context: Data, + local_user_view: LocalUserView, +) -> LemmyResult> { + let community = Community::read(&mut context.pool(), data.id).await?; + is_mod_or_admin(&mut context.pool(), &local_user_view.person, community.id).await?; + + delete_old_image(&community.icon, &context).await?; + + let form = CommunityUpdateForm { + icon: Some(None), + ..Default::default() + }; + Community::update(&mut context.pool(), community.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn delete_user_avatar( + context: Data, + local_user_view: LocalUserView, +) -> LemmyResult> { + delete_old_image(&local_user_view.person.avatar, &context).await?; + + let form = PersonUpdateForm { + avatar: Some(None), + ..Default::default() + }; + Person::update(&mut context.pool(), local_user_view.person.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn delete_user_banner( + context: Data, + local_user_view: LocalUserView, +) -> LemmyResult> { + delete_old_image(&local_user_view.person.banner, &context).await?; + + let form = PersonUpdateForm { + banner: Some(None), + ..Default::default() + }; + Person::update(&mut context.pool(), local_user_view.person.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +// TODO: get rid of delete tokens and allow deletion by admin or uploader +pub async fn delete_image( + data: Json, + context: Data, + // require login + _local_user_view: LocalUserView, +) -> LemmyResult> { + let pictrs_config = context.settings().pictrs()?; + let url = format!( + "{}image/delete/{}/{}", + pictrs_config.url, &data.token, &data.filename + ); + + context + .pictrs_client() + .delete(url) + .send() + .await? + .error_for_status()?; + + LocalImage::delete_by_alias(&mut context.pool(), &data.filename).await?; + + Ok(Json(SuccessResponse::default())) +} diff --git a/crates/routes/src/images/download.rs b/crates/routes/src/images/download.rs new file mode 100644 index 000000000..76f09a8d1 --- /dev/null +++ b/crates/routes/src/images/download.rs @@ -0,0 +1,129 @@ +use super::utils::{adapt_request, convert_header}; +use actix_web::{ + body::{BodyStream, BoxBody}, + http::StatusCode, + web::{Data, *}, + HttpRequest, + HttpResponse, + Responder, +}; +use lemmy_api_common::{ + context::LemmyContext, + image::{ImageGetParams, ImageProxyParams}, +}; +use lemmy_db_schema::source::{images::RemoteImage, local_site::LocalSite}; +use lemmy_db_views::structs::LocalUserView; +use lemmy_utils::error::LemmyResult; +use url::Url; + +pub async fn get_image( + filename: Path, + Query(params): Query, + req: HttpRequest, + local_user_view: Option, + context: Data, +) -> LemmyResult { + // block access to images if instance is private + if local_user_view.is_none() { + let local_site = LocalSite::read(&mut context.pool()).await?; + if local_site.private_instance { + return Ok(HttpResponse::Unauthorized().finish()); + } + } + let name = &filename.into_inner(); + + // If there are no query params, the URL is original + let pictrs_url = context.settings().pictrs()?.url; + let processed_url = if params.file_type.is_none() && params.max_size.is_none() { + format!("{}image/original/{}", pictrs_url, name) + } else { + let file_type = file_type(params.file_type, name); + let mut url = format!("{}image/process.{}?src={}", pictrs_url, file_type, name); + + if let Some(size) = params.max_size { + url = format!("{url}&thumbnail={size}",); + } + url + }; + + do_get_image(processed_url, req, &context).await +} + +pub async fn image_proxy( + Query(params): Query, + req: HttpRequest, + context: Data, +) -> LemmyResult, HttpResponse>> { + let url = Url::parse(¶ms.url)?; + + // Check that url corresponds to a federated image so that this can't be abused as a proxy + // for arbitrary purposes. + RemoteImage::validate(&mut context.pool(), url.clone().into()).await?; + + let pictrs_config = context.settings().pictrs()?; + let processed_url = if params.file_type.is_none() && params.max_size.is_none() { + format!("{}image/original?proxy={}", pictrs_config.url, params.url) + } else { + let file_type = file_type(params.file_type, url.as_str()); + let mut url = format!( + "{}image/process.{}?proxy={}", + pictrs_config.url, file_type, url + ); + + if let Some(size) = params.max_size { + url = format!("{url}&thumbnail={size}",); + } + url + }; + + let bypass_proxy = pictrs_config + .proxy_bypass_domains + .iter() + .any(|s| url.domain().is_some_and(|d| d == s)); + if bypass_proxy { + // Bypass proxy and redirect user to original image + Ok(Either::Left(Redirect::to(url.to_string()).respond_to(&req))) + } else { + // Proxy the image data through Lemmy + Ok(Either::Right( + do_get_image(processed_url, req, &context).await?, + )) + } +} + +pub(super) async fn do_get_image( + url: String, + req: HttpRequest, + context: &LemmyContext, +) -> LemmyResult { + let mut client_req = adapt_request(&req, url, context); + + if let Some(addr) = req.head().peer_addr { + client_req = client_req.header("X-Forwarded-For", addr.to_string()); + } + + if let Some(addr) = req.head().peer_addr { + client_req = client_req.header("X-Forwarded-For", addr.to_string()); + } + + let res = client_req.send().await?; + + if res.status() == http::StatusCode::NOT_FOUND { + return Ok(HttpResponse::NotFound().finish()); + } + + let mut client_res = HttpResponse::build(StatusCode::from_u16(res.status().as_u16())?); + + for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") { + client_res.insert_header(convert_header(name, value)); + } + + Ok(client_res.body(BodyStream::new(res.bytes_stream()))) +} + +/// Take file type from param, name, or use jpg if nothing is given +pub(super) fn file_type(file_type: Option, name: &str) -> String { + file_type + .clone() + .unwrap_or_else(|| name.split('.').last().unwrap_or("jpg").to_string()) +} diff --git a/crates/routes/src/images/mod.rs b/crates/routes/src/images/mod.rs new file mode 100644 index 000000000..aefe42831 --- /dev/null +++ b/crates/routes/src/images/mod.rs @@ -0,0 +1,22 @@ +use actix_web::web::*; +use lemmy_api_common::{context::LemmyContext, SuccessResponse}; +use lemmy_utils::error::LemmyResult; + +pub mod delete; +pub mod download; +pub mod upload; +mod utils; + +pub async fn pictrs_health(context: Data) -> LemmyResult> { + let pictrs_config = context.settings().pictrs()?; + let url = format!("{}healthz", pictrs_config.url); + + context + .pictrs_client() + .get(url) + .send() + .await? + .error_for_status()?; + + Ok(Json(SuccessResponse::default())) +} diff --git a/crates/routes/src/images/upload.rs b/crates/routes/src/images/upload.rs new file mode 100644 index 000000000..50660419b --- /dev/null +++ b/crates/routes/src/images/upload.rs @@ -0,0 +1,239 @@ +use super::utils::{adapt_request, delete_old_image, make_send}; +use actix_web::{self, web::*, HttpRequest}; +use lemmy_api_common::{ + context::LemmyContext, + image::{CommunityIdQuery, UploadImageResponse}, + request::PictrsResponse, + utils::{is_admin, is_mod_or_admin}, + LemmyErrorType, + SuccessResponse, +}; +use lemmy_db_schema::{ + source::{ + community::{Community, CommunityUpdateForm}, + images::{LocalImage, LocalImageForm}, + person::{Person, PersonUpdateForm}, + site::{Site, SiteUpdateForm}, + }, + traits::Crud, +}; +use lemmy_db_views::structs::LocalUserView; +use lemmy_utils::error::LemmyResult; +use reqwest::Body; +use std::time::Duration; +use UploadType::*; + +pub enum UploadType { + Avatar, + Banner, + Other, +} + +pub async fn upload_image( + req: HttpRequest, + body: Payload, + local_user_view: LocalUserView, + context: Data, +) -> LemmyResult> { + if context.settings().pictrs()?.image_upload_disabled { + return Err(LemmyErrorType::ImageUploadDisabled.into()); + } + + Ok(Json( + do_upload_image(req, body, Other, &local_user_view, &context).await?, + )) +} + +pub async fn upload_user_avatar( + req: HttpRequest, + body: Payload, + local_user_view: LocalUserView, + context: Data, +) -> LemmyResult> { + let image = do_upload_image(req, body, Avatar, &local_user_view, &context).await?; + delete_old_image(&local_user_view.person.avatar, &context).await?; + + let form = PersonUpdateForm { + avatar: Some(Some(image.image_url.into())), + ..Default::default() + }; + Person::update(&mut context.pool(), local_user_view.person.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn upload_user_banner( + req: HttpRequest, + body: Payload, + local_user_view: LocalUserView, + context: Data, +) -> LemmyResult> { + let image = do_upload_image(req, body, Banner, &local_user_view, &context).await?; + delete_old_image(&local_user_view.person.banner, &context).await?; + + let form = PersonUpdateForm { + banner: Some(Some(image.image_url.into())), + ..Default::default() + }; + Person::update(&mut context.pool(), local_user_view.person.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn upload_community_icon( + req: HttpRequest, + query: Query, + body: Payload, + local_user_view: LocalUserView, + context: Data, +) -> LemmyResult> { + let community: Community = Community::read(&mut context.pool(), query.id).await?; + is_mod_or_admin(&mut context.pool(), &local_user_view.person, community.id).await?; + + let image = do_upload_image(req, body, Avatar, &local_user_view, &context).await?; + delete_old_image(&community.icon, &context).await?; + + let form = CommunityUpdateForm { + icon: Some(Some(image.image_url.into())), + ..Default::default() + }; + Community::update(&mut context.pool(), community.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn upload_community_banner( + req: HttpRequest, + query: Query, + body: Payload, + local_user_view: LocalUserView, + context: Data, +) -> LemmyResult> { + let community: Community = Community::read(&mut context.pool(), query.id).await?; + is_mod_or_admin(&mut context.pool(), &local_user_view.person, community.id).await?; + + let image = do_upload_image(req, body, Banner, &local_user_view, &context).await?; + delete_old_image(&community.banner, &context).await?; + + let form = CommunityUpdateForm { + banner: Some(Some(image.image_url.into())), + ..Default::default() + }; + Community::update(&mut context.pool(), community.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn upload_site_icon( + req: HttpRequest, + body: Payload, + local_user_view: LocalUserView, + context: Data, +) -> LemmyResult> { + is_admin(&local_user_view)?; + let site = Site::read_local(&mut context.pool()).await?; + + let image = do_upload_image(req, body, Avatar, &local_user_view, &context).await?; + delete_old_image(&site.icon, &context).await?; + + let form = SiteUpdateForm { + icon: Some(Some(image.image_url.into())), + ..Default::default() + }; + Site::update(&mut context.pool(), site.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn upload_site_banner( + req: HttpRequest, + body: Payload, + local_user_view: LocalUserView, + context: Data, +) -> LemmyResult> { + is_admin(&local_user_view)?; + let site = Site::read_local(&mut context.pool()).await?; + + let image = do_upload_image(req, body, Banner, &local_user_view, &context).await?; + delete_old_image(&site.banner, &context).await?; + + let form = SiteUpdateForm { + banner: Some(Some(image.image_url.into())), + ..Default::default() + }; + Site::update(&mut context.pool(), site.id, &form).await?; + + Ok(Json(SuccessResponse::default())) +} + +pub async fn do_upload_image( + req: HttpRequest, + body: Payload, + upload_type: UploadType, + local_user_view: &LocalUserView, + context: &Data, +) -> LemmyResult { + let pictrs = context.settings().pictrs()?; + let image_url = format!("{}image", pictrs.url); + + let mut client_req = adapt_request(&req, image_url, context); + + client_req = match upload_type { + Avatar => { + let max_size = pictrs.max_avatar_size.to_string(); + client_req.query(&[ + ("resize", max_size.as_ref()), + ("allow_animation", "false"), + ("allow_video", "false"), + ]) + } + Banner => { + let max_size = pictrs.max_banner_size.to_string(); + client_req.query(&[ + ("resize", max_size.as_ref()), + ("allow_animation", "false"), + ("allow_video", "false"), + ]) + } + _ => client_req, + }; + if let Some(addr) = req.head().peer_addr { + client_req = client_req.header("X-Forwarded-For", addr.to_string()) + }; + let res = client_req + .timeout(Duration::from_secs(pictrs.upload_timeout)) + .body(Body::wrap_stream(make_send(body))) + .send() + .await? + .error_for_status()?; + + let mut images = res.json::().await?; + for image in &images.files { + // Pictrs allows uploading multiple images in a single request. Lemmy doesnt need this, + // but still a user may upload multiple and so we need to store all links in db for + // to allow deletion via web ui. + let form = LocalImageForm { + local_user_id: Some(local_user_view.local_user.id), + pictrs_alias: image.file.to_string(), + pictrs_delete_token: image.delete_token.to_string(), + }; + + let protocol_and_hostname = context.settings().get_protocol_and_hostname(); + let thumbnail_url = image.image_url(&protocol_and_hostname)?; + + // Also store the details for the image + let details_form = image.details.build_image_details_form(&thumbnail_url); + LocalImage::create(&mut context.pool(), &form, &details_form).await?; + } + let image = images + .files + .pop() + .ok_or(LemmyErrorType::InvalidImageUpload)?; + + let url = image.image_url(&context.settings().get_protocol_and_hostname())?; + Ok(UploadImageResponse { + image_url: url, + filename: image.file, + delete_token: image.delete_token, + }) +} diff --git a/crates/routes/src/images/utils.rs b/crates/routes/src/images/utils.rs new file mode 100644 index 000000000..80108360c --- /dev/null +++ b/crates/routes/src/images/utils.rs @@ -0,0 +1,111 @@ +use actix_web::{ + http::{ + header::{HeaderName, ACCEPT_ENCODING, HOST}, + Method, + }, + web::Data, + HttpRequest, +}; +use futures::stream::{Stream, StreamExt}; +use http::HeaderValue; +use lemmy_api_common::{context::LemmyContext, request::delete_image_from_pictrs}; +use lemmy_db_schema::{newtypes::DbUrl, source::images::LocalImage}; +use lemmy_utils::{error::LemmyResult, REQWEST_TIMEOUT}; +use reqwest_middleware::RequestBuilder; + +pub(super) fn adapt_request( + request: &HttpRequest, + url: String, + context: &LemmyContext, +) -> RequestBuilder { + // remove accept-encoding header so that pictrs doesn't compress the response + const INVALID_HEADERS: &[HeaderName] = &[ACCEPT_ENCODING, HOST]; + + let client_request = context + .pictrs_client() + .request(convert_method(request.method()), url) + .timeout(REQWEST_TIMEOUT); + + request + .headers() + .iter() + .fold(client_request, |client_req, (key, value)| { + if INVALID_HEADERS.contains(key) { + client_req + } else { + // TODO: remove as_str and as_bytes conversions after actix-web upgrades to http 1.0 + client_req.header(key.as_str(), value.as_bytes()) + } + }) +} + +pub(super) fn make_send(mut stream: S) -> impl Stream + Send + Unpin + 'static +where + S: Stream + Unpin + 'static, + S::Item: Send, +{ + // NOTE: the 8 here is arbitrary + let (tx, rx) = tokio::sync::mpsc::channel(8); + + // NOTE: spawning stream into a new task can potentially hit this bug: + // - https://github.com/actix/actix-web/issues/1679 + // + // Since 4.0.0-beta.2 this issue is incredibly less frequent. I have not personally reproduced it. + // That said, it is still technically possible to encounter. + actix_web::rt::spawn(async move { + while let Some(res) = stream.next().await { + if tx.send(res).await.is_err() { + break; + } + } + }); + + SendStream { rx } +} + +struct SendStream { + rx: tokio::sync::mpsc::Receiver, +} + +impl Stream for SendStream +where + T: Send, +{ + type Item = T; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.rx).poll_recv(cx) + } +} + +// TODO: remove these conversions after actix-web upgrades to http 1.0 +#[allow(clippy::expect_used)] +pub(super) fn convert_method(method: &Method) -> http::Method { + http::Method::from_bytes(method.as_str().as_bytes()).expect("method can be converted") +} + +pub(super) fn convert_header<'a>( + name: &'a http::HeaderName, + value: &'a HeaderValue, +) -> (&'a str, &'a [u8]) { + (name.as_str(), value.as_bytes()) +} + +/// When adding a new avatar, banner or similar image, delete the old one. +pub(super) async fn delete_old_image( + old_image: &Option, + context: &Data, +) -> LemmyResult<()> { + if let Some(old_image) = old_image { + let image = LocalImage::delete_by_url(&mut context.pool(), old_image) + .await + .ok(); + if let Some(image) = image { + delete_image_from_pictrs(&image.pictrs_alias, &image.pictrs_delete_token, context).await?; + } + } + Ok(()) +} diff --git a/crates/utils/src/error.rs b/crates/utils/src/error.rs index f1ce64d49..1b3b36c27 100644 --- a/crates/utils/src/error.rs +++ b/crates/utils/src/error.rs @@ -23,7 +23,6 @@ pub enum LemmyErrorType { CouldntUpdateComment, CouldntUpdatePrivateMessage, CannotLeaveAdmin, - // TODO: also remove the translations of unused errors PictrsResponseError(String), PictrsPurgeResponseError(String), ImageUrlMissingPathSegments, @@ -31,6 +30,8 @@ pub enum LemmyErrorType { PictrsApiKeyNotProvided, NoContentTypeHeader, NotAnImageType, + InvalidImageUpload, + ImageUploadDisabled, NotAModOrAdmin, NotTopMod, NotLoggedIn, diff --git a/crates/utils/src/settings/mod.rs b/crates/utils/src/settings/mod.rs index 923e36ce6..ecd0a9b55 100644 --- a/crates/utils/src/settings/mod.rs +++ b/crates/utils/src/settings/mod.rs @@ -97,7 +97,7 @@ impl Settings { WEBFINGER_REGEX.clone() } - pub fn pictrs_config(&self) -> LemmyResult { + pub fn pictrs(&self) -> LemmyResult { self .pictrs .clone() diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index 15cc2f008..4e5152acc 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -98,6 +98,20 @@ pub struct PictrsConfig { /// Resize post thumbnails to this maximum width/height. #[default(512)] pub max_thumbnail_size: u32, + + /// Maximum size for user avatar, community icon and site icon. + #[default(512)] + pub max_avatar_size: u32, + + /// Maximum size for user, community and site banner. Larger images are downscaled to fit + /// into a square of this size. + #[default(1024)] + pub max_banner_size: u32, + + /// Prevent users from uploading images for posts or embedding in markdown. Avatars, icons and + /// banners can still be uploaded. + #[default(false)] + pub image_upload_disabled: bool, } #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document, PartialEq)] @@ -113,7 +127,7 @@ pub enum PictrsImageMode { /// This behaviour matches Lemmy 0.18. StoreLinkPreviews, /// If enabled, all images from remote domains are rewritten to pass through - /// `/api/v4/image_proxy`, including embedded images in markdown. Images are stored temporarily + /// `/api/v4/image/proxy`, including embedded images in markdown. Images are stored temporarily /// in pict-rs for caching. This improves privacy as users don't expose their IP to untrusted /// servers, and decreases load on other servers. However it increases bandwidth use for the /// local server. diff --git a/crates/utils/src/utils/markdown/image_links.rs b/crates/utils/src/utils/markdown/image_links.rs index 7914452ff..d22860f67 100644 --- a/crates/utils/src/utils/markdown/image_links.rs +++ b/crates/utils/src/utils/markdown/image_links.rs @@ -18,7 +18,7 @@ pub fn markdown_rewrite_image_links(mut src: String) -> (String, Vec) { // If link points to remote domain, replace with proxied link if parsed.domain() != Some(&SETTINGS.hostname) { let mut proxied = format!( - "{}/api/v4/image_proxy?url={}", + "{}/api/v4/image/proxy?url={}", SETTINGS.get_protocol_and_hostname(), encode(url), ); @@ -116,7 +116,7 @@ mod tests { ( "remote image proxied", "![link](http://example.com/image.jpg)", - "![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)", + "![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)", ), ( "local image unproxied", @@ -126,7 +126,7 @@ mod tests { ( "multiple image links", "![link](http://example.com/image1.jpg) ![link](http://example.com/image2.jpg)", - "![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)", + "![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)", ), ( "empty link handled", @@ -136,7 +136,7 @@ mod tests { ( "empty label handled", "![](http://example.com/image.jpg)", - "![](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" + "![](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" ), ( "invalid image link removed", @@ -146,12 +146,12 @@ mod tests { ( "label with nested markdown handled", "![a *b* c](http://example.com/image.jpg)", - "![a *b* c](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" + "![a *b* c](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" ), ( "custom emoji support", r#"![party-blob](https://www.hexbear.net/pictrs/image/83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#, - r#"![party-blob](https://lemmy-alpha/api/v4/image_proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"# + r#"![party-blob](https://lemmy-alpha/api/v4/image/proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"# ) ]; diff --git a/crates/utils/src/utils/markdown/mod.rs b/crates/utils/src/utils/markdown/mod.rs index 58f03d910..3c761143c 100644 --- a/crates/utils/src/utils/markdown/mod.rs +++ b/crates/utils/src/utils/markdown/mod.rs @@ -155,7 +155,7 @@ mod tests { ( "remote image proxied", "![link](http://example.com/image.jpg)", - "![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)", + "![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)", ), ( "local image unproxied", @@ -165,7 +165,7 @@ mod tests { ( "multiple image links", "![link](http://example.com/image1.jpg) ![link](http://example.com/image2.jpg)", - "![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)", + "![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)", ), ( "empty link handled", @@ -175,7 +175,7 @@ mod tests { ( "empty label handled", "![](http://example.com/image.jpg)", - "![](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" + "![](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" ), ( "invalid image link removed", @@ -185,12 +185,12 @@ mod tests { ( "label with nested markdown handled", "![a *b* c](http://example.com/image.jpg)", - "![a *b* c](https://lemmy-alpha/api/v4/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" + "![a *b* c](https://lemmy-alpha/api/v4/image/proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)" ), ( "custom emoji support", r#"![party-blob](https://www.hexbear.net/pictrs/image/83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#, - r#"![party-blob](https://lemmy-alpha/api/v4/image_proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"# + r#"![party-blob](https://lemmy-alpha/api/v4/image/proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"# ) ]; diff --git a/src/api_routes_v3.rs b/src/api_routes_v3.rs index 5e8fb741d..574646309 100644 --- a/src/api_routes_v3.rs +++ b/src/api_routes_v3.rs @@ -77,7 +77,6 @@ use lemmy_api::{ unread_count::get_unread_registration_application_count, }, }, - sitemap::get_sitemap, }; use lemmy_api_crud::{ comment::{ @@ -124,252 +123,261 @@ use lemmy_apub::api::{ search::search, user_settings_backup::{export_settings, import_settings}, }; -use lemmy_routes::images::image_proxy; +use lemmy_routes::images::{ + delete::delete_image, + download::{get_image, image_proxy}, + pictrs_health, + upload::upload_image, +}; use lemmy_utils::rate_limit::RateLimitCell; // Deprecated, use api v4 instead. -// When removing api v3, we also need to rewrite all links in database with -// `/api/v3/image_proxy` to use `/api/v4/image_proxy` instead. +// When removing api v3, make sure to keep `/api/v3/image_proxy` as it is still used in old posts. pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) { - cfg.service( - scope("/api/v3") - .route("/image_proxy", get().to(image_proxy)) - // Site - .service( - scope("/site") - .wrap(rate_limit.message()) - .route("", get().to(get_site_v3)) - // Admin Actions - .route("", post().to(create_site)) - .route("", put().to(update_site)) - .route("/block", post().to(user_block_instance)), - ) - .service( - resource("/modlog") - .wrap(rate_limit.message()) - .route(get().to(get_mod_log)), - ) - .service( - resource("/search") - .wrap(rate_limit.search()) - .route(get().to(search)), - ) - .service( - resource("/resolve_object") - .wrap(rate_limit.message()) - .route(get().to(resolve_object)), - ) - // Community - .service( - resource("/community") - .guard(guard::Post()) - .wrap(rate_limit.register()) - .route(post().to(create_community)), - ) - .service( - scope("/community") - .wrap(rate_limit.message()) - .route("", get().to(get_community)) - .route("", put().to(update_community)) - .route("/hide", put().to(hide_community)) - .route("/list", get().to(list_communities)) - .route("/follow", post().to(follow_community)) - .route("/block", post().to(user_block_community)) - .route("/delete", post().to(delete_community)) - // Mod Actions - .route("/remove", post().to(remove_community)) - .route("/transfer", post().to(transfer_community)) - .route("/ban_user", post().to(ban_from_community)) - .route("/mod", post().to(add_mod_to_community)), - ) - .service( - scope("/federated_instances") - .wrap(rate_limit.message()) - .route("", get().to(get_federated_instances)), - ) - // Post - .service( - // Handle POST to /post separately to add the post() rate limitter - resource("/post") - .guard(guard::Post()) - .wrap(rate_limit.post()) - .route(post().to(create_post)), - ) - .service( - scope("/post") - .wrap(rate_limit.message()) - .route("", get().to(get_post)) - .route("", put().to(update_post)) - .route("/delete", post().to(delete_post)) - .route("/remove", post().to(remove_post)) - .route("/mark_as_read", post().to(mark_post_as_read)) - .route("/hide", post().to(hide_post)) - .route("/lock", post().to(lock_post)) - .route("/feature", post().to(feature_post)) - .route("/list", get().to(list_posts)) - .route("/like", post().to(like_post)) - .route("/like/list", get().to(list_post_likes)) - .route("/save", put().to(save_post)) - .route("/report", post().to(create_post_report)) - .route("/report/resolve", put().to(resolve_post_report)) - .route("/site_metadata", get().to(get_link_metadata)), - ) - // Comment - .service( - // Handle POST to /comment separately to add the comment() rate limitter - resource("/comment") - .guard(guard::Post()) - .wrap(rate_limit.comment()) - .route(post().to(create_comment)), - ) - .service( - scope("/comment") - .wrap(rate_limit.message()) - .route("", get().to(get_comment)) - .route("", put().to(update_comment)) - .route("/delete", post().to(delete_comment)) - .route("/remove", post().to(remove_comment)) - .route("/mark_as_read", post().to(mark_reply_as_read)) - .route("/distinguish", post().to(distinguish_comment)) - .route("/like", post().to(like_comment)) - .route("/like/list", get().to(list_comment_likes)) - .route("/save", put().to(save_comment)) - .route("/list", get().to(list_comments)) - .route("/report", post().to(create_comment_report)) - .route("/report/resolve", put().to(resolve_comment_report)), - ) - // Private Message - .service( - scope("/private_message") - .wrap(rate_limit.message()) - .route("/list", get().to(get_private_message)) - .route("", post().to(create_private_message)) - .route("", put().to(update_private_message)) - .route("/delete", post().to(delete_private_message)) - .route("/mark_as_read", post().to(mark_pm_as_read)) - .route("/report", post().to(create_pm_report)) - .route("/report/resolve", put().to(resolve_pm_report)), - ) - // User - .service( - // Account action, I don't like that it's in /user maybe /accounts - // Handle /user/register separately to add the register() rate limiter - resource("/user/register") - .guard(guard::Post()) - .wrap(rate_limit.register()) - .route(post().to(register)), - ) - // User - .service( - // Handle /user/login separately to add the register() rate limiter - // TODO: pretty annoying way to apply rate limits for register and login, we should - // group them under a common path so that rate limit is only applied once (eg under - // /account). - resource("/user/login") - .guard(guard::Post()) - .wrap(rate_limit.register()) - .route(post().to(login)), - ) - .service( - resource("/user/password_reset") - .wrap(rate_limit.register()) - .route(post().to(reset_password)), - ) - .service( - // Handle captcha separately - resource("/user/get_captcha") - .wrap(rate_limit.post()) - .route(get().to(get_captcha)), - ) - .service( - resource("/user/export_settings") - .wrap(rate_limit.import_user_settings()) - .route(get().to(export_settings)), - ) - .service( - resource("/user/import_settings") - .wrap(rate_limit.import_user_settings()) - .route(post().to(import_settings)), - ) - // TODO, all the current account related actions under /user need to get moved here eventually - .service( - scope("/account") - .wrap(rate_limit.message()) - .route("/list_media", get().to(list_media)), - ) - // User actions - .service( - scope("/user") - .wrap(rate_limit.message()) - .route("", get().to(read_person)) - .route("/mention", get().to(list_mentions)) - .route( - "/mention/mark_as_read", - post().to(mark_person_mention_as_read), - ) - .route("/replies", get().to(list_replies)) - // Admin action. I don't like that it's in /user - .route("/ban", post().to(ban_from_site)) - .route("/banned", get().to(list_banned_users)) - .route("/block", post().to(user_block_person)) - // TODO Account actions. I don't like that they're in /user maybe /accounts - .route("/logout", post().to(logout)) - .route("/delete_account", post().to(delete_account)) - .route("/password_change", post().to(change_password_after_reset)) - // TODO mark_all_as_read feels off being in this section as well - .route("/mark_all_as_read", post().to(mark_all_notifications_read)) - .route("/save_user_settings", put().to(save_user_settings)) - .route("/change_password", put().to(change_password)) - .route("/report_count", get().to(report_count)) - .route("/unread_count", get().to(unread_count)) - .route("/verify_email", post().to(verify_email)) - .route("/leave_admin", post().to(leave_admin)) - .route("/totp/generate", post().to(generate_totp_secret)) - .route("/totp/update", post().to(update_totp)) - .route("/list_logins", get().to(list_logins)) - .route("/validate_auth", get().to(validate_auth)), - ) - // Admin Actions - .service( - scope("/admin") - .wrap(rate_limit.message()) - .route("/add", post().to(add_admin)) - .route( - "/registration_application/count", - get().to(get_unread_registration_application_count), - ) - .route( - "/registration_application/list", - get().to(list_registration_applications), - ) - .route( - "/registration_application/approve", - put().to(approve_registration_application), - ) - .route( - "/registration_application", - get().to(get_registration_application), - ) - .route("/list_all_media", get().to(list_all_media)) - .service( - scope("/purge") - .route("/person", post().to(purge_person)) - .route("/community", post().to(purge_community)) - .route("/post", post().to(purge_post)) - .route("/comment", post().to(purge_comment)), - ), - ) - .service( - scope("/custom_emoji") - .wrap(rate_limit.message()) - .route("", post().to(create_custom_emoji)) - .route("", put().to(update_custom_emoji)) - .route("/delete", post().to(delete_custom_emoji)), - ), - ); - cfg.service( - scope("/sitemap.xml") - .wrap(rate_limit.message()) - .route("", get().to(get_sitemap)), - ); + cfg + .service( + resource("/pictrs/image") + .wrap(rate_limit.image()) + .route(post().to(upload_image)), + ) + .service(resource("/pictrs/image/{filename}").route(get().to(get_image))) + .service(resource("/pictrs/image/delete/{token}/{filename}").route(get().to(delete_image))) + .service(resource("/pictrs/healthz").route(get().to(pictrs_health))) + .service( + scope("/api/v3") + .route("/image_proxy", get().to(image_proxy)) + // Site + .service( + scope("/site") + .wrap(rate_limit.message()) + .route("", get().to(get_site_v3)) + // Admin Actions + .route("", post().to(create_site)) + .route("", put().to(update_site)) + .route("/block", post().to(user_block_instance)), + ) + .service( + resource("/modlog") + .wrap(rate_limit.message()) + .route(get().to(get_mod_log)), + ) + .service( + resource("/search") + .wrap(rate_limit.search()) + .route(get().to(search)), + ) + .service( + resource("/resolve_object") + .wrap(rate_limit.message()) + .route(get().to(resolve_object)), + ) + // Community + .service( + resource("/community") + .guard(guard::Post()) + .wrap(rate_limit.register()) + .route(post().to(create_community)), + ) + .service( + scope("/community") + .wrap(rate_limit.message()) + .route("", get().to(get_community)) + .route("", put().to(update_community)) + .route("/hide", put().to(hide_community)) + .route("/list", get().to(list_communities)) + .route("/follow", post().to(follow_community)) + .route("/block", post().to(user_block_community)) + .route("/delete", post().to(delete_community)) + // Mod Actions + .route("/remove", post().to(remove_community)) + .route("/transfer", post().to(transfer_community)) + .route("/ban_user", post().to(ban_from_community)) + .route("/mod", post().to(add_mod_to_community)), + ) + .service( + scope("/federated_instances") + .wrap(rate_limit.message()) + .route("", get().to(get_federated_instances)), + ) + // Post + .service( + // Handle POST to /post separately to add the post() rate limitter + resource("/post") + .guard(guard::Post()) + .wrap(rate_limit.post()) + .route(post().to(create_post)), + ) + .service( + scope("/post") + .wrap(rate_limit.message()) + .route("", get().to(get_post)) + .route("", put().to(update_post)) + .route("/delete", post().to(delete_post)) + .route("/remove", post().to(remove_post)) + .route("/mark_as_read", post().to(mark_post_as_read)) + .route("/hide", post().to(hide_post)) + .route("/lock", post().to(lock_post)) + .route("/feature", post().to(feature_post)) + .route("/list", get().to(list_posts)) + .route("/like", post().to(like_post)) + .route("/like/list", get().to(list_post_likes)) + .route("/save", put().to(save_post)) + .route("/report", post().to(create_post_report)) + .route("/report/resolve", put().to(resolve_post_report)) + .route("/site_metadata", get().to(get_link_metadata)), + ) + // Comment + .service( + // Handle POST to /comment separately to add the comment() rate limitter + resource("/comment") + .guard(guard::Post()) + .wrap(rate_limit.comment()) + .route(post().to(create_comment)), + ) + .service( + scope("/comment") + .wrap(rate_limit.message()) + .route("", get().to(get_comment)) + .route("", put().to(update_comment)) + .route("/delete", post().to(delete_comment)) + .route("/remove", post().to(remove_comment)) + .route("/mark_as_read", post().to(mark_reply_as_read)) + .route("/distinguish", post().to(distinguish_comment)) + .route("/like", post().to(like_comment)) + .route("/like/list", get().to(list_comment_likes)) + .route("/save", put().to(save_comment)) + .route("/list", get().to(list_comments)) + .route("/report", post().to(create_comment_report)) + .route("/report/resolve", put().to(resolve_comment_report)), + ) + // Private Message + .service( + scope("/private_message") + .wrap(rate_limit.message()) + .route("/list", get().to(get_private_message)) + .route("", post().to(create_private_message)) + .route("", put().to(update_private_message)) + .route("/delete", post().to(delete_private_message)) + .route("/mark_as_read", post().to(mark_pm_as_read)) + .route("/report", post().to(create_pm_report)) + .route("/report/resolve", put().to(resolve_pm_report)), + ) + // User + .service( + // Account action, I don't like that it's in /user maybe /accounts + // Handle /user/register separately to add the register() rate limiter + resource("/user/register") + .guard(guard::Post()) + .wrap(rate_limit.register()) + .route(post().to(register)), + ) + // User + .service( + // Handle /user/login separately to add the register() rate limiter + // TODO: pretty annoying way to apply rate limits for register and login, we should + // group them under a common path so that rate limit is only applied once (eg under + // /account). + resource("/user/login") + .guard(guard::Post()) + .wrap(rate_limit.register()) + .route(post().to(login)), + ) + .service( + resource("/user/password_reset") + .wrap(rate_limit.register()) + .route(post().to(reset_password)), + ) + .service( + // Handle captcha separately + resource("/user/get_captcha") + .wrap(rate_limit.post()) + .route(get().to(get_captcha)), + ) + .service( + resource("/user/export_settings") + .wrap(rate_limit.import_user_settings()) + .route(get().to(export_settings)), + ) + .service( + resource("/user/import_settings") + .wrap(rate_limit.import_user_settings()) + .route(post().to(import_settings)), + ) + // TODO, all the current account related actions under /user need to get moved here + // eventually + .service( + scope("/account") + .wrap(rate_limit.message()) + .route("/list_media", get().to(list_media)), + ) + // User actions + .service( + scope("/user") + .wrap(rate_limit.message()) + .route("", get().to(read_person)) + .route("/mention", get().to(list_mentions)) + .route( + "/mention/mark_as_read", + post().to(mark_person_mention_as_read), + ) + .route("/replies", get().to(list_replies)) + // Admin action. I don't like that it's in /user + .route("/ban", post().to(ban_from_site)) + .route("/banned", get().to(list_banned_users)) + .route("/block", post().to(user_block_person)) + // TODO Account actions. I don't like that they're in /user maybe /accounts + .route("/logout", post().to(logout)) + .route("/delete_account", post().to(delete_account)) + .route("/password_change", post().to(change_password_after_reset)) + // TODO mark_all_as_read feels off being in this section as well + .route("/mark_all_as_read", post().to(mark_all_notifications_read)) + .route("/save_user_settings", put().to(save_user_settings)) + .route("/change_password", put().to(change_password)) + .route("/report_count", get().to(report_count)) + .route("/unread_count", get().to(unread_count)) + .route("/verify_email", post().to(verify_email)) + .route("/leave_admin", post().to(leave_admin)) + .route("/totp/generate", post().to(generate_totp_secret)) + .route("/totp/update", post().to(update_totp)) + .route("/list_logins", get().to(list_logins)) + .route("/validate_auth", get().to(validate_auth)), + ) + // Admin Actions + .service( + scope("/admin") + .wrap(rate_limit.message()) + .route("/add", post().to(add_admin)) + .route( + "/registration_application/count", + get().to(get_unread_registration_application_count), + ) + .route( + "/registration_application/list", + get().to(list_registration_applications), + ) + .route( + "/registration_application/approve", + put().to(approve_registration_application), + ) + .route( + "/registration_application", + get().to(get_registration_application), + ) + .route("/list_all_media", get().to(list_all_media)) + .service( + scope("/purge") + .route("/person", post().to(purge_person)) + .route("/community", post().to(purge_community)) + .route("/post", post().to(purge_post)) + .route("/comment", post().to(purge_comment)), + ), + ) + .service( + scope("/custom_emoji") + .wrap(rate_limit.message()) + .route("", post().to(create_custom_emoji)) + .route("", put().to(update_custom_emoji)) + .route("/delete", post().to(delete_custom_emoji)), + ), + ); } diff --git a/src/api_routes_v4.rs b/src/api_routes_v4.rs index 9f2b8d289..02eb11cd7 100644 --- a/src/api_routes_v4.rs +++ b/src/api_routes_v4.rs @@ -88,7 +88,6 @@ use lemmy_api::{ unread_count::get_unread_registration_application_count, }, }, - sitemap::get_sitemap, }; use lemmy_api_crud::{ comment::{ @@ -152,20 +151,44 @@ use lemmy_apub::api::{ search::search, user_settings_backup::{export_settings, import_settings}, }; -use lemmy_routes::images::image_proxy; +use lemmy_routes::images::{ + delete::{ + delete_community_banner, + delete_community_icon, + delete_image, + delete_site_banner, + delete_site_icon, + delete_user_avatar, + delete_user_banner, + }, + download::{get_image, image_proxy}, + pictrs_health, + upload::{ + upload_community_banner, + upload_community_icon, + upload_image, + upload_site_banner, + upload_site_icon, + upload_user_avatar, + upload_user_banner, + }, +}; use lemmy_utils::rate_limit::RateLimitCell; pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) { cfg.service( scope("/api/v4") .wrap(rate_limit.message()) - .route("/image_proxy", get().to(image_proxy)) // Site .service( scope("/site") .route("", get().to(get_site_v4)) .route("", post().to(create_site)) - .route("", put().to(update_site)), + .route("", put().to(update_site)) + .route("/icon", post().to(upload_site_icon)) + .route("/icon", delete().to(delete_site_icon)) + .route("/banner", post().to(upload_site_banner)) + .route("/banner", delete().to(delete_site_banner)), ) .route("/modlog", get().to(get_mod_log)) .service( @@ -195,6 +218,10 @@ pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) { .route("/transfer", post().to(transfer_community)) .route("/ban_user", post().to(ban_from_community)) .route("/mod", post().to(add_mod_to_community)) + .route("/icon", post().to(upload_community_icon)) + .route("/icon", delete().to(delete_community_icon)) + .route("/banner", post().to(upload_community_banner)) + .route("/banner", delete().to(delete_community_banner)) .service( scope("/pending_follows") .route("/count", get().to(get_pending_follows_count)) @@ -313,6 +340,10 @@ pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) { .route("/unread_count", get().to(unread_count)) .route("/list_logins", get().to(list_logins)) .route("/validate_auth", get().to(validate_auth)) + .route("/avatar", post().to(upload_user_avatar)) + .route("/avatar", delete().to(delete_user_avatar)) + .route("/banner", post().to(upload_user_banner)) + .route("/banner", delete().to(delete_user_banner)) .service( scope("/block") .route("/person", post().to(user_block_person)) @@ -388,6 +419,17 @@ pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) { .wrap(rate_limit.register()) .route("/authenticate", post().to(authenticate_with_oauth)), ) - .route("/sitemap.xml", get().to(get_sitemap)), + .service( + scope("/image") + .service( + resource("") + .wrap(rate_limit.image()) + .route(post().to(upload_image)) + .route(delete().to(delete_image)), + ) + .route("/proxy", get().to(image_proxy)) + .route("/health", get().to(pictrs_health)) + .route("/{filename}", get().to(get_image)), + ), ); } diff --git a/src/lib.rs b/src/lib.rs index 2f4af9ea3..bd84d0264 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,13 +11,14 @@ use actix_cors::Cors; use actix_web::{ dev::{ServerHandle, ServiceResponse}, middleware::{self, Condition, ErrorHandlerResponse, ErrorHandlers}, - web::Data, + web::{get, scope, Data}, App, HttpResponse, HttpServer, }; use actix_web_prom::PrometheusMetricsBuilder; use clap::{Parser, Subcommand}; +use lemmy_api::sitemap::get_sitemap; use lemmy_api_common::{ context::LemmyContext, lemmy_db_views::structs::SiteView, @@ -36,7 +37,7 @@ use lemmy_apub::{ }; use lemmy_db_schema::{schema_setup, source::secret::Secret, utils::build_db_pool}; use lemmy_federate::{Opts, SendManager}; -use lemmy_routes::{feeds, images, nodeinfo, webfinger}; +use lemmy_routes::{feeds, nodeinfo, webfinger}; use lemmy_utils::{ error::{LemmyErrorType, LemmyResult}, rate_limit::RateLimitCell, @@ -194,9 +195,13 @@ pub async fn start_lemmy_server(args: CmdArgs) -> LemmyResult<()> { let client = ClientBuilder::new(client_builder(&SETTINGS).build()?) .with(TracingMiddleware::default()) .build(); + let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?) + .with(TracingMiddleware::default()) + .build(); let context = LemmyContext::create( pool.clone(), client.clone(), + pictrs_client, secret.clone(), rate_limit_cell.clone(), ); @@ -329,17 +334,12 @@ fn create_http_server( .build() .map_err(|e| LemmyErrorType::Unknown(format!("Should always be buildable: {e}")))?; - let context: LemmyContext = federation_config.deref().clone(); - let rate_limit_cell = federation_config.rate_limit_cell().clone(); - - // Pictrs cannot use proxy - let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?) - .with(TracingMiddleware::default()) - .build(); - // Create Http server let bind = (settings.bind, settings.port); let server = HttpServer::new(move || { + let context: LemmyContext = federation_config.deref().clone(); + let rate_limit_cell = federation_config.rate_limit_cell().clone(); + let cors_config = cors_config(&settings); let app = App::new() .wrap(middleware::Logger::new( @@ -372,8 +372,12 @@ fn create_http_server( } }) .configure(feeds::config) - .configure(|cfg| images::config(cfg, pictrs_client.clone(), &rate_limit_cell)) .configure(nodeinfo::config) + .service( + scope("/sitemap.xml") + .wrap(rate_limit_cell.message()) + .route("", get().to(get_sitemap)), + ) }) .disable_signals() .bind(bind)?