From 55b8aced9d7c8b1245bc6f835d82639b0566fc6f Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 17 Dec 2024 16:23:43 +0100 Subject: [PATCH] split files into upload, download --- config/defaults.hjson | 3 + crates/api/src/site/leave_admin.rs | 1 + crates/api_common/src/utils.rs | 7 +- crates/api_crud/src/site/read.rs | 2 +- crates/routes/src/images/download.rs | 113 +++++++++++++++++++++++ crates/routes/src/images/mod.rs | 111 ++-------------------- crates/routes/src/images/person.rs | 36 -------- crates/routes/src/images/upload.rs | 132 +++++++++++++++++++++++++++ crates/routes/src/images/utils.rs | 122 ++----------------------- crates/utils/src/settings/structs.rs | 4 +- src/api_routes_v3.rs | 7 +- src/api_routes_v4.rs | 6 +- 12 files changed, 277 insertions(+), 267 deletions(-) create mode 100644 crates/routes/src/images/download.rs delete mode 100644 crates/routes/src/images/person.rs create mode 100644 crates/routes/src/images/upload.rs diff --git a/config/defaults.hjson b/config/defaults.hjson index aaf2a94d8..d6c24cce8 100644 --- a/config/defaults.hjson +++ b/config/defaults.hjson @@ -72,6 +72,9 @@ # Otherwise we have to use crop, or use max_width/max_height which throws error # if image is larger. max_banner_size: 512 + # Prevent users from uploading images for posts or embedding in markdown. Avatars, icons and + # banners can still be uploaded. + image_upload_disabled: false } # Email sending configuration. All options except login/password are mandatory email: { diff --git a/crates/api/src/site/leave_admin.rs b/crates/api/src/site/leave_admin.rs index fde258dd2..042009d24 100644 --- a/crates/api/src/site/leave_admin.rs +++ b/crates/api/src/site/leave_admin.rs @@ -76,5 +76,6 @@ pub async fn leave_admin( blocked_urls, tagline, my_user: None, + image_upload_disabled: context.settings().pictrs()?.image_upload_disabled, })) } diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 3daa34107..27b7ad531 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -1128,12 +1128,7 @@ async fn proxy_image_link_internal( /// Rewrite a link to go through `/api/v4/image_proxy` endpoint. This is only for remote urls and /// if image_proxy setting is enabled. pub async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult { - proxy_image_link_internal( - link, - context.settings().pictrs()?.image_mode(), - context, - ) - .await + proxy_image_link_internal(link, context.settings().pictrs()?.image_mode(), context).await } pub async fn proxy_image_link_opt_api( diff --git a/crates/api_crud/src/site/read.rs b/crates/api_crud/src/site/read.rs index b618c0fa2..64d2237a0 100644 --- a/crates/api_crud/src/site/read.rs +++ b/crates/api_crud/src/site/read.rs @@ -69,6 +69,6 @@ async fn read_site(context: &LemmyContext) -> LemmyResult { tagline, oauth_providers: Some(oauth_providers), admin_oauth_providers: Some(admin_oauth_providers), - image_upload_disabled: context.settings().pictrs()?.disable_image_upload, + image_upload_disabled: context.settings().pictrs()?.image_upload_disabled, }) } diff --git a/crates/routes/src/images/download.rs b/crates/routes/src/images/download.rs new file mode 100644 index 000000000..b54b72f1c --- /dev/null +++ b/crates/routes/src/images/download.rs @@ -0,0 +1,113 @@ +use super::utils::{adapt_request, convert_header, file_type}; +use actix_web::{ + body::{BodyStream, BoxBody}, + http::StatusCode, + web::{Data, *}, + HttpRequest, + HttpResponse, + Responder, +}; +use lemmy_api_common::{ + context::LemmyContext, + image::{ImageGetParams, ImageProxyParams}, +}; +use lemmy_db_schema::source::{images::RemoteImage, local_site::LocalSite}; +use lemmy_db_views::structs::LocalUserView; +use lemmy_utils::error::LemmyResult; +use url::Url; + +pub async fn get_image( + filename: Path, + Query(params): Query, + req: HttpRequest, + context: Data, + local_user_view: Option, +) -> LemmyResult { + // block access to images if instance is private and unauthorized, public + let local_site = LocalSite::read(&mut context.pool()).await?; + if local_site.private_instance && local_user_view.is_none() { + return Ok(HttpResponse::Unauthorized().finish()); + } + let name = &filename.into_inner(); + + // If there are no query params, the URL is original + let pictrs_url = context.settings().pictrs()?.url; + let processed_url = if params.file_type.is_none() && params.max_size.is_none() { + format!("{}image/original/{}", pictrs_url, name) + } else { + let file_type = file_type(params.file_type, name); + let mut url = format!("{}image/process.{}?src={}", pictrs_url, file_type, name); + + if let Some(size) = params.max_size { + url = format!("{url}&thumbnail={size}",); + } + url + }; + + do_get_image(processed_url, req).await +} +pub async fn image_proxy( + Query(params): Query, + req: HttpRequest, + context: Data, +) -> LemmyResult, HttpResponse>> { + let url = Url::parse(¶ms.url)?; + + // Check that url corresponds to a federated image so that this can't be abused as a proxy + // for arbitrary purposes. + RemoteImage::validate(&mut context.pool(), url.clone().into()).await?; + + let pictrs_config = context.settings().pictrs()?; + let processed_url = if params.file_type.is_none() && params.max_size.is_none() { + format!("{}image/original?proxy={}", pictrs_config.url, params.url) + } else { + let file_type = file_type(params.file_type, url.as_str()); + let mut url = format!( + "{}image/process.{}?proxy={}", + pictrs_config.url, file_type, url + ); + + if let Some(size) = params.max_size { + url = format!("{url}&thumbnail={size}",); + } + url + }; + + let bypass_proxy = pictrs_config + .proxy_bypass_domains + .iter() + .any(|s| url.domain().is_some_and(|d| d == s)); + if bypass_proxy { + // Bypass proxy and redirect user to original image + Ok(Either::Left(Redirect::to(url.to_string()).respond_to(&req))) + } else { + // Proxy the image data through Lemmy + Ok(Either::Right(do_get_image(processed_url, req).await?)) + } +} + +pub(super) async fn do_get_image(url: String, req: HttpRequest) -> LemmyResult { + let mut client_req = adapt_request(&req, url); + + if let Some(addr) = req.head().peer_addr { + client_req = client_req.header("X-Forwarded-For", addr.to_string()); + } + + if let Some(addr) = req.head().peer_addr { + client_req = client_req.header("X-Forwarded-For", addr.to_string()); + } + + let res = client_req.send().await?; + + if res.status() == http::StatusCode::NOT_FOUND { + return Ok(HttpResponse::NotFound().finish()); + } + + let mut client_res = HttpResponse::build(StatusCode::from_u16(res.status().as_u16())?); + + for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") { + client_res.insert_header(convert_header(name, value)); + } + + Ok(client_res.body(BodyStream::new(res.bytes_stream()))) +} diff --git a/crates/routes/src/images/mod.rs b/crates/routes/src/images/mod.rs index 8e5a54734..52d3d89df 100644 --- a/crates/routes/src/images/mod.rs +++ b/crates/routes/src/images/mod.rs @@ -1,73 +1,14 @@ -use actix_web::{body::BoxBody, web::*, HttpRequest, HttpResponse, Responder}; -use lemmy_api_common::{ - context::LemmyContext, - image::{DeleteImageParams, ImageGetParams, ImageProxyParams, UploadImageResponse}, - LemmyErrorType, - SuccessResponse, -}; -use lemmy_db_schema::source::{ - images::{LocalImage, RemoteImage}, - local_site::LocalSite, -}; +use actix_web::web::*; +use lemmy_api_common::{context::LemmyContext, image::DeleteImageParams, SuccessResponse}; +use lemmy_db_schema::source::images::LocalImage; use lemmy_db_views::structs::LocalUserView; use lemmy_utils::error::LemmyResult; -use url::Url; -use utils::{do_get_image, do_upload_image, file_type, UploadType, PICTRS_CLIENT}; +use utils::PICTRS_CLIENT; -pub mod person; +pub mod download; +pub mod upload; mod utils; -pub async fn upload_image( - req: HttpRequest, - body: Payload, - local_user_view: LocalUserView, - context: Data, -) -> LemmyResult> { - if context.settings().pictrs()?.disable_image_upload { - return Err(LemmyErrorType::ImageUploadDisabled.into()); - } - - let image = do_upload_image(req, body, UploadType::Other, &local_user_view, &context).await?; - - let image_url = image.image_url(&context.settings().get_protocol_and_hostname())?; - Ok(Json(UploadImageResponse { - image_url, - filename: image.file, - delete_token: image.delete_token, - })) -} - -pub async fn get_image( - filename: Path, - Query(params): Query, - req: HttpRequest, - context: Data, - local_user_view: Option, -) -> LemmyResult { - // block access to images if instance is private and unauthorized, public - let local_site = LocalSite::read(&mut context.pool()).await?; - if local_site.private_instance && local_user_view.is_none() { - return Ok(HttpResponse::Unauthorized().finish()); - } - let name = &filename.into_inner(); - - // If there are no query params, the URL is original - let pictrs_url = context.settings().pictrs()?.url; - let processed_url = if params.file_type.is_none() && params.max_size.is_none() { - format!("{}image/original/{}", pictrs_url, name) - } else { - let file_type = file_type(params.file_type, name); - let mut url = format!("{}image/process.{}?src={}", pictrs_url, file_type, name); - - if let Some(size) = params.max_size { - url = format!("{url}&thumbnail={size}",); - } - url - }; - - do_get_image(processed_url, req).await -} - pub async fn delete_image( data: Json, context: Data, @@ -95,43 +36,3 @@ pub async fn pictrs_health(context: Data) -> LemmyResult, - req: HttpRequest, - context: Data, -) -> LemmyResult, HttpResponse>> { - let url = Url::parse(¶ms.url)?; - - // Check that url corresponds to a federated image so that this can't be abused as a proxy - // for arbitrary purposes. - RemoteImage::validate(&mut context.pool(), url.clone().into()).await?; - - let pictrs_config = context.settings().pictrs()?; - let processed_url = if params.file_type.is_none() && params.max_size.is_none() { - format!("{}image/original?proxy={}", pictrs_config.url, params.url) - } else { - let file_type = file_type(params.file_type, url.as_str()); - let mut url = format!( - "{}image/process.{}?proxy={}", - pictrs_config.url, file_type, url - ); - - if let Some(size) = params.max_size { - url = format!("{url}&thumbnail={size}",); - } - url - }; - - let bypass_proxy = pictrs_config - .proxy_bypass_domains - .iter() - .any(|s| url.domain().is_some_and(|d| d == s)); - if bypass_proxy { - // Bypass proxy and redirect user to original image - Ok(Either::Left(Redirect::to(url.to_string()).respond_to(&req))) - } else { - // Proxy the image data through Lemmy - Ok(Either::Right(do_get_image(processed_url, req).await?)) - } -} diff --git a/crates/routes/src/images/person.rs b/crates/routes/src/images/person.rs deleted file mode 100644 index edac1e41b..000000000 --- a/crates/routes/src/images/person.rs +++ /dev/null @@ -1,36 +0,0 @@ -use super::utils::{delete_old_image, do_upload_image, UploadType}; -use actix_web::{self, web::*, HttpRequest}; -use lemmy_api_common::{context::LemmyContext, SuccessResponse}; -use lemmy_db_schema::{ - source::person::{Person, PersonUpdateForm}, - traits::Crud, -}; -use lemmy_db_views::structs::LocalUserView; -use lemmy_utils::error::LemmyResult; -use url::Url; - -pub async fn upload_avatar( - req: HttpRequest, - body: Payload, - local_user_view: LocalUserView, - context: Data, -) -> LemmyResult> { - let image = do_upload_image(req, body, UploadType::Avatar, &local_user_view, &context).await?; - - delete_old_image(&local_user_view.person.avatar, &context).await?; - - let avatar = format!( - "{}/api/v4/image/{}", - context.settings().get_protocol_and_hostname(), - image.file - ); - let avatar = Some(Some(Url::parse(&avatar)?.into())); - let person_form = PersonUpdateForm { - avatar, - ..Default::default() - }; - - Person::update(&mut context.pool(), local_user_view.person.id, &person_form).await?; - - Ok(Json(SuccessResponse::default())) -} diff --git a/crates/routes/src/images/upload.rs b/crates/routes/src/images/upload.rs new file mode 100644 index 000000000..c009aae26 --- /dev/null +++ b/crates/routes/src/images/upload.rs @@ -0,0 +1,132 @@ +use super::utils::{adapt_request, delete_old_image, make_send}; +use actix_web::{self, web::*, HttpRequest}; +use lemmy_api_common::{ + context::LemmyContext, + image::UploadImageResponse, + request::{PictrsFile, PictrsResponse}, + LemmyErrorType, + SuccessResponse, +}; +use lemmy_db_schema::{ + source::{ + images::{LocalImage, LocalImageForm}, + person::{Person, PersonUpdateForm}, + }, + traits::Crud, +}; +use lemmy_db_views::structs::LocalUserView; +use lemmy_utils::error::LemmyResult; +use reqwest::Body; +use std::time::Duration; +use url::Url; + +pub async fn upload_image( + req: HttpRequest, + body: Payload, + local_user_view: LocalUserView, + context: Data, +) -> LemmyResult> { + if context.settings().pictrs()?.image_upload_disabled { + return Err(LemmyErrorType::ImageUploadDisabled.into()); + } + + let image = do_upload_image(req, body, UploadType::Other, &local_user_view, &context).await?; + + let image_url = image.image_url(&context.settings().get_protocol_and_hostname())?; + Ok(Json(UploadImageResponse { + image_url, + filename: image.file, + delete_token: image.delete_token, + })) +} + +pub async fn upload_avatar( + req: HttpRequest, + body: Payload, + local_user_view: LocalUserView, + context: Data, +) -> LemmyResult> { + let image = do_upload_image(req, body, UploadType::Avatar, &local_user_view, &context).await?; + + delete_old_image(&local_user_view.person.avatar, &context).await?; + + let avatar = format!( + "{}/api/v4/image/{}", + context.settings().get_protocol_and_hostname(), + image.file + ); + let avatar = Some(Some(Url::parse(&avatar)?.into())); + let person_form = PersonUpdateForm { + avatar, + ..Default::default() + }; + + Person::update(&mut context.pool(), local_user_view.person.id, &person_form).await?; + + Ok(Json(SuccessResponse::default())) +} +pub enum UploadType { + Avatar, + Other, +} + +pub async fn do_upload_image( + req: HttpRequest, + body: Payload, + upload_type: UploadType, + local_user_view: &LocalUserView, + context: &Data, +) -> LemmyResult { + let pictrs_config = context.settings().pictrs()?; + let image_url = format!("{}image", pictrs_config.url); + + let mut client_req = adapt_request(&req, image_url); + + client_req = match upload_type { + UploadType::Avatar => { + let max_size = context.settings().pictrs()?.max_avatar_size.to_string(); + client_req.query(&[ + ("resize", max_size.as_ref()), + ("allow_animation", "false"), + ("allow_video", "false"), + ]) + } + // TODO: same as above but using `max_banner_size` + // UploadType::Banner => {} + _ => client_req, + }; + if let Some(addr) = req.head().peer_addr { + client_req = client_req.header("X-Forwarded-For", addr.to_string()) + }; + let res = client_req + .timeout(Duration::from_secs(pictrs_config.upload_timeout)) + .body(Body::wrap_stream(make_send(body))) + .send() + .await? + .error_for_status()?; + + let mut images = res.json::().await?; + for image in &images.files { + // Pictrs allows uploading multiple images in a single request. Lemmy doesnt need this, + // but still a user may upload multiple and so we need to store all links in db for + // to allow deletion via web ui. + let form = LocalImageForm { + local_user_id: Some(local_user_view.local_user.id), + pictrs_alias: image.file.to_string(), + pictrs_delete_token: image.delete_token.to_string(), + }; + + let protocol_and_hostname = context.settings().get_protocol_and_hostname(); + let thumbnail_url = image.image_url(&protocol_and_hostname)?; + + // Also store the details for the image + let details_form = image.details.build_image_details_form(&thumbnail_url); + LocalImage::create(&mut context.pool(), &form, &details_form).await?; + } + let image = images + .files + .pop() + .ok_or(LemmyErrorType::InvalidImageUpload)?; + + Ok(image) +} diff --git a/crates/routes/src/images/utils.rs b/crates/routes/src/images/utils.rs index 50e0b8fd4..040734a8b 100644 --- a/crates/routes/src/images/utils.rs +++ b/crates/routes/src/images/utils.rs @@ -1,31 +1,22 @@ use actix_web::{ - body::BodyStream, http::{ header::{HeaderName, ACCEPT_ENCODING, HOST}, Method, - StatusCode, }, - web::{Data, Payload}, + web::Data, HttpRequest, - HttpResponse, }; use futures::stream::{Stream, StreamExt}; use http::HeaderValue; use lemmy_api_common::{ context::LemmyContext, - request::{client_builder, delete_image_from_pictrs, PictrsFile, PictrsResponse}, - LemmyErrorType, + request::{client_builder, delete_image_from_pictrs}, }; -use lemmy_db_schema::{ - newtypes::DbUrl, - source::images::{LocalImage, LocalImageForm}, -}; -use lemmy_db_views::structs::LocalUserView; +use lemmy_db_schema::{newtypes::DbUrl, source::images::LocalImage}; use lemmy_utils::{error::LemmyResult, settings::SETTINGS, REQWEST_TIMEOUT}; -use reqwest::Body; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, RequestBuilder}; use reqwest_tracing::TracingMiddleware; -use std::{sync::LazyLock, time::Duration}; +use std::sync::LazyLock; // Pictrs cannot use proxy #[allow(clippy::expect_used)] @@ -40,7 +31,7 @@ pub(super) static PICTRS_CLIENT: LazyLock = LazyLock::new( .build() }); -fn adapt_request(request: &HttpRequest, url: String) -> RequestBuilder { +pub(super) fn adapt_request(request: &HttpRequest, url: String) -> RequestBuilder { // remove accept-encoding header so that pictrs doesn't compress the response const INVALID_HEADERS: &[HeaderName] = &[ACCEPT_ENCODING, HOST]; @@ -61,7 +52,7 @@ fn adapt_request(request: &HttpRequest, url: String) -> RequestBuilder { }) } -fn make_send(mut stream: S) -> impl Stream + Send + Unpin + 'static +pub(super) fn make_send(mut stream: S) -> impl Stream + Send + Unpin + 'static where S: Stream + Unpin + 'static, S::Item: Send, @@ -109,106 +100,13 @@ pub(super) fn convert_method(method: &Method) -> http::Method { http::Method::from_bytes(method.as_str().as_bytes()).expect("method can be converted") } -fn convert_header<'a>(name: &'a http::HeaderName, value: &'a HeaderValue) -> (&'a str, &'a [u8]) { +pub(super) fn convert_header<'a>( + name: &'a http::HeaderName, + value: &'a HeaderValue, +) -> (&'a str, &'a [u8]) { (name.as_str(), value.as_bytes()) } -pub(super) enum UploadType { - Avatar, - Other, -} - -pub(super) async fn do_upload_image( - req: HttpRequest, - body: Payload, - upload_type: UploadType, - local_user_view: &LocalUserView, - context: &Data, -) -> LemmyResult { - let pictrs_config = context.settings().pictrs()?; - let image_url = format!("{}image", pictrs_config.url); - - let mut client_req = adapt_request(&req, image_url); - - client_req = match upload_type { - UploadType::Avatar => { - let max_size = context - .settings() - .pictrs()? - .max_avatar_size - .to_string(); - client_req.query(&[ - ("resize", max_size.as_ref()), - ("allow_animation", "false"), - ("allow_video", "false"), - ]) - } - // TODO: same as above but using `max_banner_size` - // UploadType::Banner => {} - _ => client_req, - }; - if let Some(addr) = req.head().peer_addr { - client_req = client_req.header("X-Forwarded-For", addr.to_string()) - }; - let res = client_req - .timeout(Duration::from_secs(pictrs_config.upload_timeout)) - .body(Body::wrap_stream(make_send(body))) - .send() - .await? - .error_for_status()?; - - let mut images = res.json::().await?; - for image in &images.files { - // Pictrs allows uploading multiple images in a single request. Lemmy doesnt need this, - // but still a user may upload multiple and so we need to store all links in db for - // to allow deletion via web ui. - let form = LocalImageForm { - local_user_id: Some(local_user_view.local_user.id), - pictrs_alias: image.file.to_string(), - pictrs_delete_token: image.delete_token.to_string(), - }; - - let protocol_and_hostname = context.settings().get_protocol_and_hostname(); - let thumbnail_url = image.image_url(&protocol_and_hostname)?; - - // Also store the details for the image - let details_form = image.details.build_image_details_form(&thumbnail_url); - LocalImage::create(&mut context.pool(), &form, &details_form).await?; - } - let image = images - .files - .pop() - .ok_or(LemmyErrorType::InvalidImageUpload)?; - - Ok(image) -} - -pub(super) async fn do_get_image(url: String, req: HttpRequest) -> LemmyResult { - let mut client_req = adapt_request(&req, url); - - if let Some(addr) = req.head().peer_addr { - client_req = client_req.header("X-Forwarded-For", addr.to_string()); - } - - if let Some(addr) = req.head().peer_addr { - client_req = client_req.header("X-Forwarded-For", addr.to_string()); - } - - let res = client_req.send().await?; - - if res.status() == http::StatusCode::NOT_FOUND { - return Ok(HttpResponse::NotFound().finish()); - } - - let mut client_res = HttpResponse::build(StatusCode::from_u16(res.status().as_u16())?); - - for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") { - client_res.insert_header(convert_header(name, value)); - } - - Ok(client_res.body(BodyStream::new(res.bytes_stream()))) -} - /// When adding a new avatar, banner or similar image, delete the old one. pub(super) async fn delete_old_image( old_image: &Option, diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index 041980d2e..b9c38f203 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -116,11 +116,11 @@ pub struct PictrsConfig { /// if image is larger. #[default(512)] pub max_banner_size: u32, - + /// Prevent users from uploading images for posts or embedding in markdown. Avatars, icons and /// banners can still be uploaded. #[default(false)] - pub disable_image_upload: bool, + pub image_upload_disabled: bool, } #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document, PartialEq)] diff --git a/src/api_routes_v3.rs b/src/api_routes_v3.rs index e7cb82883..59b7de610 100644 --- a/src/api_routes_v3.rs +++ b/src/api_routes_v3.rs @@ -134,7 +134,12 @@ use lemmy_apub::api::{ search::search, user_settings_backup::{export_settings, import_settings}, }; -use lemmy_routes::images::{delete_image, get_image, image_proxy, pictrs_health, upload_image}; +use lemmy_routes::images::{ + delete_image, + download::{get_image, image_proxy}, + pictrs_health, + upload::upload_image, +}; use lemmy_utils::rate_limit::RateLimitCell; // Deprecated, use api v4 instead. diff --git a/src/api_routes_v4.rs b/src/api_routes_v4.rs index 99ae86fbc..4bfe06a6a 100644 --- a/src/api_routes_v4.rs +++ b/src/api_routes_v4.rs @@ -161,11 +161,9 @@ use lemmy_apub::api::{ }; use lemmy_routes::images::{ delete_image, - get_image, - image_proxy, - person::upload_avatar, + download::{get_image, image_proxy}, pictrs_health, - upload_image, + upload::{upload_avatar, upload_image}, }; use lemmy_utils::rate_limit::RateLimitCell;