Proxy pictrs requests through Lemmy (fixes #371) (#77)

fix check_only value for image rate limit

Fix image rate limit

Add rate limit for image uploads

Proxy pictrs requests through Lemmy (fixes #371)

Co-authored-by: Felix Ableitner <me@nutomic.com>
Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/77

ansible/templates/nginx.conf

@@ -74,18 +74,6 @@ server {
       return 301 /pictrs/image/$1;
     }
 
-    # pict-rs images
-    location /pictrs {
-      location /pictrs/image {
-        proxy_pass http://0.0.0.0:8537/image;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header Host $host;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      }
-      # Block the import
-      return 403;
-    }
-
     location /iframely/ {
       proxy_pass http://0.0.0.0:8061/;
       proxy_set_header X-Real-IP $remote_addr;

docker/dev/docker-compose.yml

@@ -21,7 +21,8 @@ services:
   postgres:
     image: postgres:12-alpine
     ports:
-      - "127.0.0.1:5432:5432"
+      # use a different port so it doesnt conflict with postgres running on the host
+      - "127.0.0.1:5433:5432"
     environment:
       - POSTGRES_USER=lemmy
       - POSTGRES_PASSWORD=password

docker/federation/nginx.conf

@@ -26,18 +26,6 @@ http {
             proxy_set_header Connection "upgrade";
         }
 
-        # pict-rs images
-        location /pictrs {
-          location /pictrs/image {
-            proxy_pass http://pictrs:8080/image;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header Host $host;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-          }
-          # Block the import
-          return 403;
-        }
-
         location /iframely/ {
             proxy_pass http://iframely:80/;
             proxy_set_header X-Real-IP $remote_addr;
@@ -69,18 +57,6 @@ http {
             proxy_set_header Connection "upgrade";
         }
 
-        # pict-rs images
-        location /pictrs {
-          location /pictrs/image {
-            proxy_pass http://pictrs:8080/image;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header Host $host;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-          }
-          # Block the import
-          return 403;
-        }
-
         location /iframely/ {
             proxy_pass http://iframely:80/;
             proxy_set_header X-Real-IP $remote_addr;
@@ -112,18 +88,6 @@ http {
             proxy_set_header Connection "upgrade";
         }
 
-        # pict-rs images
-        location /pictrs {
-          location /pictrs/image {
-            proxy_pass http://pictrs:8080/image;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header Host $host;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-          }
-          # Block the import
-          return 403;
-        }
-
         location /iframely/ {
             proxy_pass http://iframely:80/;
             proxy_set_header X-Real-IP $remote_addr;

docker/lemmy.hjson

@@ -2,6 +2,15 @@
   # for more info about the config, check out the documentation
   # https://dev.lemmy.ml/docs/administration_configuration.html
 
+  setup: {
+    # username for the admin user
+    admin_username: "lemmy"
+    # password for the admin user
+    admin_password: "lemmy"
+    # name of the site (can be changed later)
+    site_name: "lemmy-test"
+  }
+
   # the domain name of your instance (eg "dev.lemmy.ml")
   hostname: "my_domain"
   # address where lemmy should listen for incoming requests

server/config/defaults.hjson

@@ -35,6 +35,8 @@
   jwt_secret: "changeme"
   # The location of the frontend
   front_end_dir: "../ui/dist"
+  # address where pictrs is available
+  pictrs_url: "http://pictrs:8080"
   # rate limits for various user actions, by user ip
   rate_limit: {
     # maximum number of messages created in interval
@@ -49,6 +51,10 @@
     register: 3
     # interval length for registration limit
     register_per_second: 3600
+    # maximum number of image uploads in interval
+    image: 6
+    # interval length for image uploads
+    image_per_second: 3600
   }
   # settings related to activitypub federation
   federation: {

server/lemmy_utils/src/settings.rs

@@ -14,6 +14,7 @@ pub struct Settings {
   pub port: u16,
   pub jwt_secret: String,
   pub front_end_dir: String,
+  pub pictrs_url: String,
   pub rate_limit: RateLimitConfig,
   pub email: Option<EmailConfig>,
   pub federation: Federation,
@@ -36,6 +37,8 @@ pub struct RateLimitConfig {
   pub post_per_second: i32,
   pub register: i32,
   pub register_per_second: i32,
+  pub image: i32,
+  pub image_per_second: i32,
 }
 
 #[derive(Debug, Deserialize, Clone)]

server/src/main.rs

@@ -27,7 +27,7 @@ use lemmy_server::{
   blocking,
   code_migrations::run_advanced_migrations,
   rate_limit::{rate_limiter::RateLimiter, RateLimit},
-  routes::{api, federation, feeds, index, nodeinfo, webfinger},
+  routes::*,
   websocket::server::*,
   LemmyError,
 };
@@ -91,9 +91,10 @@ async fn main() -> Result<(), LemmyError> {
       .data(server.clone())
       .data(Client::default())
       // The routes
-      .configure(move |cfg| api::config(cfg, &rate_limiter))
+      .configure(|cfg| api::config(cfg, &rate_limiter))
       .configure(federation::config)
       .configure(feeds::config)
+      .configure(|cfg| images::config(cfg, &rate_limiter))
       .configure(index::config)
       .configure(nodeinfo::config)
       .configure(webfinger::config)

server/src/rate_limit/mod.rs

@@ -45,6 +45,10 @@ impl RateLimit {
     self.kind(RateLimitType::Register)
   }
 
+  pub fn image(&self) -> RateLimited {
+    self.kind(RateLimitType::Image)
+  }
+
   fn kind(&self, type_: RateLimitType) -> RateLimited {
     RateLimited {
       rate_limiter: self.rate_limiter.clone(),
@@ -101,6 +105,15 @@ impl RateLimited {
             true,
           )?;
         }
+        RateLimitType::Image => {
+          limiter.check_rate_limit_full(
+            self.type_,
+            &ip_addr,
+            rate_limit.image,
+            rate_limit.image_per_second,
+            false,
+          )?;
+        }
       };
     }
 

server/src/rate_limit/rate_limiter.rs

@@ -15,6 +15,7 @@ pub enum RateLimitType {
   Message,
   Register,
   Post,
+  Image,
 }
 
 /// Rate limiting based on rate type and IP addr

server/src/routes/images.rs

@@ -0,0 +1,146 @@
+use crate::rate_limit::RateLimit;
+use actix::clock::Duration;
+use actix_web::{body::BodyStream, http::StatusCode, *};
+use awc::Client;
+use lemmy_utils::settings::Settings;
+use serde::{Deserialize, Serialize};
+
+const THUMBNAIL_SIZES: &[u64] = &[256];
+
+pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimit) {
+  let client = Client::build()
+    .header("User-Agent", "pict-rs-frontend, v0.1.0")
+    .timeout(Duration::from_secs(30))
+    .finish();
+
+  cfg
+    .data(client)
+    .service(
+      web::resource("/pictrs/image")
+        .wrap(rate_limit.image())
+        .route(web::post().to(upload)),
+    )
+    .service(web::resource("/pictrs/image/{filename}").route(web::get().to(full_res)))
+    .service(
+      web::resource("/pictrs/image/thumbnail{size}/{filename}").route(web::get().to(thumbnail)),
+    )
+    .service(web::resource("/pictrs/image/delete/{token}/{filename}").route(web::get().to(delete)));
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Image {
+  file: String,
+  delete_token: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Images {
+  msg: String,
+  files: Option<Vec<Image>>,
+}
+
+async fn upload(
+  req: HttpRequest,
+  body: web::Payload,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  // TODO: check auth and rate limit here
+
+  let mut res = client
+    .request_from(format!("{}/image", Settings::get().pictrs_url), req.head())
+    .if_some(req.head().peer_addr, |addr, req| {
+      req.header("X-Forwarded-For", addr.to_string())
+    })
+    .send_stream(body)
+    .await?;
+
+  let images = res.json::<Images>().await?;
+
+  Ok(HttpResponse::build(res.status()).json(images))
+}
+
+async fn full_res(
+  filename: web::Path<String>,
+  req: HttpRequest,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  let url = format!(
+    "{}/image/{}",
+    Settings::get().pictrs_url,
+    &filename.into_inner()
+  );
+  image(url, req, client).await
+}
+
+async fn thumbnail(
+  parts: web::Path<(u64, String)>,
+  req: HttpRequest,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  let (size, file) = parts.into_inner();
+
+  if THUMBNAIL_SIZES.contains(&size) {
+    let url = format!(
+      "{}/image/thumbnail{}/{}",
+      Settings::get().pictrs_url,
+      size,
+      &file
+    );
+
+    return image(url, req, client).await;
+  }
+
+  Ok(HttpResponse::NotFound().finish())
+}
+
+async fn image(
+  url: String,
+  req: HttpRequest,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  let res = client
+    .request_from(url, req.head())
+    .if_some(req.head().peer_addr, |addr, req| {
+      req.header("X-Forwarded-For", addr.to_string())
+    })
+    .no_decompress()
+    .send()
+    .await?;
+
+  if res.status() == StatusCode::NOT_FOUND {
+    return Ok(HttpResponse::NotFound().finish());
+  }
+
+  let mut client_res = HttpResponse::build(res.status());
+
+  for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") {
+    client_res.header(name.clone(), value.clone());
+  }
+
+  Ok(client_res.body(BodyStream::new(res)))
+}
+
+async fn delete(
+  components: web::Path<(String, String)>,
+  req: HttpRequest,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  let (token, file) = components.into_inner();
+
+  let url = format!(
+    "{}/image/delete/{}/{}",
+    Settings::get().pictrs_url,
+    &token,
+    &file
+  );
+  let res = client
+    .request_from(url, req.head())
+    .if_some(req.head().peer_addr, |addr, req| {
+      req.header("X-Forwarded-For", addr.to_string())
+    })
+    .no_decompress()
+    .send()
+    .await?;
+
+  Ok(HttpResponse::build(res.status()).body(BodyStream::new(res)))
+}

server/src/routes/mod.rs

@@ -1,6 +1,7 @@
 pub mod api;
 pub mod federation;
 pub mod feeds;
+pub mod images;
 pub mod index;
 pub mod nodeinfo;
 pub mod webfinger;