Make object storage with minio work

.gitignore

@@ -2,3 +2,4 @@
 /data
 /docker/dev/volumes
 /client-examples/javascript/node_modules
+/docker/object-storage/storage

docker/object-storage/Dockerfile

@@ -0,0 +1,23 @@
+FROM archlinux:latest
+
+ARG UID=1000
+ARG GID=1000
+
+RUN \
+ pacman -Syu --noconfirm \
+        perl-image-exiftool \
+        imagemagick \
+        ffmpeg && \
+ groupadd -g 1000 app && \
+ useradd -m \
+        -d /opt/app \
+        -u $UID \
+        -g $GID \
+        app
+
+COPY ./pict-rs.toml /etc/pict-rs.toml
+
+ENV PATH=$PATH:/usr/bin/vendor_perl
+
+WORKDIR /opt/app
+USER app

docker/object-storage/dev.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+ARCH=${1:-amd64}
+
+export USER_ID=$(id -u)
+export GROUP_ID=$(id -g)
+
+sudo docker-compose build --pull
+sudo docker-compose up -d minio
+sudo docker-compose up -d pictrs_proxy
+sudo docker-compose run --service-ports --use-aliases pictrs
+sudo docker-compose down

docker/object-storage/docker-compose.yml

@@ -0,0 +1,35 @@
+version: '3.3'
+
+services:
+  pictrs:
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+      args:
+        UID: "${USER_ID:-1000}"
+        GID: "${GROUP_ID:-1000}"
+    ports:
+      - "8080:8080"
+    links:
+      - "minio:pict-rs.minio"
+    stdin_open: true
+    tty: true
+    volumes:
+      - ./storage/pict-rs:/mnt
+      - ../../:/opt/app
+
+  pictrs_proxy:
+    image: asonix/pictrs-proxy:latest
+    ports:
+      - "8081:8081"
+    environment:
+      - PICTRS_PROXY_UPSTREAM=http://pictrs:8080
+
+  minio:
+    image: quay.io/minio/minio
+    command: server /mnt --console-address ":9001"
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - ./storage/minio:/mnt

docker/object-storage/pict-rs.toml

@@ -0,0 +1,9 @@
+path = '/mnt'
+addr = '0.0.0.0:8080'
+
+[store]
+type = 's3_store'
+bucket_name = 'pict-rs'
+region = 'http://minio:9000'
+access_key = '09ODZ3BGBISV4U92JLIM'
+secret_key = 'j35YE9RrxhBP0dpiD5mmdXRXvPkEJR4k6zK12q3o'

pict-rs.toml

@@ -41,8 +41,8 @@ max_image_width = 10000 # in Pixels
 # default: 10,000
 max_image_height = 10000 # in Pixels
 # environment variable: PICTRS_MAX_IMAGE_AREA
-# default: 40,000
+# default: 40,000,000
-max_image_area = 40000 # in Pixels
+max_image_area = 40000000 # in Pixels
 
 ## Optional: skip image validation on the import endpoint
 # environment variable: PICTRS_SKIP_VALIDATE_IMPORTS

src/config.rs

@@ -182,7 +182,7 @@ impl Defaults {
             max_file_size: 40,
             max_image_width: 10_000,
             max_image_height: 10_000,
-            max_image_area: 40_000,
+            max_image_area: 40_000_000,
             store: Store::FileStore { path: None },
         }
     }

src/store/file_store.rs

@@ -54,6 +54,7 @@ impl Store for FileStore {
     type Identifier = FileId;
     type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
 
+    #[tracing::instrument(skip(reader))]
     async fn save_async_read<Reader>(
         &self,
         reader: &mut Reader,
@@ -71,6 +72,7 @@ impl Store for FileStore {
         self.file_id_from_path(path)
     }
 
+    #[tracing::instrument(skip(bytes))]
     async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
         let path = self.next_file()?;
 
@@ -82,6 +84,7 @@ impl Store for FileStore {
         self.file_id_from_path(path)
     }
 
+    #[tracing::instrument]
     async fn to_stream(
         &self,
         identifier: &Self::Identifier,
@@ -98,6 +101,7 @@ impl Store for FileStore {
         Ok(Box::pin(stream))
     }
 
+    #[tracing::instrument(skip(writer))]
     async fn read_into<Writer>(
         &self,
         identifier: &Self::Identifier,
@@ -113,6 +117,7 @@ impl Store for FileStore {
         Ok(())
     }
 
+    #[tracing::instrument]
     async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Self::Error> {
         let path = self.path_from_file_id(identifier);
 
@@ -121,6 +126,7 @@ impl Store for FileStore {
         Ok(len)
     }
 
+    #[tracing::instrument]
     async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error> {
         let path = self.path_from_file_id(identifier);
 

src/store/object_store.rs

@@ -59,6 +59,7 @@ impl Store for ObjectStore {
     type Identifier = ObjectId;
     type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
 
+    #[tracing::instrument(skip(reader))]
     async fn save_async_read<Reader>(
         &self,
         reader: &mut Reader,
@@ -73,6 +74,7 @@ impl Store for ObjectStore {
         Ok(ObjectId::from_string(path))
     }
 
+    #[tracing::instrument(skip(bytes))]
     async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
         let path = self.next_file()?;
 
@@ -81,6 +83,7 @@ impl Store for ObjectStore {
         Ok(ObjectId::from_string(path))
     }
 
+    #[tracing::instrument]
     async fn to_stream(
         &self,
         identifier: &Self::Identifier,
@@ -99,6 +102,7 @@ impl Store for ObjectStore {
         Ok(Box::pin(io_error(response.bytes_stream())))
     }
 
+    #[tracing::instrument(skip(writer))]
     async fn read_into<Writer>(
         &self,
         identifier: &Self::Identifier,
@@ -121,6 +125,7 @@ impl Store for ObjectStore {
         Ok(())
     }
 
+    #[tracing::instrument]
     async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Self::Error> {
         let path = identifier.as_str();
 
@@ -130,6 +135,7 @@ impl Store for ObjectStore {
         Ok(length as u64)
     }
 
+    #[tracing::instrument]
     async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error> {
         let path = identifier.as_str();
 
@@ -155,9 +161,15 @@ impl ObjectStore {
         Ok(ObjectStore {
             path_gen,
             settings_tree,
-            bucket: Bucket::new(
+            bucket: Bucket::new_with_path_style(
                 bucket_name,
-                region,
+                match region {
+                    Region::Custom { endpoint, .. } => Region::Custom {
+                        region: String::from(""),
+                        endpoint,
+                    },
+                    region => region,
+                },
                 Credentials {
                     access_key,
                     secret_key,
@@ -181,7 +193,7 @@ impl ObjectStore {
         let filename = Uuid::new_v4().to_string();
         let path = self.next_directory()?.to_strings().join("/");
 
-        Ok(format!("/{}/{}", path, filename))
+        Ok(format!("{}/{}", path, filename))
     }
 }