Add endpoint for downloading remote images

Cargo.lock

@@ -30,8 +30,11 @@ dependencies = [
  "futures-util",
  "http",
  "log",
+ "rustls",
+ "tokio-rustls",
  "trust-dns-proto",
  "trust-dns-resolver",
+ "webpki",
 ]
 
 [[package]]
@@ -73,8 +76,9 @@ dependencies = [
  "actix-rt",
  "actix-service",
  "actix-threadpool",
+ "actix-tls",
  "actix-utils",
- "base64",
+ "base64 0.12.1",
  "bitflags",
  "brotli2",
  "bytes",
@@ -236,6 +240,10 @@ dependencies = [
  "either",
  "futures",
  "log",
+ "rustls",
+ "tokio-rustls",
+ "webpki",
+ "webpki-roots",
 ]
 
 [[package]]
@@ -286,6 +294,7 @@ dependencies = [
  "mime",
  "pin-project",
  "regex",
+ "rustls",
  "serde",
  "serde_json",
  "serde_urlencoded",
@@ -389,7 +398,7 @@ dependencies = [
  "actix-http",
  "actix-rt",
  "actix-service",
- "base64",
+ "base64 0.12.1",
  "bytes",
  "derive_more",
  "futures-core",
@@ -397,6 +406,7 @@ dependencies = [
  "mime",
  "percent-encoding",
  "rand",
+ "rustls",
  "serde",
  "serde_json",
  "serde_urlencoded",
@@ -421,6 +431,12 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1"
 
+[[package]]
+name = "base64"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
+
 [[package]]
 name = "base64"
 version = "0.12.1"
@@ -1047,6 +1063,15 @@ dependencies = [
  "rayon",
 ]
 
+[[package]]
+name = "js-sys"
+version = "0.3.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177"
+dependencies = [
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "kernel32-sys"
 version = "0.2.2"
@@ -1330,6 +1355,7 @@ dependencies = [
  "log",
  "mime",
  "rand",
+ "serde",
  "serde_json",
  "sha2",
  "sled",
@@ -1548,6 +1574,21 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "ring"
+version = "0.16.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06b3fefa4f12272808f809a0af618501fdaba41a58963c5fb72238ab0be09603"
+dependencies = [
+ "cc",
+ "libc",
+ "once_cell",
+ "spin",
+ "untrusted",
+ "web-sys",
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.16"
@@ -1563,6 +1604,19 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1"
+dependencies = [
+ "base64 0.11.0",
+ "log",
+ "ring",
+ "sct",
+ "webpki",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.5"
@@ -1581,6 +1635,16 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "sct"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "semver"
 version = "0.9.0"
@@ -1720,6 +1784,12 @@ dependencies = [
  "winapi 0.3.8",
 ]
 
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
 [[package]]
 name = "standback"
 version = "0.2.9"
@@ -1962,6 +2032,18 @@ dependencies = [
  "winapi 0.3.8",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4"
+dependencies = [
+ "futures-core",
+ "rustls",
+ "tokio",
+ "webpki",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.2.0"
@@ -2088,6 +2170,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
 
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
 [[package]]
 name = "url"
 version = "2.1.1"
@@ -2171,6 +2259,35 @@ version = "0.2.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd"
 
+[[package]]
+name = "web-sys"
+version = "0.3.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "webpki"
+version = "0.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8eff4b7516a57307f9349c64bf34caa34b940b66fed4b2fb3136cb7386e5739"
+dependencies = [
+ "webpki",
+]
+
 [[package]]
 name = "widestring"
 version = "0.4.0"

Cargo.toml

@@ -14,7 +14,7 @@ edition = "2018"
 actix-form-data = { git = "https://git.asonix.dog/Aardwolf/actix-form-data" }
 actix-fs = { git = "https://git.asonix.dog/asonix/actix-fs" }
 actix-rt = "1.1.1"
-actix-web = "3.0.0-alpha.2"
+actix-web = { version = "3.0.0-alpha.2", features = ["rustls"] }
 anyhow = "1.0"
 bytes = "0.5"
 env_logger = "0.7"
@@ -23,6 +23,7 @@ image = "0.23.4"
 log = "0.4"
 mime = "0.3.1"
 rand = "0.7.3"
+serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sha2 = "0.8.2"
 sled = "0.32.0-rc1"

README.md

@@ -55,6 +55,8 @@ pict-rs offers four endpoints:
         "msg": "ok"
     }
     ```
+- `GET /image/download?url=...` Download an image from a remote server, returning the same JSON
+    payload as the `POST` endpoint
 - `GET /image/{file}` for getting a full-resolution image. `file` here is the `file` key from the
     `/image` endpoint's JSON
 - `GET /image/{size}/{file}` where `size` is a positive integer. This endpoint is for accessing

src/error.rs

@@ -43,6 +43,24 @@ pub enum UploadError {
 
     #[error("Uploaded content could not be validated as an image")]
     InvalidImage(image::error::ImageError),
+
+    #[error("Unsupported image format")]
+    UnsupportedFormat,
+
+    #[error("Unable to download image, bad response {0}")]
+    Download(actix_web::http::StatusCode),
+
+    #[error("Unable to download image, {0}")]
+    Payload(#[from] actix_web::client::PayloadError),
+
+    #[error("Unable to send request, {0}")]
+    SendRequest(String),
+}
+
+impl From<actix_web::client::SendRequestError> for UploadError {
+    fn from(e: actix_web::client::SendRequestError) -> Self {
+        UploadError::SendRequest(e.to_string())
+    }
 }
 
 impl From<sled::transaction::TransactionError<UploadError>> for UploadError {

src/main.rs

@@ -1,5 +1,6 @@
 use actix_form_data::{Field, Form, Value};
 use actix_web::{
+    client::Client,
     guard,
     http::header::{CacheControl, CacheDirective},
     middleware::{Compress, Logger},
@@ -100,12 +101,43 @@ async fn upload(
             let delete_token = manager.delete_token(saved_as.to_owned()).await?;
             files.push(serde_json::json!({
                 "file": saved_as,
-                "delete_token": delete_token,
+                "delete_token": delete_token
             }));
         }
     }
 
-    Ok(HttpResponse::Created().json(serde_json::json!({ "msg": "ok", "files": files })))
+    Ok(HttpResponse::Created().json(serde_json::json!({
+        "msg": "ok",
+        "files": files
+    })))
+}
+
+/// download an image from a URL
+async fn download(
+    client: web::Data<Client>,
+    manager: web::Data<UploadManager>,
+    query: web::Query<UrlQuery>,
+) -> Result<HttpResponse, UploadError> {
+    let mut res = client.get(&query.url).send().await?;
+
+    if !res.status().is_success() {
+        return Err(UploadError::Download(res.status()));
+    }
+
+    let fut = res.body().limit(40 * MEGABYTES);
+
+    let stream = Box::pin(futures::stream::once(fut));
+
+    let alias = manager.upload(stream).await?;
+    let delete_token = manager.delete_token(alias.clone()).await?;
+
+    Ok(HttpResponse::Created().json(serde_json::json!({
+        "msg": "ok",
+        "files": [{
+            "file": alias,
+            "delete_token": delete_token
+        }]
+    })))
 }
 
 async fn delete(
@@ -234,6 +266,11 @@ where
         .streaming(stream.err_into())
 }
 
+#[derive(Debug, serde::Deserialize)]
+struct UrlQuery {
+    url: String,
+}
+
 #[actix_rt::main]
 async fn main() -> Result<(), anyhow::Error> {
     let config = Config::from_args();
@@ -250,18 +287,29 @@ async fn main() -> Result<(), anyhow::Error> {
         .max_file_size(40 * MEGABYTES)
         .field(
             "images",
-            Field::array(Field::file(move |filename, content_type, stream| {
+            Field::array(Field::file(move |_, _, stream| {
                 let manager = manager2.clone();
 
-                async move { manager.upload(filename, content_type, stream).await }
+                async move {
+                    manager.upload(stream).await.map(|alias| {
+                        let mut path = PathBuf::new();
+                        path.push(alias);
+                        Some(path)
+                    })
+                }
             })),
         );
 
     HttpServer::new(move || {
+        let client = Client::build()
+            .header("User-Agent", "pict-rs v0.1.0-master")
+            .finish();
+
         App::new()
             .wrap(Logger::default())
             .wrap(Compress::default())
             .data(manager.clone())
+            .data(client)
             .service(
                 web::scope("/image")
                     .service(
@@ -270,6 +318,7 @@ async fn main() -> Result<(), anyhow::Error> {
                             .wrap(form.clone())
                             .route(web::post().to(upload)),
                     )
+                    .service(web::resource("/download").route(web::get().to(download)))
                     .service(web::resource("/{filename}").route(web::get().to(serve)))
                     .service(
                         web::resource("/delete/{delete_token}/{filename}")

src/upload_manager.rs

@@ -18,7 +18,7 @@ struct UploadManagerInner {
     db: sled::Db,
 }
 
-type UploadStream = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, actix_form_data::Error>>>>;
+type UploadStream<E> = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, E>>>>;
 
 enum Dup {
     Exists,
@@ -176,16 +176,10 @@ impl UploadManager {
     }
 
     /// Upload the file, discarding bytes if it's already present, or saving if it's new
-    pub(crate) async fn upload(
+    pub(crate) async fn upload<E>(&self, mut stream: UploadStream<E>) -> Result<String, UploadError>
-        &self,
+    where
-        _filename: String,
+        UploadError: From<E>,
-        content_type: mime::Mime,
+    {
-        mut stream: UploadStream,
-    ) -> Result<Option<PathBuf>, UploadError> {
-        if ACCEPTED_MIMES.iter().all(|valid| *valid != content_type) {
-            return Err(UploadError::ContentType(content_type));
-        }
-
         let (img, format) = {
             // -- READ IN BYTES FROM CLIENT --
             let mut bytes = bytes::BytesMut::new();
@@ -208,7 +202,11 @@ impl UploadManager {
             .format
             .as_ref()
             .map(|f| (f.to_image_format(), f.to_mime()))
-            .unwrap_or((format, content_type));
+            .unwrap_or((format.clone(), valid_format(format)?));
+
+        if ACCEPTED_MIMES.iter().all(|valid| *valid != content_type) {
+            return Err(UploadError::ContentType(content_type));
+        }
 
         let bytes: bytes::Bytes = {
             let mut bytes = std::io::Cursor::new(vec![]);
@@ -226,9 +224,7 @@ impl UploadManager {
 
         // bail early with alias to existing file if this is a duplicate
         if dup.exists() {
-            let mut path = PathBuf::new();
+            return Ok(alias);
-            path.push(alias);
-            return Ok(Some(path));
         }
 
         // -- WRITE NEW FILE --
@@ -238,9 +234,7 @@ impl UploadManager {
         safe_save_file(real_path, bytes).await?;
 
         // Return alias to file
-        let mut path = PathBuf::new();
+        Ok(alias)
-        path.push(alias);
-        Ok(Some(path))
     }
 
     pub(crate) async fn from_alias(&self, alias: String) -> Result<String, UploadError> {
@@ -440,3 +434,13 @@ fn alias_id_key(alias: &str) -> String {
 fn delete_key(alias: &str) -> String {
     format!("{}/delete", alias)
 }
+
+fn valid_format(format: image::ImageFormat) -> Result<mime::Mime, UploadError> {
+    match format {
+        image::ImageFormat::Jpeg => Ok(mime::IMAGE_JPEG),
+        image::ImageFormat::Png => Ok(mime::IMAGE_PNG),
+        image::ImageFormat::Gif => Ok(mime::IMAGE_GIF),
+        image::ImageFormat::Bmp => Ok(mime::IMAGE_BMP),
+        _ => Err(UploadError::UnsupportedFormat),
+    }
+}