Add endpoint for downloading remote images

2024-12-22 03:11:24 +00:00 · 2020-06-07 10:59:58 -05:00 · 2020-06-07 10:59:58 -05:00 · fc1ae4be49
commit fc1ae4be49
parent 1b526f4c1f
6 changed files with 216 additions and 25 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -30,8 +30,11 @@ dependencies = [
 "futures-util",
 "http",
 "log",
+ "rustls",
+ "tokio-rustls",
 "trust-dns-proto",
 "trust-dns-resolver",
+ "webpki",
 ]

 [[package]]
@ -73,8 +76,9 @@ dependencies = [
 "actix-rt",
 "actix-service",
 "actix-threadpool",
+ "actix-tls",
 "actix-utils",
- "base64",
+ "base64 0.12.1",
 "bitflags",
 "brotli2",
 "bytes",
@ -236,6 +240,10 @@ dependencies = [
 "either",
 "futures",
 "log",
+ "rustls",
+ "tokio-rustls",
+ "webpki",
+ "webpki-roots",
 ]

 [[package]]
@ -286,6 +294,7 @@ dependencies = [
 "mime",
 "pin-project",
 "regex",
+ "rustls",
 "serde",
 "serde_json",
 "serde_urlencoded",
@ -389,7 +398,7 @@ dependencies = [
 "actix-http",
 "actix-rt",
 "actix-service",
- "base64",
+ "base64 0.12.1",
 "bytes",
 "derive_more",
 "futures-core",
@ -397,6 +406,7 @@ dependencies = [
 "mime",
 "percent-encoding",
 "rand",
+ "rustls",
 "serde",
 "serde_json",
 "serde_urlencoded",
@ -421,6 +431,12 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1"

+[[package]]
+name = "base64"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
+
 [[package]]
 name = "base64"
 version = "0.12.1"
@ -1047,6 +1063,15 @@ dependencies = [
 "rayon",
 ]

+[[package]]
+name = "js-sys"
+version = "0.3.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177"
+dependencies = [
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "kernel32-sys"
 version = "0.2.2"
@ -1330,6 +1355,7 @@ dependencies = [
 "log",
 "mime",
 "rand",
+ "serde",
 "serde_json",
 "sha2",
 "sled",
@ -1548,6 +1574,21 @@ dependencies = [
 "quick-error",
 ]

+[[package]]
+name = "ring"
+version = "0.16.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06b3fefa4f12272808f809a0af618501fdaba41a58963c5fb72238ab0be09603"
+dependencies = [
+ "cc",
+ "libc",
+ "once_cell",
+ "spin",
+ "untrusted",
+ "web-sys",
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.16"
@ -1563,6 +1604,19 @@ dependencies = [
 "semver",
 ]

+[[package]]
+name = "rustls"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1"
+dependencies = [
+ "base64 0.11.0",
+ "log",
+ "ring",
+ "sct",
+ "webpki",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.5"
@ -1581,6 +1635,16 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

+[[package]]
+name = "sct"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "semver"
 version = "0.9.0"
@ -1720,6 +1784,12 @@ dependencies = [
 "winapi 0.3.8",
 ]

+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
 [[package]]
 name = "standback"
 version = "0.2.9"
@ -1962,6 +2032,18 @@ dependencies = [
 "winapi 0.3.8",
 ]

+[[package]]
+name = "tokio-rustls"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4"
+dependencies = [
+ "futures-core",
+ "rustls",
+ "tokio",
+ "webpki",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.2.0"
@ -2088,6 +2170,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"

+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
 [[package]]
 name = "url"
 version = "2.1.1"
@ -2171,6 +2259,35 @@ version = "0.2.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd"

+[[package]]
+name = "web-sys"
+version = "0.3.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "webpki"
+version = "0.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8eff4b7516a57307f9349c64bf34caa34b940b66fed4b2fb3136cb7386e5739"
+dependencies = [
+ "webpki",
+]
+
 [[package]]
 name = "widestring"
 version = "0.4.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -14,7 +14,7 @@ edition = "2018"
 actix-form-data = { git = "https://git.asonix.dog/Aardwolf/actix-form-data" }
 actix-fs = { git = "https://git.asonix.dog/asonix/actix-fs" }
 actix-rt = "1.1.1"
-actix-web = "3.0.0-alpha.2"
+actix-web = { version = "3.0.0-alpha.2", features = ["rustls"] }
 anyhow = "1.0"
 bytes = "0.5"
 env_logger = "0.7"
@ -23,6 +23,7 @@ image = "0.23.4"
 log = "0.4"
 mime = "0.3.1"
 rand = "0.7.3"
+serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sha2 = "0.8.2"
 sled = "0.32.0-rc1"
--- a/README.md
+++ b/README.md
@ -55,6 +55,8 @@ pict-rs offers four endpoints:
        "msg": "ok"
    }
    ```
+- `GET /image/download?url=...` Download an image from a remote server, returning the same JSON
+    payload as the `POST` endpoint
 - `GET /image/{file}` for getting a full-resolution image. `file` here is the `file` key from the
    `/image` endpoint's JSON
 - `GET /image/{size}/{file}` where `size` is a positive integer. This endpoint is for accessing
--- a/src/error.rs
+++ b/src/error.rs
@ -43,6 +43,24 @@ pub enum UploadError {

    #[error("Uploaded content could not be validated as an image")]
    InvalidImage(image::error::ImageError),
+
+    #[error("Unsupported image format")]
+    UnsupportedFormat,
+
+    #[error("Unable to download image, bad response {0}")]
+    Download(actix_web::http::StatusCode),
+
+    #[error("Unable to download image, {0}")]
+    Payload(#[from] actix_web::client::PayloadError),
+
+    #[error("Unable to send request, {0}")]
+    SendRequest(String),
+}
+
+impl From<actix_web::client::SendRequestError> for UploadError {
+    fn from(e: actix_web::client::SendRequestError) -> Self {
+        UploadError::SendRequest(e.to_string())
+    }
 }

 impl From<sled::transaction::TransactionError<UploadError>> for UploadError {
--- a/src/main.rs
+++ b/src/main.rs
@ -1,5 +1,6 @@
 use actix_form_data::{Field, Form, Value};
 use actix_web::{
+    client::Client,
    guard,
    http::header::{CacheControl, CacheDirective},
    middleware::{Compress, Logger},
@ -100,12 +101,43 @@ async fn upload(
            let delete_token = manager.delete_token(saved_as.to_owned()).await?;
            files.push(serde_json::json!({
                "file": saved_as,
-                "delete_token": delete_token,
+                "delete_token": delete_token
            }));
        }
    }

-    Ok(HttpResponse::Created().json(serde_json::json!({ "msg": "ok", "files": files })))
+    Ok(HttpResponse::Created().json(serde_json::json!({
+        "msg": "ok",
+        "files": files
+    })))
+}
+
+/// download an image from a URL
+async fn download(
+    client: web::Data<Client>,
+    manager: web::Data<UploadManager>,
+    query: web::Query<UrlQuery>,
+) -> Result<HttpResponse, UploadError> {
+    let mut res = client.get(&query.url).send().await?;
+
+    if !res.status().is_success() {
+        return Err(UploadError::Download(res.status()));
+    }
+
+    let fut = res.body().limit(40 * MEGABYTES);
+
+    let stream = Box::pin(futures::stream::once(fut));
+
+    let alias = manager.upload(stream).await?;
+    let delete_token = manager.delete_token(alias.clone()).await?;
+
+    Ok(HttpResponse::Created().json(serde_json::json!({
+        "msg": "ok",
+        "files": [{
+            "file": alias,
+            "delete_token": delete_token
+        }]
+    })))
 }

 async fn delete(
@ -234,6 +266,11 @@ where
        .streaming(stream.err_into())
 }

+#[derive(Debug, serde::Deserialize)]
+struct UrlQuery {
+    url: String,
+}
+
 #[actix_rt::main]
 async fn main() -> Result<(), anyhow::Error> {
    let config = Config::from_args();
@ -250,18 +287,29 @@ async fn main() -> Result<(), anyhow::Error> {
        .max_file_size(40 * MEGABYTES)
        .field(
            "images",
-            Field::array(Field::file(move |filename, content_type, stream| {
+            Field::array(Field::file(move |_, _, stream| {
                let manager = manager2.clone();

-                async move { manager.upload(filename, content_type, stream).await }
+                async move {
+                    manager.upload(stream).await.map(|alias| {
+                        let mut path = PathBuf::new();
+                        path.push(alias);
+                        Some(path)
+                    })
+                }
            })),
        );

    HttpServer::new(move || {
+        let client = Client::build()
+            .header("User-Agent", "pict-rs v0.1.0-master")
+            .finish();
+
        App::new()
            .wrap(Logger::default())
            .wrap(Compress::default())
            .data(manager.clone())
+            .data(client)
            .service(
                web::scope("/image")
                    .service(
@ -270,6 +318,7 @@ async fn main() -> Result<(), anyhow::Error> {
                            .wrap(form.clone())
                            .route(web::post().to(upload)),
                    )
+                    .service(web::resource("/download").route(web::get().to(download)))
                    .service(web::resource("/{filename}").route(web::get().to(serve)))
                    .service(
                        web::resource("/delete/{delete_token}/{filename}")
--- a/src/upload_manager.rs
+++ b/src/upload_manager.rs
@ -18,7 +18,7 @@ struct UploadManagerInner {
    db: sled::Db,
 }

-type UploadStream = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, actix_form_data::Error>>>>;
+type UploadStream<E> = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, E>>>>;

 enum Dup {
    Exists,
@ -176,16 +176,10 @@ impl UploadManager {
    }

    /// Upload the file, discarding bytes if it's already present, or saving if it's new
-    pub(crate) async fn upload(
-        &self,
-        _filename: String,
-        content_type: mime::Mime,
-        mut stream: UploadStream,
-    ) -> Result<Option<PathBuf>, UploadError> {
-        if ACCEPTED_MIMES.iter().all(|valid| *valid != content_type) {
-            return Err(UploadError::ContentType(content_type));
-        }
-
+    pub(crate) async fn upload<E>(&self, mut stream: UploadStream<E>) -> Result<String, UploadError>
+    where
+        UploadError: From<E>,
+    {
        let (img, format) = {
            // -- READ IN BYTES FROM CLIENT --
            let mut bytes = bytes::BytesMut::new();
@ -208,7 +202,11 @@ impl UploadManager {
            .format
            .as_ref()
            .map(|f| (f.to_image_format(), f.to_mime()))
-            .unwrap_or((format, content_type));
+            .unwrap_or((format.clone(), valid_format(format)?));
+
+        if ACCEPTED_MIMES.iter().all(|valid| *valid != content_type) {
+            return Err(UploadError::ContentType(content_type));
+        }

        let bytes: bytes::Bytes = {
            let mut bytes = std::io::Cursor::new(vec![]);
@ -226,9 +224,7 @@ impl UploadManager {

        // bail early with alias to existing file if this is a duplicate
        if dup.exists() {
-            let mut path = PathBuf::new();
-            path.push(alias);
-            return Ok(Some(path));
+            return Ok(alias);
        }

        // -- WRITE NEW FILE --
@ -238,9 +234,7 @@ impl UploadManager {
        safe_save_file(real_path, bytes).await?;

        // Return alias to file
-        let mut path = PathBuf::new();
-        path.push(alias);
-        Ok(Some(path))
+        Ok(alias)
    }

    pub(crate) async fn from_alias(&self, alias: String) -> Result<String, UploadError> {
@ -440,3 +434,13 @@ fn alias_id_key(alias: &str) -> String {
 fn delete_key(alias: &str) -> String {
    format!("{}/delete", alias)
 }
+
+fn valid_format(format: image::ImageFormat) -> Result<mime::Mime, UploadError> {
+    match format {
+        image::ImageFormat::Jpeg => Ok(mime::IMAGE_JPEG),
+        image::ImageFormat::Png => Ok(mime::IMAGE_PNG),
+        image::ImageFormat::Gif => Ok(mime::IMAGE_GIF),
+        image::ImageFormat::Bmp => Ok(mime::IMAGE_BMP),
+        _ => Err(UploadError::UnsupportedFormat),
+    }
+}