From c0a20588d32a31cf310bcfaa250ecd5abf9c4497 Mon Sep 17 00:00:00 2001 From: asonix Date: Thu, 10 Dec 2020 12:49:10 -0600 Subject: [PATCH] Add content type, width, height to Details, add details endpoints --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 19 +++++- src/main.rs | 154 ++++++++++++++++++++++++++++-------------- src/upload_manager.rs | 108 +++++++++++++++++++++++++++-- 5 files changed, 229 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e7a6f2a..20f8ef8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1449,7 +1449,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pict-rs" -version = "0.3.0-alpha.0" +version = "0.3.0-alpha.1" dependencies = [ "actix-form-data", "actix-fs", diff --git a/Cargo.toml b/Cargo.toml index b3a0fca..2217d51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pict-rs" description = "A simple image hosting service" -version = "0.3.0-alpha.0" +version = "0.3.0-alpha.1" authors = ["asonix "] license = "AGPL-3.0" readme = "README.md" diff --git a/README.md b/README.md index ade539e..ad14e6c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ _a simple image hosting service_ ## Usage ### Running ``` -pict-rs 0.3.0-alpha.0 +pict-rs 0.3.0-alpha.1 USAGE: pict-rs [FLAGS] [OPTIONS] --path @@ -105,6 +105,21 @@ pict-rs offers the following endpoints: payload as the `POST` endpoint - `GET /image/original/{file}` for getting a full-resolution image. `file` here is the `file` key from the `/image` endpoint's JSON +- `GET /image/details/original/{file}` for getting the details of a full-resolution image. + The returned JSON is structured like so: + ```json + { + "width": 800, + "height": 537, + "content_type": "image/webp", + "created_at": [ + 2020, + 345, + 67376, + 394363487 + ] + } + ``` - `GET /image/process.{ext}?src={file}&...` get a file with transformations applied. existing transformations include - `identity=true`: apply no changes @@ -121,6 +136,8 @@ pict-rs offers the following endpoints: GET /image/process.jpg?src=asdf.png&thumbnail=256&blur=3.0 ``` which would create a 256x256px JPEG thumbnail and blur it +- `GET /image/details/process.{ext}?src={file}&...` for getting the details of a processed image. + The returned JSON is the same format as listed for the full-resolution details endpoint. - `DELETE /image/delete/{delete_token}/{file}` or `GET /image/delete/{delete_token}/{file}` to delete a file, where `delete_token` and `file` are from the `/image` endpoint's JSON diff --git a/src/main.rs b/src/main.rs index 5fbf2dd..1e5d472 100644 --- a/src/main.rs +++ b/src/main.rs @@ -136,30 +136,6 @@ fn to_ext(mime: mime::Mime) -> Result<&'static str, UploadError> { } } -fn from_name(name: &str) -> Result { - match name - .rsplit('.') - .next() - .ok_or(UploadError::UnsupportedFormat)? - { - "jpg" => Ok(mime::IMAGE_JPEG), - "webp" => Ok(image_webp()), - "png" => Ok(mime::IMAGE_PNG), - "mp4" => Ok(video_mp4()), - "gif" => Ok(mime::IMAGE_GIF), - _ => Err(UploadError::UnsupportedFormat), - } -} - -fn from_ext(ext: &str) -> Result { - match ext { - "jpg" => Ok(mime::IMAGE_JPEG), - "png" => Ok(mime::IMAGE_PNG), - "webp" => Ok(image_webp()), - _ => Err(UploadError::UnsupportedFormat), - } -} - /// Handle responding to succesful uploads #[instrument(skip(value, manager))] async fn upload( @@ -244,14 +220,12 @@ async fn delete( type ProcessQuery = Vec<(String, String)>; -/// Process files -#[instrument(skip(manager, whitelist))] -async fn process( +async fn prepare_process( query: web::Query, - ext: web::Path, - manager: web::Data, - whitelist: web::Data>>, -) -> Result { + ext: &str, + manager: &UploadManager, + whitelist: &Option>, +) -> Result<(processor::ProcessChain, Format, String, PathBuf), UploadError> { let (alias, operations) = query .into_inner() @@ -282,8 +256,6 @@ async fn process( let chain = self::processor::build_chain(&operations); - let ext = ext.into_inner(); - let content_type = from_ext(&ext)?; let format = ext .parse::() .map_err(|_| UploadError::UnsupportedFormat)?; @@ -291,6 +263,36 @@ async fn process( let base = manager.image_dir(); let thumbnail_path = self::processor::build_path(base, &chain, processed_name); + Ok((chain, format, name, thumbnail_path)) +} + +async fn process_details( + query: web::Query, + ext: web::Path, + manager: web::Data, + whitelist: web::Data>>, +) -> Result { + let (_, _, name, thumbnail_path) = + prepare_process(query, ext.as_str(), &manager, &whitelist).await?; + + let details = manager.variant_details(thumbnail_path, name).await?; + + let details = details.ok_or(UploadError::NoFiles)?; + + Ok(HttpResponse::Ok().json(details)) +} + +/// Process files +#[instrument(skip(manager, whitelist))] +async fn process( + query: web::Query, + ext: web::Path, + manager: web::Data, + whitelist: web::Data>>, +) -> Result { + let (chain, format, name, thumbnail_path) = + prepare_process(query, ext.as_str(), &manager, &whitelist).await?; + // If the thumbnail doesn't exist, we need to create it let thumbnail_exists = if let Err(e) = actix_fs::metadata(thumbnail_path.clone()).await { if e.kind() != Some(std::io::ErrorKind::NotFound) { @@ -339,16 +341,27 @@ async fn process( let path2 = thumbnail_path.clone(); let img_bytes2 = img_bytes.clone(); + let store_details = details.is_none(); + let details = if let Some(details) = details { + details + } else { + let details = Details::from_bytes(&img_bytes)?; + manager + .store_variant_details(path2.clone(), name.clone(), &details) + .await?; + details + }; + // Save the file in another task, we want to return the thumbnail now debug!("Spawning storage task"); let span = Span::current(); - let store_details = details.is_none(); + let details2 = details.clone(); actix_rt::spawn(async move { let entered = span.enter(); if store_details { debug!("Storing details"); if let Err(e) = manager - .store_variant_details(path2.clone(), name.clone()) + .store_variant_details(path2.clone(), name.clone(), &details2) .await { error!("Error storing details, {}", e); @@ -366,30 +379,60 @@ async fn process( drop(entered); }); - let details = details.unwrap_or(Details::now()); - return Ok(srv_response( Box::pin(futures::stream::once(async { Ok(img_bytes) as Result<_, UploadError> })), - content_type, + details.content_type(), 7 * DAYS, details.system_time(), )); } - let stream = actix_fs::read_to_stream(thumbnail_path).await?; + let details = if let Some(details) = details { + details + } else { + let details = Details::from_path(thumbnail_path.clone()).await?; + manager + .store_variant_details(thumbnail_path.clone(), name, &details) + .await?; + details + }; - let details = details.unwrap_or(Details::now()); + let stream = actix_fs::read_to_stream(thumbnail_path).await?; Ok(srv_response( stream, - content_type, + details.content_type(), 7 * DAYS, details.system_time(), )) } +/// Fetch file details +async fn details( + alias: web::Path, + manager: web::Data, +) -> Result { + let name = manager.from_alias(alias.into_inner()).await?; + let mut path = manager.image_dir(); + path.push(name.clone()); + + let details = manager.variant_details(path.clone(), name.clone()).await?; + + let details = if let Some(details) = details { + details + } else { + let new_details = Details::from_path(path.clone()).await?; + manager + .store_variant_details(path.clone(), name, &new_details) + .await?; + new_details + }; + + Ok(HttpResponse::Ok().json(details)) +} + /// Serve files #[instrument(skip(manager))] async fn serve( @@ -397,23 +440,26 @@ async fn serve( manager: web::Data, ) -> Result { let name = manager.from_alias(alias.into_inner()).await?; - let content_type = from_name(&name)?; let mut path = manager.image_dir(); path.push(name.clone()); let details = manager.variant_details(path.clone(), name.clone()).await?; - if details.is_none() { - manager.store_variant_details(path.clone(), name).await?; - } - - let details = details.unwrap_or(Details::now()); + let details = if let Some(details) = details { + details + } else { + let details = Details::from_path(path.clone()).await?; + manager + .store_variant_details(path.clone(), name, &details) + .await?; + details + }; let stream = actix_fs::read_to_stream(path).await?; Ok(srv_response( stream, - content_type, + details.content_type(), 7 * DAYS, details.system_time(), )) @@ -604,7 +650,17 @@ async fn main() -> Result<(), anyhow::Error> { .route(web::get().to(delete)), ) .service(web::resource("/original/{filename}").route(web::get().to(serve))) - .service(web::resource("/process.{ext}").route(web::get().to(process))), + .service(web::resource("/process.{ext}").route(web::get().to(process))) + .service( + web::scope("/details") + .service( + web::resource("/original/{filename}").route(web::get().to(details)), + ) + .service( + web::resource("/process.{ext}") + .route(web::get().to(process_details)), + ), + ), ) .service( web::scope("/internal") diff --git a/src/upload_manager.rs b/src/upload_manager.rs index 086dd8a..3a7e398 100644 --- a/src/upload_manager.rs +++ b/src/upload_manager.rs @@ -46,18 +46,114 @@ impl std::fmt::Debug for UploadManager { type UploadStream = Pin>>>; -#[derive(serde::Deserialize, serde::Serialize)] +#[derive(Clone)] +pub(crate) struct Serde { + inner: T, +} + +impl Serde { + pub(crate) fn new(inner: T) -> Self { + Serde { inner } + } +} + +mod my_serde { + impl serde::Serialize for super::Serde + where + T: std::fmt::Display, + { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let s = self.inner.to_string(); + serde::Serialize::serialize(s.as_str(), serializer) + } + } + + impl<'de, T> serde::Deserialize<'de> for super::Serde + where + T: std::str::FromStr, + ::Err: std::fmt::Display, + { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s: String = serde::Deserialize::deserialize(deserializer)?; + let inner = s + .parse::() + .map_err(|e| serde::de::Error::custom(e.to_string()))?; + + Ok(super::Serde { inner }) + } + } +} + +#[derive(Clone, serde::Deserialize, serde::Serialize)] pub(crate) struct Details { + width: usize, + height: usize, + content_type: Serde, created_at: time::OffsetDateTime, } +fn mime_from_media_type(media_type: rexiv2::MediaType) -> mime::Mime { + match media_type { + rexiv2::MediaType::Jpeg => mime::IMAGE_JPEG, + rexiv2::MediaType::Png => mime::IMAGE_PNG, + rexiv2::MediaType::Gif => mime::IMAGE_GIF, + rexiv2::MediaType::Other(s) if s == "image/webp" => s.parse::().unwrap(), + rexiv2::MediaType::Other(s) if s == "video/mp4" || s == "video/quicktime" => { + "video/mp4".parse::().unwrap() + } + _ => mime::APPLICATION_OCTET_STREAM, + } +} + impl Details { - pub(crate) fn now() -> Self { + pub(crate) fn from_bytes(bytes: &[u8]) -> Result { + let metadata = rexiv2::Metadata::new_from_buffer(bytes)?; + let mime_type = mime_from_media_type(metadata.get_media_type()?); + let width = metadata.get_pixel_width(); + let height = metadata.get_pixel_height(); + let details = Details::now(width as usize, height as usize, mime_type); + Ok(details) + } + + pub(crate) async fn from_path(path: PathBuf) -> Result { + let (mime_type, width, height) = web::block(move || { + rexiv2::Metadata::new_from_path(&path).and_then(|metadata| { + metadata + .get_media_type() + .map(mime_from_media_type) + .map(|mime_type| { + ( + mime_type, + metadata.get_pixel_width(), + metadata.get_pixel_height(), + ) + }) + }) + }) + .await?; + + Ok(Details::now(width as usize, height as usize, mime_type)) + } + + fn now(width: usize, height: usize, content_type: mime::Mime) -> Self { Details { + width, + height, + content_type: Serde::new(content_type), created_at: time::OffsetDateTime::now_utc(), } } + pub(crate) fn content_type(&self) -> mime::Mime { + self.content_type.inner.clone() + } + pub(crate) fn system_time(&self) -> std::time::SystemTime { self.created_at.into() } @@ -184,7 +280,10 @@ impl UploadManager { let main_tree = self.inner.main_tree.clone(); debug!("Getting details"); let opt = match web::block(move || main_tree.get(key)).await? { - Some(ivec) => Some(serde_json::from_slice(&ivec)?), + Some(ivec) => match serde_json::from_slice(&ivec) { + Ok(details) => Some(details), + Err(_) => None, + }, None => None, }; debug!("Got details"); @@ -196,6 +295,7 @@ impl UploadManager { &self, path: PathBuf, filename: String, + details: &Details, ) -> Result<(), UploadError> { let path_string = path.to_str().ok_or(UploadError::Path)?.to_string(); @@ -207,7 +307,7 @@ impl UploadManager { let key = variant_details_key(&hash, &path_string); let main_tree = self.inner.main_tree.clone(); - let details_value = serde_json::to_string(&Details::now())?; + let details_value = serde_json::to_string(details)?; debug!("Storing details"); web::block(move || main_tree.insert(key, details_value.as_bytes())).await?; debug!("Stored details");